From e5e7d5f250701de0ebfc7f64f4014615564eb703 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 3 Apr 2023 16:09:50 +0200 Subject: [PATCH 01/77] add rules for perfect foresight --- Snakefile | 5 + rules/solve_perfect.smk | 223 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 rules/solve_perfect.smk diff --git a/Snakefile b/Snakefile index 621e4e9d..d067a339 100644 --- a/Snakefile +++ b/Snakefile @@ -65,6 +65,11 @@ if config["foresight"] == "myopic": include: "rules/solve_myopic.smk" +if config["foresight"] == "perfect": + + include: "rules/solve_perfect.smk" + + rule purge: message: "Purging generated resources, results and docs. Downloads are kept." diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk new file mode 100644 index 00000000..9ced1c76 --- /dev/null +++ b/rules/solve_perfect.smk @@ -0,0 +1,223 @@ +# SPDX-FileCopyrightText: : 2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +rule add_existing_baseyear: + input: + overrides="data/override_component_attrs", + network=RESULTS + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + powerplants=RESOURCES + "powerplants.csv", + busmap_s=RESOURCES + "busmap_elec_s{simpl}.csv", + busmap=RESOURCES + "busmap_elec_s{simpl}_{clusters}.csv", + clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + costs="data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), + cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", + cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", + existing_heating="data/existing_infrastructure/existing_heating_raw.csv", + existing_solar="data/existing_infrastructure/solar_capacity_IRENA.csv", + existing_onwind="data/existing_infrastructure/onwind_capacity_IRENA.csv", + existing_offwind="data/existing_infrastructure/offwind_capacity_IRENA.csv", + output: + RESULTS + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + wildcard_constraints: + planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear + threads: 1 + resources: + mem_mb=2000, + log: + LOGS + + "add_existing_baseyear_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + benchmark: + ( + BENCHMARKS + + "add_existing_baseyear/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ) + conda: + "../envs/environment.yaml" + script: + "../scripts/add_existing_baseyear.py" + + +rule add_brownfield: + input: + overrides="data/override_component_attrs", + network=RESULTS + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + network_p=solved_previous_horizon, #solved network at previous time step + costs="data/costs_{planning_horizons}.csv", + cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", + cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", + output: + RESULTS + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + threads: 4 + resources: + mem_mb=10000, + log: + LOGS + + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", + benchmark: + ( + BENCHMARKS + + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ) + conda: + "../envs/environment.yaml" + script: + "../scripts/add_brownfield.py" + + +ruleorder: add_existing_baseyear > add_brownfield + + +rule prepare_perfect_foresight: + input: + **{ + f"network_{year}": RDIR + + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_" + + f"{year}.nc" + for year in config["scenario"]["planning_horizons"][1:] + }, + brownfield_network=lambda w: ( + RDIR + + "/prenetworks-brownfield/" + + "elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_" + + "{}.nc".format(str(config["scenario"]["planning_horizons"][0])) + ), + overrides="data/override_component_attrs", + output: + RDIR + + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc", + threads: 2 + resources: + mem_mb=10000, + script: + "../scripts/prepare_perfect_foresight.py" + log: + LOGS + + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log", + benchmark: + ( + BENCHMARKS + + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}" + ) + conda: + "../envs/environment.yaml" + script: + "../scripts/prepare_perfect_foresight.py" + + +rule solve_network_perfect: + input: + overrides="data/override_component_attrs", + network=RDIR + + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc", + costs="data/costs_2030.csv", + config=SDIR + "/configs/config.yaml", + output: + RDIR + + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc", + shadow: + "shallow" + log: + solver=RDIR + + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years_solver.log", + python=RDIR + + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years_python.log", + memory=RDIR + + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years_memory.log", + threads: 4 + resources: + mem_mb=config["solving"]["mem"], + benchmark: + ( + RDIR + + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years" + ) + script: + "../scripts/solve_network.py" + conda: + "../envs/environment.yaml" + + +rule make_summary_perfect: + input: + **{ + f"networks_{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}": RDIR + + f"/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc" + for simpl in config["scenario"]["simpl"] + for clusters in config["scenario"]["clusters"] + for opts in config["scenario"]["opts"] + for sector_opts in config["scenario"]["sector_opts"] + for lv in config["scenario"]["lv"] + }, + costs="data/costs_2020.csv", + overrides="data/override_component_attrs", + output: + nodal_costs="results" + "/" + config["run"] + "/csvs/nodal_costs.csv", + nodal_capacities="results" + "/" + config["run"] + "/csvs/nodal_capacities.csv", + nodal_cfs="results" + "/" + config["run"] + "/csvs/nodal_cfs.csv", + cfs="results" + "/" + config["run"] + "/csvs/cfs.csv", + costs="results" + "/" + config["run"] + "/csvs/costs.csv", + capacities="results" + "/" + config["run"] + "/csvs/capacities.csv", + curtailment="results" + "/" + config["run"] + "/csvs/curtailment.csv", + capital_cost="results" + "/" + config["run"] + "/csvs/capital_cost.csv", + energy="results" + "/" + config["run"] + "/csvs/energy.csv", + supply="results" + "/" + config["run"] + "/csvs/supply.csv", + supply_energy="results" + "/" + config["run"] + "/csvs/supply_energy.csv", + prices="results" + "/" + config["run"] + "/csvs/prices.csv", + weighted_prices="results" + "/" + config["run"] + "/csvs/weighted_prices.csv", + market_values="results" + "/" + config["run"] + "/csvs/market_values.csv", + price_statistics="results" + "/" + config["run"] + "/csvs/price_statistics.csv", + metrics="results" + "/" + config["run"] + "/csvs/metrics.csv", + co2_emissions="results" + "/" + config["run"] + "/csvs/co2_emissions.csv", + threads: 2 + resources: + mem_mb=10000, + script: + "../scripts/make_summary_perfect.py" + log: + LOGS + "make_summary_perfect{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log", + benchmark: + ( + BENCHMARKS + + "make_summary_perfect{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}" + ) + conda: + "../envs/environment.yaml" + + +rule plot_summary_perfect: + input: + **{ + f"costs_{year}": f"data/costs_{year}.csv" + for year in config["scenario"]["planning_horizons"] + }, + costs_csv="results" + "/" + config["run"] + "/csvs/costs.csv", + energy="results" + "/" + config["run"] + "/csvs/energy.csv", + balances="results" + "/" + config["run"] + "/csvs/supply_energy.csv", + eea="data/eea/UNFCCC_v24.csv", + countries="results" + "/" + config["run"] + "/csvs/nodal_capacities.csv", + co2_emissions="results" + "/" + config["run"] + "/csvs/co2_emissions.csv", + capacities="results" + "/" + config["run"] + "/csvs/capacities.csv", + capital_cost="results" + "/" + config["run"] + "/csvs/capital_cost.csv", + output: + costs1="results" + "/" + config["run"] + "/graphs/costs.pdf", + costs2="results" + "/" + config["run"] + "/graphs/costs2.pdf", + costs3="results" + "/" + config["run"] + "/graphs/total_costs_per_year.pdf", + # energy="results" + '/' + config['run'] + '/graphs/energy.pdf', + balances="results" + "/" + config["run"] + "/graphs/balances-energy.pdf", + co2_emissions="results" + "/" + config["run"] + "/graphs/carbon_budget_plot.pdf", + capacities="results" + "/" + config["run"] + "/graphs/capacities.pdf", + threads: 2 + resources: + mem_mb=10000, + script: + "../scripts/plot_summary_perfect.py" + log: + LOGS + "plot_summary_perfect.log", + benchmark: + (BENCHMARKS + "plot_summary_perfect") + conda: + "../envs/environment.yaml" From 00544ee6f9c6c2dd65b508d0508924a0c796655d Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 3 Apr 2023 16:39:59 +0200 Subject: [PATCH 02/77] add lifetimes for perfect --- scripts/prepare_sector_network.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index da6eab72..ee32b8bd 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1130,6 +1130,7 @@ def add_storage_and_grids(n, costs): e_cyclic=True, carrier="H2 Store", capital_cost=h2_capital_cost, + lifetime=costs.at["hydrogen storage tank type 1 including compressor", "lifetime"], ) if options["gas_network"] or options["H2_retrofit"]: @@ -1483,6 +1484,7 @@ def add_land_transport(n, costs): carrier="BEV charger", p_max_pu=avail_profile[nodes], efficiency=options.get("bev_charge_efficiency", 0.9), + lifetime=1, # These were set non-zero to find LU infeasibility when availability = 0.25 # p_nom_extendable=True, # p_nom_min=p_nom, @@ -1500,6 +1502,7 @@ def add_land_transport(n, costs): carrier="V2G", p_max_pu=avail_profile[nodes], efficiency=options.get("bev_charge_efficiency", 0.9), + lifetime=1, ) if electric_share > 0 and options["bev_dsm"]: @@ -1520,6 +1523,7 @@ def add_land_transport(n, costs): e_nom=e_nom, e_max_pu=1, e_min_pu=dsm_profile[nodes], + lifetime=1, ) if fuel_cell_share > 0: @@ -2019,6 +2023,7 @@ def add_heat(n, costs): country=ct, capital_cost=capital_cost[strength] * options["retrofitting"]["cost_factor"], + lifetime=50, ) @@ -2113,6 +2118,7 @@ def add_biomass(n, costs): e_nom=biogas_potentials_spatial, marginal_cost=costs.at["biogas", "fuel"], e_initial=biogas_potentials_spatial, + lifetime=1, ) n.madd( @@ -2123,6 +2129,7 @@ def add_biomass(n, costs): e_nom=solid_biomass_potentials_spatial, marginal_cost=costs.at["solid biomass", "fuel"], e_initial=solid_biomass_potentials_spatial, + lifetime=1, ) n.madd( @@ -3280,7 +3287,7 @@ if __name__ == "__main__": spatial = define_spatial(pop_layout.index, options) - if snakemake.config["foresight"] == "myopic": + if snakemake.config["foresight"] in ['myopic', 'perfect']: add_lifetime_wind_solar(n, costs) conventional = snakemake.config["existing_capacities"]["conventional_carriers"] @@ -3395,11 +3402,11 @@ if __name__ == "__main__": if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.config["foresight"] == "myopic") and ( + first_year_multi = (snakemake.config["foresight"] in ['myopic', 'perfect']) and ( snakemake.config["scenario"]["planning_horizons"][0] == investment_year ) - if options.get("cluster_heat_buses", False) and not first_year_myopic: + if options.get("cluster_heat_buses", False) and not first_year_multi: cluster_heat_buses(n) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) From 952a6ba1b5c9cf54275cae4317ac703768ef61b4 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 3 Apr 2023 16:41:15 +0200 Subject: [PATCH 03/77] add scripts for perfect foresight --- scripts/make_summary_perfect.py | 793 +++++++++++++++++++++++++++ scripts/prepare_perfect_foresight.py | 270 +++++++++ 2 files changed, 1063 insertions(+) create mode 100644 scripts/make_summary_perfect.py create mode 100644 scripts/prepare_perfect_foresight.py diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py new file mode 100644 index 00000000..8f21db76 --- /dev/null +++ b/scripts/make_summary_perfect.py @@ -0,0 +1,793 @@ +from six import iteritems + +import pandas as pd + +import numpy as np + +import pypsa + +from pypsa.descriptors import ( + nominal_attrs, + get_active_assets, +) + +from helper import override_component_attrs + +from prepare_sector_network import prepare_costs + +from make_summary import (assign_carriers, assign_locations, + calculate_cfs, calculate_nodal_cfs, + calculate_nodal_costs) + +idx = pd.IndexSlice + +opt_name = {"Store": "e", "Line": "s", "Transformer": "s"} + +def calculate_costs(n, label, costs): + + investments = n.investment_periods + cols = pd.MultiIndex.from_product( + [ + costs.columns.levels[0], + costs.columns.levels[1], + costs.columns.levels[2], + investments, + ], + names=costs.columns.names[:3] + ["year"], + ) + costs = costs.reindex(cols, axis=1) + + for c in n.iterate_components( + n.branch_components | n.controllable_one_port_components ^ {"Load"} + ): + capital_costs = c.df.capital_cost * c.df[opt_name.get(c.name, "p") + "_nom_opt"] + active = pd.concat( + [ + get_active_assets(n, c.name, inv_p).rename(inv_p) + for inv_p in investments + ], + axis=1, + ).astype(int) + capital_costs = active.mul(capital_costs, axis=0) + discount = ( + n.investment_period_weightings["objective_weightings"] + / n.investment_period_weightings["time_weightings"] + ) + capital_costs_grouped = ( + capital_costs.groupby(c.df.carrier).sum().mul(discount) + ) + + capital_costs_grouped = pd.concat([capital_costs_grouped], keys=["capital"]) + capital_costs_grouped = pd.concat([capital_costs_grouped], keys=[c.list_name]) + + costs = costs.reindex(capital_costs_grouped.index.union(costs.index)) + + costs.loc[capital_costs_grouped.index, label] = capital_costs_grouped.values + + if c.name == "Link": + p = ( + c.pnl.p0.multiply(n.snapshot_weightings.generator_weightings, axis=0) + .groupby(level=0) + .sum() + ) + elif c.name == "Line": + continue + elif c.name == "StorageUnit": + p_all = c.pnl.p.multiply(n.snapshot_weightings.store_weightings, axis=0) + p_all[p_all < 0.0] = 0.0 + p = p_all.groupby(level=0).sum() + else: + p = ( + round(c.pnl.p, ndigits=2) + .multiply(n.snapshot_weightings.generator_weightings, axis=0) + .groupby(level=0) + .sum() + ) + + # correct sequestration cost + if c.name == "Store": + items = c.df.index[ + (c.df.carrier == "co2 stored") & (c.df.marginal_cost <= -100.0) + ] + c.df.loc[items, "marginal_cost"] = -20.0 + + marginal_costs = p.mul(c.df.marginal_cost).T + # marginal_costs = active.mul(marginal_costs, axis=0) + marginal_costs_grouped = ( + marginal_costs.groupby(c.df.carrier).sum().mul(discount) + ) + + marginal_costs_grouped = pd.concat([marginal_costs_grouped], keys=["marginal"]) + marginal_costs_grouped = pd.concat([marginal_costs_grouped], keys=[c.list_name]) + + costs = costs.reindex(marginal_costs_grouped.index.union(costs.index)) + + costs.loc[marginal_costs_grouped.index, label] = marginal_costs_grouped.values + + # add back in all hydro + # costs.loc[("storage_units","capital","hydro"),label] = (0.01)*2e6*n.storage_units.loc[n.storage_units.group=="hydro","p_nom"].sum() + # costs.loc[("storage_units","capital","PHS"),label] = (0.01)*2e6*n.storage_units.loc[n.storage_units.group=="PHS","p_nom"].sum() + # costs.loc[("generators","capital","ror"),label] = (0.02)*3e6*n.generators.loc[n.generators.group=="ror","p_nom"].sum() + + return costs + + +def calculate_cumulative_cost(): + planning_horizons = snakemake.config["scenario"]["planning_horizons"] + + cumulative_cost = pd.DataFrame( + index=df["costs"].sum().index, + columns=pd.Series(data=np.arange(0, 0.1, 0.01), name="social discount rate"), + ) + + # discount cost and express them in money value of planning_horizons[0] + for r in cumulative_cost.columns: + cumulative_cost[r] = [ + df["costs"].sum()[index] / ((1 + r) ** (index[-1] - planning_horizons[0])) + for index in cumulative_cost.index + ] + + # integrate cost throughout the transition path + for r in cumulative_cost.columns: + for cluster in cumulative_cost.index.get_level_values(level=0).unique(): + for lv in cumulative_cost.index.get_level_values(level=1).unique(): + for sector_opts in cumulative_cost.index.get_level_values( + level=2 + ).unique(): + cumulative_cost.loc[ + (cluster, lv, sector_opts, "cumulative cost"), r + ] = np.trapz( + cumulative_cost.loc[ + idx[cluster, lv, sector_opts, planning_horizons], r + ].values, + x=planning_horizons, + ) + + return cumulative_cost + + +def calculate_nodal_capacities(n, label, nodal_capacities): + # Beware this also has extraneous locations for country (e.g. biomass) or continent-wide (e.g. fossil gas/oil) stuff + for c in n.iterate_components( + n.branch_components | n.controllable_one_port_components ^ {"Load"} + ): + nodal_capacities_c = c.df.groupby(["location", "carrier"])[ + opt_name.get(c.name, "p") + "_nom_opt" + ].sum() + index = pd.MultiIndex.from_tuples( + [(c.list_name,) + t for t in nodal_capacities_c.index.to_list()] + ) + nodal_capacities = nodal_capacities.reindex(index.union(nodal_capacities.index)) + nodal_capacities.loc[index, label] = nodal_capacities_c.values + + return nodal_capacities + + +def calculate_capacities(n, label, capacities): + + investments = n.investment_periods + cols = pd.MultiIndex.from_product( + [ + capacities.columns.levels[0], + capacities.columns.levels[1], + capacities.columns.levels[2], + investments, + ], + names=capacities.columns.names[:3] + ["year"], + ) + capacities = capacities.reindex(cols, axis=1) + + for c in n.iterate_components( + n.branch_components | n.controllable_one_port_components ^ {"Load"} + ): + active = pd.concat( + [ + get_active_assets(n, c.name, inv_p).rename(inv_p) + for inv_p in investments + ], + axis=1, + ).astype(int) + caps = c.df[opt_name.get(c.name, "p") + "_nom_opt"] + caps = active.mul(caps, axis=0) + capacities_grouped = ( + caps.groupby(c.df.carrier) + .sum() + .drop("load", errors="ignore") + ) + capacities_grouped = pd.concat([capacities_grouped], keys=[c.list_name]) + + capacities = capacities.reindex( + capacities_grouped.index.union(capacities.index) + ) + + capacities.loc[capacities_grouped.index, label] = capacities_grouped.values + + return capacities + + +def calculate_curtailment(n, label, curtailment): + + avail = ( + n.generators_t.p_max_pu.multiply(n.generators.p_nom_opt) + .sum() + .groupby(n.generators.carrier) + .sum() + ) + used = n.generators_t.p.sum().groupby(n.generators.carrier).sum() + + curtailment[label] = (((avail - used) / avail) * 100).round(3) + + return curtailment + + +def calculate_energy(n, label, energy): + + for c in n.iterate_components(n.one_port_components | n.branch_components): + + if c.name in n.one_port_components: + c_energies = ( + c.pnl.p.multiply(n.snapshot_weightings, axis=0) + .sum() + .multiply(c.df.sign) + .groupby(c.df.carrier) + .sum() + ) + else: + c_energies = pd.Series(0.0, c.df.carrier.unique()) + for port in [col[3:] for col in c.df.columns if col[:3] == "bus"]: + totals = c.pnl["p" + port].multiply(n.snapshot_weightings, axis=0).sum() + # remove values where bus is missing (bug in nomopyomo) + no_bus = c.df.index[c.df["bus" + port] == ""] + totals.loc[no_bus] = n.component_attrs[c.name].loc[ + "p" + port, "default" + ] + c_energies -= totals.groupby(c.df.carrier).sum() + + c_energies = pd.concat([c_energies], keys=[c.list_name]) + + energy = energy.reindex(c_energies.index.union(energy.index)) + + energy.loc[c_energies.index, label] = c_energies + + return energy + + +def calculate_supply(n, label, supply): + """calculate the max dispatch of each component at the buses aggregated by carrier""" + + bus_carriers = n.buses.carrier.unique() + + for i in bus_carriers: + bus_map = n.buses.carrier == i + bus_map.at[""] = False + + for c in n.iterate_components(n.one_port_components): + + items = c.df.index[c.df.bus.map(bus_map).fillna(False)] + + if len(items) == 0: + continue + + s = ( + c.pnl.p[items] + .max() + .multiply(c.df.loc[items, "sign"]) + .groupby(c.df.loc[items, "carrier"]) + .sum() + ) + s = pd.concat([s], keys=[c.list_name]) + s = pd.concat([s], keys=[i]) + + supply = supply.reindex(s.index.union(supply.index)) + supply.loc[s.index, label] = s + + for c in n.iterate_components(n.branch_components): + + for end in [col[3:] for col in c.df.columns if col[:3] == "bus"]: + + items = c.df.index[c.df["bus" + end].map(bus_map, na_action=False)] + + if len(items) == 0: + continue + + # lots of sign compensation for direction and to do maximums + s = (-1) ** (1 - int(end)) * ( + (-1) ** int(end) * c.pnl["p" + end][items] + ).max().groupby(c.df.loc[items, "carrier"]).sum() + s.index = s.index + end + s = pd.concat([s], keys=[c.list_name]) + s = pd.concat([s], keys=[i]) + + supply = supply.reindex(s.index.union(supply.index)) + supply.loc[s.index, label] = s + + return supply + + +def calculate_supply_energy(n, label, supply_energy): + """calculate the total energy supply/consuption of each component at the buses aggregated by carrier""" + + investments = n.investment_periods + cols = pd.MultiIndex.from_product( + [ + supply_energy.columns.levels[0], + supply_energy.columns.levels[1], + supply_energy.columns.levels[2], + investments, + ], + names=supply_energy.columns.names[:3] + ["year"], + ) + supply_energy = supply_energy.reindex(cols, axis=1) + + bus_carriers = n.buses.carrier.unique() + + for i in bus_carriers: + bus_map = n.buses.carrier == i + bus_map.at[""] = False + + for c in n.iterate_components(n.one_port_components): + + items = c.df.index[c.df.bus.map(bus_map).fillna(False)] + + if len(items) == 0: + continue + + if c.name == "Generator": + weightings = n.snapshot_weightings.generator_weightings + else: + weightings = n.snapshot_weightings.store_weightings + + if i in ["oil", "co2", "H2"]: + if c.name=="Load": + c.df.loc[items, "carrier"] = [load.split("-202")[0] for load in items] + if i=="oil" and c.name=="Generator": + c.df.loc[items, "carrier"] = "imported oil" + s = ( + c.pnl.p[items] + .multiply(weightings, axis=0) + .groupby(level=0) + .sum() + .multiply(c.df.loc[items, "sign"]) + .groupby(c.df.loc[items, "carrier"], axis=1) + .sum() + .T + ) + s = pd.concat([s], keys=[c.list_name]) + s = pd.concat([s], keys=[i]) + + supply_energy = supply_energy.reindex( + s.index.union(supply_energy.index, sort=False) + ) + supply_energy.loc[s.index, label] = s.values + + for c in n.iterate_components(n.branch_components): + + for end in [col[3:] for col in c.df.columns if col[:3] == "bus"]: + + items = c.df.index[c.df["bus" + str(end)].map(bus_map, na_action=False)] + + if len(items) == 0: + continue + + s = ( + (-1) + * c.pnl["p" + end] + .reindex(items, axis=1) + .multiply(n.snapshot_weightings.objective_weightings, axis=0) + .groupby(level=0) + .sum() + .groupby(c.df.loc[items, "carrier"], axis=1) + .sum() + ).T + s.index = s.index + end + s = pd.concat([s], keys=[c.list_name]) + s = pd.concat([s], keys=[i]) + + supply_energy = supply_energy.reindex( + s.index.union(supply_energy.index, sort=False) + ) + + supply_energy.loc[s.index, label] = s.values + + return supply_energy + + +def calculate_metrics(n, label, metrics): + + metrics = metrics.reindex( + pd.Index( + [ + "line_volume", + "line_volume_limit", + "line_volume_AC", + "line_volume_DC", + "line_volume_shadow", + "co2_shadow", + ] + ).union(metrics.index) + ) + + metrics.at["line_volume_DC", label] = (n.links.length * n.links.p_nom_opt)[ + n.links.carrier == "DC" + ].sum() + metrics.at["line_volume_AC", label] = (n.lines.length * n.lines.s_nom_opt).sum() + metrics.at["line_volume", label] = metrics.loc[ + ["line_volume_AC", "line_volume_DC"], label + ].sum() + + if hasattr(n, "line_volume_limit"): + metrics.at["line_volume_limit", label] = n.line_volume_limit + metrics.at["line_volume_shadow", label] = n.line_volume_limit_dual + + if "CO2Limit" in n.global_constraints.index: + metrics.at["co2_shadow", label] = n.global_constraints.at["CO2Limit", "mu"] + + return metrics + + +def calculate_prices(n, label, prices): + + prices = prices.reindex(prices.index.union(n.buses.carrier.unique())) + + # WARNING: this is time-averaged, see weighted_prices for load-weighted average + prices[label] = ( + n.buses_t.marginal_price.mean() + .groupby(n.buses.carrier) + .mean() + ) + + return prices + + +def calculate_weighted_prices(n, label, weighted_prices): + # Warning: doesn't include storage units as loads + + weighted_prices = weighted_prices.reindex( + pd.Index( + [ + "electricity", + "heat", + "space heat", + "urban heat", + "space urban heat", + "gas", + "H2", + ] + ) + ) + + link_loads = { + "electricity": [ + "heat pump", + "resistive heater", + "battery charger", + "H2 Electrolysis", + ], + "heat": ["water tanks charger"], + "urban heat": ["water tanks charger"], + "space heat": [], + "space urban heat": [], + "gas": ["OCGT", "gas boiler", "CHP electric", "CHP heat"], + "H2": ["Sabatier", "H2 Fuel Cell"], + } + + for carrier in link_loads: + + if carrier == "electricity": + suffix = "" + elif carrier[:5] == "space": + suffix = carrier[5:] + else: + suffix = " " + carrier + + buses = n.buses.index[n.buses.index.str[2:] == suffix] + + if buses.empty: + continue + + if carrier in ["H2", "gas"]: + load = pd.DataFrame(index=n.snapshots, columns=buses, data=0.0) + else: + load = n.loads_t.p_set.reindex(buses, axis=1) + + for tech in link_loads[carrier]: + + names = n.links.index[n.links.index.to_series().str[-len(tech) :] == tech] + + if names.empty: + continue + + load += ( + n.links_t.p0[names].groupby(n.links.loc[names, "bus0"], axis=1).sum() + ) + + # Add H2 Store when charging + # if carrier == "H2": + # stores = n.stores_t.p[buses+ " Store"].groupby(n.stores.loc[buses+ " Store","bus"],axis=1).sum(axis=1) + # stores[stores > 0.] = 0. + # load += -stores + + weighted_prices.loc[carrier, label] = ( + load * n.buses_t.marginal_price[buses] + ).sum().sum() / load.sum().sum() + + if carrier[:5] == "space": + print(load * n.buses_t.marginal_price[buses]) + + return weighted_prices + + +def calculate_market_values(n, label, market_values): + # Warning: doesn't include storage units + + carrier = "AC" + + buses = n.buses.index[n.buses.carrier == carrier] + + ## First do market value of generators ## + + generators = n.generators.index[n.buses.loc[n.generators.bus, "carrier"] == carrier] + + techs = n.generators.loc[generators, "carrier"].value_counts().index + + market_values = market_values.reindex(market_values.index.union(techs)) + + for tech in techs: + gens = generators[n.generators.loc[generators, "carrier"] == tech] + + dispatch = ( + n.generators_t.p[gens] + .groupby(n.generators.loc[gens, "bus"], axis=1) + .sum() + .reindex(columns=buses, fill_value=0.0) + ) + + revenue = dispatch * n.buses_t.marginal_price[buses] + + market_values.at[tech, label] = revenue.sum().sum() / dispatch.sum().sum() + + ## Now do market value of links ## + + for i in ["0", "1"]: + all_links = n.links.index[n.buses.loc[n.links["bus" + i], "carrier"] == carrier] + + techs = n.links.loc[all_links, "carrier"].value_counts().index + + market_values = market_values.reindex(market_values.index.union(techs)) + + for tech in techs: + links = all_links[n.links.loc[all_links, "carrier"] == tech] + + dispatch = ( + n.links_t["p" + i][links] + .groupby(n.links.loc[links, "bus" + i], axis=1) + .sum() + .reindex(columns=buses, fill_value=0.0) + ) + + revenue = dispatch * n.buses_t.marginal_price[buses] + + market_values.at[tech, label] = revenue.sum().sum() / dispatch.sum().sum() + + return market_values + + +def calculate_price_statistics(n, label, price_statistics): + + price_statistics = price_statistics.reindex( + price_statistics.index.union( + pd.Index(["zero_hours", "mean", "standard_deviation"]) + ) + ) + + buses = n.buses.index[n.buses.carrier == "AC"] + + threshold = 0.1 # higher than phoney marginal_cost of wind/solar + + df = pd.DataFrame(data=0.0, columns=buses, index=n.snapshots) + + df[n.buses_t.marginal_price[buses] < threshold] = 1.0 + + price_statistics.at["zero_hours", label] = df.sum().sum() / ( + df.shape[0] * df.shape[1] + ) + + price_statistics.at["mean", label] = ( + n.buses_t.marginal_price[buses].unstack().mean() + ) + + price_statistics.at["standard_deviation", label] = ( + n.buses_t.marginal_price[buses].unstack().std() + ) + + return price_statistics + + +def calculate_co2_emissions(n, label, df): + + carattr = "co2_emissions" + emissions = n.carriers.query(f"{carattr} != 0")[carattr] + + if emissions.empty: + return + + weightings = n.snapshot_weightings.mul( + n.investment_period_weightings["time_weightings"] + .reindex(n.snapshots) + .fillna(method="bfill") + .fillna(1.0), + axis=0, + ) + + # generators + gens = n.generators.query("carrier in @emissions.index") + if not gens.empty: + em_pu = gens.carrier.map(emissions) / gens.efficiency + em_pu = ( + weightings["generator_weightings"].to_frame("weightings") + @ em_pu.to_frame("weightings").T + ) + emitted = n.generators_t.p[gens.index].mul(em_pu) + + emitted_grouped = ( + emitted.groupby(level=0) + .sum() + .groupby(n.generators.carrier, axis=1) + .sum() + .T + ) + + df = df.reindex(emitted_grouped.index.union(df.index)) + + df.loc[emitted_grouped.index, label] = emitted_grouped.values + + if any(n.stores.carrier == "co2"): + co2_i = n.stores[n.stores.carrier == "co2"].index + df[label] = n.stores_t.e.groupby(level=0).last()[co2_i].iloc[:, 0] + + return df + + + + +def calculate_cumulative_capacities(n, label, cum_cap): + # TODO + + investments = n.investment_periods + cols = pd.MultiIndex.from_product( + [ + cum_cap.columns.levels[0], + cum_cap.columns.levels[1], + cum_cap.columns.levels[2], + investments, + ], + names=cum_cap.columns.names[:3] + ["year"], + ) + cum_cap = cum_cap.reindex(cols, axis=1) + + learn_i = n.carriers[n.carriers.learning_rate != 0].index + + for c, attr in nominal_attrs.items(): + if "carrier" not in n.df(c) or n.df(c).empty: + continue + caps = ( + n.df(c)[n.df(c).carrier.isin(learn_i)] + .groupby([n.df(c).carrier, n.df(c).build_year])[ + opt_name.get(c, "p") + "_nom_opt" + ] + .sum() + ) + + if caps.empty: + continue + + caps = round( + caps.unstack().reindex(columns=investments).fillna(0).cumsum(axis=1) + ) + cum_cap = cum_cap.reindex(caps.index.union(cum_cap.index)) + + cum_cap.loc[caps.index, label] = caps.values + + return cum_cap + + + + + +outputs = [ + "nodal_costs", + "nodal_capacities", + "nodal_cfs", + "cfs", + "costs", + "capacities", + "curtailment", + "energy", + "supply", + "supply_energy", + "prices", + "weighted_prices", + "price_statistics", + "market_values", + "metrics", + "co2_emissions", + "cumulative_capacities", +] + + +def make_summaries(networks_dict): + + columns = pd.MultiIndex.from_tuples( + networks_dict.keys(), names=["cluster", "lv", "opt", "year"] + ) + df = {} + + for output in outputs: + df[output] = pd.DataFrame(columns=columns, dtype=float) + + overrides = override_component_attrs(snakemake.input.overrides) + for label, filename in iteritems(networks_dict): + print(label, filename) + try: + n = pypsa.Network( + filename, override_component_attrs=overrides + ) + except OSError: + print(label, " not solved yet.") + continue + # del networks_dict[label] + + if not hasattr(n, "objective"): + print(label, " not solved correctly. Check log if infeasible or unbounded.") + continue + assign_carriers(n) + assign_locations(n) + + for output in outputs: + df[output] = globals()["calculate_" + output](n, label, df[output]) + + return df + + +def to_csv(df): + + for key in df: + df[key] = df[key].apply(lambda x: pd.to_numeric(x)) + df[key].to_csv(snakemake.output[key]) + + +#%% +if __name__ == "__main__": + # Detect running outside of snakemake and mock snakemake for testing + if "snakemake" not in globals(): + from helper import mock_snakemake + snakemake = mock_snakemake('make_summary_perfect') + + networks_dict = { + (clusters, lv, opts+sector_opts) : + snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc' \ + for simpl in snakemake.config['scenario']['simpl'] \ + for clusters in snakemake.config['scenario']['clusters'] \ + for opts in snakemake.config['scenario']['opts'] \ + for sector_opts in snakemake.config['scenario']['sector_opts'] \ + for lv in snakemake.config['scenario']['lv'] \ + } + + + print(networks_dict) + + Nyears = 1 + + costs_db = prepare_costs( + snakemake.input.costs, + snakemake.config["costs"]["USD2013_to_EUR2013"], + snakemake.config["costs"]["discountrate"], + Nyears, + snakemake.config["costs"]["lifetime"], + ) + + df = make_summaries(networks_dict) + + df["metrics"].loc["total costs"] = df["costs"].sum().groupby(level=[0, 1, 2]).sum() + + to_csv(df) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py new file mode 100644 index 00000000..54e60337 --- /dev/null +++ b/scripts/prepare_perfect_foresight.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Concats pypsa networks of single investment periods to one network. + +Created on Tue Aug 16 10:40:41 2022 + +@author: lisa +""" +import pypsa +import pandas as pd +from helper import override_component_attrs, update_config_with_sector_opts +from pypsa.io import import_components_from_dataframe +from add_existing_baseyear import add_build_year_to_new_assets +from six import iterkeys +from pypsa.descriptors import expand_series +import re +import logging +logger = logging.getLogger(__name__) + +# helper functions --------------------------------------------------- +def get_missing(df, n, c): + """Get in network n missing assets of df for component c. + + Input: + df: pandas DataFrame, static values of pypsa components + n : pypsa Network to which new assets should be added + c : string, pypsa component.list_name (e.g. "generators") + Return: + pd.DataFrame with static values of missing assets + """ + df_final = getattr(n, c) + missing_i = df.index.difference(df_final.index) + return df.loc[missing_i] + + +def get_social_discount(t, r=0.01): + """Calculate for a given time t the social discount.""" + return 1 / (1 + r) ** t + + +def get_investment_weighting(time_weighting, r=0.01): + """Define cost weighting. + + Returns cost weightings depending on the the time_weighting (pd.Series) + and the social discountrate r + """ + end = time_weighting.cumsum() + start = time_weighting.cumsum().shift().fillna(0) + return pd.concat([start, end], axis=1).apply( + lambda x: sum([get_social_discount(t, r) for t in range(int(x[0]), int(x[1]))]), + axis=1, + ) + +# -------------------------------------------------------------------- +def concat_networks(years): + """Concat given pypsa networks and adds build_year. + + Return: + n : pypsa.Network for the whole planning horizon + + """ + + # input paths of sector coupling networks + network_paths = [snakemake.input.brownfield_network] + [ + snakemake.input[f"network_{year}"] for year in years[1:]] + # final concatenated network + overrides = override_component_attrs(snakemake.input.overrides) + n = pypsa.Network(override_component_attrs=overrides) + + + # iterate over single year networks and concat to perfect foresight network + for i, network_path in enumerate(network_paths): + year = years[i] + network = pypsa.Network(network_path, override_component_attrs=overrides) + network.lines["carrier"] = "AC" + add_build_year_to_new_assets(network, year) + + # static ---------------------------------- + # (1) add buses and carriers + for component in network.iterate_components(["Bus", "Carrier"]): + df_year = component.df + # get missing assets + missing = get_missing(df_year, n, component.list_name) + import_components_from_dataframe(n, missing, component.name) + # (2) add generators, links, stores and loads + for component in network.iterate_components( + ["Generator", "Link", "Store", "Load", "Line", "StorageUnit"] + ): + + df_year = component.df.copy() + missing = get_missing(df_year, n, component.list_name) + + import_components_from_dataframe(n, missing, component.name) + + # time variant -------------------------------------------------- + network_sns = pd.MultiIndex.from_product([[year], network.snapshots]) + snapshots = n.snapshots.drop("now", errors="ignore").union(network_sns) + n.set_snapshots(snapshots) + + for component in network.iterate_components(): + pnl = getattr(n, component.list_name + "_t") + for k in iterkeys(component.pnl): + pnl_year = component.pnl[k].copy().reindex(snapshots, level=1) + if pnl_year.empty and ~(component.name=="Load" and k=="p_set"): continue + if component.name == "Load": + static_load = network.loads.loc[network.loads.p_set != 0] + static_load_t = expand_series( + static_load.p_set, network_sns + ).T + pnl_year = pd.concat([pnl_year.reindex(network_sns), + static_load_t], axis=1) + columns = (pnl[k].columns.union(pnl_year.columns)).unique() + pnl[k] = pnl[k].reindex(columns=columns) + pnl[k].loc[pnl_year.index, pnl_year.columns] = pnl_year + + else: + # this is to avoid adding multiple times assets with infinit lifetime as ror + cols = pnl_year.columns.difference(pnl[k].columns) + pnl[k] = pd.concat([pnl[k], pnl_year[cols]], axis=1) + + + n.snapshot_weightings.loc[year,:] = network.snapshot_weightings.values + # set investment periods + n.investment_periods = n.snapshots.levels[0] + # weighting of the investment period -> assuming last period same weighting as the period before + time_w = n.investment_periods.to_series().diff().shift(-1).fillna(method="ffill") + n.investment_period_weightings["years"] = time_w + # set objective weightings + objective_w = get_investment_weighting(n.investment_period_weightings["years"], + social_discountrate) + n.investment_period_weightings["objective"] = objective_w + # all former static loads are now time-dependent -> set static = 0 + n.loads["p_set"] = 0 + + return n + +def adjust_stores(n): + """Make sure that stores still behave cyclic over one year and not whole + modelling horizon.""" + # cylclic constraint + cyclic_i = n.stores[n.stores.e_cyclic].index + n.stores.loc[cyclic_i, "e_cyclic_per_period"] = True + n.stores.loc[cyclic_i, "e_cyclic"] = False + # non cyclic store assumptions + non_cyclic_store = ["co2", "co2 stored", "solid biomass", "biogas", "Li ion"] + co2_i = n.stores[n.stores.carrier.isin(non_cyclic_store)].index + n.stores.loc[co2_i, "e_cyclic_per_period"] = False + n.stores.loc[co2_i, "e_cyclic"] = False + + return n + +def set_phase_out(n, carrier, ct, phase_out_year): + """Set planned phase outs for given carrier,country (ct) and planned year + of phase out (phase_out_year).""" + df = n.links[(n.links.carrier.isin(carrier))& (n.links.bus1.str[:2]==ct)] + # assets which are going to be phased out before end of their lifetime + assets_i = df[df[["build_year", "lifetime"]].sum(axis=1) > phase_out_year].index + build_year = n.links.loc[assets_i, "build_year"] + # adjust lifetime + n.links.loc[assets_i, "lifetime"] = (phase_out_year - build_year).astype(float) + +def set_all_phase_outs(n): + # TODO move this to a csv or to the config + planned= [(["nuclear"], "DE", 2022), + (["nuclear"], "BE", 2025), + (["nuclear"], "ES", 2027), + (["coal", "lignite"], "DE", 2038), + (["coal", "lignite"], "ES", 2027), + (["coal", "lignite"], "FR", 2022), + (["coal", "lignite"], "GB", 2024), + (["coal", "lignite"], "IT", 2025)] + for carrier, ct, phase_out_year in planned: + set_phase_out(n, carrier,ct, phase_out_year) + # remove assets which are already phased out + remove_i = n.links[n.links[["build_year", "lifetime"]].sum(axis=1) Date: Mon, 3 Apr 2023 17:39:47 +0200 Subject: [PATCH 04/77] adjust to new pypsa-eur syntax --- rules/collect.smk | 7 ++ rules/solve_perfect.smk | 136 +++++++++++++-------------- scripts/prepare_perfect_foresight.py | 37 ++++---- 3 files changed, 92 insertions(+), 88 deletions(-) diff --git a/rules/collect.smk b/rules/collect.smk index 611b099c..e17384e0 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -65,6 +65,13 @@ rule solve_sector_networks: **config["scenario"] ), +rule solve_sector_networks_perfect: + input: + expand( + RESULTS + + "/postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + **config["scenario"] + ), rule plot_networks: input: diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 9ced1c76..20631c46 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -68,32 +68,27 @@ rule add_brownfield: "../scripts/add_brownfield.py" -ruleorder: add_existing_baseyear > add_brownfield - - rule prepare_perfect_foresight: input: **{ - f"network_{year}": RDIR - + "/prenetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_" + f"network_{year}": RESULTS + + "/prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + f"{year}.nc" for year in config["scenario"]["planning_horizons"][1:] }, brownfield_network=lambda w: ( - RDIR + RESULTS + "/prenetworks-brownfield/" - + "elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_" + + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + "{}.nc".format(str(config["scenario"]["planning_horizons"][0])) ), overrides="data/override_component_attrs", output: - RDIR - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc", + RESULTS + + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", threads: 2 resources: mem_mb=10000, - script: - "../scripts/prepare_perfect_foresight.py" log: LOGS + "prepare_perfect_foresight{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log", @@ -108,84 +103,84 @@ rule prepare_perfect_foresight: "../scripts/prepare_perfect_foresight.py" -rule solve_network_perfect: +rule solve_sector_network_perfect: input: overrides="data/override_component_attrs", - network=RDIR - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc", + network=RESULTS + + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", costs="data/costs_2030.csv", - config=SDIR + "/configs/config.yaml", + config=RESULTS + "configs/config.yaml", output: - RDIR - + "/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc", - shadow: - "shallow" - log: - solver=RDIR - + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years_solver.log", - python=RDIR - + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years_python.log", - memory=RDIR - + "/logs/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years_memory.log", + RESULTS + + "/postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", threads: 4 resources: mem_mb=config["solving"]["mem"], + shadow: + "shallow" + log: + solver=RESULTS + + "/logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_solver.log", + python=RESULTS + + "/logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_python.log", + memory=RESULTS + + "/logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log", benchmark: ( - RDIR - + "/benchmarks/solve_network/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years" + BENCHMARKS + + "solve_sector_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years}" ) - script: - "../scripts/solve_network.py" conda: "../envs/environment.yaml" + script: + "../scripts/solve_network.py" rule make_summary_perfect: input: **{ - f"networks_{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}": RDIR - + f"/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc" + f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS + + f"/postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc" for simpl in config["scenario"]["simpl"] for clusters in config["scenario"]["clusters"] for opts in config["scenario"]["opts"] for sector_opts in config["scenario"]["sector_opts"] - for lv in config["scenario"]["lv"] + for ll in config["scenario"]["ll"] }, costs="data/costs_2020.csv", overrides="data/override_component_attrs", output: - nodal_costs="results" + "/" + config["run"] + "/csvs/nodal_costs.csv", - nodal_capacities="results" + "/" + config["run"] + "/csvs/nodal_capacities.csv", - nodal_cfs="results" + "/" + config["run"] + "/csvs/nodal_cfs.csv", - cfs="results" + "/" + config["run"] + "/csvs/cfs.csv", - costs="results" + "/" + config["run"] + "/csvs/costs.csv", - capacities="results" + "/" + config["run"] + "/csvs/capacities.csv", - curtailment="results" + "/" + config["run"] + "/csvs/curtailment.csv", - capital_cost="results" + "/" + config["run"] + "/csvs/capital_cost.csv", - energy="results" + "/" + config["run"] + "/csvs/energy.csv", - supply="results" + "/" + config["run"] + "/csvs/supply.csv", - supply_energy="results" + "/" + config["run"] + "/csvs/supply_energy.csv", - prices="results" + "/" + config["run"] + "/csvs/prices.csv", - weighted_prices="results" + "/" + config["run"] + "/csvs/weighted_prices.csv", - market_values="results" + "/" + config["run"] + "/csvs/market_values.csv", - price_statistics="results" + "/" + config["run"] + "/csvs/price_statistics.csv", - metrics="results" + "/" + config["run"] + "/csvs/metrics.csv", - co2_emissions="results" + "/" + config["run"] + "/csvs/co2_emissions.csv", + nodal_costs=RESULTS + "/csvs/nodal_costs.csv", + nodal_capacities=RESULTS + "/csvs/nodal_capacities.csv", + nodal_cfs=RESULTS + "/csvs/nodal_cfs.csv", + cfs=RESULTS + "/csvs/cfs.csv", + costs=RESULTS + "/csvs/costs.csv", + capacities=RESULTS + "/csvs/capacities.csv", + curtailment=RESULTS + "/csvs/curtailment.csv", + capital_cost=RESULTS + "/csvs/capital_cost.csv", + energy=RESULTS + "/csvs/energy.csv", + supply=RESULTS + "/csvs/supply.csv", + supply_energy=RESULTS + "/csvs/supply_energy.csv", + prices=RESULTS + "/csvs/prices.csv", + weighted_prices=RESULTS + "/csvs/weighted_prices.csv", + market_values=RESULTS + "/csvs/market_values.csv", + price_statistics=RESULTS + "/csvs/price_statistics.csv", + metrics=RESULTS + "/csvs/metrics.csv", + co2_emissions=RESULTS + "/csvs/co2_emissions.csv", threads: 2 resources: mem_mb=10000, - script: - "../scripts/make_summary_perfect.py" log: - LOGS + "make_summary_perfect{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}.log", + LOGS + "make_summary_perfect.log", benchmark: ( BENCHMARKS - + "make_summary_perfect{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}" + + "make_summary_perfect" ) conda: "../envs/environment.yaml" + script: + "../scripts/make_summary_perfect.py" rule plot_summary_perfect: @@ -194,30 +189,33 @@ rule plot_summary_perfect: f"costs_{year}": f"data/costs_{year}.csv" for year in config["scenario"]["planning_horizons"] }, - costs_csv="results" + "/" + config["run"] + "/csvs/costs.csv", - energy="results" + "/" + config["run"] + "/csvs/energy.csv", - balances="results" + "/" + config["run"] + "/csvs/supply_energy.csv", + costs_csv=RESULTS + "/csvs/costs.csv", + energy=RESULTS + "/csvs/energy.csv", + balances=RESULTS + "/csvs/supply_energy.csv", eea="data/eea/UNFCCC_v24.csv", - countries="results" + "/" + config["run"] + "/csvs/nodal_capacities.csv", - co2_emissions="results" + "/" + config["run"] + "/csvs/co2_emissions.csv", - capacities="results" + "/" + config["run"] + "/csvs/capacities.csv", - capital_cost="results" + "/" + config["run"] + "/csvs/capital_cost.csv", + countries=RESULTS + "/csvs/nodal_capacities.csv", + co2_emissions=RESULTS + "/csvs/co2_emissions.csv", + capacities=RESULTS + "/csvs/capacities.csv", + capital_cost=RESULTS + "/csvs/capital_cost.csv", output: - costs1="results" + "/" + config["run"] + "/graphs/costs.pdf", - costs2="results" + "/" + config["run"] + "/graphs/costs2.pdf", - costs3="results" + "/" + config["run"] + "/graphs/total_costs_per_year.pdf", + costs1=RESULTS + "/graphs/costs.pdf", + costs2=RESULTS + "/graphs/costs2.pdf", + costs3=RESULTS + "/graphs/total_costs_per_year.pdf", # energy="results" + '/' + config['run'] + '/graphs/energy.pdf', - balances="results" + "/" + config["run"] + "/graphs/balances-energy.pdf", - co2_emissions="results" + "/" + config["run"] + "/graphs/carbon_budget_plot.pdf", - capacities="results" + "/" + config["run"] + "/graphs/capacities.pdf", + balances=RESULTS + "/graphs/balances-energy.pdf", + co2_emissions=RESULTS + "/graphs/carbon_budget_plot.pdf", + capacities=RESULTS + "/graphs/capacities.pdf", threads: 2 resources: mem_mb=10000, - script: - "../scripts/plot_summary_perfect.py" log: LOGS + "plot_summary_perfect.log", benchmark: (BENCHMARKS + "plot_summary_perfect") conda: "../envs/environment.yaml" + script: + "../scripts/plot_summary_perfect.py" + + +ruleorder: add_existing_baseyear > add_brownfield diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 54e60337..06c8ffb7 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -1,15 +1,15 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + """ Concats pypsa networks of single investment periods to one network. - -Created on Tue Aug 16 10:40:41 2022 - -@author: lisa """ + import pypsa import pandas as pd -from helper import override_component_attrs, update_config_with_sector_opts +from _helpers import override_component_attrs, update_config_with_sector_opts from pypsa.io import import_components_from_dataframe from add_existing_baseyear import add_build_year_to_new_assets from six import iterkeys @@ -179,23 +179,22 @@ def set_all_phase_outs(n): def set_carbon_constraints(n, opts): """Add global constraints for carbon emissions.""" - budget = ( - snakemake.config["co2_budget"]["1p7"] * 1e9 - ) # budget for + 1.7 Celsius for Europe + budget = None for o in opts: # other budgets m = re.match(r"^\d+p\d$", o, re.IGNORECASE) if m is not None: budget = snakemake.config["co2_budget"][m.group(0)] * 1e9 - logger.info("add carbon budget of {}".format(budget)) - n.add( - "GlobalConstraint", - "Budget", - type="Co2constraint", - carrier_attribute="co2_emissions", - sense="<=", - constant=budget, - ) + if budget!=None: + logger.info("add carbon budget of {}".format(budget)) + n.add( + "GlobalConstraint", + "Budget", + type="Co2constraint", + carrier_attribute="co2_emissions", + sense="<=", + constant=budget, + ) if not "noco2neutral" in opts: logger.info("Add carbon neutrality constraint.") @@ -232,7 +231,7 @@ def set_carbon_constraints(n, opts): #%% if __name__ == "__main__": if 'snakemake' not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake( 'prepare_perfect_foresight', simpl='', From d836970004c6d4022c2015c01437ca922fcc50f1 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 10:39:16 +0200 Subject: [PATCH 05/77] always add load shedding with 1e2 --- scripts/solve_network.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index cfb95bfe..3456dba9 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -157,9 +157,9 @@ def prepare_network(n, solve_opts=None, config=None): # TODO: retrieve color and nice name from config n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding") buses_i = n.buses.query("carrier == 'AC'").index - if not np.isscalar(load_shedding): + #if not np.isscalar(load_shedding): # TODO: do not scale via sign attribute (use Eur/MWh instead of Eur/kWh) - load_shedding = 1e2 # Eur/kWh + load_shedding = 1e2 # Eur/kWh n.madd( "Generator", @@ -626,20 +626,20 @@ def solve_network(n, config, opts="", **kwargs): return n - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake( - "solve_sector_network", - configfiles="test/config.overnight.yaml", + "solve_sector_network_perfect", + configfiles="config.yaml", simpl="", opts="", - clusters="5", - ll="v1.5", - sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1", - planning_horizons="2030", + clusters="37", + ll="v1.0", + sector_opts="cb40ex0-8760H-T-H-B-I-A-solar+p3-dist1", + # planning_horizons="2030", ) configure_logging(snakemake) if "sector_opts" in snakemake.wildcards.keys(): From b9c58db0ba91bb06cdbec53addb4088ca91cf33f Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 10:41:21 +0200 Subject: [PATCH 06/77] remove slash --- rules/solve_perfect.smk | 74 ++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 20631c46..0f7ad49e 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -72,13 +72,13 @@ rule prepare_perfect_foresight: input: **{ f"network_{year}": RESULTS - + "/prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + f"{year}.nc" for year in config["scenario"]["planning_horizons"][1:] }, brownfield_network=lambda w: ( RESULTS - + "/prenetworks-brownfield/" + + "prenetworks-brownfield/" + "elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_" + "{}.nc".format(str(config["scenario"]["planning_horizons"][0])) ), @@ -107,12 +107,12 @@ rule solve_sector_network_perfect: input: overrides="data/override_component_attrs", network=RESULTS - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", costs="data/costs_2030.csv", config=RESULTS + "configs/config.yaml", output: RESULTS - + "/postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", threads: 4 resources: mem_mb=config["solving"]["mem"], @@ -120,11 +120,11 @@ rule solve_sector_network_perfect: "shallow" log: solver=RESULTS - + "/logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_solver.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_solver.log", python=RESULTS - + "/logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_python.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_python.log", memory=RESULTS - + "/logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log", + + "logs/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_memory.log", benchmark: ( BENCHMARKS @@ -150,23 +150,23 @@ rule make_summary_perfect: costs="data/costs_2020.csv", overrides="data/override_component_attrs", output: - nodal_costs=RESULTS + "/csvs/nodal_costs.csv", - nodal_capacities=RESULTS + "/csvs/nodal_capacities.csv", - nodal_cfs=RESULTS + "/csvs/nodal_cfs.csv", - cfs=RESULTS + "/csvs/cfs.csv", - costs=RESULTS + "/csvs/costs.csv", - capacities=RESULTS + "/csvs/capacities.csv", - curtailment=RESULTS + "/csvs/curtailment.csv", - capital_cost=RESULTS + "/csvs/capital_cost.csv", - energy=RESULTS + "/csvs/energy.csv", - supply=RESULTS + "/csvs/supply.csv", - supply_energy=RESULTS + "/csvs/supply_energy.csv", - prices=RESULTS + "/csvs/prices.csv", - weighted_prices=RESULTS + "/csvs/weighted_prices.csv", - market_values=RESULTS + "/csvs/market_values.csv", - price_statistics=RESULTS + "/csvs/price_statistics.csv", - metrics=RESULTS + "/csvs/metrics.csv", - co2_emissions=RESULTS + "/csvs/co2_emissions.csv", + nodal_costs=RESULTS + "csvs/nodal_costs.csv", + nodal_capacities=RESULTS + "csvs/nodal_capacities.csv", + nodal_cfs=RESULTS + "csvs/nodal_cfs.csv", + cfs=RESULTS + "csvs/cfs.csv", + costs=RESULTS + "csvs/costs.csv", + capacities=RESULTS + "csvs/capacities.csv", + curtailment=RESULTS + "csvs/curtailment.csv", + capital_cost=RESULTS + "csvs/capital_cost.csv", + energy=RESULTS + "csvs/energy.csv", + supply=RESULTS + "csvs/supply.csv", + supply_energy=RESULTS + "csvs/supply_energy.csv", + prices=RESULTS + "csvs/prices.csv", + weighted_prices=RESULTS + "csvs/weighted_prices.csv", + market_values=RESULTS + "csvs/market_values.csv", + price_statistics=RESULTS + "csvs/price_statistics.csv", + metrics=RESULTS + "csvs/metrics.csv", + co2_emissions=RESULTS + "csvs/co2_emissions.csv", threads: 2 resources: mem_mb=10000, @@ -189,22 +189,22 @@ rule plot_summary_perfect: f"costs_{year}": f"data/costs_{year}.csv" for year in config["scenario"]["planning_horizons"] }, - costs_csv=RESULTS + "/csvs/costs.csv", - energy=RESULTS + "/csvs/energy.csv", - balances=RESULTS + "/csvs/supply_energy.csv", + costs_csv=RESULTS + "csvs/costs.csv", + energy=RESULTS + "csvs/energy.csv", + balances=RESULTS + "csvs/supply_energy.csv", eea="data/eea/UNFCCC_v24.csv", - countries=RESULTS + "/csvs/nodal_capacities.csv", - co2_emissions=RESULTS + "/csvs/co2_emissions.csv", - capacities=RESULTS + "/csvs/capacities.csv", - capital_cost=RESULTS + "/csvs/capital_cost.csv", + countries=RESULTS + "csvs/nodal_capacities.csv", + co2_emissions=RESULTS + "csvs/co2_emissions.csv", + capacities=RESULTS + "csvs/capacities.csv", + capital_cost=RESULTS + "csvs/capital_cost.csv", output: - costs1=RESULTS + "/graphs/costs.pdf", - costs2=RESULTS + "/graphs/costs2.pdf", - costs3=RESULTS + "/graphs/total_costs_per_year.pdf", + costs1=RESULTS + "graphs/costs.pdf", + costs2=RESULTS + "graphs/costs2.pdf", + costs3=RESULTS + "graphs/total_costs_per_year.pdf", # energy="results" + '/' + config['run'] + '/graphs/energy.pdf', - balances=RESULTS + "/graphs/balances-energy.pdf", - co2_emissions=RESULTS + "/graphs/carbon_budget_plot.pdf", - capacities=RESULTS + "/graphs/capacities.pdf", + balances=RESULTS + "graphs/balances-energy.pdf", + co2_emissions=RESULTS + "graphs/carbon_budget_plot.pdf", + capacities=RESULTS + "graphs/capacities.pdf", threads: 2 resources: mem_mb=10000, From b50954aee49f83a4184e57cefdff656e26591af1 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 15:43:02 +0200 Subject: [PATCH 07/77] remove snakemake from function --- scripts/prepare_sector_network.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index ee32b8bd..f462b159 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -197,17 +197,17 @@ def get(item, investment_year=None): def co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year + countries, input_eurostat, input_eea, opts, emissions_scope, report_year, year ): """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.config["energy"]["emissions"] - eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) + + eea_co2 = build_eea_co2(input_eea, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.config["energy"]["eurostat_report_year"] + if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -228,7 +228,7 @@ def co2_emissions_year( # TODO: move to own rule with sector-opts wildcard? -def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): +def build_carbon_budget(o, input_eurostat, input_eea, fn, emissions_scope, report_year): """ Distribute carbon budget following beta or exponential transition path. """ @@ -246,12 +246,12 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries = snakemake.config["countries"] e_1990 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year=1990 + countries, input_eurostat, input_eea, opts, emissions_scope, report_year, year=1990 ) # emissions at the beginning of the path (last year available 2018) e_0 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year=2018 + countries, input_eurostat, input_eea, opts, emissions_scope, report_year, year=2018 ) planning_horizons = snakemake.config["scenario"]["planning_horizons"] @@ -3369,7 +3369,7 @@ if __name__ == "__main__": emissions_scope = snakemake.config["energy"]["emissions"] report_year = snakemake.config["energy"]["eurostat_report_year"] build_carbon_budget( - o, snakemake.input.eurostat, fn, emissions_scope, report_year + o, snakemake.input.eurostat, snakemake.input.co2, fn, emissions_scope, report_year ) co2_cap = pd.read_csv(fn, index_col=0).squeeze() limit = co2_cap.loc[investment_year] From 87135fd95066a71972f593d3f09c99e63b06d0b8 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 15:44:01 +0200 Subject: [PATCH 08/77] remove more / --- rules/collect.smk | 2 +- rules/solve_perfect.smk | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/collect.smk b/rules/collect.smk index e17384e0..b76e8ced 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -69,7 +69,7 @@ rule solve_sector_networks_perfect: input: expand( RESULTS - + "/postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", **config["scenario"] ), diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 0f7ad49e..2bdb59b8 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -85,7 +85,7 @@ rule prepare_perfect_foresight: overrides="data/override_component_attrs", output: RESULTS - + "/prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", threads: 2 resources: mem_mb=10000, @@ -140,7 +140,7 @@ rule make_summary_perfect: input: **{ f"networks_{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}": RESULTS - + f"/postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc" + + f"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc" for simpl in config["scenario"]["simpl"] for clusters in config["scenario"]["clusters"] for opts in config["scenario"]["opts"] From 461ea66185342127620b839a16873aa168c8e724 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 15:44:31 +0200 Subject: [PATCH 09/77] add eea co2 emissions as input --- rules/postprocess.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index d5095358..dc9b2ea0 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -120,6 +120,7 @@ rule plot_summary: energy=RESULTS + "csvs/energy.csv", balances=RESULTS + "csvs/supply_energy.csv", eurostat=input_eurostat, + co2="data/eea/UNFCCC_v23.csv", output: costs=RESULTS + "graphs/costs.pdf", energy=RESULTS + "graphs/energy.pdf", From f2ff8beae83690ac5cc466aa1381b5d913b2b3e4 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 15:45:07 +0200 Subject: [PATCH 10/77] fix plot bugs --- scripts/plot_summary.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index af028116..b771f079 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -41,6 +41,8 @@ def rename_techs(label): "ground heat pump", "resistive heater", "Fischer-Tropsch", + "land transport fuel cell", + "land transport oil" ] rename_if_contains_dict = { @@ -268,7 +270,6 @@ def plot_balances(): i for i in balances_df.index.levels[0] if i not in co2_carriers ] - fig, ax = plt.subplots(figsize=(12, 8)) for k, v in balances.items(): df = balances_df.loc[v] @@ -280,7 +281,7 @@ def plot_balances(): # remove trailing link ports df.index = [ i[:-1] - if ((i not in ["co2", "NH3"]) and (i[-1:] in ["0", "1", "2", "3"])) + if ((i not in ["co2", "NH3", "H2"]) and (i[-1:] in ["0", "1", "2", "3"])) else i for i in df.index ] @@ -314,6 +315,8 @@ def plot_balances(): new_columns = df.columns.sort_values() + fig, ax = plt.subplots(figsize=(12, 8)) + df.loc[new_index, new_columns].T.plot( kind="bar", ax=ax, @@ -437,7 +440,7 @@ def historical_emissions(countries): return emissions -def plot_carbon_budget_distribution(input_eurostat): +def plot_carbon_budget_distribution(input_eurostat, input_eea): """ Plot historical carbon emissions in the EU and decarbonization path. """ @@ -452,6 +455,20 @@ def plot_carbon_budget_distribution(input_eurostat): plt.rcParams["xtick.labelsize"] = 20 plt.rcParams["ytick.labelsize"] = 20 + + path_cb = "results/" + snakemake.params.RDIR + "csvs/" + countries = snakemake.config["countries"] + emissions_scope = snakemake.config["energy"]["emissions"] + # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK + report_year = snakemake.config["energy"]["eurostat_report_year"] + e_1990 = co2_emissions_year( + countries, input_eurostat, input_eea, opts, emissions_scope, + report_year, year=1990 + ) + + + CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) + plt.figure(figsize=(10, 7)) gs1 = gridspec.GridSpec(1, 1) ax1 = plt.subplot(gs1[0, 0]) @@ -459,10 +476,6 @@ def plot_carbon_budget_distribution(input_eurostat): ax1.set_ylim([0, 5]) ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1]) - path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.config["countries"] - e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) - CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) ax1.plot(e_1990 * CO2_CAP[o], linewidth=3, color="dodgerblue", label=None) @@ -540,7 +553,7 @@ def plot_carbon_budget_distribution(input_eurostat): path_cb_plot = "results/" + snakemake.params.RDIR + "/graphs/" plt.savefig(path_cb_plot + "carbon_budget_plot.pdf", dpi=300) - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -561,4 +574,5 @@ if __name__ == "__main__": opts = sector_opts.split("-") for o in opts: if "cb" in o: - plot_carbon_budget_distribution(snakemake.input.eurostat) + plot_carbon_budget_distribution(snakemake.input.eurostat, + snakemake.input.co2) From 5989c287c4523ea57a1249a9a03d8509bedc5b43 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 15:45:35 +0200 Subject: [PATCH 11/77] adjust summary functions --- scripts/make_summary_perfect.py | 105 +++++++++++--------------------- 1 file changed, 34 insertions(+), 71 deletions(-) diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 8f21db76..579c0402 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -1,3 +1,15 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT + +""" +Create summary CSV files for all scenario runs with perfect foresight +including costs, capacities, capacity factors, curtailment, energy balances, +prices and other metrics. +""" + + from six import iteritems import pandas as pd @@ -11,7 +23,7 @@ from pypsa.descriptors import ( get_active_assets, ) -from helper import override_component_attrs +from _helpers import override_component_attrs from prepare_sector_network import prepare_costs @@ -50,8 +62,8 @@ def calculate_costs(n, label, costs): ).astype(int) capital_costs = active.mul(capital_costs, axis=0) discount = ( - n.investment_period_weightings["objective_weightings"] - / n.investment_period_weightings["time_weightings"] + n.investment_period_weightings["objective"] + / n.investment_period_weightings["years"] ) capital_costs_grouped = ( capital_costs.groupby(c.df.carrier).sum().mul(discount) @@ -66,20 +78,20 @@ def calculate_costs(n, label, costs): if c.name == "Link": p = ( - c.pnl.p0.multiply(n.snapshot_weightings.generator_weightings, axis=0) + c.pnl.p0.multiply(n.snapshot_weightings.generators, axis=0) .groupby(level=0) .sum() ) elif c.name == "Line": continue elif c.name == "StorageUnit": - p_all = c.pnl.p.multiply(n.snapshot_weightings.store_weightings, axis=0) + p_all = c.pnl.p.multiply(n.snapshot_weightings.stores, axis=0) p_all[p_all < 0.0] = 0.0 p = p_all.groupby(level=0).sum() else: p = ( round(c.pnl.p, ndigits=2) - .multiply(n.snapshot_weightings.generator_weightings, axis=0) + .multiply(n.snapshot_weightings.generators, axis=0) .groupby(level=0) .sum() ) @@ -285,7 +297,7 @@ def calculate_supply(n, label, supply): for end in [col[3:] for col in c.df.columns if col[:3] == "bus"]: - items = c.df.index[c.df["bus" + end].map(bus_map, na_action=False)] + items = c.df.index[c.df["bus" + end].map(bus_map).fillna(False)] if len(items) == 0: continue @@ -333,9 +345,9 @@ def calculate_supply_energy(n, label, supply_energy): continue if c.name == "Generator": - weightings = n.snapshot_weightings.generator_weightings + weightings = n.snapshot_weightings.generators else: - weightings = n.snapshot_weightings.store_weightings + weightings = n.snapshot_weightings.stores if i in ["oil", "co2", "H2"]: if c.name=="Load": @@ -364,7 +376,7 @@ def calculate_supply_energy(n, label, supply_energy): for end in [col[3:] for col in c.df.columns if col[:3] == "bus"]: - items = c.df.index[c.df["bus" + str(end)].map(bus_map, na_action=False)] + items = c.df.index[c.df["bus" + str(end)].map(bus_map).fillna(False)] if len(items) == 0: continue @@ -373,7 +385,7 @@ def calculate_supply_energy(n, label, supply_energy): (-1) * c.pnl["p" + end] .reindex(items, axis=1) - .multiply(n.snapshot_weightings.objective_weightings, axis=0) + .multiply(n.snapshot_weightings.objective, axis=0) .groupby(level=0) .sum() .groupby(c.df.loc[items, "carrier"], axis=1) @@ -593,11 +605,11 @@ def calculate_price_statistics(n, label, price_statistics): ) price_statistics.at["mean", label] = ( - n.buses_t.marginal_price[buses].unstack().mean() + n.buses_t.marginal_price[buses].mean().mean() ) price_statistics.at["standard_deviation", label] = ( - n.buses_t.marginal_price[buses].unstack().std() + n.buses_t.marginal_price[buses].droplevel(0).unstack().std() ) return price_statistics @@ -612,7 +624,7 @@ def calculate_co2_emissions(n, label, df): return weightings = n.snapshot_weightings.mul( - n.investment_period_weightings["time_weightings"] + n.investment_period_weightings["years"] .reindex(n.snapshots) .fillna(method="bfill") .fillna(1.0), @@ -624,7 +636,7 @@ def calculate_co2_emissions(n, label, df): if not gens.empty: em_pu = gens.carrier.map(emissions) / gens.efficiency em_pu = ( - weightings["generator_weightings"].to_frame("weightings") + weightings["generators"].to_frame("weightings") @ em_pu.to_frame("weightings").T ) emitted = n.generators_t.p[gens.index].mul(em_pu) @@ -648,52 +660,6 @@ def calculate_co2_emissions(n, label, df): return df - - -def calculate_cumulative_capacities(n, label, cum_cap): - # TODO - - investments = n.investment_periods - cols = pd.MultiIndex.from_product( - [ - cum_cap.columns.levels[0], - cum_cap.columns.levels[1], - cum_cap.columns.levels[2], - investments, - ], - names=cum_cap.columns.names[:3] + ["year"], - ) - cum_cap = cum_cap.reindex(cols, axis=1) - - learn_i = n.carriers[n.carriers.learning_rate != 0].index - - for c, attr in nominal_attrs.items(): - if "carrier" not in n.df(c) or n.df(c).empty: - continue - caps = ( - n.df(c)[n.df(c).carrier.isin(learn_i)] - .groupby([n.df(c).carrier, n.df(c).build_year])[ - opt_name.get(c, "p") + "_nom_opt" - ] - .sum() - ) - - if caps.empty: - continue - - caps = round( - caps.unstack().reindex(columns=investments).fillna(0).cumsum(axis=1) - ) - cum_cap = cum_cap.reindex(caps.index.union(cum_cap.index)) - - cum_cap.loc[caps.index, label] = caps.values - - return cum_cap - - - - - outputs = [ "nodal_costs", "nodal_capacities", @@ -711,14 +677,13 @@ outputs = [ "market_values", "metrics", "co2_emissions", - "cumulative_capacities", ] def make_summaries(networks_dict): columns = pd.MultiIndex.from_tuples( - networks_dict.keys(), names=["cluster", "lv", "opt", "year"] + networks_dict.keys(), names=["cluster", "lv", "opt"] ) df = {} @@ -760,30 +725,28 @@ def to_csv(df): if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if "snakemake" not in globals(): - from helper import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake('make_summary_perfect') networks_dict = { (clusters, lv, opts+sector_opts) : - snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec_s{simpl}_{clusters}_lv{lv}_{opts}_{sector_opts}_brownfield_all_years.nc' \ + "results/" + snakemake.config['run']["name"] + f'/postnetworks/elec_s{simpl}_{clusters}_l{lv}_{opts}_{sector_opts}_brownfield_all_years.nc' \ for simpl in snakemake.config['scenario']['simpl'] \ for clusters in snakemake.config['scenario']['clusters'] \ for opts in snakemake.config['scenario']['opts'] \ for sector_opts in snakemake.config['scenario']['sector_opts'] \ - for lv in snakemake.config['scenario']['lv'] \ + for lv in snakemake.config['scenario']['ll'] \ } print(networks_dict) - Nyears = 1 + nyears = 1 costs_db = prepare_costs( snakemake.input.costs, - snakemake.config["costs"]["USD2013_to_EUR2013"], - snakemake.config["costs"]["discountrate"], - Nyears, - snakemake.config["costs"]["lifetime"], + snakemake.config["costs"], + nyears, ) df = make_summaries(networks_dict) From f0d6d2be07eb6905652e42c6a5a7fb5e60f95e45 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 15:47:03 +0200 Subject: [PATCH 12/77] add multi-deacde --- scripts/solve_network.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 3456dba9..17a33602 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -156,7 +156,7 @@ def prepare_network(n, solve_opts=None, config=None): # http://journal.frontiersin.org/article/10.3389/fenrg.2015.00055/full # TODO: retrieve color and nice name from config n.add("Carrier", "load", color="#dd2e23", nice_name="Load shedding") - buses_i = n.buses.query("carrier == 'AC'").index + buses_i = n.buses.index # query("carrier == 'AC'").index #if not np.isscalar(load_shedding): # TODO: do not scale via sign attribute (use Eur/MWh instead of Eur/kWh) load_shedding = 1e2 # Eur/kWh @@ -589,6 +589,7 @@ def solve_network(n, config, opts="", **kwargs): track_iterations = cf_solving.get("track_iterations", False) min_iterations = cf_solving.get("min_iterations", 4) max_iterations = cf_solving.get("max_iterations", 6) + multi_horizon = True if config["foresight"] == "perfect" else False # add to network for extra_functionality n.config = config @@ -603,6 +604,7 @@ def solve_network(n, config, opts="", **kwargs): status, condition = n.optimize( solver_name=solver_name, extra_functionality=extra_functionality, + multi_investment_periods=multi_horizon, **solver_options, **kwargs, ) @@ -613,6 +615,7 @@ def solve_network(n, config, opts="", **kwargs): min_iterations=min_iterations, max_iterations=max_iterations, extra_functionality=extra_functionality, + multi_investment_periods=multi_horizon, **solver_options, **kwargs, ) @@ -633,12 +636,12 @@ if __name__ == "__main__": snakemake = mock_snakemake( "solve_sector_network_perfect", - configfiles="config.yaml", + #configfiles="config.yaml", simpl="", opts="", clusters="37", ll="v1.0", - sector_opts="cb40ex0-8760H-T-H-B-I-A-solar+p3-dist1", + sector_opts="cb40ex0-2190H-T-H-B-solar+p3-dist1", # planning_horizons="2030", ) configure_logging(snakemake) From c1cced44e0f37693458dd6510881db8aeff11634 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 18:17:27 +0200 Subject: [PATCH 13/77] add social discount rate --- config.default.yaml | 391 ++++++++++++++++++++++---------------------- 1 file changed, 197 insertions(+), 194 deletions(-) diff --git a/config.default.yaml b/config.default.yaml index 5218bc22..132fc1fb 100755 --- a/config.default.yaml +++ b/config.default.yaml @@ -570,6 +570,7 @@ costs: year: 2030 version: v0.5.0 rooftop_share: 0.14 # based on the potentials, assuming (0.1 kW/m2 and 10 m2/person) + social_discountrate: 0.02 fill_values: FOM: 0 VOM: 0 @@ -763,197 +764,199 @@ plotting: lines: "Transmission Lines" ror: "Run of River" - tech_colors: - # wind - onwind: "#235ebc" - onshore wind: "#235ebc" - offwind: "#6895dd" - offshore wind: "#6895dd" - offwind-ac: "#6895dd" - offshore wind (AC): "#6895dd" - offshore wind ac: "#6895dd" - offwind-dc: "#74c6f2" - offshore wind (DC): "#74c6f2" - offshore wind dc: "#74c6f2" - # water - hydro: '#298c81' - hydro reservoir: '#298c81' - ror: '#3dbfb0' - run of river: '#3dbfb0' - hydroelectricity: '#298c81' - PHS: '#51dbcc' - hydro+PHS: "#08ad97" - wave: '#a7d4cf' - # solar - solar: "#f9d002" - solar PV: "#f9d002" - solar thermal: '#ffbf2b' - solar rooftop: '#ffea80' - # gas - OCGT: '#e0986c' - OCGT marginal: '#e0986c' - OCGT-heat: '#e0986c' - gas boiler: '#db6a25' - gas boilers: '#db6a25' - gas boiler marginal: '#db6a25' - gas: '#e05b09' - fossil gas: '#e05b09' - natural gas: '#e05b09' - CCGT: '#a85522' - CCGT marginal: '#a85522' - allam: '#B98F76' - gas for industry co2 to atmosphere: '#692e0a' - gas for industry co2 to stored: '#8a3400' - gas for industry: '#853403' - gas for industry CC: '#692e0a' - gas pipeline: '#ebbca0' - gas pipeline new: '#a87c62' - # oil - oil: '#c9c9c9' - oil boiler: '#adadad' - agriculture machinery oil: '#949494' - shipping oil: "#808080" - land transport oil: '#afafaf' - # nuclear - Nuclear: '#ff8c00' - Nuclear marginal: '#ff8c00' - nuclear: '#ff8c00' - uranium: '#ff8c00' - # coal - Coal: '#545454' - coal: '#545454' - Coal marginal: '#545454' - solid: '#545454' - Lignite: '#826837' - lignite: '#826837' - Lignite marginal: '#826837' - # biomass - biogas: '#e3d37d' - biomass: '#baa741' - solid biomass: '#baa741' - solid biomass transport: '#baa741' - solid biomass for industry: '#7a6d26' - solid biomass for industry CC: '#47411c' - solid biomass for industry co2 from atmosphere: '#736412' - solid biomass for industry co2 to stored: '#47411c' - biomass boiler: '#8A9A5B' - biomass to liquid: '#32CD32' - BioSNG: '#123456' - # power transmission - lines: '#6c9459' - transmission lines: '#6c9459' - electricity distribution grid: '#97ad8c' - # electricity demand - Electric load: '#110d63' - electric demand: '#110d63' - electricity: '#110d63' - industry electricity: '#2d2a66' - industry new electricity: '#2d2a66' - agriculture electricity: '#494778' - # battery + EVs - battery: '#ace37f' - battery storage: '#ace37f' - home battery: '#80c944' - home battery storage: '#80c944' - BEV charger: '#baf238' - V2G: '#e5ffa8' - land transport EV: '#baf238' - Li ion: '#baf238' - # hot water storage - water tanks: '#e69487' - hot water storage: '#e69487' - hot water charging: '#e69487' - hot water discharging: '#e69487' - # heat demand - Heat load: '#cc1f1f' - heat: '#cc1f1f' - heat demand: '#cc1f1f' - rural heat: '#ff5c5c' - central heat: '#cc1f1f' - decentral heat: '#750606' - low-temperature heat for industry: '#8f2727' - process heat: '#ff0000' - agriculture heat: '#d9a5a5' - # heat supply - heat pumps: '#2fb537' - heat pump: '#2fb537' - air heat pump: '#36eb41' - ground heat pump: '#2fb537' - Ambient: '#98eb9d' - CHP: '#8a5751' - CHP CC: '#634643' - CHP heat: '#8a5751' - CHP electric: '#8a5751' - district heating: '#e8beac' - resistive heater: '#d8f9b8' - retrofitting: '#8487e8' - building retrofitting: '#8487e8' - # hydrogen - H2 for industry: "#f073da" - H2 for shipping: "#ebaee0" - H2: '#bf13a0' - hydrogen: '#bf13a0' - SMR: '#870c71' - SMR CC: '#4f1745' - H2 liquefaction: '#d647bd' - hydrogen storage: '#bf13a0' - H2 storage: '#bf13a0' - land transport fuel cell: '#6b3161' - H2 pipeline: '#f081dc' - H2 pipeline retrofitted: '#ba99b5' - H2 Fuel Cell: '#c251ae' - H2 Electrolysis: '#ff29d9' - # ammonia - NH3: '#46caf0' - ammonia: '#46caf0' - ammonia store: '#00ace0' - ammonia cracker: '#87d0e6' - Haber-Bosch: '#076987' - # syngas - Sabatier: '#9850ad' - methanation: '#c44ce6' - methane: '#c44ce6' - helmeth: '#e899ff' - # synfuels - Fischer-Tropsch: '#25c49a' - liquid: '#25c49a' - kerosene for aviation: '#a1ffe6' - naphtha for industry: '#57ebc4' - methanolisation: '#83d6d5' - methanol: '#468c8b' - shipping methanol: '#468c8b' - # co2 - CC: '#f29dae' - CCS: '#f29dae' - CO2 sequestration: '#f29dae' - DAC: '#ff5270' - co2 stored: '#f2385a' - co2: '#f29dae' - co2 vent: '#ffd4dc' - CO2 pipeline: '#f5627f' - # emissions - process emissions CC: '#000000' - process emissions: '#222222' - process emissions to stored: '#444444' - process emissions to atmosphere: '#888888' - oil emissions: '#aaaaaa' - shipping oil emissions: "#555555" - shipping methanol emissions: '#666666' - land transport oil emissions: '#777777' - agriculture machinery oil emissions: '#333333' - # other - shipping: '#03a2ff' - power-to-heat: '#2fb537' - power-to-gas: '#c44ce6' - power-to-H2: '#ff29d9' - power-to-liquid: '#25c49a' - gas-to-power/heat: '#ee8340' - waste: '#e3d37d' - other: '#000000' - geothermal: '#ba91b1' - AC-AC: "#70af1d" - AC line: "#70af1d" - links: "#8a1caf" - HVDC links: "#8a1caf" - DC-DC: "#8a1caf" - DC link: "#8a1caf" +tech_colors: + # wind + onwind: "#235ebc" + onshore wind: "#235ebc" + offwind: "#6895dd" + offshore wind: "#6895dd" + offwind-ac: "#6895dd" + offshore wind (AC): "#6895dd" + offshore wind ac: "#6895dd" + offwind-dc: "#74c6f2" + offshore wind (DC): "#74c6f2" + offshore wind dc: "#74c6f2" + # water + hydro: '#298c81' + hydro reservoir: '#298c81' + ror: '#3dbfb0' + run of river: '#3dbfb0' + hydroelectricity: '#298c81' + PHS: '#51dbcc' + hydro+PHS: "#08ad97" + wave: '#a7d4cf' + # solar + solar: "#f9d002" + solar PV: "#f9d002" + solar thermal: '#ffbf2b' + solar rooftop: '#ffea80' + # gas + OCGT: '#e0986c' + OCGT marginal: '#e0986c' + OCGT-heat: '#e0986c' + gas boiler: '#db6a25' + gas boilers: '#db6a25' + gas boiler marginal: '#db6a25' + gas: '#e05b09' + fossil gas: '#e05b09' + natural gas: '#e05b09' + CCGT: '#a85522' + CCGT marginal: '#a85522' + allam: '#B98F76' + gas for industry co2 to atmosphere: '#692e0a' + gas for industry co2 to stored: '#8a3400' + gas for industry: '#853403' + gas for industry CC: '#692e0a' + gas pipeline: '#ebbca0' + gas pipeline new: '#a87c62' + # oil + oil: '#c9c9c9' + imported oil: '#c9c9c9' + oil boiler: '#717171' + agriculture machinery oil: '#dddddd' + shipping oil: "#989898" + land transport oil: '#afafaf' + # nuclear + Nuclear: '#ff8c00' + Nuclear marginal: '#ff8c00' + nuclear: '#ff8c00' + uranium: '#ff8c00' + # coal + Coal: '#545454' + coal: '#545454' + Coal marginal: '#545454' + solid: '#545454' + Lignite: '#826837' + lignite: '#826837' + Lignite marginal: '#826837' + # biomass + biogas: '#e3d37d' + biomass: '#baa741' + solid biomass: '#baa741' + solid biomass transport: '#baa741' + solid biomass for industry: '#7a6d26' + solid biomass for industry CC: '#47411c' + solid biomass for industry co2 from atmosphere: '#736412' + solid biomass for industry co2 to stored: '#47411c' + biomass boiler: '#8A9A5B' + biomass to liquid: '#32CD32' + BioSNG: '#123456' + # power transmission + lines: '#6c9459' + transmission lines: '#6c9459' + electricity distribution grid: '#97ad8c' + # electricity demand + Electric load: '#110d63' + electric demand: '#110d63' + electricity: '#110d63' + industry electricity: '#4c47ab' + industry new electricity: '#2d2a66' + agriculture electricity: '#494778' + # battery + EVs + battery: '#ace37f' + battery storage: '#ace37f' + home battery: '#80c944' + home battery storage: '#80c944' + BEV charger: '#baf238' + V2G: '#e5ffa8' + land transport EV: '#baf238' + Li ion: '#baf238' + # hot water storage + water tanks: '#e69487' + hot water storage: '#e69487' + hot water charging: '#e69487' + hot water discharging: '#e69487' + # heat demand + Heat load: '#cc1f1f' + heat: '#cc1f1f' + heat demand: '#cc1f1f' + rural heat: '#ff5c5c' + central heat: '#cc1f1f' + decentral heat: '#750606' + low-temperature heat for industry: '#8f2727' + process heat: '#ff0000' + agriculture heat: '#d9a5a5' + # heat supply + heat pumps: '#2fb537' + heat pump: '#2fb537' + air heat pump: '#36eb41' + ground heat pump: '#2fb537' + Ambient: '#98eb9d' + CHP: '#8a5751' + CHP CC: '#634643' + CHP heat: '#8a5751' + CHP electric: '#8a5751' + district heating: '#e8beac' + resistive heater: '#d8f9b8' + retrofitting: '#8487e8' + building retrofitting: '#8487e8' + # hydrogen + H2 for industry: "#f073da" + H2 for shipping: "#ebaee0" + H2: '#bf13a0' + hydrogen: '#bf13a0' + SMR: '#870c71' + SMR CC: '#4f1745' + H2 liquefaction: '#d647bd' + hydrogen storage: '#bf13a0' + H2 storage: '#bf13a0' + land transport fuel cell: '#6b3161' + H2 pipeline: '#f081dc' + H2 pipeline retrofitted: '#ba99b5' + H2 Fuel Cell: '#c251ae' + H2 Electrolysis: '#ff29d9' + # ammonia + NH3: '#46caf0' + ammonia: '#46caf0' + ammonia store: '#00ace0' + ammonia cracker: '#87d0e6' + Haber-Bosch: '#076987' + # syngas + Sabatier: '#9850ad' + methanation: '#c44ce6' + methane: '#c44ce6' + helmeth: '#e899ff' + # synfuels + Fischer-Tropsch: '#25c49a' + liquid: '#25c49a' + kerosene for aviation: '#a1ffe6' + naphtha for industry: '#57ebc4' + methanolisation: '#83d6d5' + methanol: '#468c8b' + shipping methanol: '#468c8b' + # co2 + CC: '#f29dae' + CCS: '#f29dae' + CO2 sequestration: '#f29dae' + DAC: '#ff5270' + co2 stored: '#f2385a' + co2: '#f29dae' + co2 vent: '#ffd4dc' + CO2 pipeline: '#f5627f' + # emissions + process emissions CC: '#000000' + process emissions: '#222222' + process emissions to stored: '#444444' + process emissions to atmosphere: '#888888' + oil emissions: '#aaaaaa' + shipping oil emissions: "#555555" + shipping methanol emissions: '#666666' + land transport oil emissions: '#777777' + agriculture machinery oil emissions: '#333333' + # other + shipping: '#03a2ff' + power-to-heat: '#2fb537' + power-to-gas: '#c44ce6' + power-to-H2: '#ff29d9' + power-to-liquid: '#25c49a' + gas-to-power/heat: '#ee8340' + waste: '#e3d37d' + other: '#000000' + geothermal: '#ba91b1' + AC-AC: "#70af1d" + AC line: "#70af1d" + links: "#8a1caf" + HVDC links: "#8a1caf" + DC-DC: "#8a1caf" + DC link: "#8a1caf" + load: "red" From 1a7e9c241afd82e0dcf32e869f3903ca3135e5b5 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 18:17:52 +0200 Subject: [PATCH 14/77] remove output capital cost --- rules/solve_perfect.smk | 1 - 1 file changed, 1 deletion(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 2bdb59b8..41a1fd0b 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -157,7 +157,6 @@ rule make_summary_perfect: costs=RESULTS + "csvs/costs.csv", capacities=RESULTS + "csvs/capacities.csv", curtailment=RESULTS + "csvs/curtailment.csv", - capital_cost=RESULTS + "csvs/capital_cost.csv", energy=RESULTS + "csvs/energy.csv", supply=RESULTS + "csvs/supply.csv", supply_energy=RESULTS + "csvs/supply_energy.csv", From 91d2a4a05246bb44e37d5a516de51917300aad1b Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 18:18:53 +0200 Subject: [PATCH 15/77] adjust plotting --- scripts/make_summary_perfect.py | 2 +- scripts/plot_summary.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 579c0402..73ce2d40 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -730,7 +730,7 @@ if __name__ == "__main__": networks_dict = { (clusters, lv, opts+sector_opts) : - "results/" + snakemake.config['run']["name"] + f'/postnetworks/elec_s{simpl}_{clusters}_l{lv}_{opts}_{sector_opts}_brownfield_all_years.nc' \ + "results/" + snakemake.config['run']["name"] + f'postnetworks/elec_s{simpl}_{clusters}_l{lv}_{opts}_{sector_opts}_brownfield_all_years.nc' \ for simpl in snakemake.config['scenario']['simpl'] \ for clusters in snakemake.config['scenario']['clusters'] \ for opts in snakemake.config['scenario']['opts'] \ diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index b771f079..8d811e53 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -42,7 +42,9 @@ def rename_techs(label): "resistive heater", "Fischer-Tropsch", "land transport fuel cell", - "land transport oil" + "land transport oil", + "H2 for industry", + "shipping oil" ] rename_if_contains_dict = { @@ -216,6 +218,11 @@ def plot_energy(): logger.info(f"Total energy of {round(df.sum()[0])} TWh/a") + if df.empty: + fig, ax = plt.subplots(figsize=(12, 8)) + fig.savefig(snakemake.output.energy, bbox_inches="tight") + return + new_index = preferred_order.intersection(df.index).append( df.index.difference(preferred_order) ) From a72a620a60ea69119dea30087d414ab252ed068d Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 5 Apr 2023 18:19:35 +0200 Subject: [PATCH 16/77] add global constraints in concat, adjust bio stores --- scripts/prepare_perfect_foresight.py | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 06c8ffb7..17fc5257 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -52,6 +52,21 @@ def get_investment_weighting(time_weighting, r=0.01): axis=1, ) +def add_year_to_constraints(n, baseyear): + """ + Parameters + ---------- + n : pypsa.Network + baseyear : int + year in which optimized assets are built + """ + + for c in n.iterate_components(["GlobalConstraint"]): + + c.df["investment_period"] = baseyear + c.df.rename(index=lambda x: x + "-" + str(baseyear), inplace=True) + + # -------------------------------------------------------------------- def concat_networks(years): """Concat given pypsa networks and adds build_year. @@ -121,6 +136,12 @@ def concat_networks(years): n.snapshot_weightings.loc[year,:] = network.snapshot_weightings.values + + # (3) global constraints + for component in network.iterate_components(["GlobalConstraint"]): + add_year_to_constraints(n, year) + import_components_from_dataframe(n, component.df.index, component.name) + # set investment periods n.investment_periods = n.snapshots.levels[0] # weighting of the investment period -> assuming last period same weighting as the period before @@ -132,6 +153,7 @@ def concat_networks(years): n.investment_period_weightings["objective"] = objective_w # all former static loads are now time-dependent -> set static = 0 n.loads["p_set"] = 0 + n.loads_t.p_set.fillna(0,inplace=True) return n @@ -147,6 +169,11 @@ def adjust_stores(n): co2_i = n.stores[n.stores.carrier.isin(non_cyclic_store)].index n.stores.loc[co2_i, "e_cyclic_per_period"] = False n.stores.loc[co2_i, "e_cyclic"] = False + # e_initial at beginning of each investment period + e_initial_store = ["solid biomass", "biogas"] + co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index + n.stores.loc[co2_i, "e_initial"] *= 10 + n.stores.loc[co2_i, "e_nom"] *= 10 return n @@ -236,9 +263,9 @@ if __name__ == "__main__": 'prepare_perfect_foresight', simpl='', opts="", - clusters="45", - lv=1.0, - sector_opts='1p7-365H-T-H-B-I-A-solar+p3-dist1', + clusters="37", + ll=1.0, + sector_opts='cb40ex0-2190H-T-H-B-solar+p3-dist1', ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) From 5f554ab28fa326ddb140a655733d6adcd9a3cc65 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Apr 2023 13:57:26 +0000 Subject: [PATCH 17/77] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/collect.smk | 2 + rules/solve_perfect.smk | 5 +- scripts/make_summary_perfect.py | 120 +++++++------------ scripts/plot_summary.py | 23 ++-- scripts/prepare_perfect_foresight.py | 169 +++++++++++++++------------ scripts/prepare_sector_network.py | 31 ++++- scripts/solve_network.py | 11 +- 7 files changed, 188 insertions(+), 173 deletions(-) diff --git a/rules/collect.smk b/rules/collect.smk index b76e8ced..1d7e21e5 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -65,6 +65,7 @@ rule solve_sector_networks: **config["scenario"] ), + rule solve_sector_networks_perfect: input: expand( @@ -73,6 +74,7 @@ rule solve_sector_networks_perfect: **config["scenario"] ), + rule plot_networks: input: expand( diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 41a1fd0b..49561c36 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -172,10 +172,7 @@ rule make_summary_perfect: log: LOGS + "make_summary_perfect.log", benchmark: - ( - BENCHMARKS - + "make_summary_perfect" - ) + (BENCHMARKS + "make_summary_perfect") conda: "../envs/environment.yaml" script: diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 73ce2d40..4f38b87a 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -4,39 +4,33 @@ # SPDX-License-Identifier: MIT """ -Create summary CSV files for all scenario runs with perfect foresight -including costs, capacities, capacity factors, curtailment, energy balances, -prices and other metrics. +Create summary CSV files for all scenario runs with perfect foresight including +costs, capacities, capacity factors, curtailment, energy balances, prices and +other metrics. """ -from six import iteritems - -import pandas as pd - import numpy as np - +import pandas as pd import pypsa - -from pypsa.descriptors import ( - nominal_attrs, - get_active_assets, -) - from _helpers import override_component_attrs - +from make_summary import ( + assign_carriers, + assign_locations, + calculate_cfs, + calculate_nodal_cfs, + calculate_nodal_costs, +) from prepare_sector_network import prepare_costs - -from make_summary import (assign_carriers, assign_locations, - calculate_cfs, calculate_nodal_cfs, - calculate_nodal_costs) +from pypsa.descriptors import get_active_assets, nominal_attrs +from six import iteritems idx = pd.IndexSlice opt_name = {"Store": "e", "Line": "s", "Transformer": "s"} -def calculate_costs(n, label, costs): +def calculate_costs(n, label, costs): investments = n.investment_periods cols = pd.MultiIndex.from_product( [ @@ -65,9 +59,7 @@ def calculate_costs(n, label, costs): n.investment_period_weightings["objective"] / n.investment_period_weightings["years"] ) - capital_costs_grouped = ( - capital_costs.groupby(c.df.carrier).sum().mul(discount) - ) + capital_costs_grouped = capital_costs.groupby(c.df.carrier).sum().mul(discount) capital_costs_grouped = pd.concat([capital_costs_grouped], keys=["capital"]) capital_costs_grouped = pd.concat([capital_costs_grouped], keys=[c.list_name]) @@ -176,7 +168,6 @@ def calculate_nodal_capacities(n, label, nodal_capacities): def calculate_capacities(n, label, capacities): - investments = n.investment_periods cols = pd.MultiIndex.from_product( [ @@ -202,9 +193,7 @@ def calculate_capacities(n, label, capacities): caps = c.df[opt_name.get(c.name, "p") + "_nom_opt"] caps = active.mul(caps, axis=0) capacities_grouped = ( - caps.groupby(c.df.carrier) - .sum() - .drop("load", errors="ignore") + caps.groupby(c.df.carrier).sum().drop("load", errors="ignore") ) capacities_grouped = pd.concat([capacities_grouped], keys=[c.list_name]) @@ -218,7 +207,6 @@ def calculate_capacities(n, label, capacities): def calculate_curtailment(n, label, curtailment): - avail = ( n.generators_t.p_max_pu.multiply(n.generators.p_nom_opt) .sum() @@ -233,9 +221,7 @@ def calculate_curtailment(n, label, curtailment): def calculate_energy(n, label, energy): - for c in n.iterate_components(n.one_port_components | n.branch_components): - if c.name in n.one_port_components: c_energies = ( c.pnl.p.multiply(n.snapshot_weightings, axis=0) @@ -265,7 +251,10 @@ def calculate_energy(n, label, energy): def calculate_supply(n, label, supply): - """calculate the max dispatch of each component at the buses aggregated by carrier""" + """ + Calculate the max dispatch of each component at the buses aggregated by + carrier. + """ bus_carriers = n.buses.carrier.unique() @@ -274,7 +263,6 @@ def calculate_supply(n, label, supply): bus_map.at[""] = False for c in n.iterate_components(n.one_port_components): - items = c.df.index[c.df.bus.map(bus_map).fillna(False)] if len(items) == 0: @@ -294,9 +282,7 @@ def calculate_supply(n, label, supply): supply.loc[s.index, label] = s for c in n.iterate_components(n.branch_components): - for end in [col[3:] for col in c.df.columns if col[:3] == "bus"]: - items = c.df.index[c.df["bus" + end].map(bus_map).fillna(False)] if len(items) == 0: @@ -317,7 +303,10 @@ def calculate_supply(n, label, supply): def calculate_supply_energy(n, label, supply_energy): - """calculate the total energy supply/consuption of each component at the buses aggregated by carrier""" + """ + Calculate the total energy supply/consuption of each component at the buses + aggregated by carrier. + """ investments = n.investment_periods cols = pd.MultiIndex.from_product( @@ -338,7 +327,6 @@ def calculate_supply_energy(n, label, supply_energy): bus_map.at[""] = False for c in n.iterate_components(n.one_port_components): - items = c.df.index[c.df.bus.map(bus_map).fillna(False)] if len(items) == 0: @@ -350,9 +338,11 @@ def calculate_supply_energy(n, label, supply_energy): weightings = n.snapshot_weightings.stores if i in ["oil", "co2", "H2"]: - if c.name=="Load": - c.df.loc[items, "carrier"] = [load.split("-202")[0] for load in items] - if i=="oil" and c.name=="Generator": + if c.name == "Load": + c.df.loc[items, "carrier"] = [ + load.split("-202")[0] for load in items + ] + if i == "oil" and c.name == "Generator": c.df.loc[items, "carrier"] = "imported oil" s = ( c.pnl.p[items] @@ -373,9 +363,7 @@ def calculate_supply_energy(n, label, supply_energy): supply_energy.loc[s.index, label] = s.values for c in n.iterate_components(n.branch_components): - for end in [col[3:] for col in c.df.columns if col[:3] == "bus"]: - items = c.df.index[c.df["bus" + str(end)].map(bus_map).fillna(False)] if len(items) == 0: @@ -405,7 +393,6 @@ def calculate_supply_energy(n, label, supply_energy): def calculate_metrics(n, label, metrics): - metrics = metrics.reindex( pd.Index( [ @@ -438,15 +425,10 @@ def calculate_metrics(n, label, metrics): def calculate_prices(n, label, prices): - prices = prices.reindex(prices.index.union(n.buses.carrier.unique())) # WARNING: this is time-averaged, see weighted_prices for load-weighted average - prices[label] = ( - n.buses_t.marginal_price.mean() - .groupby(n.buses.carrier) - .mean() - ) + prices[label] = n.buses_t.marginal_price.mean().groupby(n.buses.carrier).mean() return prices @@ -484,7 +466,6 @@ def calculate_weighted_prices(n, label, weighted_prices): } for carrier in link_loads: - if carrier == "electricity": suffix = "" elif carrier[:5] == "space": @@ -503,7 +484,6 @@ def calculate_weighted_prices(n, label, weighted_prices): load = n.loads_t.p_set.reindex(buses, axis=1) for tech in link_loads[carrier]: - names = n.links.index[n.links.index.to_series().str[-len(tech) :] == tech] if names.empty: @@ -585,7 +565,6 @@ def calculate_market_values(n, label, market_values): def calculate_price_statistics(n, label, price_statistics): - price_statistics = price_statistics.reindex( price_statistics.index.union( pd.Index(["zero_hours", "mean", "standard_deviation"]) @@ -604,9 +583,7 @@ def calculate_price_statistics(n, label, price_statistics): df.shape[0] * df.shape[1] ) - price_statistics.at["mean", label] = ( - n.buses_t.marginal_price[buses].mean().mean() - ) + price_statistics.at["mean", label] = n.buses_t.marginal_price[buses].mean().mean() price_statistics.at["standard_deviation", label] = ( n.buses_t.marginal_price[buses].droplevel(0).unstack().std() @@ -616,7 +593,6 @@ def calculate_price_statistics(n, label, price_statistics): def calculate_co2_emissions(n, label, df): - carattr = "co2_emissions" emissions = n.carriers.query(f"{carattr} != 0")[carattr] @@ -642,11 +618,7 @@ def calculate_co2_emissions(n, label, df): emitted = n.generators_t.p[gens.index].mul(em_pu) emitted_grouped = ( - emitted.groupby(level=0) - .sum() - .groupby(n.generators.carrier, axis=1) - .sum() - .T + emitted.groupby(level=0).sum().groupby(n.generators.carrier, axis=1).sum().T ) df = df.reindex(emitted_grouped.index.union(df.index)) @@ -681,7 +653,6 @@ outputs = [ def make_summaries(networks_dict): - columns = pd.MultiIndex.from_tuples( networks_dict.keys(), names=["cluster", "lv", "opt"] ) @@ -694,9 +665,7 @@ def make_summaries(networks_dict): for label, filename in iteritems(networks_dict): print(label, filename) try: - n = pypsa.Network( - filename, override_component_attrs=overrides - ) + n = pypsa.Network(filename, override_component_attrs=overrides) except OSError: print(label, " not solved yet.") continue @@ -715,33 +684,32 @@ def make_summaries(networks_dict): def to_csv(df): - for key in df: df[key] = df[key].apply(lambda x: pd.to_numeric(x)) df[key].to_csv(snakemake.output[key]) -#%% +# %% if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake('make_summary_perfect') + + snakemake = mock_snakemake("make_summary_perfect") networks_dict = { - (clusters, lv, opts+sector_opts) : - "results/" + snakemake.config['run']["name"] + f'postnetworks/elec_s{simpl}_{clusters}_l{lv}_{opts}_{sector_opts}_brownfield_all_years.nc' \ - for simpl in snakemake.config['scenario']['simpl'] \ - for clusters in snakemake.config['scenario']['clusters'] \ - for opts in snakemake.config['scenario']['opts'] \ - for sector_opts in snakemake.config['scenario']['sector_opts'] \ - for lv in snakemake.config['scenario']['ll'] \ + (clusters, lv, opts + sector_opts): "results/" + + snakemake.config["run"]["name"] + + f"postnetworks/elec_s{simpl}_{clusters}_l{lv}_{opts}_{sector_opts}_brownfield_all_years.nc" + for simpl in snakemake.config["scenario"]["simpl"] + for clusters in snakemake.config["scenario"]["clusters"] + for opts in snakemake.config["scenario"]["opts"] + for sector_opts in snakemake.config["scenario"]["sector_opts"] + for lv in snakemake.config["scenario"]["ll"] } - print(networks_dict) - nyears = 1 costs_db = prepare_costs( snakemake.input.costs, diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 8d811e53..9f6e55b4 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -44,7 +44,7 @@ def rename_techs(label): "land transport fuel cell", "land transport oil", "H2 for industry", - "shipping oil" + "shipping oil", ] rename_if_contains_dict = { @@ -277,7 +277,6 @@ def plot_balances(): i for i in balances_df.index.levels[0] if i not in co2_carriers ] - for k, v in balances.items(): df = balances_df.loc[v] df = df.groupby(df.index.get_level_values(2)).sum() @@ -462,18 +461,21 @@ def plot_carbon_budget_distribution(input_eurostat, input_eea): plt.rcParams["xtick.labelsize"] = 20 plt.rcParams["ytick.labelsize"] = 20 - path_cb = "results/" + snakemake.params.RDIR + "csvs/" countries = snakemake.config["countries"] emissions_scope = snakemake.config["energy"]["emissions"] # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK report_year = snakemake.config["energy"]["eurostat_report_year"] e_1990 = co2_emissions_year( - countries, input_eurostat, input_eea, opts, emissions_scope, - report_year, year=1990 + countries, + input_eurostat, + input_eea, + opts, + emissions_scope, + report_year, + year=1990, ) - CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) plt.figure(figsize=(10, 7)) @@ -483,7 +485,6 @@ def plot_carbon_budget_distribution(input_eurostat, input_eea): ax1.set_ylim([0, 5]) ax1.set_xlim([1990, snakemake.config["scenario"]["planning_horizons"][-1] + 1]) - ax1.plot(e_1990 * CO2_CAP[o], linewidth=3, color="dodgerblue", label=None) emissions = historical_emissions(countries) @@ -560,7 +561,8 @@ def plot_carbon_budget_distribution(input_eurostat, input_eea): path_cb_plot = "results/" + snakemake.params.RDIR + "/graphs/" plt.savefig(path_cb_plot + "carbon_budget_plot.pdf", dpi=300) -#%% + +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -581,5 +583,6 @@ if __name__ == "__main__": opts = sector_opts.split("-") for o in opts: if "cb" in o: - plot_carbon_budget_distribution(snakemake.input.eurostat, - snakemake.input.co2) + plot_carbon_budget_distribution( + snakemake.input.eurostat, snakemake.input.co2 + ) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 17fc5257..ba07e178 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -7,43 +7,50 @@ Concats pypsa networks of single investment periods to one network. """ -import pypsa -import pandas as pd -from _helpers import override_component_attrs, update_config_with_sector_opts -from pypsa.io import import_components_from_dataframe -from add_existing_baseyear import add_build_year_to_new_assets -from six import iterkeys -from pypsa.descriptors import expand_series -import re import logging +import re + +import pandas as pd +import pypsa +from _helpers import override_component_attrs, update_config_with_sector_opts +from add_existing_baseyear import add_build_year_to_new_assets +from pypsa.descriptors import expand_series +from pypsa.io import import_components_from_dataframe +from six import iterkeys + logger = logging.getLogger(__name__) + # helper functions --------------------------------------------------- def get_missing(df, n, c): - """Get in network n missing assets of df for component c. + """ + Get in network n missing assets of df for component c. - Input: - df: pandas DataFrame, static values of pypsa components - n : pypsa Network to which new assets should be added - c : string, pypsa component.list_name (e.g. "generators") - Return: - pd.DataFrame with static values of missing assets - """ + Input: + df: pandas DataFrame, static values of pypsa components + n : pypsa Network to which new assets should be added + c : string, pypsa component.list_name (e.g. "generators") + Return: + pd.DataFrame with static values of missing assets + """ df_final = getattr(n, c) missing_i = df.index.difference(df_final.index) return df.loc[missing_i] def get_social_discount(t, r=0.01): - """Calculate for a given time t the social discount.""" + """ + Calculate for a given time t the social discount. + """ return 1 / (1 + r) ** t def get_investment_weighting(time_weighting, r=0.01): - """Define cost weighting. + """ + Define cost weighting. - Returns cost weightings depending on the the time_weighting (pd.Series) - and the social discountrate r + Returns cost weightings depending on the the time_weighting + (pd.Series) and the social discountrate r """ end = time_weighting.cumsum() start = time_weighting.cumsum().shift().fillna(0) @@ -52,6 +59,7 @@ def get_investment_weighting(time_weighting, r=0.01): axis=1, ) + def add_year_to_constraints(n, baseyear): """ Parameters @@ -62,28 +70,27 @@ def add_year_to_constraints(n, baseyear): """ for c in n.iterate_components(["GlobalConstraint"]): - c.df["investment_period"] = baseyear c.df.rename(index=lambda x: x + "-" + str(baseyear), inplace=True) # -------------------------------------------------------------------- def concat_networks(years): - """Concat given pypsa networks and adds build_year. + """ + Concat given pypsa networks and adds build_year. - Return: - n : pypsa.Network for the whole planning horizon - - """ + Return: + n : pypsa.Network for the whole planning horizon + """ # input paths of sector coupling networks network_paths = [snakemake.input.brownfield_network] + [ - snakemake.input[f"network_{year}"] for year in years[1:]] + snakemake.input[f"network_{year}"] for year in years[1:] + ] # final concatenated network overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(override_component_attrs=overrides) - # iterate over single year networks and concat to perfect foresight network for i, network_path in enumerate(network_paths): year = years[i] @@ -102,7 +109,6 @@ def concat_networks(years): for component in network.iterate_components( ["Generator", "Link", "Store", "Load", "Line", "StorageUnit"] ): - df_year = component.df.copy() missing = get_missing(df_year, n, component.list_name) @@ -117,14 +123,14 @@ def concat_networks(years): pnl = getattr(n, component.list_name + "_t") for k in iterkeys(component.pnl): pnl_year = component.pnl[k].copy().reindex(snapshots, level=1) - if pnl_year.empty and ~(component.name=="Load" and k=="p_set"): continue + if pnl_year.empty and ~(component.name == "Load" and k == "p_set"): + continue if component.name == "Load": static_load = network.loads.loc[network.loads.p_set != 0] - static_load_t = expand_series( - static_load.p_set, network_sns - ).T - pnl_year = pd.concat([pnl_year.reindex(network_sns), - static_load_t], axis=1) + static_load_t = expand_series(static_load.p_set, network_sns).T + pnl_year = pd.concat( + [pnl_year.reindex(network_sns), static_load_t], axis=1 + ) columns = (pnl[k].columns.union(pnl_year.columns)).unique() pnl[k] = pnl[k].reindex(columns=columns) pnl[k].loc[pnl_year.index, pnl_year.columns] = pnl_year @@ -134,8 +140,7 @@ def concat_networks(years): cols = pnl_year.columns.difference(pnl[k].columns) pnl[k] = pd.concat([pnl[k], pnl_year[cols]], axis=1) - - n.snapshot_weightings.loc[year,:] = network.snapshot_weightings.values + n.snapshot_weightings.loc[year, :] = network.snapshot_weightings.values # (3) global constraints for component in network.iterate_components(["GlobalConstraint"]): @@ -148,18 +153,22 @@ def concat_networks(years): time_w = n.investment_periods.to_series().diff().shift(-1).fillna(method="ffill") n.investment_period_weightings["years"] = time_w # set objective weightings - objective_w = get_investment_weighting(n.investment_period_weightings["years"], - social_discountrate) + objective_w = get_investment_weighting( + n.investment_period_weightings["years"], social_discountrate + ) n.investment_period_weightings["objective"] = objective_w # all former static loads are now time-dependent -> set static = 0 n.loads["p_set"] = 0 - n.loads_t.p_set.fillna(0,inplace=True) + n.loads_t.p_set.fillna(0, inplace=True) return n + def adjust_stores(n): - """Make sure that stores still behave cyclic over one year and not whole - modelling horizon.""" + """ + Make sure that stores still behave cyclic over one year and not whole + modelling horizon. + """ # cylclic constraint cyclic_i = n.stores[n.stores.e_cyclic].index n.stores.loc[cyclic_i, "e_cyclic_per_period"] = True @@ -177,42 +186,50 @@ def adjust_stores(n): return n + def set_phase_out(n, carrier, ct, phase_out_year): - """Set planned phase outs for given carrier,country (ct) and planned year - of phase out (phase_out_year).""" - df = n.links[(n.links.carrier.isin(carrier))& (n.links.bus1.str[:2]==ct)] + """ + Set planned phase outs for given carrier,country (ct) and planned year of + phase out (phase_out_year). + """ + df = n.links[(n.links.carrier.isin(carrier)) & (n.links.bus1.str[:2] == ct)] # assets which are going to be phased out before end of their lifetime assets_i = df[df[["build_year", "lifetime"]].sum(axis=1) > phase_out_year].index build_year = n.links.loc[assets_i, "build_year"] # adjust lifetime n.links.loc[assets_i, "lifetime"] = (phase_out_year - build_year).astype(float) + def set_all_phase_outs(n): # TODO move this to a csv or to the config - planned= [(["nuclear"], "DE", 2022), - (["nuclear"], "BE", 2025), - (["nuclear"], "ES", 2027), - (["coal", "lignite"], "DE", 2038), - (["coal", "lignite"], "ES", 2027), - (["coal", "lignite"], "FR", 2022), - (["coal", "lignite"], "GB", 2024), - (["coal", "lignite"], "IT", 2025)] + planned = [ + (["nuclear"], "DE", 2022), + (["nuclear"], "BE", 2025), + (["nuclear"], "ES", 2027), + (["coal", "lignite"], "DE", 2038), + (["coal", "lignite"], "ES", 2027), + (["coal", "lignite"], "FR", 2022), + (["coal", "lignite"], "GB", 2024), + (["coal", "lignite"], "IT", 2025), + ] for carrier, ct, phase_out_year in planned: - set_phase_out(n, carrier,ct, phase_out_year) + set_phase_out(n, carrier, ct, phase_out_year) # remove assets which are already phased out - remove_i = n.links[n.links[["build_year", "lifetime"]].sum(axis=1) Date: Tue, 22 Aug 2023 10:23:16 +0200 Subject: [PATCH 18/77] add *.orig to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0adf0ae6..e126f9c5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__ *dconf gurobi.log .vscode +*.orig /bak /resources From 890c80e0472810204288e413dba37a8aa93896ce Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 22 Aug 2023 11:00:25 +0200 Subject: [PATCH 19/77] remove overrides --- scripts/prepare_perfect_foresight.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index ba07e178..75a7d7a9 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -12,7 +12,7 @@ import re import pandas as pd import pypsa -from _helpers import override_component_attrs, update_config_with_sector_opts +from _helpers import update_config_with_sector_opts from add_existing_baseyear import add_build_year_to_new_assets from pypsa.descriptors import expand_series from pypsa.io import import_components_from_dataframe @@ -88,13 +88,12 @@ def concat_networks(years): snakemake.input[f"network_{year}"] for year in years[1:] ] # final concatenated network - overrides = override_component_attrs(snakemake.input.overrides) - n = pypsa.Network(override_component_attrs=overrides) + n = pypsa.Network() # iterate over single year networks and concat to perfect foresight network for i, network_path in enumerate(network_paths): year = years[i] - network = pypsa.Network(network_path, override_component_attrs=overrides) + network = pypsa.Network(network_path) network.lines["carrier"] = "AC" add_build_year_to_new_assets(network, year) @@ -286,8 +285,8 @@ if __name__ == "__main__": simpl="", opts="", clusters="37", - ll=1.0, - sector_opts="cb40ex0-2190H-T-H-B-solar+p3-dist1", + ll="v1.0", + sector_opts="Co2L0-8760H-T-H-B-I-A-solar+p3-dist1", ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) From e5673731aba7af23579377048b67e79f5eaa4072 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 22 Aug 2023 11:01:00 +0200 Subject: [PATCH 20/77] make plot_network conditional --- rules/postprocess.smk | 97 ++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ae95b201..ca1d6fac 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -7,56 +7,57 @@ localrules: copy_config, copy_conda_env, +if config["foresight"] != "perfect": + rule plot_network: + params: + foresight=config["foresight"], + plotting=config["plotting"], + input: + network=RESULTS + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", + regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + output: + map=RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", + today=RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}-today.pdf", + threads: 2 + resources: + mem_mb=10000, + benchmark: + ( + BENCHMARKS + + "plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" + ) + conda: + "../envs/environment.yaml" + script: + "../scripts/plot_network.py" -rule plot_network: - params: - foresight=config["foresight"], - plotting=config["plotting"], - input: - network=RESULTS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - output: - map=RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", - today=RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}-today.pdf", - threads: 2 - resources: - mem_mb=10000, - benchmark: - ( +if config["foresight"] == "perfect": + rule plot_network: + params: + foresight=config["foresight"], + plotting=config["plotting"], + input: + network=RESULTS + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", + regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", + output: + map=RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{year}.pdf", + today=RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}-today.pdf", + threads: 2 + resources: + mem_mb=10000, + benchmark: BENCHMARKS - + "plot_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) - conda: - "../envs/environment.yaml" - script: - "../scripts/plot_network.py" - -rule plot_network_perfect: - params: - foresight=config["foresight"], - plotting=config["plotting"], - input: - network=RESULTS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", - output: - map=RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{year}.pdf", - today=RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}-today.pdf", - threads: 2 - resources: - mem_mb=10000, - benchmark: - BENCHMARKS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}_brownfield_all_years_benchmark.txt", - conda: - "../envs/environment.yaml" - script: - "../scripts/plot_network.py" + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}_brownfield_all_years_benchmark.txt", + conda: + "../envs/environment.yaml" + script: + "../scripts/plot_network.py" From c6294767213da745930c6526655426d7a675fef4 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 22 Aug 2023 11:01:51 +0200 Subject: [PATCH 21/77] adapt to new config path --- rules/solve_perfect.smk | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 162921e2..77ee40f4 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -2,6 +2,11 @@ # # SPDX-License-Identifier: MIT rule add_existing_baseyear: + params: + baseyear=config["scenario"]["planning_horizons"][0], + sector=config["sector"], + existing_capacities=config["existing_capacities"], + costs=config["costs"], input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -39,6 +44,10 @@ rule add_existing_baseyear: rule add_brownfield: + params: + H2_retrofit=config["sector"]["H2_retrofit"], + H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], + threshold_capacity=config["existing_capacities"]["threshold_capacity"], input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -105,7 +114,7 @@ rule solve_sector_network_perfect: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", costs="data/costs_2030.csv", - config=RESULTS + "configs/config.yaml", + config=RESULTS + "config.yaml", output: RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", From a21b1f2d10c919f6a024f62708229a8b7002691e Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 23 Aug 2023 13:23:27 +0200 Subject: [PATCH 22/77] add params --- rules/solve_perfect.smk | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 77ee40f4..93c5096a 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -110,6 +110,13 @@ rule prepare_perfect_foresight: rule solve_sector_network_perfect: + params: + solving=config["solving"], + foresight=config["foresight"], + planning_horizons=config["scenario"]["planning_horizons"], + co2_sequestration_potential=config["sector"].get( + "co2_sequestration_potential", 200 + ), input: network=RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", From f47ce1c432b6e558feb7e040f7449e0c34323988 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 23 Aug 2023 13:24:25 +0200 Subject: [PATCH 23/77] add lifetime --- scripts/add_electricity.py | 1 + scripts/prepare_sector_network.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 56375800..c485aa99 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -406,6 +406,7 @@ def attach_wind_and_solar( capital_cost=capital_cost, efficiency=costs.at[supcar, "efficiency"], p_max_pu=ds["profile"].transpose("time", "bus").to_pandas(), + lifetime=costs.at[supcar, "lifetime"], ) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 5ff10109..37667ac5 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1150,6 +1150,7 @@ def add_storage_and_grids(n, costs): e_cyclic=True, carrier="H2 Store", capital_cost=h2_capital_cost, + lifetime=costs.at["hydrogen storage tank type 1 including compressor", "lifetime"], ) if options["gas_network"] or options["H2_retrofit"]: @@ -3273,20 +3274,20 @@ def set_temporal_aggregation(n, opts, solver_name): break return n - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake( "prepare_sector_network", - configfiles="test/config.overnight.yaml", + # configfiles="test/config.overnight.yaml", simpl="", opts="", - clusters="5", - ll="v1.5", - sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1", - planning_horizons="2030", + clusters="37", + ll="v1.0", + sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", + planning_horizons="2020", ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -3319,7 +3320,7 @@ if __name__ == "__main__": spatial = define_spatial(pop_layout.index, options) - if snakemake.params.foresight == "myopic": + if snakemake.params.foresight in ["myopic", "perfect"]: add_lifetime_wind_solar(n, costs) conventional = snakemake.params.conventional_carriers @@ -3434,7 +3435,7 @@ if __name__ == "__main__": if options["electricity_grid_connection"]: add_electricity_grid_connection(n, costs) - first_year_myopic = (snakemake.params.foresight == "myopic") and ( + first_year_myopic = (snakemake.params.foresight in ["myopic", "perfect"]) and ( snakemake.params.planning_horizons[0] == investment_year ) From 20bf1cdaf568ff42d20aaf754235aaf1449388b6 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 23 Aug 2023 13:26:08 +0200 Subject: [PATCH 24/77] deal with global constraints --- scripts/prepare_perfect_foresight.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 75a7d7a9..99534104 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -143,8 +143,8 @@ def concat_networks(years): # (3) global constraints for component in network.iterate_components(["GlobalConstraint"]): - add_year_to_constraints(n, year) - import_components_from_dataframe(n, component.df.index, component.name) + add_year_to_constraints(network, year) + import_components_from_dataframe(n, component.df, component.name) # set investment periods n.investment_periods = n.snapshots.levels[0] @@ -275,6 +275,19 @@ def set_carbon_constraints(n, opts): return n +def adjust_lvlimit(n): + c = "GlobalConstraint" + cols = ['carrier_attribute', 'sense', "constant", "type"] + glc_type = "transmission_volume_expansion_limit" + if (n.df(c)[n.df(c).type==glc_type][cols].nunique()==1).all(): + glc = n.df(c)[n.df(c).type==glc_type][cols].iloc[[0]] + glc.index = pd.Index(["lv_limit"]) + remove_i = n.df(c)[n.df(c).type==glc_type].index + n.mremove(c, remove_i) + import_components_from_dataframe(n, glc, c) + + return n + # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -286,7 +299,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="Co2L0-8760H-T-H-B-I-A-solar+p3-dist1", + sector_opts="2p0-co2min-8760H-T-H-B-I-A-solar+p3-dist1", ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) @@ -306,7 +319,9 @@ if __name__ == "__main__": # concat prenetworks of planning horizon to single network ------------ n = concat_networks(years) - + + # adjust global constraints lv limit if the same for all years + n = adjust_lvlimit(n) # set phase outs set_all_phase_outs(n) # adjust stores to multi period investment From 6585e2c623e395d16ac37fe646921b0cb72b9971 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 23 Aug 2023 13:26:27 +0200 Subject: [PATCH 25/77] adjust summary and plotting --- scripts/make_summary_perfect.py | 39 ++++++++++++++++++++------------- scripts/plot_summary.py | 20 +++++++++++------ 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 4f38b87a..c1cce820 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -13,7 +13,6 @@ other metrics. import numpy as np import pandas as pd import pypsa -from _helpers import override_component_attrs from make_summary import ( assign_carriers, assign_locations, @@ -221,31 +220,43 @@ def calculate_curtailment(n, label, curtailment): def calculate_energy(n, label, energy): + investments = n.investment_periods + cols = pd.MultiIndex.from_product( + [ + energy.columns.levels[0], + energy.columns.levels[1], + energy.columns.levels[2], + investments, + ], + names=energy.columns.names[:3] + ["year"], + ) + energy = energy.reindex(cols, axis=1) + for c in n.iterate_components(n.one_port_components | n.branch_components): if c.name in n.one_port_components: c_energies = ( - c.pnl.p.multiply(n.snapshot_weightings, axis=0) - .sum() + c.pnl.p.multiply(n.snapshot_weightings.generators, axis=0) + .groupby(level=0).sum() .multiply(c.df.sign) - .groupby(c.df.carrier) + .groupby(c.df.carrier, axis=1) .sum() ) else: - c_energies = pd.Series(0.0, c.df.carrier.unique()) + c_energies = pd.DataFrame(0.0, columns=c.df.carrier.unique(), index=n.investment_periods) for port in [col[3:] for col in c.df.columns if col[:3] == "bus"]: - totals = c.pnl["p" + port].multiply(n.snapshot_weightings, axis=0).sum() + totals = c.pnl["p" + port].multiply(n.snapshot_weightings.generators, axis=0).groupby(level=0).sum() # remove values where bus is missing (bug in nomopyomo) no_bus = c.df.index[c.df["bus" + port] == ""] - totals.loc[no_bus] = n.component_attrs[c.name].loc[ + totals[no_bus] = float(n.component_attrs[c.name].loc[ "p" + port, "default" - ] - c_energies -= totals.groupby(c.df.carrier).sum() + ]) + c_energies -= totals.groupby(c.df.carrier, axis=1).sum() - c_energies = pd.concat([c_energies], keys=[c.list_name]) + c_energies = pd.concat([c_energies.T], keys=[c.list_name]) energy = energy.reindex(c_energies.index.union(energy.index)) - energy.loc[c_energies.index, label] = c_energies + energy.loc[c_energies.index, label] = c_energies.values return energy @@ -599,7 +610,7 @@ def calculate_co2_emissions(n, label, df): if emissions.empty: return - weightings = n.snapshot_weightings.mul( + weightings = n.snapshot_weightings.generators.mul( n.investment_period_weightings["years"] .reindex(n.snapshots) .fillna(method="bfill") @@ -661,11 +672,10 @@ def make_summaries(networks_dict): for output in outputs: df[output] = pd.DataFrame(columns=columns, dtype=float) - overrides = override_component_attrs(snakemake.input.overrides) for label, filename in iteritems(networks_dict): print(label, filename) try: - n = pypsa.Network(filename, override_component_attrs=overrides) + n = pypsa.Network(filename) except OSError: print(label, " not solved yet.") continue @@ -694,7 +704,6 @@ if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("make_summary_perfect") networks_dict = { diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 14cc12af..629adc5d 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -49,6 +49,10 @@ def rename_techs(label): # "H2 Fuel Cell": "hydrogen storage", # "H2 pipeline": "hydrogen storage", "battery": "battery storage", + "H2 for industry": "H2 for industry", + "land transport fuel cell": "land transport fuel cell", + "land transport oil": "land transport oil", + "oil shipping": "shipping oil", # "CC": "CC" } @@ -157,11 +161,11 @@ def plot_costs(): df.index.difference(preferred_order) ) - new_columns = df.sum().sort_values().index + # new_columns = df.sum().sort_values().index fig, ax = plt.subplots(figsize=(12, 8)) - df.loc[new_index, new_columns].T.plot( + df.loc[new_index].T.plot( kind="bar", ax=ax, stacked=True, @@ -222,13 +226,13 @@ def plot_energy(): df.index.difference(preferred_order) ) - new_columns = df.columns.sort_values() + # new_columns = df.columns.sort_values() fig, ax = plt.subplots(figsize=(12, 8)) - logger.debug(df.loc[new_index, new_columns]) + logger.debug(df.loc[new_index]) - df.loc[new_index, new_columns].T.plot( + df.loc[new_index].T.plot( kind="bar", ax=ax, stacked=True, @@ -272,7 +276,7 @@ def plot_balances(): i for i in balances_df.index.levels[0] if i not in co2_carriers ] - fig, ax = plt.subplots(figsize=(12, 8)) + for k, v in balances.items(): df = balances_df.loc[v] @@ -317,6 +321,8 @@ def plot_balances(): ) new_columns = df.columns.sort_values() + + fig, ax = plt.subplots(figsize=(12, 8)) df.loc[new_index, new_columns].T.plot( kind="bar", @@ -544,7 +550,7 @@ def plot_carbon_budget_distribution(input_eurostat): path_cb_plot = "results/" + snakemake.params.RDIR + "/graphs/" plt.savefig(path_cb_plot + "carbon_budget_plot.pdf", dpi=300) - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake From ba589ac7af2314acd96532804e713467d23b09a6 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 23 Aug 2023 13:27:21 +0200 Subject: [PATCH 26/77] add config for perfect foresight --- config/config.perfect.yaml | 973 +++++++++++++++++++++++++++++++++++++ 1 file changed, 973 insertions(+) create mode 100644 config/config.perfect.yaml diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml new file mode 100644 index 00000000..a9954fe7 --- /dev/null +++ b/config/config.perfect.yaml @@ -0,0 +1,973 @@ +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration +version: 0.8.1 +tutorial: false + +logging: + level: INFO + format: '%(levelname)s:%(name)s:%(message)s' + +private: + keys: + entsoe_api: + +remote: + ssh: "" + path: "" + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run +run: + name: "" + disable_progressbar: false + shared_resources: false + shared_cutouts: true + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight +foresight: perfect + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#scenario +# Wildcard docs in https://pypsa-eur.readthedocs.io/en/latest/wildcards.html +scenario: + simpl: + - '' + ll: + - v1.0 + - v1.5 + clusters: + - 37 +# - 128 +# - 256 +# - 512 + #- 1024 + opts: + - '' + sector_opts: + - 2p0-co2min-8760H-T-H-B-I-A-solar+p3-dist1 + planning_horizons: + - 2020 + - 2030 + - 2040 + - 2050 + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries +countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#snapshots +snapshots: + start: "2013-01-01" + end: "2014-01-01" + inclusive: 'left' + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable +enable: + retrieve: auto + prepare_links_p_nom: false + retrieve_databundle: true + retrieve_sector_databundle: true + retrieve_cost_data: true + build_cutout: false + retrieve_cutout: false + build_natura_raster: false + retrieve_natura_raster: true + custom_busmap: false + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget +co2_budget: + 2020: 0.701 + 2025: 0.524 + 2030: 0.297 + 2035: 0.150 + 2040: 0.071 + 2045: 0.032 + 2050: 0.000 + + # update of IPCC 6th AR compared to the 1.5SR. (discussed here: https://twitter.com/JoeriRogelj/status/1424743828339167233) + 1p5 : 34.2 # 25.7 # Budget in Gt CO2 for 1.5 for Europe, global 420 Gt, assuming per capita share + 1p6 : 43.259666 # 35 # Budget in Gt CO2 for 1.6 for Europe, global 580 Gt + 1p7 : 51.4 # 45 # Budget in Gt CO2 for 1.7 for Europe, global 800 Gt + 2p0 : 69.778 # 73.9 # Budget in Gt CO2 for 2 for Europe, global 1170 Gt + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity +electricity: + voltages: [220., 300., 380.] + gaslimit: false + co2limit: 7.75e+7 + co2base: 1.487e+9 + agg_p_nom_limits: data/agg_p_nom_minmax.csv + + operational_reserve: + activate: false + epsilon_load: 0.02 + epsilon_vres: 0.02 + contingency: 4000 + + max_hours: + battery: 6 + H2: 168 + + extendable_carriers: + Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] + StorageUnit: [] # battery, H2 + Store: [battery, H2] + Link: [] # H2 pipeline + + powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) + custom_powerplants: false + + conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] + + estimate_renewable_capacities: + enable: true + from_opsd: true + year: 2020 + expansion_limit: false + technology_mapping: + Offshore: [offwind-ac, offwind-dc] + Onshore: [onwind] + PV: [solar] + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite +atlite: + default_cutout: europe-2013-era5 + nprocesses: 4 + show_progress: false + cutouts: + # use 'base' to determine geographical bounds and time span from config + # base: + # module: era5 + europe-2013-era5: + module: era5 # in priority order + x: [-12., 35.] + y: [33., 72] + dx: 0.3 + dy: 0.3 + time: ['2013', '2013'] + europe-2013-sarah: + module: [sarah, era5] # in priority order + x: [-12., 45.] + y: [33., 65] + dx: 0.2 + dy: 0.2 + time: ['2013', '2013'] + sarah_interpolate: false + sarah_dir: + features: [influx, temperature] + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#renewable +renewable: + onwind: + cutout: europe-2013-era5 + resource: + method: wind + turbine: Vestas_V112_3MW + capacity_per_sqkm: 3 + # correction_factor: 0.93 + corine: + grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32] + distance: 1000 + distance_grid_codes: [1, 2, 3, 4, 5, 6] + natura: true + excluder_resolution: 100 + potential: simple # or conservative + clip_p_max_pu: 1.e-2 + offwind-ac: + cutout: europe-2013-era5 + resource: + method: wind + turbine: NREL_ReferenceTurbine_5MW_offshore + capacity_per_sqkm: 2 + correction_factor: 0.8855 + corine: [44, 255] + natura: true + ship_threshold: 400 + max_depth: 50 + max_shore_distance: 30000 + excluder_resolution: 200 + potential: simple # or conservative + clip_p_max_pu: 1.e-2 + offwind-dc: + cutout: europe-2013-era5 + resource: + method: wind + turbine: NREL_ReferenceTurbine_5MW_offshore + capacity_per_sqkm: 2 + correction_factor: 0.8855 + corine: [44, 255] + natura: true + ship_threshold: 400 + max_depth: 50 + min_shore_distance: 30000 + excluder_resolution: 200 + potential: simple # or conservative + clip_p_max_pu: 1.e-2 + solar: + cutout: europe-2013-sarah + resource: + method: pv + panel: CSi + orientation: + slope: 35. + azimuth: 180. + capacity_per_sqkm: 1.7 + # correction_factor: 0.854337 + corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] + natura: true + excluder_resolution: 100 + potential: simple # or conservative + clip_p_max_pu: 1.e-2 + hydro: + cutout: europe-2013-era5 + carriers: [ror, PHS, hydro] + PHS_max_hours: 6 + hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float + flatten_dispatch: false + flatten_dispatch_buffer: 0.2 + clip_min_inflow: 1.0 + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#conventional +conventional: + unit_commitment: false + dynamic_fuel_price: false + nuclear: + p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#lines +lines: + types: + 220.: "Al/St 240/40 2-bundle 220.0" + 300.: "Al/St 240/40 3-bundle 300.0" + 380.: "Al/St 240/40 4-bundle 380.0" + s_max_pu: 0.7 + s_nom_max: .inf + max_extension: .inf + length_factor: 1.25 + under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity + dynamic_line_rating: + activate: false + cutout: europe-2013-era5 + correction_factor: 0.95 + max_voltage_difference: false + max_line_rating: false + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#links +links: + p_max_pu: 1.0 + p_nom_max: .inf + max_extension: .inf + include_tyndp: true + under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#transformers +transformers: + x: 0.1 + s_nom: 2000. + type: '' + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load +load: + power_statistics: true + interpolate_limit: 3 + time_shift_for_large_gaps: 1w + manual_adjustments: true # false + scaling_factor: 1.0 + +# docs +# TODO: PyPSA-Eur merge issue in prepare_sector_network.py +# regulate what components with which carriers are kept from PyPSA-Eur; +# some technologies are removed because they are implemented differently +# (e.g. battery or H2 storage) or have different year-dependent costs +# in PyPSA-Eur-Sec +pypsa_eur: + Bus: + - AC + Link: + - DC + Generator: + - onwind + - offwind-ac + - offwind-dc + - solar + - ror + StorageUnit: + - PHS + - hydro + Store: [] + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy +energy: + energy_totals_year: 2011 + base_emissions_year: 1990 + eurostat_report_year: 2016 + emissions: CO2 + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass +biomass: + year: 2030 + scenario: ENS_Med + classes: + solid biomass: + - Agricultural waste + - Fuelwood residues + - Secondary Forestry residues - woodchips + - Sawdust + - Residues from landscape care + - Municipal waste + not included: + - Sugar from sugar beet + - Rape seed + - "Sunflower, soya seed " + - Bioethanol barley, wheat, grain maize, oats, other cereals and rye + - Miscanthus, switchgrass, RCG + - Willow + - Poplar + - FuelwoodRW + - C&P_RW + biogas: + - Manure solid, liquid + - Sludge + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solar-thermal +solar_thermal: + clearsky_model: simple # should be "simple" or "enhanced"? + orientation: + slope: 45. + azimuth: 180. + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities +existing_capacities: + grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] + grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020 + threshold_capacity: 10 + conventional_carriers: + - lignite + - coal + - oil + - uranium + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#sector +sector: + district_heating: + potential: 0.6 + progress: + 2020: 0.0 + 2030: 0.3 + 2040: 0.6 + 2050: 1.0 + district_heating_loss: 0.15 + cluster_heat_buses: false + bev_dsm_restriction_value: 0.75 + bev_dsm_restriction_time: 7 + transport_heating_deadband_upper: 20. + transport_heating_deadband_lower: 15. + ICE_lower_degree_factor: 0.375 + ICE_upper_degree_factor: 1.6 + EV_lower_degree_factor: 0.98 + EV_upper_degree_factor: 0.63 + bev_dsm: true + bev_availability: 0.5 + bev_energy: 0.05 + bev_charge_efficiency: 0.9 + bev_plug_to_wheel_efficiency: 0.2 + bev_charge_rate: 0.011 + bev_avail_max: 0.95 + bev_avail_mean: 0.8 + v2g: true + land_transport_fuel_cell_share: + 2020: 0 + 2030: 0.05 + 2040: 0.1 + 2050: 0.15 + land_transport_electric_share: + 2020: 0 + 2030: 0.25 + 2040: 0.6 + 2050: 0.85 + land_transport_ice_share: + 2020: 1 + 2030: 0.7 + 2040: 0.3 + 2050: 0 + transport_fuel_cell_efficiency: 0.5 + transport_internal_combustion_efficiency: 0.3 + agriculture_machinery_electric_share: 0 + agriculture_machinery_oil_share: 1 + agriculture_machinery_fuel_efficiency: 0.7 + agriculture_machinery_electric_efficiency: 0.3 + MWh_MeOH_per_MWh_H2: 0.8787 + MWh_MeOH_per_tCO2: 4.0321 + MWh_MeOH_per_MWh_e: 3.6907 + shipping_hydrogen_liquefaction: false + shipping_hydrogen_share: + 2020: 0 + 2030: 0 + 2040: 0 + 2050: 0 + shipping_methanol_share: + 2020: 0 + 2030: 0.3 + 2040: 0.7 + 2050: 1 + shipping_oil_share: + 2020: 1 + 2030: 0.7 + 2040: 0.3 + 2050: 0 + shipping_methanol_efficiency: 0.46 + shipping_oil_efficiency: 0.40 + aviation_demand_factor: 1. + HVC_demand_factor: 1. + time_dep_hp_cop: true + heat_pump_sink_T: 55. + reduce_space_heat_exogenously: true + reduce_space_heat_exogenously_factor: + 2020: 0.10 # this results in a space heat demand reduction of 10% + 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita + 2030: 0.09 + 2035: 0.11 + 2040: 0.16 + 2045: 0.21 + 2050: 0.29 + retrofitting: + retro_endogen: false + cost_factor: 1.0 + interest_rate: 0.04 + annualise_cost: true + tax_weighting: false + construction_index: true + tes: true + tes_tau: + decentral: 3 + central: 180 + boilers: true + oil_boilers: false + biomass_boiler: true + chp: true + micro_chp: false + solar_thermal: true + solar_cf_correction: 0.788457 # = >>> 1/1.2683 + marginal_cost_storage: 0. #1e-4 + methanation: true + helmeth: false + coal_cc: false + dac: true + co2_vent: false + allam_cycle: false + hydrogen_fuel_cell: true + hydrogen_turbine: false + SMR: true + regional_co2_sequestration_potential: + enable: false + attribute: 'conservative estimate Mt' + include_onshore: false + min_size: 3 + max_size: 25 + years_of_storage: 25 + co2_sequestration_potential: 200 + co2_sequestration_cost: 10 + co2_spatial: false + co2network: false + cc_fraction: 0.9 + hydrogen_underground_storage: true + hydrogen_underground_storage_locations: + # - onshore # more than 50 km from sea + - nearshore # within 50 km of sea + # - offshore + ammonia: false + min_part_load_fischer_tropsch: 0.9 + min_part_load_methanolisation: 0 # 0.5 + use_fischer_tropsch_waste_heat: true + use_fuel_cell_waste_heat: true + use_electrolysis_waste_heat: false + electricity_distribution_grid: true + electricity_distribution_grid_cost_factor: 1.0 + electricity_grid_connection: true + H2_network: true + gas_network: false + H2_retrofit: false + H2_retrofit_capacity_per_CH4: 0.6 + gas_network_connectivity_upgrade: 1 + gas_distribution_grid: true + gas_distribution_grid_cost_factor: 1.0 + biomass_spatial: false + biomass_transport: false + conventional_generation: + OCGT: gas + biomass_to_liquid: false + biosng: false + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry +industry: + St_primary_fraction: + 2020: 0.6 + 2025: 0.55 + 2030: 0.5 + 2035: 0.45 + 2040: 0.4 + 2045: 0.35 + 2050: 0.3 + DRI_fraction: + 2020: 0 + 2025: 0 + 2030: 0.05 + 2035: 0.2 + 2040: 0.4 + 2045: 0.7 + 2050: 1 + H2_DRI: 1.7 + elec_DRI: 0.322 + Al_primary_fraction: + 2020: 0.4 + 2025: 0.375 + 2030: 0.35 + 2035: 0.325 + 2040: 0.3 + 2045: 0.25 + 2050: 0.2 + MWh_NH3_per_tNH3: 5.166 + MWh_CH4_per_tNH3_SMR: 10.8 + MWh_elec_per_tNH3_SMR: 0.7 + MWh_H2_per_tNH3_electrolysis: 6.5 + MWh_elec_per_tNH3_electrolysis: 1.17 + MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv + NH3_process_emissions: 24.5 + petrochemical_process_emissions: 25.5 + HVC_primary_fraction: 1. + HVC_mechanical_recycling_fraction: 0. + HVC_chemical_recycling_fraction: 0. + HVC_production_today: 52. + MWh_elec_per_tHVC_mechanical_recycling: 0.547 + MWh_elec_per_tHVC_chemical_recycling: 6.9 + chlorine_production_today: 9.58 + MWh_elec_per_tCl: 3.6 + MWh_H2_per_tCl: -0.9372 + methanol_production_today: 1.5 + MWh_elec_per_tMeOH: 0.167 + MWh_CH4_per_tMeOH: 10.25 + hotmaps_locate_missing: false + reference_year: 2015 + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs +costs: + year: 2030 + social_discountrate: 0.02 + version: v0.6.0 + rooftop_share: 0.14 # based on the potentials, assuming (0.1 kW/m2 and 10 m2/person) + fill_values: + FOM: 0 + VOM: 0 + efficiency: 1 + fuel: 0 + investment: 0 + lifetime: 25 + "CO2 intensity": 0 + "discount rate": 0.07 + # Marginal and capital costs can be overwritten + # capital_cost: + # onwind: 500 + marginal_cost: + solar: 0.01 + onwind: 0.015 + offwind: 0.015 + hydro: 0. + H2: 0. + electrolysis: 0. + fuel cell: 0. + battery: 0. + battery inverter: 0. + emission_prices: + co2: 0. + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering +clustering: + simplify_network: + to_substations: false + algorithm: kmeans # choose from: [hac, kmeans] + feature: solar+onwind-time + exclude_carriers: [] + remove_stubs: true + remove_stubs_across_borders: true + cluster_network: + algorithm: kmeans + feature: solar+onwind-time + exclude_carriers: [] + consider_efficiency_classes: false + aggregation_strategies: + generators: + committable: any + ramp_limit_up: max + ramp_limit_down: max + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving +solving: + #tmpdir: "path/to/tmp" + options: + clip_p_max_pu: 1.e-2 + load_shedding: true + noisy_costs: true + skip_iterations: true + rolling_horizon: false + seed: 123 + # options that go into the optimize function + track_iterations: false + min_iterations: 4 + max_iterations: 6 + transmission_losses: 0 + linearized_unit_commitment: true + horizon: 365 + + solver: + name: gurobi + options: gurobi-default + + solver_options: + highs-default: + # refer to https://ergo-code.github.io/HiGHS/options/definitions.html#solver + threads: 4 + solver: "ipm" + run_crossover: "off" + small_matrix_value: 1e-6 + large_matrix_value: 1e9 + primal_feasibility_tolerance: 1e-5 + dual_feasibility_tolerance: 1e-5 + ipm_optimality_tolerance: 1e-4 + parallel: "on" + random_seed: 123 + gurobi-default: + threads: 4 + method: 2 # barrier + crossover: 0 + BarConvTol: 1.e-6 + Seed: 123 + AggFill: 0 + PreDual: 0 + GURO_PAR_BARDENSETHRESH: 200 + gurobi-numeric-focus: + name: gurobi + NumericFocus: 3 # Favour numeric stability over speed + method: 2 # barrier + crossover: 0 # do not use crossover + BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-4 + OptimalityTol: 1.e-4 + ObjScale: -0.5 + threads: 8 + Seed: 123 + gurobi-fallback: # Use gurobi defaults + name: gurobi + crossover: 0 + method: 2 # barrier + BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge + BarConvTol: 1.e-5 + FeasibilityTol: 1.e-5 + OptimalityTol: 1.e-5 + Seed: 123 + threads: 8 + cplex-default: + threads: 4 + lpmethod: 4 # barrier + solutiontype: 2 # non basic solution, ie no crossover + barrier.convergetol: 1.e-5 + feasopt.tolerance: 1.e-6 + cbc-default: {} # Used in CI + glpk-default: {} # Used in CI + + mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 + walltime: "12:00:00" + +# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting +plotting: + map: + boundaries: [-11, 30, 34, 71] + color_geomap: + ocean: white + land: white + eu_node_location: + x: -5.5 + y: 46. + costs_max: 1000 + costs_threshold: 1 + energy_max: 20000 + energy_min: -20000 + energy_threshold: 50. + + nice_names: + OCGT: "Open-Cycle Gas" + CCGT: "Combined-Cycle Gas" + offwind-ac: "Offshore Wind (AC)" + offwind-dc: "Offshore Wind (DC)" + onwind: "Onshore Wind" + solar: "Solar" + PHS: "Pumped Hydro Storage" + hydro: "Reservoir & Dam" + battery: "Battery Storage" + H2: "Hydrogen Storage" + lines: "Transmission Lines" + ror: "Run of River" + ac: "AC" + dc: "DC" + + tech_colors: + # wind + onwind: "#235ebc" + onshore wind: "#235ebc" + offwind: "#6895dd" + offshore wind: "#6895dd" + offwind-ac: "#6895dd" + offshore wind (AC): "#6895dd" + offshore wind ac: "#6895dd" + offwind-dc: "#74c6f2" + offshore wind (DC): "#74c6f2" + offshore wind dc: "#74c6f2" + # water + hydro: '#298c81' + hydro reservoir: '#298c81' + ror: '#3dbfb0' + run of river: '#3dbfb0' + hydroelectricity: '#298c81' + PHS: '#51dbcc' + hydro+PHS: "#08ad97" + wave: '#a7d4cf' + # solar + solar: "#f9d002" + solar PV: "#f9d002" + solar thermal: '#ffbf2b' + residential rural solar thermal: '#f1c069' + services rural solar thermal: '#eabf61' + residential urban decentral solar thermal: '#e5bc5a' + services urban decentral solar thermal: '#dfb953' + urban central solar thermal: '#d7b24c' + solar rooftop: '#ffea80' + # gas + OCGT: '#e0986c' + OCGT marginal: '#e0986c' + OCGT-heat: '#e0986c' + gas boiler: '#db6a25' + gas boilers: '#db6a25' + gas boiler marginal: '#db6a25' + residential rural gas boiler: '#d4722e' + residential urban decentral gas boiler: '#cb7a36' + services rural gas boiler: '#c4813f' + services urban decentral gas boiler: '#ba8947' + urban central gas boiler: '#b0904f' + gas: '#e05b09' + fossil gas: '#e05b09' + natural gas: '#e05b09' + biogas to gas: '#e36311' + CCGT: '#a85522' + CCGT marginal: '#a85522' + allam: '#B98F76' + gas for industry co2 to atmosphere: '#692e0a' + gas for industry co2 to stored: '#8a3400' + gas for industry: '#853403' + gas for industry CC: '#692e0a' + gas pipeline: '#ebbca0' + gas pipeline new: '#a87c62' + # oil + oil: '#c9c9c9' + imported oil: '#c9c9c9' + oil boiler: '#adadad' + residential rural oil boiler: '#a9a9a9' + services rural oil boiler: '#a5a5a5' + residential urban decentral oil boiler: '#a1a1a1' + urban central oil boiler: '#9d9d9d' + services urban decentral oil boiler: '#999999' + agriculture machinery oil: '#949494' + shipping oil: "#808080" + land transport oil: '#afafaf' + # nuclear + Nuclear: '#ff8c00' + Nuclear marginal: '#ff8c00' + nuclear: '#ff8c00' + uranium: '#ff8c00' + # coal + Coal: '#545454' + coal: '#545454' + Coal marginal: '#545454' + solid: '#545454' + Lignite: '#826837' + lignite: '#826837' + Lignite marginal: '#826837' + # biomass + biogas: '#e3d37d' + biomass: '#baa741' + solid biomass: '#baa741' + solid biomass transport: '#baa741' + solid biomass for industry: '#7a6d26' + solid biomass for industry CC: '#47411c' + solid biomass for industry co2 from atmosphere: '#736412' + solid biomass for industry co2 to stored: '#47411c' + urban central solid biomass CHP: '#9d9042' + urban central solid biomass CHP CC: '#6c5d28' + biomass boiler: '#8A9A5B' + residential rural biomass boiler: '#a1a066' + residential urban decentral biomass boiler: '#b0b87b' + services rural biomass boiler: '#c6cf98' + services urban decentral biomass boiler: '#dde5b5' + biomass to liquid: '#32CD32' + BioSNG: '#123456' + # power transmission + lines: '#6c9459' + transmission lines: '#6c9459' + electricity distribution grid: '#97ad8c' + low voltage: '#97ad8c' + # electricity demand + Electric load: '#110d63' + electric demand: '#110d63' + electricity: '#110d63' + industry electricity: '#2d2a66' + industry new electricity: '#2d2a66' + agriculture electricity: '#494778' + # battery + EVs + battery: '#ace37f' + battery storage: '#ace37f' + battery charger: '#88a75b' + battery discharger: '#5d4e29' + home battery: '#80c944' + home battery storage: '#80c944' + home battery charger: '#5e8032' + home battery discharger: '#3c5221' + BEV charger: '#baf238' + V2G: '#e5ffa8' + land transport EV: '#baf238' + Li ion: '#baf238' + # hot water storage + water tanks: '#e69487' + residential rural water tanks: '#f7b7a3' + services rural water tanks: '#f3afa3' + residential urban decentral water tanks: '#f2b2a3' + services urban decentral water tanks: '#f1b4a4' + urban central water tanks: '#e9977d' + hot water storage: '#e69487' + hot water charging: '#e8998b' + urban central water tanks charger: '#b57a67' + residential rural water tanks charger: '#b4887c' + residential urban decentral water tanks charger: '#b39995' + services rural water tanks charger: '#b3abb0' + services urban decentral water tanks charger: '#b3becc' + hot water discharging: '#e99c8e' + urban central water tanks discharger: '#b9816e' + residential rural water tanks discharger: '#ba9685' + residential urban decentral water tanks discharger: '#baac9e' + services rural water tanks discharger: '#bbc2b8' + services urban decentral water tanks discharger: '#bdd8d3' + # heat demand + Heat load: '#cc1f1f' + heat: '#cc1f1f' + heat demand: '#cc1f1f' + rural heat: '#ff5c5c' + residential rural heat: '#ff7c7c' + services rural heat: '#ff9c9c' + central heat: '#cc1f1f' + urban central heat: '#d15959' + decentral heat: '#750606' + residential urban decentral heat: '#a33c3c' + services urban decentral heat: '#cc1f1f' + low-temperature heat for industry: '#8f2727' + process heat: '#ff0000' + agriculture heat: '#d9a5a5' + # heat supply + heat pumps: '#2fb537' + heat pump: '#2fb537' + air heat pump: '#36eb41' + residential urban decentral air heat pump: '#48f74f' + services urban decentral air heat pump: '#5af95d' + urban central air heat pump: '#6cfb6b' + ground heat pump: '#2fb537' + residential rural ground heat pump: '#48f74f' + services rural ground heat pump: '#5af95d' + Ambient: '#98eb9d' + CHP: '#8a5751' + urban central gas CHP: '#8d5e56' + CHP CC: '#634643' + urban central gas CHP CC: '#6e4e4c' + CHP heat: '#8a5751' + CHP electric: '#8a5751' + district heating: '#e8beac' + resistive heater: '#d8f9b8' + residential rural resistive heater: '#bef5b5' + residential urban decentral resistive heater: '#b2f1a9' + services rural resistive heater: '#a5ed9d' + services urban decentral resistive heater: '#98e991' + urban central resistive heater: '#8cdf85' + retrofitting: '#8487e8' + building retrofitting: '#8487e8' + # hydrogen + H2 for industry: "#f073da" + H2 for shipping: "#ebaee0" + H2: '#bf13a0' + hydrogen: '#bf13a0' + SMR: '#870c71' + SMR CC: '#4f1745' + H2 liquefaction: '#d647bd' + hydrogen storage: '#bf13a0' + H2 Store: '#bf13a0' + H2 storage: '#bf13a0' + land transport fuel cell: '#6b3161' + H2 pipeline: '#f081dc' + H2 pipeline retrofitted: '#ba99b5' + H2 Fuel Cell: '#c251ae' + H2 fuel cell: '#c251ae' + H2 turbine: '#991f83' + H2 Electrolysis: '#ff29d9' + H2 electrolysis: '#ff29d9' + # ammonia + NH3: '#46caf0' + ammonia: '#46caf0' + ammonia store: '#00ace0' + ammonia cracker: '#87d0e6' + Haber-Bosch: '#076987' + # syngas + Sabatier: '#9850ad' + methanation: '#c44ce6' + methane: '#c44ce6' + helmeth: '#e899ff' + # synfuels + Fischer-Tropsch: '#25c49a' + liquid: '#25c49a' + kerosene for aviation: '#a1ffe6' + naphtha for industry: '#57ebc4' + methanolisation: '#83d6d5' + methanol: '#468c8b' + shipping methanol: '#468c8b' + # co2 + CC: '#f29dae' + CCS: '#f29dae' + CO2 sequestration: '#f29dae' + DAC: '#ff5270' + co2 stored: '#f2385a' + co2: '#f29dae' + co2 vent: '#ffd4dc' + CO2 pipeline: '#f5627f' + # emissions + process emissions CC: '#000000' + process emissions: '#222222' + process emissions to stored: '#444444' + process emissions to atmosphere: '#888888' + oil emissions: '#aaaaaa' + shipping oil emissions: "#555555" + shipping methanol emissions: '#666666' + land transport oil emissions: '#777777' + agriculture machinery oil emissions: '#333333' + # other + shipping: '#03a2ff' + power-to-heat: '#2fb537' + power-to-gas: '#c44ce6' + power-to-H2: '#ff29d9' + power-to-liquid: '#25c49a' + gas-to-power/heat: '#ee8340' + waste: '#e3d37d' + other: '#000000' + geothermal: '#ba91b1' + AC: "#70af1d" + AC-AC: "#70af1d" + AC line: "#70af1d" + links: "#8a1caf" + HVDC links: "#8a1caf" + DC: "#8a1caf" + DC-DC: "#8a1caf" + DC link: "#8a1caf" From 60b1968eb07aa6a5c7c9f54ebe05ac339c136cd9 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 23 Aug 2023 13:27:40 +0200 Subject: [PATCH 27/77] check for missing buses --- scripts/add_existing_baseyear.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index b2c534e6..4e6caaaa 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -304,6 +304,17 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas bus0 = vars(spatial)[carrier[generator]].nodes if "EU" not in vars(spatial)[carrier[generator]].locations: bus0 = bus0.intersection(capacity.index + " gas") + + # check for missing bus + missing_bus = pd.Index(bus0).difference(n.buses.index) + if not missing_bus.empty: + logger.info(f"add buses {bus0}") + n.madd("Bus", + bus0, + carrier=generator, + location=vars(spatial)[carrier[generator]].locations, + unit="MWh_el" + ) already_build = n.links.index.intersection(asset_i) new_build = asset_i.difference(n.links.index) @@ -605,13 +616,13 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_existing_baseyear", - configfiles="config/test/config.myopic.yaml", + # configfiles="config/test/config.myopic.yaml", simpl="", - clusters="5", - ll="v1.5", + clusters="37", + ll="v1.0", opts="", - sector_opts="24H-T-H-B-I-A-solar+p3-dist1", - planning_horizons=2030, + sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", + planning_horizons=2020, ) logging.basicConfig(level=snakemake.config["logging"]["level"]) From a77f0f6ab17e7c367fd132b3f423155a2e877e48 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Jul 2023 16:02:14 +0000 Subject: [PATCH 28/77] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/make_summary_perfect.py | 1 - scripts/prepare_perfect_foresight.py | 1 - 2 files changed, 2 deletions(-) diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index c1cce820..3ca2c170 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT - """ Create summary CSV files for all scenario runs with perfect foresight including costs, capacities, capacity factors, curtailment, energy balances, prices and diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 99534104..81ee7285 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors # # SPDX-License-Identifier: MIT - """ Concats pypsa networks of single investment periods to one network. """ From 8c5ca05eb49875356a5a942de2724c02be87ec79 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 08:42:24 +0200 Subject: [PATCH 29/77] add carbon constraint to perfect --- scripts/solve_network.py | 57 ++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8c344233..5cc7d877 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -113,7 +113,7 @@ def _add_land_use_constraint_m(n, planning_horizons, config): n.generators.p_nom_max.clip(lower=0, inplace=True) -def add_co2_sequestration_limit(n, limit=200): +def add_co2_sequestration_limit(n, config, limit=200): """ Add a global constraint on the amount of Mt CO2 that can be sequestered. """ @@ -126,17 +126,51 @@ def add_co2_sequestration_limit(n, limit=200): continue limit = float(o[o.find("seq") + 3 :]) * 1e6 break - - n.add( + + if config["foresight"] == "perfect": + periods = n.investment_periods + names = pd.Index([f"co2_sequestration_limit-{period}" for period in periods]) + else: + periods = [np.nan] + names = pd.Index(["co2_sequestration_limit"]) + + n.madd( "GlobalConstraint", - "co2_sequestration_limit", + names, sense="<=", constant=limit, type="primary_energy", carrier_attribute="co2_absorptions", + investment_period=periods, ) +def add_carbon_neutral_constraint(n, snapshots): + glcs = n.global_constraints.query('type == "co2_limit"') + if glcs.empty: + return + for name, glc in glcs.iterrows(): + rhs = glc.constant + carattr = glc.carrier_attribute + emissions = n.carriers.query(f"{carattr} != 0")[carattr] + + if emissions.empty: + continue + + # stores + n.stores["carrier"] = n.stores.bus.map(n.buses.carrier) + stores = n.stores.query("carrier in @emissions.index and not e_cyclic") + time_valid = int(glc.loc["investment_period"]) + if not stores.empty: + last = n.snapshot_weightings.reset_index().groupby("period").last() + last_i = last.set_index([last.index, last.timestep]).index + final_e = n.model["Store-e"].loc[last_i, stores.index] + time_i = pd.IndexSlice[time_valid, :] + lhs = final_e.loc[time_i,:] - final_e.shift(snapshot=1).loc[time_i,:] + + n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") + + def prepare_network( n, solve_opts=None, @@ -199,7 +233,7 @@ def prepare_network( if n.stores.carrier.eq("co2 stored").any(): limit = co2_sequestration_potential - add_co2_sequestration_limit(n, limit=limit) + add_co2_sequestration_limit(n, config, limit=limit) return n @@ -591,6 +625,7 @@ def extra_functionality(n, snapshots): add_EQ_constraints(n, o) add_battery_constraints(n) add_pipe_retrofit_constraint(n) + add_carbon_neutral_constraint(n, snapshots) def solve_network(n, config, solving, opts="", **kwargs): @@ -643,19 +678,19 @@ def solve_network(n, config, solving, opts="", **kwargs): return n - +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake( - "solve_sector_network", - configfiles="test/config.overnight.yaml", + "solve_sector_network_perfect", + configfiles="config/config.perfect.yaml", simpl="", opts="", - clusters="5", - ll="v1.5", - sector_opts="CO2L0-24H-T-H-B-I-A-solar+p3-dist1", + clusters="37", + ll="v1.0", + sector_opts="4380H-T-H-B-I-A-solar+p3-dist1", planning_horizons="2030", ) configure_logging(snakemake) From 4b073b0573dbd8c126e7e45f2badc9fa4cc4ce36 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 08:42:52 +0200 Subject: [PATCH 30/77] add co2 store lifetime --- config/config.perfect.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index a9954fe7..9a41f196 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -45,7 +45,7 @@ scenario: opts: - '' sector_opts: - - 2p0-co2min-8760H-T-H-B-I-A-solar+p3-dist1 + - 4380H-T-H-B-I-A-solar+p3-dist1 planning_horizons: - 2020 - 2030 @@ -468,6 +468,7 @@ sector: years_of_storage: 25 co2_sequestration_potential: 200 co2_sequestration_cost: 10 + co2_sequestration_lifetime: 50 co2_spatial: false co2network: false cc_fraction: 0.9 From 445216ae03eba4530ab2fce08d6e7a7a91e750f2 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 08:43:29 +0200 Subject: [PATCH 31/77] adjust co2 glcs --- scripts/prepare_perfect_foresight.py | 46 +++++++++++++++++----------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 81ee7285..66af8ed1 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -179,8 +179,12 @@ def adjust_stores(n): # e_initial at beginning of each investment period e_initial_store = ["solid biomass", "biogas"] co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index - n.stores.loc[co2_i, "e_initial"] *= 10 - n.stores.loc[co2_i, "e_nom"] *= 10 + n.stores.loc[co2_i, "e_initial_per_period"] = True + # n.stores.loc[co2_i, "e_initial"] *= 10 + # n.stores.loc[co2_i, "e_nom"] *= 10 + e_initial_store = ["co2 stored"] + co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index + n.stores.loc[co2_i, "e_initial_per_period"] = True return n @@ -232,23 +236,17 @@ def set_carbon_constraints(n, opts): n.add( "GlobalConstraint", "Budget", - type="Co2constraint", + type="Co2Budget", carrier_attribute="co2_emissions", sense="<=", constant=budget, ) - if not "noco2neutral" in opts: - logger.info("Add carbon neutrality constraint.") - n.add( - "GlobalConstraint", - "Co2neutral", - type="Co2constraint", - carrier_attribute="co2_emissions", - investment_period=n.snapshots.levels[0][-1], - sense="<=", - constant=0, - ) + + else: + e_initial_store = ["co2 stored"] + co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index + n.stores.loc[co2_i, "e_initial_per_period"] = True # set minimum CO2 emission constraint to avoid too fast reduction if "co2min" in opts: emissions_1990 = 4.53693 @@ -263,14 +261,14 @@ def set_carbon_constraints(n, opts): ) n.add( "GlobalConstraint", - "Co2min", - type="Co2constraint", + f"Co2Min-{first_year}", + type="Co2min", carrier_attribute="co2_emissions", sense=">=", investment_period=first_year, constant=co2min * 1e9 * time_weightings, ) - + return n @@ -287,6 +285,15 @@ def adjust_lvlimit(n): return n + +def adjust_CO2_glc(n): + c = "GlobalConstraint" + glc_name = "CO2Limit" + glc_type = "primary_energy" + mask = (n.df(c).index.str.contains(glc_name)) & (n.df(c).type==glc_type) + n.df(c).loc[mask, "type"] = "co2_limit" + + return n # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -298,7 +305,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="2p0-co2min-8760H-T-H-B-I-A-solar+p3-dist1", + sector_opts="4380H-T-H-B-I-A-solar+p3-dist1", ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) @@ -321,6 +328,9 @@ if __name__ == "__main__": # adjust global constraints lv limit if the same for all years n = adjust_lvlimit(n) + + # adjust global constraints CO2 limit + n = adjust_CO2_glc(n) # set phase outs set_all_phase_outs(n) # adjust stores to multi period investment From 3679687ed75d07124f11f55dff6514bb62e6f2fd Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 08:43:50 +0200 Subject: [PATCH 32/77] add co2 store lifetime --- scripts/prepare_sector_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 37667ac5..9ddccb19 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -570,6 +570,7 @@ def add_co2_tracking(n, options): capital_cost=options["co2_sequestration_cost"], carrier="co2 stored", bus=spatial.co2.nodes, + lifetime=options["co2_sequestration_lifetime"], ) n.add("Carrier", "co2 stored") From e41357e3d5a20500d4dfeb2b28f10cbe03b39339 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 14:19:34 +0200 Subject: [PATCH 33/77] update carbon plot --- scripts/plot_summary.py | 142 ++++++++++++++++++---------------------- 1 file changed, 64 insertions(+), 78 deletions(-) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 629adc5d..376ac9df 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -365,8 +365,7 @@ def historical_emissions(countries): """ # https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16 # downloaded 201228 (modified by EEA last on 201221) - fn = "data/eea/UNFCCC_v23.csv" - df = pd.read_csv(fn, encoding="latin-1") + df = pd.read_csv(snakemake.input.co2, encoding="latin-1", low_memory=False) df.loc[df["Year"] == "1985-1987", "Year"] = 1986 df["Year"] = df["Year"].astype(int) df = df.set_index( @@ -390,16 +389,24 @@ def historical_emissions(countries): e["waste management"] = "5 - Waste management" e["other"] = "6 - Other Sector" e["indirect"] = "ind_CO2 - Indirect CO2" - e["total wL"] = "Total (with LULUCF)" - e["total woL"] = "Total (without LULUCF)" + # e["total wL"] = "Total (with LULUCF)" + # e["total woL"] = "Total (without LULUCF)" + + e["other LULUCF"] = "4.H - Other LULUCF" + pol = ["CO2"] # ["All greenhouse gases - (CO2 equivalent)"] if "GB" in countries: countries.remove("GB") countries.append("UK") - year = np.arange(1990, 2018).tolist() - + year = df.index.levels[0][df.index.levels[0]>=1990] + + missing = pd.Index(countries).difference(df.index.levels[2]) + if not missing.empty: + logger.warning(f"The following countries are missing and not considered when plotting historic CO2 emissions: {missing}") + countries = pd.Index(df.index.levels[2]).intersection(countries) + idx = pd.IndexSlice co2_totals = ( df.loc[idx[year, e.values, countries, pol], "emissions"] @@ -462,100 +469,80 @@ def plot_carbon_budget_distribution(input_eurostat): plt.rcParams["xtick.labelsize"] = 20 plt.rcParams["ytick.labelsize"] = 20 + # historic emissions + countries = snakemake.params.countries + e_1990 = co2_emissions_year( + countries, input_eurostat, opts, snakemake, year=1990 + ) + emissions = historical_emissions(countries) + # add other years https://sdi.eea.europa.eu/data/0569441f-2853-4664-a7cd-db969ef54de0 + emissions.loc[2019] = 2.971372 + emissions.loc[2020] = 2.691958 + emissions.loc[2020] = 2.869355 + + + if snakemake.config["foresight"] == "myopic": + path_cb = "results/" + snakemake.params.RDIR + "/csvs/" + co2_cap = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)[["cb"]] + co2_cap *= e_1990 + else: + supply_energy = pd.read_csv(snakemake.input.balances, + index_col=[0,1,2], header=[0,1,2, 3]) + co2_cap = supply_energy.loc["co2"].droplevel(0).drop("co2").sum().unstack().T/1e9 + co2_cap.rename(index=lambda x: int(x), inplace=True) + plt.figure(figsize=(10, 7)) gs1 = gridspec.GridSpec(1, 1) ax1 = plt.subplot(gs1[0, 0]) - ax1.set_ylabel("CO$_2$ emissions (Gt per year)", fontsize=22) - ax1.set_ylim([0, 5]) + ax1.set_ylabel("CO$_2$ emissions \n [Gt per year]", fontsize=22) + # ax1.set_ylim([0, 5]) ax1.set_xlim([1990, snakemake.params.planning_horizons[-1] + 1]) - path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - countries = snakemake.params.countries - e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) - CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) - - ax1.plot(e_1990 * CO2_CAP[o], linewidth=3, color="dodgerblue", label=None) - - emissions = historical_emissions(countries) - ax1.plot(emissions, color="black", linewidth=3, label=None) - # plot committed and uder-discussion targets + # plot committed and under-discussion targets # (notice that historical emissions include all countries in the # network, but targets refer to EU) - ax1.plot( - [2020], - [0.8 * emissions[1990]], - marker="*", - markersize=12, - markerfacecolor="black", - markeredgecolor="black", - ) ax1.plot( - [2030], - [0.45 * emissions[1990]], - marker="*", - markersize=12, - markerfacecolor="white", - markeredgecolor="black", - ) + [2030], + [0.45 * emissions[1990]], + marker="*", + markersize=12, + markerfacecolor="black", + markeredgecolor="black", + ) + + ax1.plot( - [2030], - [0.6 * emissions[1990]], - marker="*", - markersize=12, - markerfacecolor="black", - markeredgecolor="black", - ) + [2050], + [0.0 * emissions[1990]], + marker="*", + markersize=12, + markerfacecolor="black", + markeredgecolor="black", + label="EU commited target", + ) - ax1.plot( - [2050, 2050], - [x * emissions[1990] for x in [0.2, 0.05]], - color="gray", - linewidth=2, - marker="_", - alpha=0.5, - ) - - ax1.plot( - [2050], - [0.01 * emissions[1990]], - marker="*", - markersize=12, - markerfacecolor="white", - linewidth=0, - markeredgecolor="black", - label="EU under-discussion target", - zorder=10, - clip_on=False, - ) - - ax1.plot( - [2050], - [0.125 * emissions[1990]], - "ro", - marker="*", - markersize=12, - markerfacecolor="black", - markeredgecolor="black", - label="EU committed target", - ) + for col in co2_cap.columns: + ax1.plot(co2_cap[col], linewidth=3, label=col) + ax1.legend( fancybox=True, fontsize=18, loc=(0.01, 0.01), facecolor="white", frameon=True ) + + plt.grid(axis="y") - path_cb_plot = "results/" + snakemake.params.RDIR + "/graphs/" - plt.savefig(path_cb_plot + "carbon_budget_plot.pdf", dpi=300) + plt.savefig(snakemake.output.co2_emissions, bbox_inches="tight") #%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("plot_summary") + snakemake = mock_snakemake("plot_summary_perfect") logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -569,6 +556,5 @@ if __name__ == "__main__": for sector_opts in snakemake.params.sector_opts: opts = sector_opts.split("-") - for o in opts: - if "cb" in o: - plot_carbon_budget_distribution(snakemake.input.eurostat) + if any(["cb" in o for o in opts]) or (snakemake.config["foresight"]=="perfect"): + plot_carbon_budget_distribution(snakemake.input.eurostat) From c10afdc857dbd5195f8f3dae0635abf932ad9976 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 14:20:14 +0200 Subject: [PATCH 34/77] adjust co2_emission fct --- scripts/prepare_sector_network.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 9ddccb19..207fa263 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -192,17 +192,17 @@ def get(item, investment_year=None): def co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year + countries, input_eurostat, opts, snakemake, year ): """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.params.energy["emissions"] + emissions_scope = snakemake.config["energy"]["emissions"] eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.params.energy["eurostat_report_year"] + report_year = snakemake.config["energy"]["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -241,12 +241,12 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries = snakemake.params.countries e_1990 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year=1990 + countries, input_eurostat, opts, snakemake, year=1990 ) # emissions at the beginning of the path (last year available 2018) e_0 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year=2018 + countries, input_eurostat, opts, snakemake, year=2018, ) planning_horizons = snakemake.params.planning_horizons From 49ec50171b2777c8b97164cd4d45baeeb2739fd5 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 14:21:02 +0200 Subject: [PATCH 35/77] fix bug network dict --- scripts/make_summary_perfect.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 3ca2c170..714f55b8 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -704,10 +704,13 @@ if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake snakemake = mock_snakemake("make_summary_perfect") - + + run = snakemake.config["run"]["name"] + if run!="": run += "/" + networks_dict = { (clusters, lv, opts + sector_opts): "results/" - + snakemake.config["run"]["name"] + + run + f"postnetworks/elec_s{simpl}_{clusters}_l{lv}_{opts}_{sector_opts}_brownfield_all_years.nc" for simpl in snakemake.config["scenario"]["simpl"] for clusters in snakemake.config["scenario"]["clusters"] From 4fb2116968cb0e0cb01dd88fa3c824405526d7e7 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 14:21:32 +0200 Subject: [PATCH 36/77] set shared_resoruces true --- config/config.perfect.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index 9a41f196..fcee4972 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -20,9 +20,9 @@ remote: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - name: "" + name: "test_co2limit" disable_progressbar: false - shared_resources: false + shared_resources: true shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight From f2b01fbbe03eab25e663d26575c45b3460117be6 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 14:43:36 +0200 Subject: [PATCH 37/77] plot summary with usual script --- rules/solve_perfect.smk | 34 +--------------------------------- scripts/plot_summary.py | 9 ++++----- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 93c5096a..bfd441f1 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -190,39 +190,7 @@ rule make_summary_perfect: "../scripts/make_summary_perfect.py" -rule plot_summary_perfect: - input: - **{ - f"costs_{year}": f"data/costs_{year}.csv" - for year in config["scenario"]["planning_horizons"] - }, - costs_csv=RESULTS + "csvs/costs.csv", - energy=RESULTS + "csvs/energy.csv", - balances=RESULTS + "csvs/supply_energy.csv", - eea="data/eea/UNFCCC_v24.csv", - countries=RESULTS + "csvs/nodal_capacities.csv", - co2_emissions=RESULTS + "csvs/co2_emissions.csv", - capacities=RESULTS + "csvs/capacities.csv", - capital_cost=RESULTS + "csvs/capital_cost.csv", - output: - costs1=RESULTS + "graphs/costs.pdf", - costs2=RESULTS + "graphs/costs2.pdf", - costs3=RESULTS + "graphs/total_costs_per_year.pdf", - # energy="results" + '/' + config['run'] + '/graphs/energy.pdf', - balances=RESULTS + "graphs/balances-energy.pdf", - co2_emissions=RESULTS + "graphs/carbon_budget_plot.pdf", - capacities=RESULTS + "graphs/capacities.pdf", - threads: 2 - resources: - mem_mb=10000, - log: - LOGS + "plot_summary_perfect.log", - benchmark: - (BENCHMARKS + "plot_summary_perfect") - conda: - "../envs/environment.yaml" - script: - "../scripts/plot_summary_perfect.py" + ruleorder: add_existing_baseyear > add_brownfield diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 376ac9df..8405bd8f 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -356,7 +356,6 @@ def plot_balances(): fig.savefig(snakemake.output.balances[:-10] + k + ".pdf", bbox_inches="tight") - plt.cla() def historical_emissions(countries): @@ -478,7 +477,7 @@ def plot_carbon_budget_distribution(input_eurostat): # add other years https://sdi.eea.europa.eu/data/0569441f-2853-4664-a7cd-db969ef54de0 emissions.loc[2019] = 2.971372 emissions.loc[2020] = 2.691958 - emissions.loc[2020] = 2.869355 + emissions.loc[2021] = 2.869355 if snakemake.config["foresight"] == "myopic": @@ -534,15 +533,15 @@ def plot_carbon_budget_distribution(input_eurostat): ) plt.grid(axis="y") - - plt.savefig(snakemake.output.co2_emissions, bbox_inches="tight") + path = snakemake.output.balances.split("balances")[0] + "carbon_budget.pdf" + plt.savefig(path, bbox_inches="tight") #%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("plot_summary_perfect") + snakemake = mock_snakemake("plot_summary") logging.basicConfig(level=snakemake.config["logging"]["level"]) From be23b5c56a31e383003d3849a55bbdeffb6e4428 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 15:32:23 +0200 Subject: [PATCH 38/77] add network plot for perfect --- rules/postprocess.smk | 12 +-- scripts/plot_network.py | 183 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 174 insertions(+), 21 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index ca1d6fac..32775220 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -44,16 +44,18 @@ if config["foresight"] == "perfect": + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", regions=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", output: - map=RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{year}.pdf", - today=RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}-today.pdf", + **{ + f"map_{year}": RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_" + + f"{year}.pdf" + for year in config["scenario"]["planning_horizons"] + }, threads: 2 resources: mem_mb=10000, benchmark: BENCHMARKS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{year}_brownfield_all_years_benchmark.txt", + + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark", conda: "../envs/environment.yaml" script: diff --git a/scripts/plot_network.py b/scripts/plot_network.py index ae1d0e0a..3203f5ab 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -24,7 +24,7 @@ from make_summary import assign_carriers from plot_summary import preferred_order, rename_techs from pypsa.plot import add_legend_circles, add_legend_lines, add_legend_patches -plt.style.use(["ggplot", "matplotlibrc"]) +plt.style.use(["ggplot"]) def rename_techs_tyndp(tech): @@ -913,6 +913,153 @@ def plot_series(network, carrier="AC", name="test"): ) +def plot_map_perfect(network, components=["Link", "Store", "StorageUnit", "Generator"], + bus_size_factor=1.7e10): + + n = network.copy() + assign_location(n) + # Drop non-electric buses so they don't clutter the plot + n.buses.drop(n.buses.index[n.buses.carrier != "AC"], inplace=True) + # investment periods + investments = n.snapshots.levels[0] + + costs = {} + for comp in components: + df_c = n.df(comp) + if df_c.empty: continue + df_c["nice_group"] = df_c.carrier.map(rename_techs_tyndp) + + attr = "e_nom_opt" if comp == "Store" else "p_nom_opt" + + active = pd.concat( + [ + n.get_active_assets(comp, inv_p).rename(inv_p) + for inv_p in investments + ], + axis=1, + ).astype(int) + capital_cost = n.df(comp)[attr] * n.df(comp).capital_cost + capital_cost_t = ((active.mul(capital_cost, axis=0)) + .groupby([n.df(comp).location, + n.df(comp).nice_group]).sum()) + + capital_cost_t.drop("load", level=1, inplace=True, errors="ignore") + + costs[comp] = capital_cost_t + + costs = pd.concat(costs).groupby(level=[1,2]).sum() + costs.drop(costs[costs.sum(axis=1)==0].index, inplace=True) + + new_columns = (preferred_order.intersection(costs.index.levels[1]) + .append(costs.index.levels[1].difference(preferred_order))) + costs = costs.reindex(new_columns, level=1) + + for item in new_columns: + if item not in snakemake.config['plotting']['tech_colors']: + print("Warning!",item,"not in config/plotting/tech_colors, assign random color") + snakemake.config['plotting']['tech_colors'] = "pink" + + n.links.drop(n.links.index[(n.links.carrier != "DC") & ( + n.links.carrier != "B2B")], inplace=True) + + # drop non-bus + to_drop = costs.index.levels[0].symmetric_difference(n.buses.index) + if len(to_drop) != 0: + print("dropping non-buses", to_drop) + costs.drop(to_drop, level=0, inplace=True, axis=0, errors="ignore") + + # make sure they are removed from index + costs.index = pd.MultiIndex.from_tuples(costs.index.values) + + # PDF has minimum width, so set these to zero + line_lower_threshold = 500. + line_upper_threshold = 1e4 + linewidth_factor = 2e3 + ac_color = "gray" + dc_color = "m" + + line_widths = n.lines.s_nom_opt + link_widths = n.links.p_nom_opt + linewidth_factor = 2e3 + line_lower_threshold = 0. + title = "Today's transmission" + + line_widths[line_widths < line_lower_threshold] = 0. + link_widths[link_widths < line_lower_threshold] = 0. + + line_widths[line_widths > line_upper_threshold] = line_upper_threshold + link_widths[link_widths > line_upper_threshold] = line_upper_threshold + + for year in costs.columns: + + fig, ax = plt.subplots(subplot_kw={"projection": ccrs.PlateCarree()}) + fig.set_size_inches(7, 6) + fig.suptitle(year) + + n.plot( + bus_sizes=costs[year] / bus_size_factor, + bus_colors=snakemake.config['plotting']['tech_colors'], + line_colors=ac_color, + link_colors=dc_color, + line_widths=line_widths / linewidth_factor, + link_widths=link_widths / linewidth_factor, + ax=ax, **map_opts + ) + + sizes = [20, 10, 5] + labels = [f"{s} bEUR/a" for s in sizes] + sizes = [s / bus_size_factor * 1e9 for s in sizes] + + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0.01, 1.06), + labelspacing=0.8, + frameon=False, + handletextpad=0, + title="system cost", + ) + + add_legend_circles( + ax, + sizes, + labels, + srid=n.srid, + patch_kw=dict(facecolor="lightgrey"), + legend_kw=legend_kw, + ) + + sizes = [10, 5] + labels = [f"{s} GW" for s in sizes] + scale = 1e3 / linewidth_factor + sizes = [s * scale for s in sizes] + + legend_kw = dict( + loc="upper left", + bbox_to_anchor=(0.27, 1.06), + frameon=False, + labelspacing=0.8, + handletextpad=1, + title=title, + ) + + add_legend_lines( + ax, sizes, labels, patch_kw=dict(color="lightgrey"), legend_kw=legend_kw + ) + + legend_kw = dict( + bbox_to_anchor=(1.52, 1.04), + frameon=False, + ) + + + fig.savefig( + snakemake.output[f"map_{year}"], + transparent=True, + bbox_inches="tight" + ) + + +#%% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -921,10 +1068,9 @@ if __name__ == "__main__": "plot_network", simpl="", opts="", - clusters="5", - ll="v1.5", - sector_opts="CO2L0-1H-T-H-B-I-A-solar+p3-dist1", - planning_horizons="2030", + clusters="37", + ll="v1.0", + sector_opts="4380H-T-H-B-I-A-solar+p3-dist1", ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -937,17 +1083,22 @@ if __name__ == "__main__": if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] - - plot_map( - n, - components=["generators", "links", "stores", "storage_units"], - bus_size_factor=2e10, - transmission=False, - ) - - plot_h2_map(n, regions) - plot_ch4_map(n) - plot_map_without(n) + + if snakemake.params["foresight"] == "perfect": + plot_map_perfect(n, + components=["Link", "Store", "StorageUnit", "Generator"], + bus_size_factor=2e10) + else: + plot_map( + n, + components=["generators", "links", "stores", "storage_units"], + bus_size_factor=2e10, + transmission=False, + ) + + plot_h2_map(n, regions) + plot_ch4_map(n) + plot_map_without(n) # plot_series(n, carrier="AC", name=suffix) # plot_series(n, carrier="heat", name=suffix) From 73df3788372c444742646cb74ea478b0790f1822 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 15:48:06 +0200 Subject: [PATCH 39/77] add memory logger --- scripts/solve_network.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 5cc7d877..cb636ef5 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -35,6 +35,8 @@ import pypsa import xarray as xr from _helpers import configure_logging, update_config_with_sector_opts +from vresutils.benchmark import memory_logger + logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense @@ -717,14 +719,21 @@ if __name__ == "__main__": planning_horizons=snakemake.params.planning_horizons, co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) + + with memory_logger(filename=getattr(snakemake.log, 'memory', None), interval=30.) as mem: + + n = solve_network( + n, + config=snakemake.config, + solving=snakemake.params.solving, + opts=opts, + log_fn=snakemake.log.solver, + ) + + logger.info("Maximum memory usage: {}".format(mem.mem_usage)) + + - n = solve_network( - n, - config=snakemake.config, - solving=snakemake.params.solving, - opts=opts, - log_fn=snakemake.log.solver, - ) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) From 6b07faf7cdd8a5621567029a78b81e046773472a Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 24 Aug 2023 15:50:37 +0200 Subject: [PATCH 40/77] intermediate solution glc transmission --- scripts/solve_network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index cb636ef5..97ff2b7e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -710,6 +710,9 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) n = pypsa.Network(snakemake.input.network) + + # todo + n.remove("GlobalConstraint", "lv_limit") n = prepare_network( n, From ca0d67110ecba86373b62a6a7da06cc0b2692388 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 10:49:03 +0200 Subject: [PATCH 41/77] add land use constraint --- scripts/solve_network.py | 68 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 97ff2b7e..3ad460f3 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -32,6 +32,7 @@ import re import numpy as np import pandas as pd import pypsa +from pypsa.descriptors import nominal_attrs import xarray as xr from _helpers import configure_logging, update_config_with_sector_opts @@ -40,13 +41,72 @@ from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense - +from linopy.expressions import merge +from numpy import isnan def add_land_use_constraint(n, planning_horizons, config): if "m" in snakemake.wildcards.clusters: _add_land_use_constraint_m(n, planning_horizons, config) else: _add_land_use_constraint(n) + + +def add_land_use_constraint_perfect(n): + """Add global constraints for tech capacity limit. + """ + logger.info("Add land-use constraint for perfect foresight") + def compress_series(s): + + def process_group(group): + if group.nunique() == 1: + return pd.Series(group.iloc[0], index=[None]) + else: + return group + + return s.groupby(level=[0,1]).apply(process_group) + + def new_index_name(t): + # Convert all elements to string and filter out None values + parts = [str(x) for x in t if x is not None] + # Join with space, but use a dash for the last item if not None + return ' '.join(parts[:2]) + (f'-{parts[-1]}' if len(parts) > 2 else '') + + def check_p_min_p_max(p_nom_max): + p_nom_min = n.generators[ext_i].groupby(grouper).sum().p_nom_min + p_nom_min = p_nom_min.reindex(p_nom_max.index) + check = (p_nom_min.groupby(level=[0,1]).sum()>p_nom_max.groupby(level=[0,1]).min()) + if check.sum(): + logger.warning(f"summed p_min_pu values at node larger than technical potential {check[check].index}") + + grouper = [n.generators.carrier, n.generators.bus, n.generators.build_year] + ext_i = n.generators.p_nom_extendable + # get technical limit per node and investment period + p_nom_max = n.generators[ext_i].groupby(grouper).min().p_nom_max + # drop carriers without tech limit + p_nom_max = p_nom_max[~p_nom_max.isin([np.inf, np.nan])] + # carrier + carriers = p_nom_max.index.get_level_values(0).unique() + gen_i = n.generators[(n.generators.carrier.isin(carriers)) & + (ext_i) & + (n.generators.build_year>n.investment_periods[0]) + ].index + n.generators.loc[gen_i, "p_nom_min"] = 0 + # check minimum capacities + check_p_min_p_max(p_nom_max) + # drop multi entries in case p_nom_max stays constant in different periods + # p_nom_max = compress_series(p_nom_max) + # adjust name to fit syntax of nominal constraint per bus + df = p_nom_max.reset_index() + df["name"] = df.apply(lambda row: f"nom_max_{row['carrier']}" + + (f"_{row['build_year']}" if row['build_year'] is not None else ""), + axis=1) + + for name in df.name.unique(): + df_carrier = df[df.name==name] + bus = df_carrier.bus + n.buses.loc[bus, name] = df_carrier.p_nom_max.values + + return n def _add_land_use_constraint(n): @@ -232,6 +292,9 @@ def prepare_network( if foresight == "myopic": add_land_use_constraint(n, planning_horizons, config) + + if foresight == "perfect": + n = add_land_use_constraint_perfect(n) if n.stores.carrier.eq("co2 stored").any(): limit = co2_sequestration_potential @@ -627,7 +690,8 @@ def extra_functionality(n, snapshots): add_EQ_constraints(n, o) add_battery_constraints(n) add_pipe_retrofit_constraint(n) - add_carbon_neutral_constraint(n, snapshots) + if n._multi_invest: + add_carbon_neutral_constraint(n, snapshots) def solve_network(n, config, solving, opts="", **kwargs): From 727aab92f5b3880ff032440d3b8c4a4bb421b0dc Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 10:50:25 +0200 Subject: [PATCH 42/77] rm intermediate solution --- scripts/solve_network.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 3ad460f3..9137dd14 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -775,8 +775,6 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) - # todo - n.remove("GlobalConstraint", "lv_limit") n = prepare_network( n, From e6d779a91f0dd8896f3da5dbe37db51f3593fa7b Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 11:56:29 +0200 Subject: [PATCH 43/77] first step add max_growth --- config/config.perfect.yaml | 5 +++++ scripts/solve_network.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index fcee4972..fcbbef74 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -499,6 +499,11 @@ sector: OCGT: gas biomass_to_liquid: false biosng: false + max_growth: + onwind: 16 + solar: 28 + offwind-ac: 35 + offwind-dc 35 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 9137dd14..9a9d353e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -232,6 +232,25 @@ def add_carbon_neutral_constraint(n, snapshots): n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") + +def add_max_growth(n): + """Add maximum growth rates for different carriers""" + logger.info("set maximum growth rate of renewables.") + # solar max grow so far 28 GW in Europe https://www.iea.org/reports/renewables-2020/solar-pv + n.carriers.loc["solar", "max_growth"] = 280 * 1.3 * 1e3 # 70 * 1e3 + # onshore max grow so far 16 GW in Europe https://www.iea.org/reports/renewables-2020/wind + n.carriers.loc["onwind", "max_growth"] = 160 * 1.3 * 1e3 # 40 * 1e3 + # offshore max grow so far 3.5 GW in Europe https://windeurope.org/about-wind/statistics/offshore/european-offshore-wind-industry-key-trends-statistics-2019/ + n.carriers.loc[["offwind-ac", "offwind-dc"], "max_growth"] = 35 * 1.3 * 1e3 # 8.75 * 1e3 + + res = ["solar", "onwind", "offwind-ac", "offwind-dc"] + n.carriers.loc[res, "max_relative_growth"] = 3 + + # # heating sector + # heat_c = n.carriers[n.carriers.index.str.contains("pump")].index + # n.carriers.loc[res, "max_relative_growth"] = 2 + + return n def prepare_network( n, @@ -295,6 +314,7 @@ def prepare_network( if foresight == "perfect": n = add_land_use_constraint_perfect(n) + n = add_max_growth(n) if n.stores.carrier.eq("co2 stored").any(): limit = co2_sequestration_potential @@ -665,6 +685,7 @@ def add_pipe_retrofit_constraint(n): n.model.add_constraints(lhs == rhs, name="Link-pipe_retrofit") + def extra_functionality(n, snapshots): """ Collects supplementary constraints which will be passed to From 3fcf3f2fe65813736dba92917822e627569e2059 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 13:27:34 +0200 Subject: [PATCH 44/77] move max growth assumptions to config --- config/config.perfect.yaml | 19 ++++++++++++++----- scripts/solve_network.py | 35 +++++++++++++++++------------------ 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index fcbbef74..1787c640 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -499,11 +499,20 @@ sector: OCGT: gas biomass_to_liquid: false biosng: false - max_growth: - onwind: 16 - solar: 28 - offwind-ac: 35 - offwind-dc 35 + limit_max_growth: + enable: true + # allowing 30% larger than max historic growth + factor: 1.3 + max_growth: # unit GW + onwind: 16 # onshore max grow so far 16 GW in Europe https://www.iea.org/reports/renewables-2020/wind + solar: 28 # solar max grow so far 28 GW in Europe https://www.iea.org/reports/renewables-2020/solar-pv + offwind-ac: 35 # offshore max grow so far 3.5 GW in Europe https://windeurope.org/about-wind/statistics/offshore/european-offshore-wind-industry-key-trends-statistics-2019/ + offwind-dc: 35 + max_relative_growth: + onwind: 3 + solar: 3 + offwind-ac: 3 + offwind-dc: 3 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 9a9d353e..2960d5c3 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -87,9 +87,7 @@ def add_land_use_constraint_perfect(n): # carrier carriers = p_nom_max.index.get_level_values(0).unique() gen_i = n.generators[(n.generators.carrier.isin(carriers)) & - (ext_i) & - (n.generators.build_year>n.investment_periods[0]) - ].index + (ext_i)].index n.generators.loc[gen_i, "p_nom_min"] = 0 # check minimum capacities check_p_min_p_max(p_nom_max) @@ -233,22 +231,22 @@ def add_carbon_neutral_constraint(n, snapshots): n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") -def add_max_growth(n): - """Add maximum growth rates for different carriers""" - logger.info("set maximum growth rate of renewables.") - # solar max grow so far 28 GW in Europe https://www.iea.org/reports/renewables-2020/solar-pv - n.carriers.loc["solar", "max_growth"] = 280 * 1.3 * 1e3 # 70 * 1e3 - # onshore max grow so far 16 GW in Europe https://www.iea.org/reports/renewables-2020/wind - n.carriers.loc["onwind", "max_growth"] = 160 * 1.3 * 1e3 # 40 * 1e3 - # offshore max grow so far 3.5 GW in Europe https://windeurope.org/about-wind/statistics/offshore/european-offshore-wind-industry-key-trends-statistics-2019/ - n.carriers.loc[["offwind-ac", "offwind-dc"], "max_growth"] = 35 * 1.3 * 1e3 # 8.75 * 1e3 +def add_max_growth(n, config): + """Add maximum growth rates for different carriers + """ - res = ["solar", "onwind", "offwind-ac", "offwind-dc"] - n.carriers.loc[res, "max_relative_growth"] = 3 + opts = config["sector"]["limit_max_growth"] + # take maximum yearly difference between investment periods since historic growth is per year + factor = n.investment_period_weightings.years.max() * opts["factor"] + for carrier in opts["max_growth"].keys(): + max_per_period = opts["max_growth"][carrier] * factor + logger.info(f"set maximum growth rate per investment period of {carrier} to {max_per_period} GW.") + n.carriers.loc[carrier, "max_growth"] = max_per_period * 1e3 - # # heating sector - # heat_c = n.carriers[n.carriers.index.str.contains("pump")].index - # n.carriers.loc[res, "max_relative_growth"] = 2 + for carrier in opts["max_relative_growth"].keys(): + max_r_per_period = opts["max_relative_growth"][carrier] + logger.info(f"set maximum relative growth per investment period of {carrier} to {max_r_per_period}.") + n.carriers.loc[carrier, "max_relative_growth"] = max_r_per_period return n @@ -314,7 +312,8 @@ def prepare_network( if foresight == "perfect": n = add_land_use_constraint_perfect(n) - n = add_max_growth(n) + if config["sector"]["limit_max_growth"]["enable"]: + n = add_max_growth(n, config) if n.stores.carrier.eq("co2 stored").any(): limit = co2_sequestration_potential From d646c09d609fad8366d3f09f3b4209cfb71a9d72 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 14:16:09 +0200 Subject: [PATCH 45/77] intermediate solution back --- scripts/solve_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2960d5c3..25472112 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -795,6 +795,7 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) + n.remove("GlobalConstraint", "lv_limit") n = prepare_network( n, From 0cfd5fab589a5d61d155d31da2bae30c63aa8d0c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 15:50:26 +0200 Subject: [PATCH 46/77] add carbon budget constraint --- scripts/prepare_perfect_foresight.py | 22 ++++++++++++++----- scripts/solve_network.py | 32 +++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 66af8ed1..d3ec4966 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -185,6 +185,7 @@ def adjust_stores(n): e_initial_store = ["co2 stored"] co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index n.stores.loc[co2_i, "e_initial_per_period"] = True + return n @@ -240,13 +241,24 @@ def set_carbon_constraints(n, opts): carrier_attribute="co2_emissions", sense="<=", constant=budget, + investment_period=n.investment_periods[-1] + ) + + # drop other CO2 limits + drop_i = n.global_constraints[n.global_constraints.type=="co2_limit"].index + n.mremove("GlobalConstraint", drop_i) + + n.add( + "GlobalConstraint", + "carbon_neutral", + type="co2_limit", + carrier_attribute="co2_emissions", + sense="<=", + constant=0, + investment_period=n.investment_periods[-1] ) - else: - e_initial_store = ["co2 stored"] - co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index - n.stores.loc[co2_i, "e_initial_per_period"] = True # set minimum CO2 emission constraint to avoid too fast reduction if "co2min" in opts: emissions_1990 = 4.53693 @@ -305,7 +317,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="4380H-T-H-B-I-A-solar+p3-dist1", + sector_opts="2p0-4380H-T-H-B-I-A-solar+p3-dist1", ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 25472112..6fa1336b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -205,7 +205,7 @@ def add_co2_sequestration_limit(n, config, limit=200): ) -def add_carbon_neutral_constraint(n, snapshots): +def add_carbon_constraint(n, snapshots): glcs = n.global_constraints.query('type == "co2_limit"') if glcs.empty: return @@ -231,6 +231,32 @@ def add_carbon_neutral_constraint(n, snapshots): n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") +def add_carbon_budget_constraint(n, snapshots): + glcs = n.global_constraints.query('type == "Co2Budget"') + if glcs.empty: + return + for name, glc in glcs.iterrows(): + rhs = glc.constant + carattr = glc.carrier_attribute + emissions = n.carriers.query(f"{carattr} != 0")[carattr] + + if emissions.empty: + continue + + # stores + n.stores["carrier"] = n.stores.bus.map(n.buses.carrier) + stores = n.stores.query("carrier in @emissions.index and not e_cyclic") + time_valid = int(glc.loc["investment_period"]) + if not stores.empty: + last = n.snapshot_weightings.reset_index().groupby("period").last() + last_i = last.set_index([last.index, last.timestep]).index + final_e = n.model["Store-e"].loc[last_i, stores.index] + time_i = pd.IndexSlice[time_valid, :] + lhs = final_e.loc[time_i,:] + + n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") + + def add_max_growth(n, config): """Add maximum growth rates for different carriers """ @@ -711,7 +737,7 @@ def extra_functionality(n, snapshots): add_battery_constraints(n) add_pipe_retrofit_constraint(n) if n._multi_invest: - add_carbon_neutral_constraint(n, snapshots) + add_carbon_constraint(n, snapshots) def solve_network(n, config, solving, opts="", **kwargs): @@ -776,7 +802,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="4380H-T-H-B-I-A-solar+p3-dist1", + sector_opts="2p0-4380H-T-H-B-I-A-solar+p3-dist1", planning_horizons="2030", ) configure_logging(snakemake) From 90b85fcb87d01d29f954bd2e0c42532f1b7a97ae Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 16:20:06 +0200 Subject: [PATCH 47/77] update to tsam performance --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index c3af36bb..bc48eee2 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -55,5 +55,5 @@ dependencies: - pip: - - tsam>=1.1.0 + - git+https://github.com/fneum/tsam.git@performance - pypsa>=0.25.1 From b362c009f5f0e7a362f71f4ef9d8644eb9c32e4c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Fri, 25 Aug 2023 16:44:25 +0200 Subject: [PATCH 48/77] drop nan with SEG --- scripts/prepare_sector_network.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 207fa263..a5c0361a 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3212,7 +3212,7 @@ def apply_time_segmentation( df = pnl.copy() df.columns = pd.MultiIndex.from_product([[c.name], [attr], df.columns]) raw = pd.concat([raw, df], axis=1) - + raw = raw.dropna(axis=1) # normalise all time-dependent data annual_max = raw.max().replace(0, 1) raw = raw.div(annual_max, level=0) @@ -3282,12 +3282,11 @@ if __name__ == "__main__": snakemake = mock_snakemake( "prepare_sector_network", - # configfiles="test/config.overnight.yaml", simpl="", opts="", clusters="37", ll="v1.0", - sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", + sector_opts="60SEG-T-H-B-I-A-solar+p3-dist1", planning_horizons="2020", ) From 369eaf34573216f22acddb8ffbf263786337d999 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 28 Aug 2023 10:21:02 +0200 Subject: [PATCH 49/77] fix bug carbon budget --- scripts/solve_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 6fa1336b..49d99311 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -247,12 +247,13 @@ def add_carbon_budget_constraint(n, snapshots): n.stores["carrier"] = n.stores.bus.map(n.buses.carrier) stores = n.stores.query("carrier in @emissions.index and not e_cyclic") time_valid = int(glc.loc["investment_period"]) + weighting = n.investment_period_weightings.loc[time_valid, "years"] if not stores.empty: last = n.snapshot_weightings.reset_index().groupby("period").last() last_i = last.set_index([last.index, last.timestep]).index final_e = n.model["Store-e"].loc[last_i, stores.index] time_i = pd.IndexSlice[time_valid, :] - lhs = final_e.loc[time_i,:] + lhs = final_e.loc[time_i,:] * weighting n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") @@ -738,6 +739,7 @@ def extra_functionality(n, snapshots): add_pipe_retrofit_constraint(n) if n._multi_invest: add_carbon_constraint(n, snapshots) + add_carbon_budget_constraint(n, snapshots) def solve_network(n, config, solving, opts="", **kwargs): From abb584de453fccbabc641cf806b065d23b1de164 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 28 Aug 2023 13:31:02 +0200 Subject: [PATCH 50/77] add H2 boiler and constraint to avoid existing gas boiler back up --- config/config.perfect.yaml | 13 +++++--- scripts/prepare_perfect_foresight.py | 34 ++++++++++++++++--- scripts/solve_network.py | 50 ++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index 1787c640..2b625420 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -20,7 +20,7 @@ remote: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - name: "test_co2limit" + name: "test_h2_retrofit" disable_progressbar: false shared_resources: true shared_cutouts: true @@ -35,7 +35,7 @@ scenario: - '' ll: - v1.0 - - v1.5 +# - v1.5 clusters: - 37 # - 128 @@ -45,7 +45,9 @@ scenario: opts: - '' sector_opts: - - 4380H-T-H-B-I-A-solar+p3-dist1 + - 1p7-4380H-T-H-B-I-A-solar+p3-dist1 + - 1p5-4380H-T-H-B-I-A-solar+p3-dist1 + - 2p0-4380H-T-H-B-I-A-solar+p3-dist1 planning_horizons: - 2020 - 2030 @@ -69,7 +71,7 @@ enable: retrieve_sector_databundle: true retrieve_cost_data: true build_cutout: false - retrieve_cutout: false + retrieve_cutout: true build_natura_raster: false retrieve_natura_raster: true custom_busmap: false @@ -133,7 +135,7 @@ electricity: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite atlite: default_cutout: europe-2013-era5 - nprocesses: 4 + nprocesses: 1 show_progress: false cutouts: # use 'base' to determine geographical bounds and time span from config @@ -916,6 +918,7 @@ plotting: H2 for shipping: "#ebaee0" H2: '#bf13a0' hydrogen: '#bf13a0' + retrofitted H2 boiler: '#e5a0d9' SMR: '#870c71' SMR CC: '#4f1745' H2 liquefaction: '#d647bd' diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index d3ec4966..3bfc2931 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -306,6 +306,29 @@ def adjust_CO2_glc(n): n.df(c).loc[mask, "type"] = "co2_limit" return n + + +def add_H2_boilers(n): + c = "Link" + logger.info("Add H2 boilers.") + # existing gas boilers + mask = n.links.carrier.str.contains("gas boiler") & ~n.links.p_nom_extendable + gas_i = n.links[mask].index + df = n.links.loc[gas_i] + # adjust bus 0 + df["bus0"] = df.bus1.map(n.buses.location) + " H2" + # rename carrier and index + df["carrier"] = df.carrier.apply(lambda x: x.replace("gas boiler", "retrofitted H2 boiler")) + df.rename(index = lambda x: x.replace("gas boiler", "retrofitted H2 boiler"), inplace=True) + # todo, costs for retrofitting + df["capital_costs"] = 100 + # set existing capacity to zero + df["p_nom"] = 0 + df["p_nom_extendable"] = True + # add H2 boilers to network + import_components_from_dataframe(n, df, c) + + # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -339,14 +362,17 @@ if __name__ == "__main__": n = concat_networks(years) # adjust global constraints lv limit if the same for all years - n = adjust_lvlimit(n) - + n = adjust_lvlimit(n) # adjust global constraints CO2 limit n = adjust_CO2_glc(n) - # set phase outs - set_all_phase_outs(n) # adjust stores to multi period investment n = adjust_stores(n) + + # set phase outs + set_all_phase_outs(n) + + # add H2 boiler + add_H2_boilers(n) # set carbon constraints opts = snakemake.wildcards.sector_opts.split("-") diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 49d99311..38bee124 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -32,7 +32,7 @@ import re import numpy as np import pandas as pd import pypsa -from pypsa.descriptors import nominal_attrs +from pypsa.descriptors import nominal_attrs, get_activity_mask import xarray as xr from _helpers import configure_logging, update_config_with_sector_opts @@ -41,6 +41,7 @@ from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense +from pypsa.io import import_components_from_dataframe from linopy.expressions import merge from numpy import isnan @@ -276,7 +277,51 @@ def add_max_growth(n, config): n.carriers.loc[carrier, "max_relative_growth"] = max_r_per_period return n - + + +def add_retrofit_gas_boiler_constraint(n, snapshots): + """Allow retrofitting of existing gas boilers to H2 boilers. + """ + c = "Link" + logger.info("Add constraint for retrofitting gas boilers to H2 boilers.") + # existing gas boilers + mask = n.links.carrier.str.contains("gas boiler") & ~n.links.p_nom_extendable + gas_i = n.links[mask].index + mask = n.links.carrier.str.contains("retrofitted H2 boiler") + h2_i = n.links[mask].index + + + n.links.loc[gas_i, "p_nom_extendable"] = True + p_nom = n.links.loc[gas_i, "p_nom"] + n.links.loc[gas_i, "p_nom"] = 0 + + # heat profile + cols = n.loads_t.p_set.columns[n.loads_t.p_set.columns.str.contains("heat") + & ~n.loads_t.p_set.columns.str.contains("industry") + & ~n.loads_t.p_set.columns.str.contains("agriculture")] + profile = n.loads_t.p_set[cols].div(n.loads_t.p_set[cols].groupby(level=0).max(), level=0) + # to deal if max value is zero + profile.fillna(0, inplace=True) + profile.rename(columns=n.loads.bus.to_dict(), inplace=True) + profile = profile.reindex(columns=n.links.loc[gas_i, "bus1"]) + profile.columns = gas_i + + + rhs = profile.mul(p_nom) + + dispatch = n.model["Link-p"] + active = get_activity_mask(n, c, snapshots, gas_i) + rhs = rhs[active] + p_gas = dispatch.sel(Link=gas_i) + p_h2 = dispatch.sel(Link=h2_i) + + lhs = p_gas + p_h2 + + n.model.add_constraints(lhs == rhs, name="gas_retrofit") + + + + def prepare_network( n, solve_opts=None, @@ -740,6 +785,7 @@ def extra_functionality(n, snapshots): if n._multi_invest: add_carbon_constraint(n, snapshots) add_carbon_budget_constraint(n, snapshots) + add_retrofit_gas_boiler_constraint(n, snapshots) def solve_network(n, config, solving, opts="", **kwargs): From 4cd492bb6f10a4e50ef86de45cc42e40314958e0 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 28 Aug 2023 13:52:55 +0200 Subject: [PATCH 51/77] drop assets which reached end of lifetime --- scripts/add_existing_baseyear.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 4e6caaaa..782fde78 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -607,6 +607,10 @@ def add_heating_capacities_installed_before_baseyear( if str(grouping_year) in index and n.links.p_nom[index] < threshold ], ) + + # drop assets which are at the end of their lifetime + links_i = n.links[(n.links.build_year+n.links.lifetime<=baseyear)].index + n.mremove("Link", links_i) # %% @@ -621,7 +625,7 @@ if __name__ == "__main__": clusters="37", ll="v1.0", opts="", - sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", + sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1", planning_horizons=2020, ) From 0c42ac4bdc622c7b1214eb4296d62bf80f530d37 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 28 Aug 2023 17:27:01 +0200 Subject: [PATCH 52/77] remove min part for fischer tropsch --- config/config.perfect.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index 2b625420..217b92e9 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -20,7 +20,7 @@ remote: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - name: "test_h2_retrofit" + name: "test_methanolisation_shorterlifetime" disable_progressbar: false shared_resources: true shared_cutouts: true @@ -409,12 +409,12 @@ sector: 2040: 0 2050: 0 shipping_methanol_share: - 2020: 0 + 2020: 0.0 2030: 0.3 2040: 0.7 2050: 1 shipping_oil_share: - 2020: 1 + 2020: 1.0 2030: 0.7 2040: 0.3 2050: 0 @@ -480,7 +480,7 @@ sector: - nearshore # within 50 km of sea # - offshore ammonia: false - min_part_load_fischer_tropsch: 0.9 + min_part_load_fischer_tropsch: 0 # 0.9 min_part_load_methanolisation: 0 # 0.5 use_fischer_tropsch_waste_heat: true use_fuel_cell_waste_heat: true From c35391d594f2e202983acf3dcde4b13286f8ea43 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 29 Aug 2023 15:46:58 +0200 Subject: [PATCH 53/77] move SEG aggregation later --- scripts/prepare_perfect_foresight.py | 94 +++++++++++++++++++++++++++- scripts/prepare_sector_network.py | 15 +++-- 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 3bfc2931..0982f8a0 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -10,13 +10,13 @@ import logging import re import pandas as pd +import numpy as np import pypsa from _helpers import update_config_with_sector_opts from add_existing_baseyear import add_build_year_to_new_assets from pypsa.descriptors import expand_series from pypsa.io import import_components_from_dataframe from six import iterkeys - logger = logging.getLogger(__name__) @@ -328,7 +328,90 @@ def add_H2_boilers(n): # add H2 boilers to network import_components_from_dataframe(n, df, c) - +def apply_time_segmentation_perfect( + n, segments, solver_name="cbc", overwrite_time_dependent=True +): + """ + Aggregating time series to segments with different lengths. + + Input: + n: pypsa Network + segments: (int) number of segments in which the typical period should be + subdivided + solver_name: (str) name of solver + overwrite_time_dependent: (bool) overwrite time dependent data of pypsa network + with typical time series created by tsam + """ + try: + import tsam.timeseriesaggregation as tsam + except: + raise ModuleNotFoundError( + "Optional dependency 'tsam' not found." "Install via 'pip install tsam'" + ) + + # get all time-dependent data + columns = pd.MultiIndex.from_tuples([], names=["component", "key", "asset"]) + raw = pd.DataFrame(index=n.snapshots, columns=columns) + for c in n.iterate_components(): + for attr, pnl in c.pnl.items(): + # exclude e_min_pu which is used for SOC of EVs in the morning + if not pnl.empty and attr != "e_min_pu": + df = pnl.copy() + df.columns = pd.MultiIndex.from_product([[c.name], [attr], df.columns]) + raw = pd.concat([raw, df], axis=1) + raw = raw.dropna(axis=1) + sn_weightings = {} + + for year in raw.index.levels[0]: + logger.info(f"Find representative snapshots for {year}.") + raw_t = raw.loc[year] + # normalise all time-dependent data + annual_max = raw_t.max().replace(0, 1) + raw_t = raw_t.div(annual_max, level=0) + # get representative segments + agg = tsam.TimeSeriesAggregation( + raw_t, + hoursPerPeriod=len(raw_t), + noTypicalPeriods=1, + noSegments=int(segments), + segmentation=True, + solver=solver_name, + ) + segmented = agg.createTypicalPeriods() + + weightings = segmented.index.get_level_values("Segment Duration") + offsets = np.insert(np.cumsum(weightings[:-1]), 0, 0) + timesteps = [raw_t.index[0] + pd.Timedelta(f"{offset}h") for offset in offsets] + snapshots = pd.DatetimeIndex(timesteps) + sn_weightings[year] = pd.Series( + weightings, index=snapshots, name="weightings", dtype="float64" + ) + + sn_weightings = pd.concat(sn_weightings) + n.set_snapshots(sn_weightings.index) + n.snapshot_weightings = n.snapshot_weightings.mul(sn_weightings, axis=0) + + # overwrite time-dependent data with timeseries created by tsam + # if overwrite_time_dependent: + # values_t = segmented.mul(annual_max).set_index(snapshots) + # for component, key in values_t.columns.droplevel(2).unique(): + # n.pnl(component)[key] = values_t[component, key] + + return n + +def set_temporal_aggregation_perfect(n, opts, solver_name): + """ + Aggregate network temporally with tsam. + """ + for o in opts: + # segments with package tsam + m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) + if m is not None: + segments = int(m[1]) + logger.info(f"Use temporal segmentation with {segments} segments") + n = apply_time_segmentation_perfect(n, segments, solver_name=solver_name) + break + return n # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -340,7 +423,7 @@ if __name__ == "__main__": opts="", clusters="37", ll="v1.0", - sector_opts="2p0-4380H-T-H-B-I-A-solar+p3-dist1", + sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1", ) update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) @@ -361,6 +444,11 @@ if __name__ == "__main__": # concat prenetworks of planning horizon to single network ------------ n = concat_networks(years) + # temporal aggregate + opts = snakemake.wildcards.sector_opts.split("-") + solver_name = snakemake.config["solving"]["solver"]["name"] + n = set_temporal_aggregation_perfect(n, opts, solver_name) + # adjust global constraints lv limit if the same for all years n = adjust_lvlimit(n) # adjust global constraints CO2 limit diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a5c0361a..6b16a4e9 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -716,7 +716,6 @@ def average_every_nhours(n, offset): return m - def cycling_shift(df, steps=1): """ Cyclic shift on index of pd.Series|pd.DataFrame by number of steps. @@ -3269,10 +3268,13 @@ def set_temporal_aggregation(n, opts, solver_name): # segments with package tsam m = re.match(r"^(\d+)seg$", o, re.IGNORECASE) if m is not None: - segments = int(m[1]) - logger.info(f"Use temporal segmentation with {segments} segments") - n = apply_time_segmentation(n, segments, solver_name=solver_name) - break + if snakemake.params.foresight!="perfect": + segments = int(m[1]) + logger.info(f"Use temporal segmentation with {segments} segments") + n = apply_time_segmentation(n, segments, solver_name=solver_name) + break + else: + logger.info("Apply temporal segmentation at prepare_perfect_foresight.") return n #%% @@ -3287,7 +3289,7 @@ if __name__ == "__main__": clusters="37", ll="v1.0", sector_opts="60SEG-T-H-B-I-A-solar+p3-dist1", - planning_horizons="2020", + planning_horizons="2050", ) logging.basicConfig(level=snakemake.config["logging"]["level"]) @@ -3389,6 +3391,7 @@ if __name__ == "__main__": add_allam(n, costs) solver_name = snakemake.config["solving"]["solver"]["name"] + n = set_temporal_aggregation(n, opts, solver_name) limit_type = "config" From 22d61ad10e500c11186fc12f38c257a3e258132c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 30 Aug 2023 11:08:30 +0200 Subject: [PATCH 54/77] add line expansion with DC links --- scripts/prepare_perfect_foresight.py | 49 +++++++++++++++++++++++++--- scripts/solve_network.py | 2 -- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 0982f8a0..20c99abe 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -73,6 +73,47 @@ def add_year_to_constraints(n, baseyear): c.df.rename(index=lambda x: x + "-" + str(baseyear), inplace=True) +def hvdc_transport_model(n): + + logger.info("Convert AC lines to DC links to perform multi-decade optimisation.") + + n.madd("Link", + n.lines.index, + bus0=n.lines.bus0, + bus1=n.lines.bus1, + p_nom_extendable=True, + p_nom=n.lines.s_nom, + p_nom_min=n.lines.s_nom, + p_min_pu=-1, + efficiency=1 - 0.03 * n.lines.length / 1000, + marginal_cost=0, + carrier="DC", + length=n.lines.length, + capital_cost=n.lines.capital_cost) + + # Remove AC lines + logger.info("Removing AC lines") + lines_rm = n.lines.index + n.mremove("Line", lines_rm) + + # Set efficiency of all DC links to include losses depending on length + n.links.loc[n.links.carrier == 'DC', 'efficiency'] = 1 - 0.03 * n.links.loc[ + n.links.carrier == 'DC', 'length'] / 1000 + + +def adjust_electricity_grid(n, year, years): + n.lines["carrier"] = "AC" + links_i = n.links[n.links.carrier=="DC"].index + if n.lines.s_nom_extendable.any() or n.links.loc[links_i, "p_nom_extendable"].any(): + hvdc_transport_model(n) + links_i = n.links[n.links.carrier=="DC"].index + n.links.loc[links_i, "lifetime"] = 100 + if year!= years[0]: + n.links.loc[links_i, "p_nom_min"] = 0 + n.links.loc[links_i, "p_nom"] = 0 + + + # -------------------------------------------------------------------- def concat_networks(years): """ @@ -93,7 +134,7 @@ def concat_networks(years): for i, network_path in enumerate(network_paths): year = years[i] network = pypsa.Network(network_path) - network.lines["carrier"] = "AC" + adjust_electricity_grid(network, year, years) add_build_year_to_new_assets(network, year) # static ---------------------------------- @@ -399,7 +440,7 @@ def apply_time_segmentation_perfect( return n -def set_temporal_aggregation_perfect(n, opts, solver_name): +def set_temporal_aggregation_SEG(n, opts, solver_name): """ Aggregate network temporally with tsam. """ @@ -422,7 +463,7 @@ if __name__ == "__main__": simpl="", opts="", clusters="37", - ll="v1.0", + ll="v1.5", sector_opts="1p7-4380H-T-H-B-I-A-solar+p3-dist1", ) @@ -447,7 +488,7 @@ if __name__ == "__main__": # temporal aggregate opts = snakemake.wildcards.sector_opts.split("-") solver_name = snakemake.config["solving"]["solver"]["name"] - n = set_temporal_aggregation_perfect(n, opts, solver_name) + n = set_temporal_aggregation_SEG(n, opts, solver_name) # adjust global constraints lv limit if the same for all years n = adjust_lvlimit(n) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 38bee124..7c6be9e4 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -869,8 +869,6 @@ if __name__ == "__main__": n = pypsa.Network(snakemake.input.network) - n.remove("GlobalConstraint", "lv_limit") - n = prepare_network( n, solve_opts, From 498b22565c2398313fb5ea99df2b566239ab0cea Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 30 Aug 2023 11:14:33 +0200 Subject: [PATCH 55/77] add CI perfect --- .github/workflows/ci.yaml | 1 + config/test/config.perfect.yaml | 88 +++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 config/test/config.perfect.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2be3909..c0fb745d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,6 +83,7 @@ jobs: snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime + snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml new file mode 100644 index 00000000..4da629bc --- /dev/null +++ b/config/test/config.perfect.yaml @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +tutorial: true + +run: + name: "test-sector-perfect" + disable_progressbar: true + shared_resources: true + shared_cutouts: true + +foresight: perfect + +scenario: + ll: + - v1.5 + clusters: + - 5 + sector_opts: + - 8760H-T-H-B-I-A-solar+p3-dist1 + planning_horizons: + - 2020 + - 2030 + - 2040 + - 2050 + +countries: ['BE'] + +snapshots: + start: "2013-03-01" + end: "2013-03-08" + +electricity: + co2limit: 100.e+6 + + extendable_carriers: + Generator: [OCGT] + StorageUnit: [battery] + Store: [H2] + Link: [H2 pipeline] + + renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] + +atlite: + default_cutout: be-03-2013-era5 + cutouts: + be-03-2013-era5: + module: era5 + x: [4., 15.] + y: [46., 56.] + time: ["2013-03-01", "2013-03-08"] + +renewable: + onwind: + cutout: be-03-2013-era5 + offwind-ac: + cutout: be-03-2013-era5 + max_depth: false + offwind-dc: + cutout: be-03-2013-era5 + max_depth: false + solar: + cutout: be-03-2013-era5 + +industry: + St_primary_fraction: + 2030: 0.6 + 2040: 0.5 + 2050: 0.4 + +solving: + solver: + name: glpk + options: glpk-default + mem: 4000 + +plotting: + map: + boundaries: + eu_node_location: + x: -5.5 + y: 46. + costs_max: 1000 + costs_threshold: 0.0000001 + energy_max: + energy_min: + energy_threshold: 0.000001 From 3247fb59e0b105a7e6f29aac28ce25b98f565e94 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 30 Aug 2023 11:50:49 +0200 Subject: [PATCH 56/77] clean up --- config/config.default.yaml | 2 ++ doc/foresight.rst | 6 ++-- doc/release_notes.rst | 2 ++ scripts/prepare_perfect_foresight.py | 50 ++++++++++++++++++++++------ scripts/solve_network.py | 7 ++-- 5 files changed, 49 insertions(+), 18 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 8b88db93..b3748039 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -549,6 +549,7 @@ costs: year: 2030 version: v0.6.0 rooftop_share: 0.14 # based on the potentials, assuming (0.1 kW/m2 and 10 m2/person) + social_discountrate: 0.02 fill_values: FOM: 0 VOM: 0 @@ -893,6 +894,7 @@ plotting: H2 for shipping: "#ebaee0" H2: '#bf13a0' hydrogen: '#bf13a0' + retrofitted H2 boiler: '#e5a0d9' SMR: '#870c71' SMR CC: '#4f1745' H2 liquefaction: '#d647bd' diff --git a/doc/foresight.rst b/doc/foresight.rst index c1be3443..dd1e0ecc 100644 --- a/doc/foresight.rst +++ b/doc/foresight.rst @@ -41,10 +41,10 @@ Perfect foresight scenarios .. warning:: - Perfect foresight is currently under development and not yet implemented. + Perfect foresight is currently implemented as a first test version. -For running perfect foresight scenarios, in future versions you will be able to -set in the ``config/config.yaml``: +For running perfect foresight scenarios, you can adjust the + ``config/config.perfect.yaml``: .. code:: yaml diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b6db1b90..57bbd1f4 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,6 +14,8 @@ Upcoming Release * For industry distribution, use EPRTR as fallback if ETS data is not available. +* New feature multi-decade optimisation with perfect foresight. + PyPSA-Eur 0.8.1 (27th July 2023) ================================ diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 20c99abe..cf9d09f8 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -8,7 +8,6 @@ Concats pypsa networks of single investment periods to one network. import logging import re - import pandas as pd import numpy as np import pypsa @@ -39,7 +38,8 @@ def get_missing(df, n, c): def get_social_discount(t, r=0.01): """ - Calculate for a given time t the social discount. + Calculate for a given time t and social discount rate r [per unit] + the social discount. """ return 1 / (1 + r) ** t @@ -61,6 +61,8 @@ def get_investment_weighting(time_weighting, r=0.01): def add_year_to_constraints(n, baseyear): """ + Add investment period to global constraints and rename index. + Parameters ---------- n : pypsa.Network @@ -74,6 +76,10 @@ def add_year_to_constraints(n, baseyear): def hvdc_transport_model(n): + """ + Convert AC lines to DC links for multi-decade optimisation with + line expansion. Losses of DC links are assumed to be 3% per 1000km + """ logger.info("Convert AC lines to DC links to perform multi-decade optimisation.") @@ -102,6 +108,18 @@ def hvdc_transport_model(n): def adjust_electricity_grid(n, year, years): + """ + Add carrier to lines. Replace AC lines with DC links in case of line + expansion. Add lifetime to DC links in case of line expansion. + + Parameters + ---------- + n : pypsa.Network + year : int + year in which optimized assets are built + years: list + investment periods + """ n.lines["carrier"] = "AC" links_i = n.links[n.links.carrier=="DC"].index if n.lines.s_nom_extendable.any() or n.links.loc[links_i, "p_nom_extendable"].any(): @@ -175,7 +193,8 @@ def concat_networks(years): pnl[k].loc[pnl_year.index, pnl_year.columns] = pnl_year else: - # this is to avoid adding multiple times assets with infinit lifetime as ror + # this is to avoid adding multiple times assets with + # infinit lifetime as ror cols = pnl_year.columns.difference(pnl[k].columns) pnl[k] = pd.concat([pnl[k], pnl_year[cols]], axis=1) @@ -250,11 +269,19 @@ def set_all_phase_outs(n): (["nuclear"], "DE", 2022), (["nuclear"], "BE", 2025), (["nuclear"], "ES", 2027), - (["coal", "lignite"], "DE", 2038), + (["coal", "lignite"], "DE", 2030), (["coal", "lignite"], "ES", 2027), (["coal", "lignite"], "FR", 2022), (["coal", "lignite"], "GB", 2024), (["coal", "lignite"], "IT", 2025), + (["coal", "lignite"], "DK", 2030), + (["coal", "lignite"], "FI", 2030), + (["coal", "lignite"], "HU", 2030), + (["coal", "lignite"], "SK", 2030), + (["coal", "lignite"], "GR", 2030), + (["coal", "lignite"], "IE", 2030), + (["coal", "lignite"], "NL", 2030), + (["coal", "lignite"], "RS", 2030), ] for carrier, ct, phase_out_year in planned: set_phase_out(n, carrier, ct, phase_out_year) @@ -326,6 +353,10 @@ def set_carbon_constraints(n, opts): def adjust_lvlimit(n): + """ + Convert global constraints for single investment period to one uniform + if all attributes stay the same. + """ c = "GlobalConstraint" cols = ['carrier_attribute', 'sense', "constant", "type"] glc_type = "transmission_volume_expansion_limit" @@ -350,6 +381,10 @@ def adjust_CO2_glc(n): def add_H2_boilers(n): + """ + Gas boilers can be retrofitted to run with H2. Add H2 boilers for heating + for all existing gas boilers. + """ c = "Link" logger.info("Add H2 boilers.") # existing gas boilers @@ -369,6 +404,7 @@ def add_H2_boilers(n): # add H2 boilers to network import_components_from_dataframe(n, df, c) + def apply_time_segmentation_perfect( n, segments, solver_name="cbc", overwrite_time_dependent=True ): @@ -432,12 +468,6 @@ def apply_time_segmentation_perfect( n.set_snapshots(sn_weightings.index) n.snapshot_weightings = n.snapshot_weightings.mul(sn_weightings, axis=0) - # overwrite time-dependent data with timeseries created by tsam - # if overwrite_time_dependent: - # values_t = segmented.mul(annual_max).set_index(snapshots) - # for component, key in values_t.columns.droplevel(2).unique(): - # n.pnl(component)[key] = values_t[component, key] - return n def set_temporal_aggregation_SEG(n, opts, solver_name): diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 7c6be9e4..d0a30f74 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -32,7 +32,7 @@ import re import numpy as np import pandas as pd import pypsa -from pypsa.descriptors import nominal_attrs, get_activity_mask +from pypsa.descriptors import get_activity_mask import xarray as xr from _helpers import configure_logging, update_config_with_sector_opts @@ -41,9 +41,7 @@ from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) from pypsa.descriptors import get_switchable_as_dense as get_as_dense -from pypsa.io import import_components_from_dataframe -from linopy.expressions import merge -from numpy import isnan + def add_land_use_constraint(n, planning_horizons, config): if "m" in snakemake.wildcards.clusters: @@ -140,7 +138,6 @@ def _add_land_use_constraint(n): def _add_land_use_constraint_m(n, planning_horizons, config): # if generators clustering is lower than network clustering, land_use accounting is at generators clusters - planning_horizons = param["planning_horizons"] grouping_years = config["existing_capacities"]["grouping_years"] current_horizon = snakemake.wildcards.planning_horizons From bdaa646ed62b07c9d7fadd095d91ff5a6b9d5391 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:05:16 +0000 Subject: [PATCH 57/77] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config/config.perfect.yaml | 8 +- rules/postprocess.smk | 8 +- rules/solve_perfect.smk | 3 - scripts/add_existing_baseyear.py | 19 +-- scripts/build_industrial_distribution_key.py | 12 +- scripts/make_summary_perfect.py | 30 ++-- scripts/plot_network.py | 84 ++++++----- scripts/plot_summary.py | 78 +++++----- scripts/prepare_perfect_foresight.py | 138 ++++++++++-------- scripts/solve_network.py | 146 ++++++++++--------- 10 files changed, 287 insertions(+), 239 deletions(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index 217b92e9..a833e3c7 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -87,10 +87,10 @@ co2_budget: 2050: 0.000 # update of IPCC 6th AR compared to the 1.5SR. (discussed here: https://twitter.com/JoeriRogelj/status/1424743828339167233) - 1p5 : 34.2 # 25.7 # Budget in Gt CO2 for 1.5 for Europe, global 420 Gt, assuming per capita share - 1p6 : 43.259666 # 35 # Budget in Gt CO2 for 1.6 for Europe, global 580 Gt - 1p7 : 51.4 # 45 # Budget in Gt CO2 for 1.7 for Europe, global 800 Gt - 2p0 : 69.778 # 73.9 # Budget in Gt CO2 for 2 for Europe, global 1170 Gt + 1p5: 34.2 # 25.7 # Budget in Gt CO2 for 1.5 for Europe, global 420 Gt, assuming per capita share + 1p6: 43.259666 # 35 # Budget in Gt CO2 for 1.6 for Europe, global 580 Gt + 1p7: 51.4 # 45 # Budget in Gt CO2 for 1.7 for Europe, global 800 Gt + 2p0: 69.778 # 73.9 # Budget in Gt CO2 for 2 for Europe, global 1170 Gt # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity electricity: diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 89e8e96b..c34fe27e 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -7,7 +7,9 @@ localrules: copy_config, copy_conda_env, + if config["foresight"] != "perfect": + rule plot_network: params: foresight=config["foresight"], @@ -34,7 +36,9 @@ if config["foresight"] != "perfect": script: "../scripts/plot_network.py" + if config["foresight"] == "perfect": + rule plot_network: params: foresight=config["foresight"], @@ -55,15 +59,13 @@ if config["foresight"] == "perfect": mem_mb=10000, benchmark: BENCHMARKS - + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark", + +"postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years_benchmark" conda: "../envs/environment.yaml" script: "../scripts/plot_network.py" - - rule copy_config: params: RDIR=RDIR, diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index bfd441f1..04c6af65 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -190,7 +190,4 @@ rule make_summary_perfect: "../scripts/make_summary_perfect.py" - - - ruleorder: add_existing_baseyear > add_brownfield diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index cb66eb15..01c69997 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -304,17 +304,18 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas bus0 = vars(spatial)[carrier[generator]].nodes if "EU" not in vars(spatial)[carrier[generator]].locations: bus0 = bus0.intersection(capacity.index + " gas") - + # check for missing bus missing_bus = pd.Index(bus0).difference(n.buses.index) if not missing_bus.empty: logger.info(f"add buses {bus0}") - n.madd("Bus", - bus0, - carrier=generator, - location=vars(spatial)[carrier[generator]].locations, - unit="MWh_el" - ) + n.madd( + "Bus", + bus0, + carrier=generator, + location=vars(spatial)[carrier[generator]].locations, + unit="MWh_el", + ) already_build = n.links.index.intersection(asset_i) new_build = asset_i.difference(n.links.index) @@ -615,9 +616,9 @@ def add_heating_capacities_installed_before_baseyear( if str(grouping_year) in index and n.links.p_nom[index] < threshold ], ) - + # drop assets which are at the end of their lifetime - links_i = n.links[(n.links.build_year+n.links.lifetime<=baseyear)].index + links_i = n.links[(n.links.build_year + n.links.lifetime <= baseyear)].index n.mremove("Link", links_i) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index a0dc195b..888a1a96 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -95,16 +95,19 @@ def prepare_hotmaps_database(regions): gdf.rename(columns={"index_right": "bus"}, inplace=True) gdf["country"] = gdf.bus.str[:2] - + # the .sjoin can lead to duplicates if a geom is in two regions if gdf.index.duplicated().any(): import pycountry + # get all duplicated entries duplicated_i = gdf.index[gdf.index.duplicated()] # convert from raw data country name to iso-2-code - s = df.loc[duplicated_i, "Country"].apply(lambda x: pycountry.countries.lookup(x).alpha_2) + s = df.loc[duplicated_i, "Country"].apply( + lambda x: pycountry.countries.lookup(x).alpha_2 + ) # Get a boolean mask where gdf's country column matches s's values for the same index - mask = gdf['country'] == gdf.index.map(s) + mask = gdf["country"] == gdf.index.map(s) # Filter gdf using the mask gdf_filtered = gdf[mask] # concat not duplicated and filtered gdf @@ -160,7 +163,8 @@ def build_nodal_distribution_key(hotmaps, regions, countries): return keys -#%% + +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 714f55b8..1a3391a9 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -230,25 +230,33 @@ def calculate_energy(n, label, energy): names=energy.columns.names[:3] + ["year"], ) energy = energy.reindex(cols, axis=1) - + for c in n.iterate_components(n.one_port_components | n.branch_components): if c.name in n.one_port_components: c_energies = ( c.pnl.p.multiply(n.snapshot_weightings.generators, axis=0) - .groupby(level=0).sum() + .groupby(level=0) + .sum() .multiply(c.df.sign) .groupby(c.df.carrier, axis=1) .sum() ) else: - c_energies = pd.DataFrame(0.0, columns=c.df.carrier.unique(), index=n.investment_periods) + c_energies = pd.DataFrame( + 0.0, columns=c.df.carrier.unique(), index=n.investment_periods + ) for port in [col[3:] for col in c.df.columns if col[:3] == "bus"]: - totals = c.pnl["p" + port].multiply(n.snapshot_weightings.generators, axis=0).groupby(level=0).sum() + totals = ( + c.pnl["p" + port] + .multiply(n.snapshot_weightings.generators, axis=0) + .groupby(level=0) + .sum() + ) # remove values where bus is missing (bug in nomopyomo) no_bus = c.df.index[c.df["bus" + port] == ""] - totals[no_bus] = float(n.component_attrs[c.name].loc[ - "p" + port, "default" - ]) + totals[no_bus] = float( + n.component_attrs[c.name].loc["p" + port, "default"] + ) c_energies -= totals.groupby(c.df.carrier, axis=1).sum() c_energies = pd.concat([c_energies.T], keys=[c.list_name]) @@ -703,11 +711,13 @@ if __name__ == "__main__": # Detect running outside of snakemake and mock snakemake for testing if "snakemake" not in globals(): from _helpers import mock_snakemake + snakemake = mock_snakemake("make_summary_perfect") - + run = snakemake.config["run"]["name"] - if run!="": run += "/" - + if run != "": + run += "/" + networks_dict = { (clusters, lv, opts + sector_opts): "results/" + run diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 3203f5ab..c0b35411 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -913,9 +913,11 @@ def plot_series(network, carrier="AC", name="test"): ) -def plot_map_perfect(network, components=["Link", "Store", "StorageUnit", "Generator"], - bus_size_factor=1.7e10): - +def plot_map_perfect( + network, + components=["Link", "Store", "StorageUnit", "Generator"], + bus_size_factor=1.7e10, +): n = network.copy() assign_location(n) # Drop non-electric buses so they don't clutter the plot @@ -926,41 +928,48 @@ def plot_map_perfect(network, components=["Link", "Store", "StorageUnit", "Gener costs = {} for comp in components: df_c = n.df(comp) - if df_c.empty: continue + if df_c.empty: + continue df_c["nice_group"] = df_c.carrier.map(rename_techs_tyndp) attr = "e_nom_opt" if comp == "Store" else "p_nom_opt" active = pd.concat( - [ - n.get_active_assets(comp, inv_p).rename(inv_p) - for inv_p in investments - ], + [n.get_active_assets(comp, inv_p).rename(inv_p) for inv_p in investments], axis=1, ).astype(int) capital_cost = n.df(comp)[attr] * n.df(comp).capital_cost - capital_cost_t = ((active.mul(capital_cost, axis=0)) - .groupby([n.df(comp).location, - n.df(comp).nice_group]).sum()) + capital_cost_t = ( + (active.mul(capital_cost, axis=0)) + .groupby([n.df(comp).location, n.df(comp).nice_group]) + .sum() + ) capital_cost_t.drop("load", level=1, inplace=True, errors="ignore") costs[comp] = capital_cost_t - costs = pd.concat(costs).groupby(level=[1,2]).sum() - costs.drop(costs[costs.sum(axis=1)==0].index, inplace=True) + costs = pd.concat(costs).groupby(level=[1, 2]).sum() + costs.drop(costs[costs.sum(axis=1) == 0].index, inplace=True) - new_columns = (preferred_order.intersection(costs.index.levels[1]) - .append(costs.index.levels[1].difference(preferred_order))) + new_columns = preferred_order.intersection(costs.index.levels[1]).append( + costs.index.levels[1].difference(preferred_order) + ) costs = costs.reindex(new_columns, level=1) for item in new_columns: - if item not in snakemake.config['plotting']['tech_colors']: - print("Warning!",item,"not in config/plotting/tech_colors, assign random color") - snakemake.config['plotting']['tech_colors'] = "pink" - - n.links.drop(n.links.index[(n.links.carrier != "DC") & ( - n.links.carrier != "B2B")], inplace=True) + if item not in snakemake.config["plotting"]["tech_colors"]: + print( + "Warning!", + item, + "not in config/plotting/tech_colors, assign random color", + ) + snakemake.config["plotting"]["tech_colors"] = "pink" + + n.links.drop( + n.links.index[(n.links.carrier != "DC") & (n.links.carrier != "B2B")], + inplace=True, + ) # drop non-bus to_drop = costs.index.levels[0].symmetric_difference(n.buses.index) @@ -972,7 +981,7 @@ def plot_map_perfect(network, components=["Link", "Store", "StorageUnit", "Gener costs.index = pd.MultiIndex.from_tuples(costs.index.values) # PDF has minimum width, so set these to zero - line_lower_threshold = 500. + line_lower_threshold = 500.0 line_upper_threshold = 1e4 linewidth_factor = 2e3 ac_color = "gray" @@ -981,29 +990,29 @@ def plot_map_perfect(network, components=["Link", "Store", "StorageUnit", "Gener line_widths = n.lines.s_nom_opt link_widths = n.links.p_nom_opt linewidth_factor = 2e3 - line_lower_threshold = 0. + line_lower_threshold = 0.0 title = "Today's transmission" - line_widths[line_widths < line_lower_threshold] = 0. - link_widths[link_widths < line_lower_threshold] = 0. + line_widths[line_widths < line_lower_threshold] = 0.0 + link_widths[link_widths < line_lower_threshold] = 0.0 line_widths[line_widths > line_upper_threshold] = line_upper_threshold link_widths[link_widths > line_upper_threshold] = line_upper_threshold for year in costs.columns: - fig, ax = plt.subplots(subplot_kw={"projection": ccrs.PlateCarree()}) fig.set_size_inches(7, 6) fig.suptitle(year) n.plot( bus_sizes=costs[year] / bus_size_factor, - bus_colors=snakemake.config['plotting']['tech_colors'], + bus_colors=snakemake.config["plotting"]["tech_colors"], line_colors=ac_color, link_colors=dc_color, line_widths=line_widths / linewidth_factor, link_widths=link_widths / linewidth_factor, - ax=ax, **map_opts + ax=ax, + **map_opts, ) sizes = [20, 10, 5] @@ -1051,15 +1060,12 @@ def plot_map_perfect(network, components=["Link", "Store", "StorageUnit", "Gener frameon=False, ) - fig.savefig( - snakemake.output[f"map_{year}"], - transparent=True, - bbox_inches="tight" + snakemake.output[f"map_{year}"], transparent=True, bbox_inches="tight" ) -#%% +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -1083,11 +1089,13 @@ if __name__ == "__main__": if map_opts["boundaries"] is None: map_opts["boundaries"] = regions.total_bounds[[0, 2, 1, 3]] + [-1, 1, -1, 1] - + if snakemake.params["foresight"] == "perfect": - plot_map_perfect(n, - components=["Link", "Store", "StorageUnit", "Generator"], - bus_size_factor=2e10) + plot_map_perfect( + n, + components=["Link", "Store", "StorageUnit", "Generator"], + bus_size_factor=2e10, + ) else: plot_map( n, @@ -1095,7 +1103,7 @@ if __name__ == "__main__": bus_size_factor=2e10, transmission=False, ) - + plot_h2_map(n, regions) plot_ch4_map(n) plot_map_without(n) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 8f939a36..b297987a 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -276,8 +276,6 @@ def plot_balances(): i for i in balances_df.index.levels[0] if i not in co2_carriers ] - - for k, v in balances.items(): df = balances_df.loc[v] df = df.groupby(df.index.get_level_values(2)).sum() @@ -321,7 +319,7 @@ def plot_balances(): ) new_columns = df.columns.sort_values() - + fig, ax = plt.subplots(figsize=(12, 8)) df.loc[new_index, new_columns].T.plot( @@ -357,7 +355,6 @@ def plot_balances(): fig.savefig(snakemake.output.balances[:-10] + k + ".pdf", bbox_inches="tight") - def historical_emissions(countries): """ Read historical emissions to add them to the carbon budget plot. @@ -393,19 +390,20 @@ def historical_emissions(countries): e["other LULUCF"] = "4.H - Other LULUCF" - pol = ["CO2"] # ["All greenhouse gases - (CO2 equivalent)"] if "GB" in countries: countries.remove("GB") countries.append("UK") - year = df.index.levels[0][df.index.levels[0]>=1990] - + year = df.index.levels[0][df.index.levels[0] >= 1990] + missing = pd.Index(countries).difference(df.index.levels[2]) if not missing.empty: - logger.warning(f"The following countries are missing and not considered when plotting historic CO2 emissions: {missing}") + logger.warning( + f"The following countries are missing and not considered when plotting historic CO2 emissions: {missing}" + ) countries = pd.Index(df.index.levels[2]).intersection(countries) - + idx = pd.IndexSlice co2_totals = ( df.loc[idx[year, e.values, countries, pol], "emissions"] @@ -469,26 +467,28 @@ def plot_carbon_budget_distribution(input_eurostat): # historic emissions countries = snakemake.params.countries - e_1990 = co2_emissions_year( - countries, input_eurostat, opts, snakemake, year=1990 - ) + e_1990 = co2_emissions_year(countries, input_eurostat, opts, snakemake, year=1990) emissions = historical_emissions(countries) # add other years https://sdi.eea.europa.eu/data/0569441f-2853-4664-a7cd-db969ef54de0 emissions.loc[2019] = 2.971372 emissions.loc[2020] = 2.691958 emissions.loc[2021] = 2.869355 - - + if snakemake.config["foresight"] == "myopic": path_cb = "results/" + snakemake.params.RDIR + "/csvs/" - co2_cap = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)[["cb"]] + co2_cap = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0)[ + ["cb"] + ] co2_cap *= e_1990 else: - supply_energy = pd.read_csv(snakemake.input.balances, - index_col=[0,1,2], header=[0,1,2, 3]) - co2_cap = supply_energy.loc["co2"].droplevel(0).drop("co2").sum().unstack().T/1e9 + supply_energy = pd.read_csv( + snakemake.input.balances, index_col=[0, 1, 2], header=[0, 1, 2, 3] + ) + co2_cap = ( + supply_energy.loc["co2"].droplevel(0).drop("co2").sum().unstack().T / 1e9 + ) co2_cap.rename(index=lambda x: int(x), inplace=True) - + plt.figure(figsize=(10, 7)) gs1 = gridspec.GridSpec(1, 1) ax1 = plt.subplot(gs1[0, 0]) @@ -511,13 +511,13 @@ def plot_carbon_budget_distribution(input_eurostat): ) ax1.plot( - [2030], - [0.45 * emissions[1990]], - marker="*", - markersize=12, - markerfacecolor="black", - markeredgecolor="black", - ) + [2030], + [0.45 * emissions[1990]], + marker="*", + markersize=12, + markerfacecolor="black", + markeredgecolor="black", + ) ax1.plot( [2030], @@ -538,28 +538,28 @@ def plot_carbon_budget_distribution(input_eurostat): ) ax1.plot( - [2050], - [0.0 * emissions[1990]], - marker="*", - markersize=12, - markerfacecolor="black", - markeredgecolor="black", - label="EU commited target", - ) + [2050], + [0.0 * emissions[1990]], + marker="*", + markersize=12, + markerfacecolor="black", + markeredgecolor="black", + label="EU commited target", + ) for col in co2_cap.columns: ax1.plot(co2_cap[col], linewidth=3, label=col) - ax1.legend( fancybox=True, fontsize=18, loc=(0.01, 0.01), facecolor="white", frameon=True ) - + plt.grid(axis="y") path = snakemake.output.balances.split("balances")[0] + "carbon_budget.pdf" plt.savefig(path, bbox_inches="tight") -#%% + +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -578,5 +578,7 @@ if __name__ == "__main__": for sector_opts in snakemake.params.sector_opts: opts = sector_opts.split("-") - if any(["cb" in o for o in opts]) or (snakemake.config["foresight"]=="perfect"): + if any(["cb" in o for o in opts]) or ( + snakemake.config["foresight"] == "perfect" + ): plot_carbon_budget_distribution(snakemake.input.eurostat) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index cf9d09f8..d71deab4 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -8,14 +8,16 @@ Concats pypsa networks of single investment periods to one network. import logging import re -import pandas as pd + import numpy as np +import pandas as pd import pypsa from _helpers import update_config_with_sector_opts from add_existing_baseyear import add_build_year_to_new_assets from pypsa.descriptors import expand_series from pypsa.io import import_components_from_dataframe from six import iterkeys + logger = logging.getLogger(__name__) @@ -38,8 +40,8 @@ def get_missing(df, n, c): def get_social_discount(t, r=0.01): """ - Calculate for a given time t and social discount rate r [per unit] - the social discount. + Calculate for a given time t and social discount rate r [per unit] the + social discount. """ return 1 / (1 + r) ** t @@ -62,7 +64,7 @@ def get_investment_weighting(time_weighting, r=0.01): def add_year_to_constraints(n, baseyear): """ Add investment period to global constraints and rename index. - + Parameters ---------- n : pypsa.Network @@ -77,25 +79,29 @@ def add_year_to_constraints(n, baseyear): def hvdc_transport_model(n): """ - Convert AC lines to DC links for multi-decade optimisation with - line expansion. Losses of DC links are assumed to be 3% per 1000km + Convert AC lines to DC links for multi-decade optimisation with line + expansion. + + Losses of DC links are assumed to be 3% per 1000km """ - + logger.info("Convert AC lines to DC links to perform multi-decade optimisation.") - - n.madd("Link", - n.lines.index, - bus0=n.lines.bus0, - bus1=n.lines.bus1, - p_nom_extendable=True, - p_nom=n.lines.s_nom, - p_nom_min=n.lines.s_nom, - p_min_pu=-1, - efficiency=1 - 0.03 * n.lines.length / 1000, - marginal_cost=0, - carrier="DC", - length=n.lines.length, - capital_cost=n.lines.capital_cost) + + n.madd( + "Link", + n.lines.index, + bus0=n.lines.bus0, + bus1=n.lines.bus1, + p_nom_extendable=True, + p_nom=n.lines.s_nom, + p_nom_min=n.lines.s_nom, + p_min_pu=-1, + efficiency=1 - 0.03 * n.lines.length / 1000, + marginal_cost=0, + carrier="DC", + length=n.lines.length, + capital_cost=n.lines.capital_cost, + ) # Remove AC lines logger.info("Removing AC lines") @@ -103,15 +109,16 @@ def hvdc_transport_model(n): n.mremove("Line", lines_rm) # Set efficiency of all DC links to include losses depending on length - n.links.loc[n.links.carrier == 'DC', 'efficiency'] = 1 - 0.03 * n.links.loc[ - n.links.carrier == 'DC', 'length'] / 1000 - + n.links.loc[n.links.carrier == "DC", "efficiency"] = ( + 1 - 0.03 * n.links.loc[n.links.carrier == "DC", "length"] / 1000 + ) + def adjust_electricity_grid(n, year, years): """ Add carrier to lines. Replace AC lines with DC links in case of line expansion. Add lifetime to DC links in case of line expansion. - + Parameters ---------- n : pypsa.Network @@ -121,16 +128,15 @@ def adjust_electricity_grid(n, year, years): investment periods """ n.lines["carrier"] = "AC" - links_i = n.links[n.links.carrier=="DC"].index + links_i = n.links[n.links.carrier == "DC"].index if n.lines.s_nom_extendable.any() or n.links.loc[links_i, "p_nom_extendable"].any(): hvdc_transport_model(n) - links_i = n.links[n.links.carrier=="DC"].index + links_i = n.links[n.links.carrier == "DC"].index n.links.loc[links_i, "lifetime"] = 100 - if year!= years[0]: + if year != years[0]: n.links.loc[links_i, "p_nom_min"] = 0 n.links.loc[links_i, "p_nom"] = 0 - - + # -------------------------------------------------------------------- def concat_networks(years): @@ -245,7 +251,6 @@ def adjust_stores(n): e_initial_store = ["co2 stored"] co2_i = n.stores[n.stores.carrier.isin(e_initial_store)].index n.stores.loc[co2_i, "e_initial_per_period"] = True - return n @@ -309,13 +314,13 @@ def set_carbon_constraints(n, opts): carrier_attribute="co2_emissions", sense="<=", constant=budget, - investment_period=n.investment_periods[-1] + investment_period=n.investment_periods[-1], ) - - # drop other CO2 limits - drop_i = n.global_constraints[n.global_constraints.type=="co2_limit"].index + + # drop other CO2 limits + drop_i = n.global_constraints[n.global_constraints.type == "co2_limit"].index n.mremove("GlobalConstraint", drop_i) - + n.add( "GlobalConstraint", "carbon_neutral", @@ -323,10 +328,9 @@ def set_carbon_constraints(n, opts): carrier_attribute="co2_emissions", sense="<=", constant=0, - investment_period=n.investment_periods[-1] + investment_period=n.investment_periods[-1], ) - # set minimum CO2 emission constraint to avoid too fast reduction if "co2min" in opts: emissions_1990 = 4.53693 @@ -348,25 +352,25 @@ def set_carbon_constraints(n, opts): investment_period=first_year, constant=co2min * 1e9 * time_weightings, ) - + return n def adjust_lvlimit(n): """ - Convert global constraints for single investment period to one uniform - if all attributes stay the same. + Convert global constraints for single investment period to one uniform if + all attributes stay the same. """ c = "GlobalConstraint" - cols = ['carrier_attribute', 'sense', "constant", "type"] + cols = ["carrier_attribute", "sense", "constant", "type"] glc_type = "transmission_volume_expansion_limit" - if (n.df(c)[n.df(c).type==glc_type][cols].nunique()==1).all(): - glc = n.df(c)[n.df(c).type==glc_type][cols].iloc[[0]] + if (n.df(c)[n.df(c).type == glc_type][cols].nunique() == 1).all(): + glc = n.df(c)[n.df(c).type == glc_type][cols].iloc[[0]] glc.index = pd.Index(["lv_limit"]) - remove_i = n.df(c)[n.df(c).type==glc_type].index + remove_i = n.df(c)[n.df(c).type == glc_type].index n.mremove(c, remove_i) import_components_from_dataframe(n, glc, c) - + return n @@ -374,16 +378,17 @@ def adjust_CO2_glc(n): c = "GlobalConstraint" glc_name = "CO2Limit" glc_type = "primary_energy" - mask = (n.df(c).index.str.contains(glc_name)) & (n.df(c).type==glc_type) + mask = (n.df(c).index.str.contains(glc_name)) & (n.df(c).type == glc_type) n.df(c).loc[mask, "type"] = "co2_limit" - + return n def add_H2_boilers(n): """ - Gas boilers can be retrofitted to run with H2. Add H2 boilers for heating - for all existing gas boilers. + Gas boilers can be retrofitted to run with H2. + + Add H2 boilers for heating for all existing gas boilers. """ c = "Link" logger.info("Add H2 boilers.") @@ -394,8 +399,12 @@ def add_H2_boilers(n): # adjust bus 0 df["bus0"] = df.bus1.map(n.buses.location) + " H2" # rename carrier and index - df["carrier"] = df.carrier.apply(lambda x: x.replace("gas boiler", "retrofitted H2 boiler")) - df.rename(index = lambda x: x.replace("gas boiler", "retrofitted H2 boiler"), inplace=True) + df["carrier"] = df.carrier.apply( + lambda x: x.replace("gas boiler", "retrofitted H2 boiler") + ) + df.rename( + index=lambda x: x.replace("gas boiler", "retrofitted H2 boiler"), inplace=True + ) # todo, costs for retrofitting df["capital_costs"] = 100 # set existing capacity to zero @@ -403,8 +412,8 @@ def add_H2_boilers(n): df["p_nom_extendable"] = True # add H2 boilers to network import_components_from_dataframe(n, df, c) - - + + def apply_time_segmentation_perfect( n, segments, solver_name="cbc", overwrite_time_dependent=True ): @@ -438,8 +447,8 @@ def apply_time_segmentation_perfect( raw = pd.concat([raw, df], axis=1) raw = raw.dropna(axis=1) sn_weightings = {} - - for year in raw.index.levels[0]: + + for year in raw.index.levels[0]: logger.info(f"Find representative snapshots for {year}.") raw_t = raw.loc[year] # normalise all time-dependent data @@ -455,7 +464,7 @@ def apply_time_segmentation_perfect( solver=solver_name, ) segmented = agg.createTypicalPeriods() - + weightings = segmented.index.get_level_values("Segment Duration") offsets = np.insert(np.cumsum(weightings[:-1]), 0, 0) timesteps = [raw_t.index[0] + pd.Timedelta(f"{offset}h") for offset in offsets] @@ -463,13 +472,14 @@ def apply_time_segmentation_perfect( sn_weightings[year] = pd.Series( weightings, index=snapshots, name="weightings", dtype="float64" ) - + sn_weightings = pd.concat(sn_weightings) n.set_snapshots(sn_weightings.index) n.snapshot_weightings = n.snapshot_weightings.mul(sn_weightings, axis=0) return n + def set_temporal_aggregation_SEG(n, opts, solver_name): """ Aggregate network temporally with tsam. @@ -483,6 +493,8 @@ def set_temporal_aggregation_SEG(n, opts, solver_name): n = apply_time_segmentation_perfect(n, segments, solver_name=solver_name) break return n + + # %% if __name__ == "__main__": if "snakemake" not in globals(): @@ -514,22 +526,22 @@ if __name__ == "__main__": # concat prenetworks of planning horizon to single network ------------ n = concat_networks(years) - + # temporal aggregate opts = snakemake.wildcards.sector_opts.split("-") solver_name = snakemake.config["solving"]["solver"]["name"] n = set_temporal_aggregation_SEG(n, opts, solver_name) - + # adjust global constraints lv limit if the same for all years - n = adjust_lvlimit(n) + n = adjust_lvlimit(n) # adjust global constraints CO2 limit n = adjust_CO2_glc(n) # adjust stores to multi period investment n = adjust_stores(n) - + # set phase outs set_all_phase_outs(n) - + # add H2 boiler add_H2_boilers(n) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index fa03037f..0e4f0eb5 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -32,10 +32,9 @@ import re import numpy as np import pandas as pd import pypsa -from pypsa.descriptors import get_activity_mask import xarray as xr from _helpers import configure_logging, update_config_with_sector_opts - +from pypsa.descriptors import get_activity_mask from vresutils.benchmark import memory_logger logger = logging.getLogger(__name__) @@ -48,34 +47,40 @@ def add_land_use_constraint(n, planning_horizons, config): _add_land_use_constraint_m(n, planning_horizons, config) else: _add_land_use_constraint(n) - + def add_land_use_constraint_perfect(n): - """Add global constraints for tech capacity limit. + """ + Add global constraints for tech capacity limit. """ logger.info("Add land-use constraint for perfect foresight") + def compress_series(s): - def process_group(group): if group.nunique() == 1: return pd.Series(group.iloc[0], index=[None]) else: return group - - return s.groupby(level=[0,1]).apply(process_group) - + + return s.groupby(level=[0, 1]).apply(process_group) + def new_index_name(t): # Convert all elements to string and filter out None values parts = [str(x) for x in t if x is not None] # Join with space, but use a dash for the last item if not None - return ' '.join(parts[:2]) + (f'-{parts[-1]}' if len(parts) > 2 else '') - + return " ".join(parts[:2]) + (f"-{parts[-1]}" if len(parts) > 2 else "") + def check_p_min_p_max(p_nom_max): p_nom_min = n.generators[ext_i].groupby(grouper).sum().p_nom_min p_nom_min = p_nom_min.reindex(p_nom_max.index) - check = (p_nom_min.groupby(level=[0,1]).sum()>p_nom_max.groupby(level=[0,1]).min()) + check = ( + p_nom_min.groupby(level=[0, 1]).sum() + > p_nom_max.groupby(level=[0, 1]).min() + ) if check.sum(): - logger.warning(f"summed p_min_pu values at node larger than technical potential {check[check].index}") + logger.warning( + f"summed p_min_pu values at node larger than technical potential {check[check].index}" + ) grouper = [n.generators.carrier, n.generators.bus, n.generators.build_year] ext_i = n.generators.p_nom_extendable @@ -85,8 +90,7 @@ def add_land_use_constraint_perfect(n): p_nom_max = p_nom_max[~p_nom_max.isin([np.inf, np.nan])] # carrier carriers = p_nom_max.index.get_level_values(0).unique() - gen_i = n.generators[(n.generators.carrier.isin(carriers)) & - (ext_i)].index + gen_i = n.generators[(n.generators.carrier.isin(carriers)) & (ext_i)].index n.generators.loc[gen_i, "p_nom_min"] = 0 # check minimum capacities check_p_min_p_max(p_nom_max) @@ -94,15 +98,17 @@ def add_land_use_constraint_perfect(n): # p_nom_max = compress_series(p_nom_max) # adjust name to fit syntax of nominal constraint per bus df = p_nom_max.reset_index() - df["name"] = df.apply(lambda row: f"nom_max_{row['carrier']}" - + (f"_{row['build_year']}" if row['build_year'] is not None else ""), - axis=1) - + df["name"] = df.apply( + lambda row: f"nom_max_{row['carrier']}" + + (f"_{row['build_year']}" if row["build_year"] is not None else ""), + axis=1, + ) + for name in df.name.unique(): - df_carrier = df[df.name==name] + df_carrier = df[df.name == name] bus = df_carrier.bus n.buses.loc[bus, name] = df_carrier.p_nom_max.values - + return n @@ -187,14 +193,14 @@ def add_co2_sequestration_limit(n, config, limit=200): continue limit = float(o[o.find("seq") + 3 :]) * 1e6 break - + if config["foresight"] == "perfect": periods = n.investment_periods names = pd.Index([f"co2_sequestration_limit-{period}" for period in periods]) else: periods = [np.nan] names = pd.Index(["co2_sequestration_limit"]) - + n.madd( "GlobalConstraint", names, @@ -224,11 +230,11 @@ def add_carbon_constraint(n, snapshots): time_valid = int(glc.loc["investment_period"]) if not stores.empty: last = n.snapshot_weightings.reset_index().groupby("period").last() - last_i = last.set_index([last.index, last.timestep]).index + last_i = last.set_index([last.index, last.timestep]).index final_e = n.model["Store-e"].loc[last_i, stores.index] time_i = pd.IndexSlice[time_valid, :] - lhs = final_e.loc[time_i,:] - final_e.shift(snapshot=1).loc[time_i,:] - + lhs = final_e.loc[time_i, :] - final_e.shift(snapshot=1).loc[time_i, :] + n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") @@ -251,77 +257,83 @@ def add_carbon_budget_constraint(n, snapshots): weighting = n.investment_period_weightings.loc[time_valid, "years"] if not stores.empty: last = n.snapshot_weightings.reset_index().groupby("period").last() - last_i = last.set_index([last.index, last.timestep]).index + last_i = last.set_index([last.index, last.timestep]).index final_e = n.model["Store-e"].loc[last_i, stores.index] time_i = pd.IndexSlice[time_valid, :] - lhs = final_e.loc[time_i,:] * weighting - + lhs = final_e.loc[time_i, :] * weighting + n.model.add_constraints(lhs <= rhs, name=f"GlobalConstraint-{name}") def add_max_growth(n, config): - """Add maximum growth rates for different carriers """ - + Add maximum growth rates for different carriers. + """ + opts = config["sector"]["limit_max_growth"] # take maximum yearly difference between investment periods since historic growth is per year factor = n.investment_period_weightings.years.max() * opts["factor"] for carrier in opts["max_growth"].keys(): max_per_period = opts["max_growth"][carrier] * factor - logger.info(f"set maximum growth rate per investment period of {carrier} to {max_per_period} GW.") + logger.info( + f"set maximum growth rate per investment period of {carrier} to {max_per_period} GW." + ) n.carriers.loc[carrier, "max_growth"] = max_per_period * 1e3 - + for carrier in opts["max_relative_growth"].keys(): max_r_per_period = opts["max_relative_growth"][carrier] - logger.info(f"set maximum relative growth per investment period of {carrier} to {max_r_per_period}.") - n.carriers.loc[carrier, "max_relative_growth"] = max_r_per_period - + logger.info( + f"set maximum relative growth per investment period of {carrier} to {max_r_per_period}." + ) + n.carriers.loc[carrier, "max_relative_growth"] = max_r_per_period + return n def add_retrofit_gas_boiler_constraint(n, snapshots): - """Allow retrofitting of existing gas boilers to H2 boilers. + """ + Allow retrofitting of existing gas boilers to H2 boilers. """ c = "Link" logger.info("Add constraint for retrofitting gas boilers to H2 boilers.") # existing gas boilers mask = n.links.carrier.str.contains("gas boiler") & ~n.links.p_nom_extendable gas_i = n.links[mask].index - mask = n.links.carrier.str.contains("retrofitted H2 boiler") + mask = n.links.carrier.str.contains("retrofitted H2 boiler") h2_i = n.links[mask].index - - + n.links.loc[gas_i, "p_nom_extendable"] = True p_nom = n.links.loc[gas_i, "p_nom"] n.links.loc[gas_i, "p_nom"] = 0 - + # heat profile - cols = n.loads_t.p_set.columns[n.loads_t.p_set.columns.str.contains("heat") - & ~n.loads_t.p_set.columns.str.contains("industry") - & ~n.loads_t.p_set.columns.str.contains("agriculture")] - profile = n.loads_t.p_set[cols].div(n.loads_t.p_set[cols].groupby(level=0).max(), level=0) + cols = n.loads_t.p_set.columns[ + n.loads_t.p_set.columns.str.contains("heat") + & ~n.loads_t.p_set.columns.str.contains("industry") + & ~n.loads_t.p_set.columns.str.contains("agriculture") + ] + profile = n.loads_t.p_set[cols].div( + n.loads_t.p_set[cols].groupby(level=0).max(), level=0 + ) # to deal if max value is zero profile.fillna(0, inplace=True) profile.rename(columns=n.loads.bus.to_dict(), inplace=True) profile = profile.reindex(columns=n.links.loc[gas_i, "bus1"]) profile.columns = gas_i - - + rhs = profile.mul(p_nom) - + dispatch = n.model["Link-p"] active = get_activity_mask(n, c, snapshots, gas_i) rhs = rhs[active] p_gas = dispatch.sel(Link=gas_i) p_h2 = dispatch.sel(Link=h2_i) - - lhs = p_gas + p_h2 - - n.model.add_constraints(lhs == rhs, name="gas_retrofit") - - - + lhs = p_gas + p_h2 + + n.model.add_constraints(lhs == rhs, name="gas_retrofit") + + def prepare_network( n, solve_opts=None, @@ -381,7 +393,7 @@ def prepare_network( if foresight == "myopic": add_land_use_constraint(n, planning_horizons, config) - + if foresight == "perfect": n = add_land_use_constraint_perfect(n) if config["sector"]["limit_max_growth"]["enable"]: @@ -756,7 +768,6 @@ def add_pipe_retrofit_constraint(n): n.model.add_constraints(lhs == rhs, name="Link-pipe_retrofit") - def extra_functionality(n, snapshots): """ Collects supplementary constraints which will be passed to @@ -791,8 +802,10 @@ def extra_functionality(n, snapshots): def solve_network(n, config, solving, opts="", **kwargs): set_of_options = solving["solver"]["options"] cf_solving = solving["options"] - - kwargs["multi_investment_periods"] = True if config["foresight"] == "perfect" else False + + kwargs["multi_investment_periods"] = ( + True if config["foresight"] == "perfect" else False + ) kwargs["solver_options"] = ( solving["solver_options"][set_of_options] if set_of_options else {} ) @@ -838,7 +851,8 @@ def solve_network(n, config, solving, opts="", **kwargs): return n -#%% + +# %% if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake @@ -868,7 +882,7 @@ if __name__ == "__main__": np.random.seed(solve_opts.get("seed", 123)) n = pypsa.Network(snakemake.input.network) - + n = prepare_network( n, solve_opts, @@ -877,9 +891,10 @@ if __name__ == "__main__": planning_horizons=snakemake.params.planning_horizons, co2_sequestration_potential=snakemake.params["co2_sequestration_potential"], ) - - with memory_logger(filename=getattr(snakemake.log, 'memory', None), interval=30.) as mem: - + + with memory_logger( + filename=getattr(snakemake.log, "memory", None), interval=30.0 + ) as mem: n = solve_network( n, config=snakemake.config, @@ -887,11 +902,8 @@ if __name__ == "__main__": opts=opts, log_fn=snakemake.log.solver, ) - + logger.info("Maximum memory usage: {}".format(mem.mem_usage)) - - - n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) From c278a3159f3235d4257d53a21b0bf52ad6348bfc Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 30 Aug 2023 14:32:57 +0200 Subject: [PATCH 58/77] add benchmark back --- scripts/_benchmark.py | 245 +++++++++++++++++++++++++++++++++++++++ scripts/solve_network.py | 2 +- 2 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 scripts/_benchmark.py diff --git a/scripts/_benchmark.py b/scripts/_benchmark.py new file mode 100644 index 00000000..777ad7ac --- /dev/null +++ b/scripts/_benchmark.py @@ -0,0 +1,245 @@ +# -*- coding: utf-8 -*- + +## Copyright 2015-2017 Frankfurt Institute for Advanced Studies + +## This program is free software; you can redistribute it and/or +## modify it under the terms of the GNU General Public License as +## published by the Free Software Foundation; either version 3 of the +## License, or (at your option) any later version. + +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with this program. If not, see . + +""" +""" + +from __future__ import absolute_import +from __future__ import print_function + +import time +import sys, os + +import logging +logger = logging.getLogger(__name__) + +# TODO: provide alternative when multiprocessing is not available +try: + from multiprocessing import Process, Pipe +except ImportError: + from multiprocessing.dummy import Process, Pipe + +from memory_profiler import _get_memory, choose_backend + +# The memory logging facilities have been adapted from memory_profiler +class MemTimer(Process): + """ + Write memory consumption over a time interval to file until signaled to + stop on the pipe + """ + + def __init__(self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw): + self.monitor_pid = monitor_pid + self.interval = interval + self.pipe = pipe + self.filename = filename + self.max_usage = max_usage + self.backend = backend + + self.timestamps = kw.pop("timestamps", True) + self.include_children = kw.pop("include_children", True) + + super(MemTimer, self).__init__(*args, **kw) + + def run(self): + # get baseline memory usage + cur_mem = ( + _get_memory(self.monitor_pid, self.backend, + timestamps=self.timestamps, + include_children=self.include_children) + ) + + n_measurements = 1 + mem_usage = cur_mem if self.max_usage else [cur_mem] + + if self.filename is not None: + stream = open(self.filename, 'w') + stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem)) + stream.flush() + else: + stream = None + + self.pipe.send(0) # we're ready + stop = False + while True: + cur_mem = ( + _get_memory(self.monitor_pid, self.backend, + timestamps=self.timestamps, + include_children=self.include_children) + ) + + if stream is not None: + stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem)) + stream.flush() + + n_measurements += 1 + if not self.max_usage: + mem_usage.append(cur_mem) + else: + mem_usage = max(cur_mem, mem_usage) + + if stop: + break + stop = self.pipe.poll(self.interval) + # do one more iteration + + if stream is not None: + stream.close() + + self.pipe.send(mem_usage) + self.pipe.send(n_measurements) + +class memory_logger(object): + """ + Context manager for taking and reporting memory measurements at fixed + intervals from a separate process, for the duration of a context. + + Parameters + ---------- + filename : None|str + Name of the text file to log memory measurements, if None no log is + created (defaults to None) + interval : float + Interval between measurements (defaults to 1.) + max_usage : bool + If True, only store and report the maximum value (defaults to True) + timestamps : bool + Whether to record tuples of memory usage and timestamps; if logging to + a file timestamps are always kept (defaults to True) + include_children : bool + Whether the memory of subprocesses is to be included (default: True) + + Arguments + --------- + n_measurements : int + Number of measurements that have been taken + mem_usage : (float, float)|[(float, float)] + All memory measurements and timestamps (if timestamps was True) or only + the maximum memory usage and its timestamp + + Note + ---- + The arguments are only set after all the measurements, i.e. outside of the + with statement. + + Example + ------- + with memory_logger(filename="memory.log", max_usage=True) as mem: + # Do a lot of long running memory intensive stuff + hard_memory_bound_stuff() + + max_mem, timestamp = mem.mem_usage + """ + def __init__(self, filename=None, interval=1., max_usage=True, + timestamps=True, include_children=True): + if filename is not None: + timestamps = True + + self.filename = filename + self.interval = interval + self.max_usage = max_usage + self.timestamps = timestamps + self.include_children = include_children + + def __enter__(self): + backend = choose_backend() + + self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results + self.p = MemTimer(os.getpid(), self.interval, self.child_conn, self.filename, + backend=backend, timestamps=self.timestamps, max_usage=self.max_usage, + include_children=self.include_children) + self.p.start() + self.parent_conn.recv() # wait until memory logging in subprocess is ready + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is None: + self.parent_conn.send(0) # finish timing + + self.mem_usage = self.parent_conn.recv() + self.n_measurements = self.parent_conn.recv() + else: + self.p.terminate() + + return False + +class timer(object): + level = 0 + opened = False + + def __init__(self, name="", verbose=True): + self.name = name + self.verbose = verbose + + def __enter__(self): + if self.verbose: + if self.opened: + sys.stdout.write("\n") + + if len(self.name) > 0: + sys.stdout.write((".. " * self.level) + self.name + ": ") + sys.stdout.flush() + + self.__class__.opened = True + + self.__class__.level += 1 + + self.start = time.time() + return self + + def print_usec(self, usec): + if usec < 1000: + print("%.1f usec" % usec) + else: + msec = usec / 1000 + if msec < 1000: + print("%.1f msec" % msec) + else: + sec = msec / 1000 + print("%.1f sec" % sec) + + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.opened and self.verbose: + sys.stdout.write(".. " * self.level) + + if exc_type is None: + stop = time.time() + self.usec = usec = (stop - self.start) * 1e6 + if self.verbose: self.print_usec(usec) + elif self.verbose: + print("failed") + sys.stdout.flush() + + self.__class__.level -= 1 + if self.verbose: self.__class__.opened = False + return False + +class optional(object): + def __init__(self, variable, contextman): + self.variable = variable + self.contextman = contextman + + def __enter__(self): + if self.variable: + return self.contextman.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.variable: + return self.contextman.__exit__(exc_type, exc_val, exc_tb) + return False diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0e4f0eb5..e1b243c8 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -35,7 +35,7 @@ import pypsa import xarray as xr from _helpers import configure_logging, update_config_with_sector_opts from pypsa.descriptors import get_activity_mask -from vresutils.benchmark import memory_logger +from _benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) From 64cbfd673c0a6022d711fb0d4e10bca6f2683b94 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:33:23 +0000 Subject: [PATCH 59/77] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_benchmark.py | 77 ++++++++++++++++++++++++++-------------- scripts/solve_network.py | 2 +- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/scripts/_benchmark.py b/scripts/_benchmark.py index 777ad7ac..ba49ddc5 100644 --- a/scripts/_benchmark.py +++ b/scripts/_benchmark.py @@ -14,35 +14,38 @@ ## You should have received a copy of the GNU General Public License ## along with this program. If not, see . - -""" """ -from __future__ import absolute_import -from __future__ import print_function +""" -import time -import sys, os +from __future__ import absolute_import, print_function import logging +import os +import sys +import time + logger = logging.getLogger(__name__) # TODO: provide alternative when multiprocessing is not available try: - from multiprocessing import Process, Pipe + from multiprocessing import Pipe, Process except ImportError: from multiprocessing.dummy import Process, Pipe from memory_profiler import _get_memory, choose_backend + # The memory logging facilities have been adapted from memory_profiler class MemTimer(Process): """ Write memory consumption over a time interval to file until signaled to - stop on the pipe + stop on the pipe. """ - def __init__(self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw): + def __init__( + self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw + ): self.monitor_pid = monitor_pid self.interval = interval self.pipe = pipe @@ -57,17 +60,18 @@ class MemTimer(Process): def run(self): # get baseline memory usage - cur_mem = ( - _get_memory(self.monitor_pid, self.backend, - timestamps=self.timestamps, - include_children=self.include_children) + cur_mem = _get_memory( + self.monitor_pid, + self.backend, + timestamps=self.timestamps, + include_children=self.include_children, ) n_measurements = 1 mem_usage = cur_mem if self.max_usage else [cur_mem] if self.filename is not None: - stream = open(self.filename, 'w') + stream = open(self.filename, "w") stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem)) stream.flush() else: @@ -76,10 +80,11 @@ class MemTimer(Process): self.pipe.send(0) # we're ready stop = False while True: - cur_mem = ( - _get_memory(self.monitor_pid, self.backend, - timestamps=self.timestamps, - include_children=self.include_children) + cur_mem = _get_memory( + self.monitor_pid, + self.backend, + timestamps=self.timestamps, + include_children=self.include_children, ) if stream is not None: @@ -103,6 +108,7 @@ class MemTimer(Process): self.pipe.send(mem_usage) self.pipe.send(n_measurements) + class memory_logger(object): """ Context manager for taking and reporting memory measurements at fixed @@ -144,8 +150,15 @@ class memory_logger(object): max_mem, timestamp = mem.mem_usage """ - def __init__(self, filename=None, interval=1., max_usage=True, - timestamps=True, include_children=True): + + def __init__( + self, + filename=None, + interval=1.0, + max_usage=True, + timestamps=True, + include_children=True, + ): if filename is not None: timestamps = True @@ -159,9 +172,16 @@ class memory_logger(object): backend = choose_backend() self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results - self.p = MemTimer(os.getpid(), self.interval, self.child_conn, self.filename, - backend=backend, timestamps=self.timestamps, max_usage=self.max_usage, - include_children=self.include_children) + self.p = MemTimer( + os.getpid(), + self.interval, + self.child_conn, + self.filename, + backend=backend, + timestamps=self.timestamps, + max_usage=self.max_usage, + include_children=self.include_children, + ) self.p.start() self.parent_conn.recv() # wait until memory logging in subprocess is ready @@ -169,7 +189,7 @@ class memory_logger(object): def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: - self.parent_conn.send(0) # finish timing + self.parent_conn.send(0) # finish timing self.mem_usage = self.parent_conn.recv() self.n_measurements = self.parent_conn.recv() @@ -178,6 +198,7 @@ class memory_logger(object): return False + class timer(object): level = 0 opened = False @@ -213,7 +234,6 @@ class timer(object): sec = msec / 1000 print("%.1f sec" % sec) - def __exit__(self, exc_type, exc_val, exc_tb): if not self.opened and self.verbose: sys.stdout.write(".. " * self.level) @@ -221,15 +241,18 @@ class timer(object): if exc_type is None: stop = time.time() self.usec = usec = (stop - self.start) * 1e6 - if self.verbose: self.print_usec(usec) + if self.verbose: + self.print_usec(usec) elif self.verbose: print("failed") sys.stdout.flush() self.__class__.level -= 1 - if self.verbose: self.__class__.opened = False + if self.verbose: + self.__class__.opened = False return False + class optional(object): def __init__(self, variable, contextman): self.variable = variable diff --git a/scripts/solve_network.py b/scripts/solve_network.py index e1b243c8..2e4c3305 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,9 +33,9 @@ import numpy as np import pandas as pd import pypsa import xarray as xr +from _benchmark import memory_logger from _helpers import configure_logging, update_config_with_sector_opts from pypsa.descriptors import get_activity_mask -from _benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING) From 9d8dd81958b71879f889b90e79a9424c94aa77b4 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 30 Aug 2023 14:35:17 +0200 Subject: [PATCH 60/77] add co2 store lifetime --- config/config.default.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.default.yaml b/config/config.default.yaml index b3748039..9c5aa9b0 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -462,6 +462,7 @@ sector: years_of_storage: 25 co2_sequestration_potential: 200 co2_sequestration_cost: 10 + co2_sequestration_lifetime: 50 co2_spatial: false co2network: false cc_fraction: 0.9 From c0152c25a468e3fa585e2aff673fc2d4d54c238a Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 30 Aug 2023 15:57:47 +0200 Subject: [PATCH 61/77] first fixes for CI --- config/config.default.yaml | 30 ++++++++++++++++++++++-------- config/test/config.perfect.yaml | 4 ++-- rules/solve_perfect.smk | 1 + scripts/solve_network.py | 12 ++++++------ 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 9c5aa9b0..f0fdd8d9 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -26,7 +26,7 @@ run: shared_cutouts: true # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight -foresight: perfect +foresight: overnight # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#scenario # Wildcard docs in https://pypsa-eur.readthedocs.io/en/latest/wildcards.html @@ -34,22 +34,21 @@ scenario: simpl: - '' ll: - - v1.0 - v1.5 clusters: - 37 - 128 -# - 256 -# - 512 - #- 1024 + - 256 + - 512 + - 1024 opts: - '' sector_opts: - Co2L0-3H-T-H-B-I-A-solar+p3-dist1 planning_horizons: - - 2020 - - 2030 - - 2040 + # - 2020 + # - 2030 + # - 2040 - 2050 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries @@ -493,6 +492,20 @@ sector: OCGT: gas biomass_to_liquid: false biosng: false + limit_max_growth: + enable: false + # allowing 30% larger than max historic growth + factor: 1.3 + max_growth: # unit GW + onwind: 16 # onshore max grow so far 16 GW in Europe https://www.iea.org/reports/renewables-2020/wind + solar: 28 # solar max grow so far 28 GW in Europe https://www.iea.org/reports/renewables-2020/solar-pv + offwind-ac: 35 # offshore max grow so far 3.5 GW in Europe https://windeurope.org/about-wind/statistics/offshore/european-offshore-wind-industry-key-trends-statistics-2019/ + offwind-dc: 35 + max_relative_growth: + onwind: 3 + solar: 3 + offwind-ac: 3 + offwind-dc: 3 # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry industry: @@ -545,6 +558,7 @@ industry: hotmaps_locate_missing: false reference_year: 2015 + # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs costs: year: 2030 diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 4da629bc..788ad326 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -14,13 +14,12 @@ foresight: perfect scenario: ll: - - v1.5 + - v1.0 clusters: - 5 sector_opts: - 8760H-T-H-B-I-A-solar+p3-dist1 planning_horizons: - - 2020 - 2030 - 2040 - 2050 @@ -65,6 +64,7 @@ renewable: industry: St_primary_fraction: + 2020: 0.8 2030: 0.6 2040: 0.5 2050: 0.4 diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 04c6af65..ef4e367d 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -113,6 +113,7 @@ rule solve_sector_network_perfect: params: solving=config["solving"], foresight=config["foresight"], + sector=config["sector"], planning_horizons=config["scenario"]["planning_horizons"], co2_sequestration_potential=config["sector"].get( "co2_sequestration_potential", 200 diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 2e4c3305..e353acd8 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -270,7 +270,7 @@ def add_max_growth(n, config): Add maximum growth rates for different carriers. """ - opts = config["sector"]["limit_max_growth"] + opts = snakemake.params["sector"]["limit_max_growth"] # take maximum yearly difference between investment periods since historic growth is per year factor = n.investment_period_weightings.years.max() * opts["factor"] for carrier in opts["max_growth"].keys(): @@ -396,7 +396,7 @@ def prepare_network( if foresight == "perfect": n = add_land_use_constraint_perfect(n) - if config["sector"]["limit_max_growth"]["enable"]: + if snakemake.params["sector"]["limit_max_growth"]["enable"]: n = add_max_growth(n, config) if n.stores.carrier.eq("co2 stored").any(): @@ -859,12 +859,12 @@ if __name__ == "__main__": snakemake = mock_snakemake( "solve_sector_network_perfect", - configfiles="config/config.perfect.yaml", + configfiles="../config/test/config.perfect.yaml", simpl="", opts="", - clusters="37", - ll="v1.0", - sector_opts="2p0-4380H-T-H-B-I-A-solar+p3-dist1", + clusters="5", + ll="v1.5", + sector_opts="8760H-T-H-B-I-A-solar+p3-dist1", planning_horizons="2030", ) configure_logging(snakemake) From 457a9e23e789a3f1c7f548e3c4a7190939f48c40 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 31 Aug 2023 09:55:38 +0200 Subject: [PATCH 62/77] fixes CI perfect --- config/config.default.yaml | 1 + config/test/config.perfect.yaml | 3 +++ scripts/plot_summary.py | 14 +++++++++++++- scripts/prepare_sector_network.py | 6 ++---- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index f0fdd8d9..7bae66d1 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -778,6 +778,7 @@ plotting: gas pipeline new: '#a87c62' # oil oil: '#c9c9c9' + imported oil: '#a3a3a3' oil boiler: '#adadad' residential rural oil boiler: '#a9a9a9' services rural oil boiler: '#a5a5a5' diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 788ad326..f20586d0 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -41,6 +41,9 @@ electricity: renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] +sector: + min_part_load_fischer_tropsch: 0 + min_part_load_methanolisation: 0 atlite: default_cutout: be-03-2013-era5 cutouts: diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index b297987a..c6f8eef9 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -464,10 +464,22 @@ def plot_carbon_budget_distribution(input_eurostat): plt.rcParams["ytick.direction"] = "in" plt.rcParams["xtick.labelsize"] = 20 plt.rcParams["ytick.labelsize"] = 20 + + emissions_scope = snakemake.params.emissions_scope + report_year = snakemake.params.eurostat_report_year + input_co2 = snakemake.input.co2 # historic emissions countries = snakemake.params.countries - e_1990 = co2_emissions_year(countries, input_eurostat, opts, snakemake, year=1990) + e_1990 = co2_emissions_year( + countries, + input_eurostat, + opts, + emissions_scope, + report_year, + input_co2, + year=1990, + ) emissions = historical_emissions(countries) # add other years https://sdi.eea.europa.eu/data/0569441f-2853-4664-a7cd-db969ef54de0 emissions.loc[2019] = 2.971372 diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index cea18fdf..0f9118f4 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -191,17 +191,15 @@ def get(item, investment_year=None): def co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year + countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year ): """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.params.energy["emissions"] - eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) + eea_co2 = build_eea_co2(input_co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.params.energy["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 From 3187081ec0473ae6d7bd8b62ef7cf8b88fab78c9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 31 Aug 2023 07:56:04 +0000 Subject: [PATCH 63/77] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config/test/config.perfect.yaml | 2 +- scripts/plot_summary.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index f20586d0..c99f4122 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -42,7 +42,7 @@ electricity: renewable_carriers: [solar, onwind, offwind-ac, offwind-dc] sector: - min_part_load_fischer_tropsch: 0 + min_part_load_fischer_tropsch: 0 min_part_load_methanolisation: 0 atlite: default_cutout: be-03-2013-era5 diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index c6f8eef9..672e371b 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -464,7 +464,7 @@ def plot_carbon_budget_distribution(input_eurostat): plt.rcParams["ytick.direction"] = "in" plt.rcParams["xtick.labelsize"] = 20 plt.rcParams["ytick.labelsize"] = 20 - + emissions_scope = snakemake.params.emissions_scope report_year = snakemake.params.eurostat_report_year input_co2 = snakemake.input.co2 From 20b68b14043f572dd421b4ab312548f324f7c815 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 22 Sep 2023 17:51:17 +0200 Subject: [PATCH 64/77] readthedocs: remove apt packages --- .readthedocs.yml | 2 -- doc/requirements.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 900dba1e..35eff5de 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,8 +8,6 @@ build: os: ubuntu-22.04 tools: python: "3.11" - apt_packages: - - graphviz python: install: diff --git a/doc/requirements.txt b/doc/requirements.txt index 3e760c81..de3ed4e8 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -7,6 +7,7 @@ sphinx sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 +graphviz pypsa powerplantmatching>=0.5.5 From 713cb1a7fc92c0757eb82b66654606c6bb5b0de6 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 22 Sep 2023 18:08:06 +0200 Subject: [PATCH 65/77] follow up --- .readthedocs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 35eff5de..ef3c4b0c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,4 +12,3 @@ build: python: install: - requirements: doc/requirements.txt - system_packages: false From 9765b24c1cdc0f82ec0d800be168704c6d1e97d0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 06:40:23 +0000 Subject: [PATCH 66/77] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Snakefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Snakefile b/Snakefile index 6ca52525..83530df7 100644 --- a/Snakefile +++ b/Snakefile @@ -66,18 +66,17 @@ if config["foresight"] == "myopic": include: "rules/solve_myopic.smk" - if config["foresight"] == "perfect": include: "rules/solve_perfect.smk" + rule all: input: RESULTS + "graphs/costs.pdf", default_target: True - rule purge: run: import builtins From 22fe2448a4ab98d724b5222c81fb356472f0be24 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:14:54 +0200 Subject: [PATCH 67/77] Update scripts/solve_network.py Co-authored-by: Fabian Hofmann --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index e353acd8..25e5e70e 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -194,7 +194,7 @@ def add_co2_sequestration_limit(n, config, limit=200): limit = float(o[o.find("seq") + 3 :]) * 1e6 break - if config["foresight"] == "perfect": + if not n.investment_periods.empty: periods = n.investment_periods names = pd.Index([f"co2_sequestration_limit-{period}" for period in periods]) else: From 08f1a0f313eebe31b96d77861b0e81bca98bb066 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:22:41 +0200 Subject: [PATCH 68/77] remove comments for newer data version --- scripts/plot_summary.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 672e371b..7fe482d2 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -385,9 +385,6 @@ def historical_emissions(countries): e["waste management"] = "5 - Waste management" e["other"] = "6 - Other Sector" e["indirect"] = "ind_CO2 - Indirect CO2" - # e["total wL"] = "Total (with LULUCF)" - # e["total woL"] = "Total (without LULUCF)" - e["other LULUCF"] = "4.H - Other LULUCF" pol = ["CO2"] # ["All greenhouse gases - (CO2 equivalent)"] From 213cec729cff15c6322d2a02e1cfa658169a7cf6 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Sat, 30 Sep 2023 08:45:40 +0200 Subject: [PATCH 69/77] remove unrelated changes --- config/config.perfect.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index a833e3c7..84c10cb0 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -1,4 +1,3 @@ -# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors # # SPDX-License-Identifier: CC0-1.0 From 117a7c1f78eb413677277b61ea3556c291ce5f01 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:04:57 +0200 Subject: [PATCH 70/77] update pypsa version to 0.25.2 --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index bc48eee2..09c209a6 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -56,4 +56,4 @@ dependencies: - pip: - git+https://github.com/fneum/tsam.git@performance - - pypsa>=0.25.1 + - pypsa>=0.25.2 From 58aa3c03dc929391992b5172e1791fdb253346b4 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 4 Oct 2023 17:23:32 +0200 Subject: [PATCH 71/77] revert removal of apt_packages --- .readthedocs.yml | 2 ++ doc/requirements.txt | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index ef3c4b0c..30684052 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,6 +8,8 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - graphviz python: install: diff --git a/doc/requirements.txt b/doc/requirements.txt index de3ed4e8..3e760c81 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -7,7 +7,6 @@ sphinx sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 -graphviz pypsa powerplantmatching>=0.5.5 From f5fb307f8947e3515b475855db1a50bcf53819f2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 4 Oct 2023 17:31:46 +0200 Subject: [PATCH 72/77] fix type, expand codespell ignore list --- .pre-commit-config.yaml | 2 +- scripts/build_line_rating.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 803eb61f..94a332b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: rev: v2.2.6 hooks: - id: codespell - args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore'] # Ignore capital case words, e.g. country codes + args: ['--ignore-regex="(\b[A-Z]+\b)"', '--ignore-words-list=fom,appartment,bage,ore,setis,tabacco,berfore,vor'] # Ignore capital case words, e.g. country codes types_or: [python, rst, markdown] files: ^(scripts|doc)/ diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 7f842d43..d3a970bd 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -41,7 +41,7 @@ The following heat gains and losses are considered: - heat gain through resistive losses - heat gain through solar radiation -- heat loss through radiation of the trasnmission line +- heat loss through radiation of the transmission line - heat loss through forced convection with wind - heat loss through natural convection From 608faf773ae0e9730323be5b22bd4c0b6288dbe8 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 4 Oct 2023 18:19:08 +0200 Subject: [PATCH 73/77] renew dag for tutorial.rst --- doc/tutorial.rst | 159 ++++++++++++++++++++++------------------------- 1 file changed, 76 insertions(+), 83 deletions(-) diff --git a/doc/tutorial.rst b/doc/tutorial.rst index f0ded3fb..40c1907c 100644 --- a/doc/tutorial.rst +++ b/doc/tutorial.rst @@ -133,89 +133,82 @@ This triggers a workflow of multiple preceding jobs that depend on each rule's i graph[bgcolor=white, margin=0]; node[shape=box, style=rounded, fontname=sans, fontsize=10, penwidth=2]; edge[penwidth=2, color=grey]; - 0[label = "solve_network", color = "0.21 0.6 0.85", style="rounded"]; - 1[label = "prepare_network\nll: copt\nopts: Co2L-24H", color = "0.02 0.6 0.85", style="rounded"]; - 2[label = "add_extra_components", color = "0.37 0.6 0.85", style="rounded"]; - 3[label = "cluster_network\nclusters: 6", color = "0.39 0.6 0.85", style="rounded"]; - 4[label = "simplify_network\nsimpl: ", color = "0.11 0.6 0.85", style="rounded"]; - 5[label = "add_electricity", color = "0.23 0.6 0.85", style="rounded"]; - 6[label = "build_renewable_profiles\ntechnology: onwind", color = "0.57 0.6 0.85", style="rounded"]; - 7[label = "base_network", color = "0.09 0.6 0.85", style="rounded"]; - 8[label = "build_shapes", color = "0.41 0.6 0.85", style="rounded"]; - 9[label = "retrieve_databundle", color = "0.28 0.6 0.85", style="rounded"]; - 10[label = "retrieve_natura_raster", color = "0.62 0.6 0.85", style="rounded"]; - 11[label = "build_bus_regions", color = "0.53 0.6 0.85", style="rounded"]; - 12[label = "retrieve_cutout\ncutout: europe-2013-era5", color = "0.05 0.6 0.85", style="rounded,dashed"]; - 13[label = "build_renewable_profiles\ntechnology: offwind-ac", color = "0.57 0.6 0.85", style="rounded"]; - 14[label = "build_ship_raster", color = "0.64 0.6 0.85", style="rounded"]; - 15[label = "retrieve_ship_raster", color = "0.07 0.6 0.85", style="rounded,dashed"]; - 16[label = "retrieve_cutout\ncutout: europe-2013-sarah", color = "0.05 0.6 0.85", style="rounded,dashed"]; - 17[label = "build_renewable_profiles\ntechnology: offwind-dc", color = "0.57 0.6 0.85", style="rounded"]; - 18[label = "build_renewable_profiles\ntechnology: solar", color = "0.57 0.6 0.85", style="rounded"]; - 19[label = "build_hydro_profile", color = "0.44 0.6 0.85", style="rounded"]; - 20[label = "retrieve_cost_data", color = "0.30 0.6 0.85", style="rounded"]; - 21[label = "build_powerplants", color = "0.16 0.6 0.85", style="rounded"]; - 22[label = "build_electricity_demand", color = "0.00 0.6 0.85", style="rounded"]; - 23[label = "retrieve_electricity_demand", color = "0.34 0.6 0.85", style="rounded,dashed"]; - 1 -> 0 - 2 -> 1 - 20 -> 1 - 3 -> 2 - 20 -> 2 - 4 -> 3 - 20 -> 3 - 5 -> 4 - 20 -> 4 - 11 -> 4 - 6 -> 5 - 13 -> 5 - 17 -> 5 - 18 -> 5 - 19 -> 5 - 7 -> 5 - 20 -> 5 - 11 -> 5 - 21 -> 5 - 9 -> 5 - 22 -> 5 - 8 -> 5 - 7 -> 6 - 9 -> 6 - 10 -> 6 - 8 -> 6 - 11 -> 6 - 12 -> 6 - 8 -> 7 - 9 -> 8 - 8 -> 11 - 7 -> 11 - 7 -> 13 - 9 -> 13 - 10 -> 13 - 14 -> 13 - 8 -> 13 - 11 -> 13 - 12 -> 13 - 15 -> 14 - 12 -> 14 - 16 -> 14 - 7 -> 17 - 9 -> 17 - 10 -> 17 - 14 -> 17 - 8 -> 17 - 11 -> 17 - 12 -> 17 - 7 -> 18 - 9 -> 18 - 10 -> 18 - 8 -> 18 - 11 -> 18 - 16 -> 18 - 8 -> 19 - 12 -> 19 - 7 -> 21 - 23 -> 22 + 0[label = "solve_network", color = "0.33 0.6 0.85", style="rounded"]; + 1[label = "prepare_network\nll: copt\nopts: Co2L-24H", color = "0.03 0.6 0.85", style="rounded"]; + 2[label = "add_extra_components", color = "0.45 0.6 0.85", style="rounded"]; + 3[label = "cluster_network\nclusters: 6", color = "0.46 0.6 0.85", style="rounded"]; + 4[label = "simplify_network\nsimpl: ", color = "0.52 0.6 0.85", style="rounded"]; + 5[label = "add_electricity", color = "0.55 0.6 0.85", style="rounded"]; + 6[label = "build_renewable_profiles\ntechnology: solar", color = "0.15 0.6 0.85", style="rounded"]; + 7[label = "base_network", color = "0.37 0.6 0.85", style="rounded,dashed"]; + 8[label = "build_shapes", color = "0.07 0.6 0.85", style="rounded,dashed"]; + 9[label = "retrieve_databundle", color = "0.60 0.6 0.85", style="rounded"]; + 10[label = "retrieve_natura_raster", color = "0.42 0.6 0.85", style="rounded"]; + 11[label = "build_bus_regions", color = "0.09 0.6 0.85", style="rounded,dashed"]; + 12[label = "build_renewable_profiles\ntechnology: onwind", color = "0.15 0.6 0.85", style="rounded"]; + 13[label = "build_renewable_profiles\ntechnology: offwind-ac", color = "0.15 0.6 0.85", style="rounded"]; + 14[label = "build_ship_raster", color = "0.02 0.6 0.85", style="rounded"]; + 15[label = "retrieve_ship_raster", color = "0.40 0.6 0.85", style="rounded"]; + 16[label = "build_renewable_profiles\ntechnology: offwind-dc", color = "0.15 0.6 0.85", style="rounded"]; + 17[label = "build_line_rating", color = "0.32 0.6 0.85", style="rounded"]; + 18[label = "retrieve_cost_data\nyear: 2030", color = "0.50 0.6 0.85", style="rounded"]; + 19[label = "build_powerplants", color = "0.64 0.6 0.85", style="rounded,dashed"]; + 20[label = "build_electricity_demand", color = "0.13 0.6 0.85", style="rounded,dashed"]; + 21[label = "retrieve_electricity_demand", color = "0.31 0.6 0.85", style="rounded"]; + 22[label = "copy_config", color = "0.23 0.6 0.85", style="rounded"]; + 1 -> 0 + 22 -> 0 + 2 -> 1 + 18 -> 1 + 3 -> 2 + 18 -> 2 + 4 -> 3 + 18 -> 3 + 5 -> 4 + 18 -> 4 + 11 -> 4 + 6 -> 5 + 12 -> 5 + 13 -> 5 + 16 -> 5 + 7 -> 5 + 17 -> 5 + 18 -> 5 + 11 -> 5 + 19 -> 5 + 9 -> 5 + 20 -> 5 + 8 -> 5 + 7 -> 6 + 9 -> 6 + 10 -> 6 + 8 -> 6 + 11 -> 6 + 8 -> 7 + 9 -> 8 + 8 -> 11 + 7 -> 11 + 7 -> 12 + 9 -> 12 + 10 -> 12 + 8 -> 12 + 11 -> 12 + 7 -> 13 + 9 -> 13 + 10 -> 13 + 14 -> 13 + 8 -> 13 + 11 -> 13 + 15 -> 14 + 7 -> 16 + 9 -> 16 + 10 -> 16 + 14 -> 16 + 8 -> 16 + 11 -> 16 + 7 -> 17 + 7 -> 19 + 21 -> 20 } | From 345331c301a49087812d00ee760a8709105d7bea Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 4 Oct 2023 18:21:13 +0200 Subject: [PATCH 74/77] reinsert graphviz to requirements.txt --- .readthedocs.yml | 2 -- doc/requirements.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 30684052..ef3c4b0c 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,8 +8,6 @@ build: os: ubuntu-22.04 tools: python: "3.11" - apt_packages: - - graphviz python: install: diff --git a/doc/requirements.txt b/doc/requirements.txt index 3e760c81..de3ed4e8 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -7,6 +7,7 @@ sphinx sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 +graphviz pypsa powerplantmatching>=0.5.5 From 18e4099ece6420c12ba108093de993381080d419 Mon Sep 17 00:00:00 2001 From: Fabian Date: Thu, 5 Oct 2023 09:58:25 +0200 Subject: [PATCH 75/77] re-reconvert graphviz installation --- .readthedocs.yml | 2 ++ doc/requirements.txt | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index ef3c4b0c..30684052 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,6 +8,8 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - graphviz python: install: diff --git a/doc/requirements.txt b/doc/requirements.txt index de3ed4e8..3e760c81 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -7,7 +7,6 @@ sphinx sphinx_book_theme sphinxcontrib-bibtex myst-parser # recommark is deprecated, https://stackoverflow.com/a/71660856/13573820 -graphviz pypsa powerplantmatching>=0.5.5 From be0214be1348a89b96e1db2d99ffb2f7169a4ee6 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 5 Oct 2023 11:23:14 +0200 Subject: [PATCH 76/77] simplify config --- config/config.perfect.yaml | 957 +------------------------------------ 1 file changed, 5 insertions(+), 952 deletions(-) diff --git a/config/config.perfect.yaml b/config/config.perfect.yaml index 84c10cb0..f355763c 100644 --- a/config/config.perfect.yaml +++ b/config/config.perfect.yaml @@ -1,28 +1,8 @@ +# SPDX-FileCopyrightText: : 2017-2023 The PyPSA-Eur Authors # # SPDX-License-Identifier: CC0-1.0 - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#top-level-configuration -version: 0.8.1 -tutorial: false - -logging: - level: INFO - format: '%(levelname)s:%(name)s:%(message)s' - -private: - keys: - entsoe_api: - -remote: - ssh: "" - path: "" - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#run run: - name: "test_methanolisation_shorterlifetime" - disable_progressbar: false - shared_resources: true - shared_cutouts: true + name: "perfect" # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#foresight foresight: perfect @@ -34,18 +14,13 @@ scenario: - '' ll: - v1.0 -# - v1.5 clusters: - 37 -# - 128 -# - 256 -# - 512 - #- 1024 opts: - '' sector_opts: - - 1p7-4380H-T-H-B-I-A-solar+p3-dist1 - 1p5-4380H-T-H-B-I-A-solar+p3-dist1 + - 1p7-4380H-T-H-B-I-A-solar+p3-dist1 - 2p0-4380H-T-H-B-I-A-solar+p3-dist1 planning_horizons: - 2020 @@ -53,938 +28,16 @@ scenario: - 2040 - 2050 -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#countries -countries: ['AL', 'AT', 'BA', 'BE', 'BG', 'CH', 'CZ', 'DE', 'DK', 'EE', 'ES', 'FI', 'FR', 'GB', 'GR', 'HR', 'HU', 'IE', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'NL', 'NO', 'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK'] - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#snapshots -snapshots: - start: "2013-01-01" - end: "2014-01-01" - inclusive: 'left' - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#enable -enable: - retrieve: auto - prepare_links_p_nom: false - retrieve_databundle: true - retrieve_sector_databundle: true - retrieve_cost_data: true - build_cutout: false - retrieve_cutout: true - build_natura_raster: false - retrieve_natura_raster: true - custom_busmap: false # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#co2-budget co2_budget: - 2020: 0.701 - 2025: 0.524 - 2030: 0.297 - 2035: 0.150 - 2040: 0.071 - 2045: 0.032 - 2050: 0.000 - # update of IPCC 6th AR compared to the 1.5SR. (discussed here: https://twitter.com/JoeriRogelj/status/1424743828339167233) 1p5: 34.2 # 25.7 # Budget in Gt CO2 for 1.5 for Europe, global 420 Gt, assuming per capita share 1p6: 43.259666 # 35 # Budget in Gt CO2 for 1.6 for Europe, global 580 Gt 1p7: 51.4 # 45 # Budget in Gt CO2 for 1.7 for Europe, global 800 Gt 2p0: 69.778 # 73.9 # Budget in Gt CO2 for 2 for Europe, global 1170 Gt -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#electricity -electricity: - voltages: [220., 300., 380.] - gaslimit: false - co2limit: 7.75e+7 - co2base: 1.487e+9 - agg_p_nom_limits: data/agg_p_nom_minmax.csv - operational_reserve: - activate: false - epsilon_load: 0.02 - epsilon_vres: 0.02 - contingency: 4000 - - max_hours: - battery: 6 - H2: 168 - - extendable_carriers: - Generator: [solar, onwind, offwind-ac, offwind-dc, OCGT] - StorageUnit: [] # battery, H2 - Store: [battery, H2] - Link: [] # H2 pipeline - - powerplants_filter: (DateOut >= 2022 or DateOut != DateOut) - custom_powerplants: false - - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, lignite, geothermal, biomass] - renewable_carriers: [solar, onwind, offwind-ac, offwind-dc, hydro] - - estimate_renewable_capacities: - enable: true - from_opsd: true - year: 2020 - expansion_limit: false - technology_mapping: - Offshore: [offwind-ac, offwind-dc] - Onshore: [onwind] - PV: [solar] - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#atlite -atlite: - default_cutout: europe-2013-era5 - nprocesses: 1 - show_progress: false - cutouts: - # use 'base' to determine geographical bounds and time span from config - # base: - # module: era5 - europe-2013-era5: - module: era5 # in priority order - x: [-12., 35.] - y: [33., 72] - dx: 0.3 - dy: 0.3 - time: ['2013', '2013'] - europe-2013-sarah: - module: [sarah, era5] # in priority order - x: [-12., 45.] - y: [33., 65] - dx: 0.2 - dy: 0.2 - time: ['2013', '2013'] - sarah_interpolate: false - sarah_dir: - features: [influx, temperature] - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#renewable -renewable: - onwind: - cutout: europe-2013-era5 - resource: - method: wind - turbine: Vestas_V112_3MW - capacity_per_sqkm: 3 - # correction_factor: 0.93 - corine: - grid_codes: [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32] - distance: 1000 - distance_grid_codes: [1, 2, 3, 4, 5, 6] - natura: true - excluder_resolution: 100 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - offwind-ac: - cutout: europe-2013-era5 - resource: - method: wind - turbine: NREL_ReferenceTurbine_5MW_offshore - capacity_per_sqkm: 2 - correction_factor: 0.8855 - corine: [44, 255] - natura: true - ship_threshold: 400 - max_depth: 50 - max_shore_distance: 30000 - excluder_resolution: 200 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - offwind-dc: - cutout: europe-2013-era5 - resource: - method: wind - turbine: NREL_ReferenceTurbine_5MW_offshore - capacity_per_sqkm: 2 - correction_factor: 0.8855 - corine: [44, 255] - natura: true - ship_threshold: 400 - max_depth: 50 - min_shore_distance: 30000 - excluder_resolution: 200 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - solar: - cutout: europe-2013-sarah - resource: - method: pv - panel: CSi - orientation: - slope: 35. - azimuth: 180. - capacity_per_sqkm: 1.7 - # correction_factor: 0.854337 - corine: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 26, 31, 32] - natura: true - excluder_resolution: 100 - potential: simple # or conservative - clip_p_max_pu: 1.e-2 - hydro: - cutout: europe-2013-era5 - carriers: [ror, PHS, hydro] - PHS_max_hours: 6 - hydro_max_hours: "energy_capacity_totals_by_country" # one of energy_capacity_totals_by_country, estimate_by_large_installations or a float - flatten_dispatch: false - flatten_dispatch_buffer: 0.2 - clip_min_inflow: 1.0 - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#conventional -conventional: - unit_commitment: false - dynamic_fuel_price: false - nuclear: - p_max_pu: "data/nuclear_p_max_pu.csv" # float of file name - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#lines -lines: - types: - 220.: "Al/St 240/40 2-bundle 220.0" - 300.: "Al/St 240/40 3-bundle 300.0" - 380.: "Al/St 240/40 4-bundle 380.0" - s_max_pu: 0.7 - s_nom_max: .inf - max_extension: .inf - length_factor: 1.25 - under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity - dynamic_line_rating: - activate: false - cutout: europe-2013-era5 - correction_factor: 0.95 - max_voltage_difference: false - max_line_rating: false - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#links -links: - p_max_pu: 1.0 - p_nom_max: .inf - max_extension: .inf - include_tyndp: true - under_construction: 'zero' # 'zero': set capacity to zero, 'remove': remove, 'keep': with full capacity - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#transformers -transformers: - x: 0.1 - s_nom: 2000. - type: '' - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#load -load: - power_statistics: true - interpolate_limit: 3 - time_shift_for_large_gaps: 1w - manual_adjustments: true # false - scaling_factor: 1.0 - -# docs -# TODO: PyPSA-Eur merge issue in prepare_sector_network.py -# regulate what components with which carriers are kept from PyPSA-Eur; -# some technologies are removed because they are implemented differently -# (e.g. battery or H2 storage) or have different year-dependent costs -# in PyPSA-Eur-Sec -pypsa_eur: - Bus: - - AC - Link: - - DC - Generator: - - onwind - - offwind-ac - - offwind-dc - - solar - - ror - StorageUnit: - - PHS - - hydro - Store: [] - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy -energy: - energy_totals_year: 2011 - base_emissions_year: 1990 - eurostat_report_year: 2016 - emissions: CO2 - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#biomass -biomass: - year: 2030 - scenario: ENS_Med - classes: - solid biomass: - - Agricultural waste - - Fuelwood residues - - Secondary Forestry residues - woodchips - - Sawdust - - Residues from landscape care - - Municipal waste - not included: - - Sugar from sugar beet - - Rape seed - - "Sunflower, soya seed " - - Bioethanol barley, wheat, grain maize, oats, other cereals and rye - - Miscanthus, switchgrass, RCG - - Willow - - Poplar - - FuelwoodRW - - C&P_RW - biogas: - - Manure solid, liquid - - Sludge - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solar-thermal -solar_thermal: - clearsky_model: simple # should be "simple" or "enhanced"? - orientation: - slope: 45. - azimuth: 180. - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#existing-capacities -existing_capacities: - grouping_years_power: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020, 2025, 2030] - grouping_years_heat: [1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2019] # these should not extend 2020 - threshold_capacity: 10 - conventional_carriers: - - lignite - - coal - - oil - - uranium - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#sector sector: - district_heating: - potential: 0.6 - progress: - 2020: 0.0 - 2030: 0.3 - 2040: 0.6 - 2050: 1.0 - district_heating_loss: 0.15 - cluster_heat_buses: false - bev_dsm_restriction_value: 0.75 - bev_dsm_restriction_time: 7 - transport_heating_deadband_upper: 20. - transport_heating_deadband_lower: 15. - ICE_lower_degree_factor: 0.375 - ICE_upper_degree_factor: 1.6 - EV_lower_degree_factor: 0.98 - EV_upper_degree_factor: 0.63 - bev_dsm: true - bev_availability: 0.5 - bev_energy: 0.05 - bev_charge_efficiency: 0.9 - bev_plug_to_wheel_efficiency: 0.2 - bev_charge_rate: 0.011 - bev_avail_max: 0.95 - bev_avail_mean: 0.8 - v2g: true - land_transport_fuel_cell_share: - 2020: 0 - 2030: 0.05 - 2040: 0.1 - 2050: 0.15 - land_transport_electric_share: - 2020: 0 - 2030: 0.25 - 2040: 0.6 - 2050: 0.85 - land_transport_ice_share: - 2020: 1 - 2030: 0.7 - 2040: 0.3 - 2050: 0 - transport_fuel_cell_efficiency: 0.5 - transport_internal_combustion_efficiency: 0.3 - agriculture_machinery_electric_share: 0 - agriculture_machinery_oil_share: 1 - agriculture_machinery_fuel_efficiency: 0.7 - agriculture_machinery_electric_efficiency: 0.3 - MWh_MeOH_per_MWh_H2: 0.8787 - MWh_MeOH_per_tCO2: 4.0321 - MWh_MeOH_per_MWh_e: 3.6907 - shipping_hydrogen_liquefaction: false - shipping_hydrogen_share: - 2020: 0 - 2030: 0 - 2040: 0 - 2050: 0 - shipping_methanol_share: - 2020: 0.0 - 2030: 0.3 - 2040: 0.7 - 2050: 1 - shipping_oil_share: - 2020: 1.0 - 2030: 0.7 - 2040: 0.3 - 2050: 0 - shipping_methanol_efficiency: 0.46 - shipping_oil_efficiency: 0.40 - aviation_demand_factor: 1. - HVC_demand_factor: 1. - time_dep_hp_cop: true - heat_pump_sink_T: 55. - reduce_space_heat_exogenously: true - reduce_space_heat_exogenously_factor: - 2020: 0.10 # this results in a space heat demand reduction of 10% - 2025: 0.09 # first heat demand increases compared to 2020 because of larger floor area per capita - 2030: 0.09 - 2035: 0.11 - 2040: 0.16 - 2045: 0.21 - 2050: 0.29 - retrofitting: - retro_endogen: false - cost_factor: 1.0 - interest_rate: 0.04 - annualise_cost: true - tax_weighting: false - construction_index: true - tes: true - tes_tau: - decentral: 3 - central: 180 - boilers: true - oil_boilers: false - biomass_boiler: true - chp: true - micro_chp: false - solar_thermal: true - solar_cf_correction: 0.788457 # = >>> 1/1.2683 - marginal_cost_storage: 0. #1e-4 - methanation: true - helmeth: false - coal_cc: false - dac: true - co2_vent: false - allam_cycle: false - hydrogen_fuel_cell: true - hydrogen_turbine: false - SMR: true - regional_co2_sequestration_potential: - enable: false - attribute: 'conservative estimate Mt' - include_onshore: false - min_size: 3 - max_size: 25 - years_of_storage: 25 - co2_sequestration_potential: 200 - co2_sequestration_cost: 10 - co2_sequestration_lifetime: 50 - co2_spatial: false - co2network: false - cc_fraction: 0.9 - hydrogen_underground_storage: true - hydrogen_underground_storage_locations: - # - onshore # more than 50 km from sea - - nearshore # within 50 km of sea - # - offshore - ammonia: false - min_part_load_fischer_tropsch: 0 # 0.9 - min_part_load_methanolisation: 0 # 0.5 - use_fischer_tropsch_waste_heat: true - use_fuel_cell_waste_heat: true - use_electrolysis_waste_heat: false - electricity_distribution_grid: true - electricity_distribution_grid_cost_factor: 1.0 - electricity_grid_connection: true - H2_network: true - gas_network: false - H2_retrofit: false - H2_retrofit_capacity_per_CH4: 0.6 - gas_network_connectivity_upgrade: 1 - gas_distribution_grid: true - gas_distribution_grid_cost_factor: 1.0 - biomass_spatial: false - biomass_transport: false - conventional_generation: - OCGT: gas - biomass_to_liquid: false - biosng: false - limit_max_growth: - enable: true - # allowing 30% larger than max historic growth - factor: 1.3 - max_growth: # unit GW - onwind: 16 # onshore max grow so far 16 GW in Europe https://www.iea.org/reports/renewables-2020/wind - solar: 28 # solar max grow so far 28 GW in Europe https://www.iea.org/reports/renewables-2020/solar-pv - offwind-ac: 35 # offshore max grow so far 3.5 GW in Europe https://windeurope.org/about-wind/statistics/offshore/european-offshore-wind-industry-key-trends-statistics-2019/ - offwind-dc: 35 - max_relative_growth: - onwind: 3 - solar: 3 - offwind-ac: 3 - offwind-dc: 3 - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#industry -industry: - St_primary_fraction: - 2020: 0.6 - 2025: 0.55 - 2030: 0.5 - 2035: 0.45 - 2040: 0.4 - 2045: 0.35 - 2050: 0.3 - DRI_fraction: - 2020: 0 - 2025: 0 - 2030: 0.05 - 2035: 0.2 - 2040: 0.4 - 2045: 0.7 - 2050: 1 - H2_DRI: 1.7 - elec_DRI: 0.322 - Al_primary_fraction: - 2020: 0.4 - 2025: 0.375 - 2030: 0.35 - 2035: 0.325 - 2040: 0.3 - 2045: 0.25 - 2050: 0.2 - MWh_NH3_per_tNH3: 5.166 - MWh_CH4_per_tNH3_SMR: 10.8 - MWh_elec_per_tNH3_SMR: 0.7 - MWh_H2_per_tNH3_electrolysis: 6.5 - MWh_elec_per_tNH3_electrolysis: 1.17 - MWh_NH3_per_MWh_H2_cracker: 1.46 # https://github.com/euronion/trace/blob/44a5ff8401762edbef80eff9cfe5a47c8d3c8be4/data/efficiencies.csv - NH3_process_emissions: 24.5 - petrochemical_process_emissions: 25.5 - HVC_primary_fraction: 1. - HVC_mechanical_recycling_fraction: 0. - HVC_chemical_recycling_fraction: 0. - HVC_production_today: 52. - MWh_elec_per_tHVC_mechanical_recycling: 0.547 - MWh_elec_per_tHVC_chemical_recycling: 6.9 - chlorine_production_today: 9.58 - MWh_elec_per_tCl: 3.6 - MWh_H2_per_tCl: -0.9372 - methanol_production_today: 1.5 - MWh_elec_per_tMeOH: 0.167 - MWh_CH4_per_tMeOH: 10.25 - hotmaps_locate_missing: false - reference_year: 2015 - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#costs -costs: - year: 2030 - social_discountrate: 0.02 - version: v0.6.0 - rooftop_share: 0.14 # based on the potentials, assuming (0.1 kW/m2 and 10 m2/person) - fill_values: - FOM: 0 - VOM: 0 - efficiency: 1 - fuel: 0 - investment: 0 - lifetime: 25 - "CO2 intensity": 0 - "discount rate": 0.07 - # Marginal and capital costs can be overwritten - # capital_cost: - # onwind: 500 - marginal_cost: - solar: 0.01 - onwind: 0.015 - offwind: 0.015 - hydro: 0. - H2: 0. - electrolysis: 0. - fuel cell: 0. - battery: 0. - battery inverter: 0. - emission_prices: - co2: 0. - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#clustering -clustering: - simplify_network: - to_substations: false - algorithm: kmeans # choose from: [hac, kmeans] - feature: solar+onwind-time - exclude_carriers: [] - remove_stubs: true - remove_stubs_across_borders: true - cluster_network: - algorithm: kmeans - feature: solar+onwind-time - exclude_carriers: [] - consider_efficiency_classes: false - aggregation_strategies: - generators: - committable: any - ramp_limit_up: max - ramp_limit_down: max - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#solving -solving: - #tmpdir: "path/to/tmp" - options: - clip_p_max_pu: 1.e-2 - load_shedding: true - noisy_costs: true - skip_iterations: true - rolling_horizon: false - seed: 123 - # options that go into the optimize function - track_iterations: false - min_iterations: 4 - max_iterations: 6 - transmission_losses: 0 - linearized_unit_commitment: true - horizon: 365 - - solver: - name: gurobi - options: gurobi-default - - solver_options: - highs-default: - # refer to https://ergo-code.github.io/HiGHS/options/definitions.html#solver - threads: 4 - solver: "ipm" - run_crossover: "off" - small_matrix_value: 1e-6 - large_matrix_value: 1e9 - primal_feasibility_tolerance: 1e-5 - dual_feasibility_tolerance: 1e-5 - ipm_optimality_tolerance: 1e-4 - parallel: "on" - random_seed: 123 - gurobi-default: - threads: 4 - method: 2 # barrier - crossover: 0 - BarConvTol: 1.e-6 - Seed: 123 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - gurobi-numeric-focus: - name: gurobi - NumericFocus: 3 # Favour numeric stability over speed - method: 2 # barrier - crossover: 0 # do not use crossover - BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge - BarConvTol: 1.e-5 - FeasibilityTol: 1.e-4 - OptimalityTol: 1.e-4 - ObjScale: -0.5 - threads: 8 - Seed: 123 - gurobi-fallback: # Use gurobi defaults - name: gurobi - crossover: 0 - method: 2 # barrier - BarHomogeneous: 1 # Use homogeneous barrier if standard does not converge - BarConvTol: 1.e-5 - FeasibilityTol: 1.e-5 - OptimalityTol: 1.e-5 - Seed: 123 - threads: 8 - cplex-default: - threads: 4 - lpmethod: 4 # barrier - solutiontype: 2 # non basic solution, ie no crossover - barrier.convergetol: 1.e-5 - feasopt.tolerance: 1.e-6 - cbc-default: {} # Used in CI - glpk-default: {} # Used in CI - - mem: 30000 #memory in MB; 20 GB enough for 50+B+I+H2; 100 GB for 181+B+I+H2 - walltime: "12:00:00" - -# docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#plotting -plotting: - map: - boundaries: [-11, 30, 34, 71] - color_geomap: - ocean: white - land: white - eu_node_location: - x: -5.5 - y: 46. - costs_max: 1000 - costs_threshold: 1 - energy_max: 20000 - energy_min: -20000 - energy_threshold: 50. - - nice_names: - OCGT: "Open-Cycle Gas" - CCGT: "Combined-Cycle Gas" - offwind-ac: "Offshore Wind (AC)" - offwind-dc: "Offshore Wind (DC)" - onwind: "Onshore Wind" - solar: "Solar" - PHS: "Pumped Hydro Storage" - hydro: "Reservoir & Dam" - battery: "Battery Storage" - H2: "Hydrogen Storage" - lines: "Transmission Lines" - ror: "Run of River" - ac: "AC" - dc: "DC" - - tech_colors: - # wind - onwind: "#235ebc" - onshore wind: "#235ebc" - offwind: "#6895dd" - offshore wind: "#6895dd" - offwind-ac: "#6895dd" - offshore wind (AC): "#6895dd" - offshore wind ac: "#6895dd" - offwind-dc: "#74c6f2" - offshore wind (DC): "#74c6f2" - offshore wind dc: "#74c6f2" - # water - hydro: '#298c81' - hydro reservoir: '#298c81' - ror: '#3dbfb0' - run of river: '#3dbfb0' - hydroelectricity: '#298c81' - PHS: '#51dbcc' - hydro+PHS: "#08ad97" - wave: '#a7d4cf' - # solar - solar: "#f9d002" - solar PV: "#f9d002" - solar thermal: '#ffbf2b' - residential rural solar thermal: '#f1c069' - services rural solar thermal: '#eabf61' - residential urban decentral solar thermal: '#e5bc5a' - services urban decentral solar thermal: '#dfb953' - urban central solar thermal: '#d7b24c' - solar rooftop: '#ffea80' - # gas - OCGT: '#e0986c' - OCGT marginal: '#e0986c' - OCGT-heat: '#e0986c' - gas boiler: '#db6a25' - gas boilers: '#db6a25' - gas boiler marginal: '#db6a25' - residential rural gas boiler: '#d4722e' - residential urban decentral gas boiler: '#cb7a36' - services rural gas boiler: '#c4813f' - services urban decentral gas boiler: '#ba8947' - urban central gas boiler: '#b0904f' - gas: '#e05b09' - fossil gas: '#e05b09' - natural gas: '#e05b09' - biogas to gas: '#e36311' - CCGT: '#a85522' - CCGT marginal: '#a85522' - allam: '#B98F76' - gas for industry co2 to atmosphere: '#692e0a' - gas for industry co2 to stored: '#8a3400' - gas for industry: '#853403' - gas for industry CC: '#692e0a' - gas pipeline: '#ebbca0' - gas pipeline new: '#a87c62' - # oil - oil: '#c9c9c9' - imported oil: '#c9c9c9' - oil boiler: '#adadad' - residential rural oil boiler: '#a9a9a9' - services rural oil boiler: '#a5a5a5' - residential urban decentral oil boiler: '#a1a1a1' - urban central oil boiler: '#9d9d9d' - services urban decentral oil boiler: '#999999' - agriculture machinery oil: '#949494' - shipping oil: "#808080" - land transport oil: '#afafaf' - # nuclear - Nuclear: '#ff8c00' - Nuclear marginal: '#ff8c00' - nuclear: '#ff8c00' - uranium: '#ff8c00' - # coal - Coal: '#545454' - coal: '#545454' - Coal marginal: '#545454' - solid: '#545454' - Lignite: '#826837' - lignite: '#826837' - Lignite marginal: '#826837' - # biomass - biogas: '#e3d37d' - biomass: '#baa741' - solid biomass: '#baa741' - solid biomass transport: '#baa741' - solid biomass for industry: '#7a6d26' - solid biomass for industry CC: '#47411c' - solid biomass for industry co2 from atmosphere: '#736412' - solid biomass for industry co2 to stored: '#47411c' - urban central solid biomass CHP: '#9d9042' - urban central solid biomass CHP CC: '#6c5d28' - biomass boiler: '#8A9A5B' - residential rural biomass boiler: '#a1a066' - residential urban decentral biomass boiler: '#b0b87b' - services rural biomass boiler: '#c6cf98' - services urban decentral biomass boiler: '#dde5b5' - biomass to liquid: '#32CD32' - BioSNG: '#123456' - # power transmission - lines: '#6c9459' - transmission lines: '#6c9459' - electricity distribution grid: '#97ad8c' - low voltage: '#97ad8c' - # electricity demand - Electric load: '#110d63' - electric demand: '#110d63' - electricity: '#110d63' - industry electricity: '#2d2a66' - industry new electricity: '#2d2a66' - agriculture electricity: '#494778' - # battery + EVs - battery: '#ace37f' - battery storage: '#ace37f' - battery charger: '#88a75b' - battery discharger: '#5d4e29' - home battery: '#80c944' - home battery storage: '#80c944' - home battery charger: '#5e8032' - home battery discharger: '#3c5221' - BEV charger: '#baf238' - V2G: '#e5ffa8' - land transport EV: '#baf238' - Li ion: '#baf238' - # hot water storage - water tanks: '#e69487' - residential rural water tanks: '#f7b7a3' - services rural water tanks: '#f3afa3' - residential urban decentral water tanks: '#f2b2a3' - services urban decentral water tanks: '#f1b4a4' - urban central water tanks: '#e9977d' - hot water storage: '#e69487' - hot water charging: '#e8998b' - urban central water tanks charger: '#b57a67' - residential rural water tanks charger: '#b4887c' - residential urban decentral water tanks charger: '#b39995' - services rural water tanks charger: '#b3abb0' - services urban decentral water tanks charger: '#b3becc' - hot water discharging: '#e99c8e' - urban central water tanks discharger: '#b9816e' - residential rural water tanks discharger: '#ba9685' - residential urban decentral water tanks discharger: '#baac9e' - services rural water tanks discharger: '#bbc2b8' - services urban decentral water tanks discharger: '#bdd8d3' - # heat demand - Heat load: '#cc1f1f' - heat: '#cc1f1f' - heat demand: '#cc1f1f' - rural heat: '#ff5c5c' - residential rural heat: '#ff7c7c' - services rural heat: '#ff9c9c' - central heat: '#cc1f1f' - urban central heat: '#d15959' - decentral heat: '#750606' - residential urban decentral heat: '#a33c3c' - services urban decentral heat: '#cc1f1f' - low-temperature heat for industry: '#8f2727' - process heat: '#ff0000' - agriculture heat: '#d9a5a5' - # heat supply - heat pumps: '#2fb537' - heat pump: '#2fb537' - air heat pump: '#36eb41' - residential urban decentral air heat pump: '#48f74f' - services urban decentral air heat pump: '#5af95d' - urban central air heat pump: '#6cfb6b' - ground heat pump: '#2fb537' - residential rural ground heat pump: '#48f74f' - services rural ground heat pump: '#5af95d' - Ambient: '#98eb9d' - CHP: '#8a5751' - urban central gas CHP: '#8d5e56' - CHP CC: '#634643' - urban central gas CHP CC: '#6e4e4c' - CHP heat: '#8a5751' - CHP electric: '#8a5751' - district heating: '#e8beac' - resistive heater: '#d8f9b8' - residential rural resistive heater: '#bef5b5' - residential urban decentral resistive heater: '#b2f1a9' - services rural resistive heater: '#a5ed9d' - services urban decentral resistive heater: '#98e991' - urban central resistive heater: '#8cdf85' - retrofitting: '#8487e8' - building retrofitting: '#8487e8' - # hydrogen - H2 for industry: "#f073da" - H2 for shipping: "#ebaee0" - H2: '#bf13a0' - hydrogen: '#bf13a0' - retrofitted H2 boiler: '#e5a0d9' - SMR: '#870c71' - SMR CC: '#4f1745' - H2 liquefaction: '#d647bd' - hydrogen storage: '#bf13a0' - H2 Store: '#bf13a0' - H2 storage: '#bf13a0' - land transport fuel cell: '#6b3161' - H2 pipeline: '#f081dc' - H2 pipeline retrofitted: '#ba99b5' - H2 Fuel Cell: '#c251ae' - H2 fuel cell: '#c251ae' - H2 turbine: '#991f83' - H2 Electrolysis: '#ff29d9' - H2 electrolysis: '#ff29d9' - # ammonia - NH3: '#46caf0' - ammonia: '#46caf0' - ammonia store: '#00ace0' - ammonia cracker: '#87d0e6' - Haber-Bosch: '#076987' - # syngas - Sabatier: '#9850ad' - methanation: '#c44ce6' - methane: '#c44ce6' - helmeth: '#e899ff' - # synfuels - Fischer-Tropsch: '#25c49a' - liquid: '#25c49a' - kerosene for aviation: '#a1ffe6' - naphtha for industry: '#57ebc4' - methanolisation: '#83d6d5' - methanol: '#468c8b' - shipping methanol: '#468c8b' - # co2 - CC: '#f29dae' - CCS: '#f29dae' - CO2 sequestration: '#f29dae' - DAC: '#ff5270' - co2 stored: '#f2385a' - co2: '#f29dae' - co2 vent: '#ffd4dc' - CO2 pipeline: '#f5627f' - # emissions - process emissions CC: '#000000' - process emissions: '#222222' - process emissions to stored: '#444444' - process emissions to atmosphere: '#888888' - oil emissions: '#aaaaaa' - shipping oil emissions: "#555555" - shipping methanol emissions: '#666666' - land transport oil emissions: '#777777' - agriculture machinery oil emissions: '#333333' - # other - shipping: '#03a2ff' - power-to-heat: '#2fb537' - power-to-gas: '#c44ce6' - power-to-H2: '#ff29d9' - power-to-liquid: '#25c49a' - gas-to-power/heat: '#ee8340' - waste: '#e3d37d' - other: '#000000' - geothermal: '#ba91b1' - AC: "#70af1d" - AC-AC: "#70af1d" - AC line: "#70af1d" - links: "#8a1caf" - HVDC links: "#8a1caf" - DC: "#8a1caf" - DC-DC: "#8a1caf" - DC link: "#8a1caf" + min_part_load_fischer_tropsch: 0 + min_part_load_methanolisation: 0 From 1eed747bde6ebb55786a26d01194b45c6376db7c Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Thu, 5 Oct 2023 12:03:06 +0200 Subject: [PATCH 77/77] remove spelling mistakes --- scripts/_benchmark.py | 18 +++--------------- scripts/plot_summary.py | 2 +- scripts/prepare_perfect_foresight.py | 4 ++-- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/scripts/_benchmark.py b/scripts/_benchmark.py index ba49ddc5..4e3413e9 100644 --- a/scripts/_benchmark.py +++ b/scripts/_benchmark.py @@ -1,19 +1,7 @@ # -*- coding: utf-8 -*- - -## Copyright 2015-2017 Frankfurt Institute for Advanced Studies - -## This program is free software; you can redistribute it and/or -## modify it under the terms of the GNU General Public License as -## published by the Free Software Foundation; either version 3 of the -## License, or (at your option) any later version. - -## This program is distributed in the hope that it will be useful, -## but WITHOUT ANY WARRANTY; without even the implied warranty of -## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -## GNU General Public License for more details. - -## You should have received a copy of the GNU General Public License -## along with this program. If not, see . +# SPDX-FileCopyrightText: : 2020-2023 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT """ """ diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 7fe482d2..86f40225 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -553,7 +553,7 @@ def plot_carbon_budget_distribution(input_eurostat): markersize=12, markerfacecolor="black", markeredgecolor="black", - label="EU commited target", + label="EU committed target", ) for col in co2_cap.columns: diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index d71deab4..ca94dedc 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -200,7 +200,7 @@ def concat_networks(years): else: # this is to avoid adding multiple times assets with - # infinit lifetime as ror + # infinite lifetime as ror cols = pnl_year.columns.difference(pnl[k].columns) pnl[k] = pd.concat([pnl[k], pnl_year[cols]], axis=1) @@ -233,7 +233,7 @@ def adjust_stores(n): Make sure that stores still behave cyclic over one year and not whole modelling horizon. """ - # cylclic constraint + # cyclic constraint cyclic_i = n.stores[n.stores.e_cyclic].index n.stores.loc[cyclic_i, "e_cyclic_per_period"] = True n.stores.loc[cyclic_i, "e_cyclic"] = False