From ae825d82dcebae7ab67f591329fe8e6140a96d72 Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 26 Jan 2024 12:10:39 +0100 Subject: [PATCH 01/19] prepare_sector_network: reorder buses for dac components --- doc/release_notes.rst | 2 ++ scripts/add_electricity.py | 2 +- scripts/prepare_sector_network.py | 22 +++++++++++----------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 93d1a268..94d5c55f 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -37,6 +37,8 @@ Upcoming Release * Add the option to customise map projection in plotting config. +* The order of buses (bus0, bus1, ...) for DAC components has changed to meet the convention of the other components. Therefore, `bus0` refers to the electricity bus (input), `bus1` to the heat bus (input), 'bus2' to the CO2 atmosphere bus (input), and `bus3` to the CO2 storage bus (output). + PyPSA-Eur 0.9.0 (5th January 2024) ================================== diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c9e5abca..8a1086bd 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -288,7 +288,7 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1. ua_md_gdp = pd.read_csv(ua_md_gdp, dtype={"name": "str"}).set_index("name") - logger.info(f"Load data scaled with scalling factor {scaling}.") + logger.info(f"Load data scaled by factor {scaling}.") opsd_load *= scaling nuts3 = gpd.read_file(nuts3_shapes).set_index("index") diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index d24feb79..3ea8c8dd 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -710,27 +710,27 @@ def add_dac(n, costs): heat_buses = n.buses.index[n.buses.carrier.isin(heat_carriers)] locations = n.buses.location[heat_buses] - efficiency2 = -( + electricity_input = ( costs.at["direct air capture", "electricity-input"] + costs.at["direct air capture", "compression-electricity-input"] - ) - efficiency3 = -( + ) # MWh_el / tCO2 + heat_input = ( costs.at["direct air capture", "heat-input"] - costs.at["direct air capture", "compression-heat-output"] - ) + ) # MWh_th / tCO2 n.madd( "Link", heat_buses.str.replace(" heat", " DAC"), - bus0="co2 atmosphere", - bus1=spatial.co2.df.loc[locations, "nodes"].values, - bus2=locations.values, - bus3=heat_buses, + bus0=locations.values, + bus1=heat_buses, + bus2="co2 atmosphere", + bus3=spatial.co2.df.loc[locations, "nodes"].values, carrier="DAC", capital_cost=costs.at["direct air capture", "fixed"], - efficiency=1.0, - efficiency2=efficiency2, - efficiency3=efficiency3, + efficiency=-heat_input / electricity_input, + efficiency2=-electricity_input, + efficiency3=electricity_input, p_nom_extendable=True, lifetime=costs.at["direct air capture", "lifetime"], ) From ae1b83121d0dfc4afcd0206f1ae1b0a2cf5808be Mon Sep 17 00:00:00 2001 From: Fabian Date: Fri, 26 Jan 2024 23:20:03 +0100 Subject: [PATCH 02/19] prepare_sector: fix dac co2 input/output efficiencies --- scripts/prepare_sector_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 3ea8c8dd..52b805d4 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -729,8 +729,8 @@ def add_dac(n, costs): carrier="DAC", capital_cost=costs.at["direct air capture", "fixed"], efficiency=-heat_input / electricity_input, - efficiency2=-electricity_input, - efficiency3=electricity_input, + efficiency2=-1 / electricity_input, + efficiency3=1 / electricity_input, p_nom_extendable=True, lifetime=costs.at["direct air capture", "lifetime"], ) From 7b0356803f07267be1d4bee8b2a1c1e5f16f052b Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 29 Jan 2024 09:01:08 +0100 Subject: [PATCH 03/19] prepare_sector: adjust DAC costs --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 3fa46605..9fba84ea 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -727,7 +727,7 @@ def add_dac(n, costs): bus2="co2 atmosphere", bus3=spatial.co2.df.loc[locations, "nodes"].values, carrier="DAC", - capital_cost=costs.at["direct air capture", "fixed"], + capital_cost=costs.at["direct air capture", "fixed"] / electricity_input, efficiency=-heat_input / electricity_input, efficiency2=-1 / electricity_input, efficiency3=1 / electricity_input, From d32789bd02bc29fafd2f0a411a06ff723b10d182 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 16:53:14 +0100 Subject: [PATCH 04/19] snakefile: allow to share resources in directory ci: use test.sh script which can be executed locally --- .github/workflows/ci.yaml | 6 +----- Snakefile | 7 ++++++- config/test/config.electricity.yaml | 4 ++-- config/test/config.myopic.yaml | 4 ++-- config/test/config.overnight.yaml | 4 ++-- config/test/config.perfect.yaml | 2 +- test.sh | 8 ++++++++ 7 files changed, 22 insertions(+), 13 deletions(-) create mode 100755 test.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4fa9f280..1af8e733 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,11 +81,7 @@ jobs: key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - name: Test snakemake workflow - run: | - snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime + run: ./test.sh - name: Upload artifacts uses: actions/upload-artifact@v4.3.0 diff --git a/Snakefile b/Snakefile index 14c9e821..8391156c 100644 --- a/Snakefile +++ b/Snakefile @@ -31,7 +31,12 @@ CDIR = RDIR if not run.get("shared_cutouts") else "" LOGS = "logs/" + RDIR BENCHMARKS = "benchmarks/" + RDIR -RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/" +if not (shared_resources := run.get("shared_resources")): + RESOURCES = "resources/" + RDIR +elif isinstance(shared_resources, str): + RESOURCES = "resources/" + shared_resources + "/" +else: + RESOURCES = "resources/" RESULTS = "results/" + RDIR diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index b750bf62..22c8e8d3 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -8,14 +8,14 @@ tutorial: true run: name: "test-elec" # use this to keep track of runs with different settings disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true scenario: clusters: - 5 opts: - - Co2L-24H + - Co2L-24h countries: ['BE'] diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 2dab7b04..2e7b3e6e 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -7,7 +7,7 @@ tutorial: true run: name: "test-sector-myopic" disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true foresight: myopic @@ -18,7 +18,7 @@ scenario: clusters: - 5 sector_opts: - - 24H-T-H-B-I-A-dist1 + - 24h-T-H-B-I-A-dist1 planning_horizons: - 2030 - 2040 diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 6d1900cf..8b98fea9 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -7,7 +7,7 @@ tutorial: true run: name: "test-sector-overnight" disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true @@ -17,7 +17,7 @@ scenario: clusters: - 5 sector_opts: - - CO2L0-24H-T-H-B-I-A-dist1 + - CO2L0-24h-T-H-B-I-A-dist1 planning_horizons: - 2030 diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index f20a2c9f..6eb9113e 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -7,7 +7,7 @@ tutorial: true run: name: "test-sector-perfect" disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true foresight: perfect diff --git a/test.sh b/test.sh new file mode 100755 index 00000000..a40276b8 --- /dev/null +++ b/test.sh @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime && \ +snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime && \ +snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime && \ +snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime From 0f80e2d089088e0c65f635516cac160954ee27e2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 17:07:01 +0100 Subject: [PATCH 05/19] plot: fix params and groupby deprecation warning --- rules/postprocess.smk | 3 +++ scripts/plot_power_network.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 59f9ead7..c1dd24c0 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -31,6 +31,7 @@ if config["foresight"] != "perfect": rule plot_power_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -54,6 +55,7 @@ if config["foresight"] != "perfect": rule plot_hydrogen_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -77,6 +79,7 @@ if config["foresight"] != "perfect": rule plot_gas_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/plot_power_network.py b/scripts/plot_power_network.py index 271e638d..0e13e497 100644 --- a/scripts/plot_power_network.py +++ b/scripts/plot_power_network.py @@ -98,7 +98,7 @@ def plot_map( logger.debug(f"{comp}, {costs}") - costs = costs.groupby(costs.columns, axis=1).sum() + costs = costs.T.groupby(costs.columns).sum().T costs.drop(list(costs.columns[(costs == 0.0).all()]), axis=1, inplace=True) From 46d8ce8f1f8293c127602e9c1a87eb58fcccf1ec Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 17:10:08 +0100 Subject: [PATCH 06/19] address deprecation warnings --- scripts/add_electricity.py | 12 +++++++----- scripts/add_existing_baseyear.py | 2 +- scripts/base_network.py | 8 +++++--- scripts/build_biomass_potentials.py | 6 +++--- scripts/build_gas_network.py | 20 ++++++++------------ scripts/build_line_rating.py | 2 +- scripts/build_population_layouts.py | 3 ++- scripts/build_retro_cost.py | 8 ++++---- scripts/cluster_network.py | 4 +++- scripts/prepare_network.py | 4 ++-- scripts/solve_network.py | 8 ++++---- 11 files changed, 40 insertions(+), 37 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c9e5abca..a57edd11 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -327,7 +327,9 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1. axis=1, ) - n.madd("Load", substation_lv_i, bus=substation_lv_i, p_set=load) + n.madd( + "Load", substation_lv_i, bus=substation_lv_i, p_set=load + ) # carrier="electricity" def update_transmission_costs(n, costs, length_factor=1.0): @@ -504,8 +506,8 @@ def attach_conventional_generators( snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0 ).iloc[:, 0] bus_values = n.buses.country.map(values) - n.generators[attr].update( - n.generators.loc[idx].bus.map(bus_values).dropna() + n.generators.update( + {attr: n.generators.loc[idx].bus.map(bus_values).dropna()} ) else: # Single value affecting all generators of technology k indiscriminantely of country @@ -749,8 +751,8 @@ def attach_OPSD_renewables(n: pypsa.Network, tech_map: Dict[str, List[str]]) -> caps = caps.groupby(["bus"]).Capacity.sum() caps = caps / gens_per_bus.reindex(caps.index, fill_value=1) - n.generators.p_nom.update(gens.bus.map(caps).dropna()) - n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) + n.generators.update({"p_nom": gens.bus.map(caps).dropna()}) + n.generators.update({"p_nom_min": gens.bus.map(caps).dropna()}) def estimate_renewable_capacities( diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 46d4bc31..0235a4d7 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -48,7 +48,7 @@ def add_build_year_to_new_assets(n, baseyear): "series" ) & n.component_attrs[c.name].status.str.contains("Input") for attr in n.component_attrs[c.name].index[selection]: - c.pnl[attr].rename(columns=rename, inplace=True) + c.pnl[attr] = c.pnl[attr].rename(columns=rename) def add_existing_renewables(df_agg): diff --git a/scripts/base_network.py b/scripts/base_network.py index 2fa31f98..1de3ef96 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -138,7 +138,9 @@ def _load_buses_from_eg(eg_buses, europe_shape, config_elec): ) buses["carrier"] = buses.pop("dc").map({True: "DC", False: "AC"}) - buses["under_construction"] = buses["under_construction"].fillna(False).astype(bool) + buses["under_construction"] = buses.under_construction.where( + lambda s: s.notnull(), False + ).astype(bool) # remove all buses outside of all countries including exclusive economic zones (offshore) europe_shape = gpd.read_file(europe_shape).loc[0, "geometry"] @@ -525,9 +527,9 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes): gb = buses.loc[substation_b].groupby( ["x", "y"], as_index=False, group_keys=False, sort=False ) - bus_map_low = gb.apply(prefer_voltage, "min") + bus_map_low = gb.apply(prefer_voltage, "min", include_groups=False) lv_b = (bus_map_low == bus_map_low.index).reindex(buses.index, fill_value=False) - bus_map_high = gb.apply(prefer_voltage, "max") + bus_map_high = gb.apply(prefer_voltage, "max", include_groups=False) hv_b = (bus_map_high == bus_map_high.index).reindex(buses.index, fill_value=False) onshore_b = pd.Series(False, buses.index) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index b6cbbfbf..6b5cb147 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -132,14 +132,14 @@ def disaggregate_nuts0(bio): pop = build_nuts_population_data() # get population in nuts2 - pop_nuts2 = pop.loc[pop.index.str.len() == 4] + pop_nuts2 = pop.loc[pop.index.str.len() == 4].copy() by_country = pop_nuts2.total.groupby(pop_nuts2.ct).sum() - pop_nuts2.loc[:, "fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country) + pop_nuts2["fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country) # distribute nuts0 data to nuts2 by population bio_nodal = bio.loc[pop_nuts2.ct] bio_nodal.index = pop_nuts2.index - bio_nodal = bio_nodal.mul(pop_nuts2.fraction, axis=0) + bio_nodal = bio_nodal.mul(pop_nuts2.fraction, axis=0).astype(float) # update inplace bio.update(bio_nodal) diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 8df7744d..13cd75ba 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -114,12 +114,10 @@ def prepare_dataset( df["p_nom_diameter"] = df.diameter_mm.apply(diameter_to_capacity) ratio = df.p_nom / df.p_nom_diameter not_nordstream = df.max_pressure_bar < 220 - df.p_nom.update( - df.p_nom_diameter.where( - (df.p_nom <= 500) - | ((ratio > correction_threshold_p_nom) & not_nordstream) - | ((ratio < 1 / correction_threshold_p_nom) & not_nordstream) - ) + df["p_nom"] = df.p_nom_diameter.where( + (df.p_nom <= 500) + | ((ratio > correction_threshold_p_nom) & not_nordstream) + | ((ratio < 1 / correction_threshold_p_nom) & not_nordstream) ) # lines which have way too discrepant line lengths @@ -130,12 +128,10 @@ def prepare_dataset( axis=1, ) ratio = df.eval("length / length_haversine") - df["length"].update( - df.length_haversine.where( - (df["length"] < 20) - | (ratio > correction_threshold_length) - | (ratio < 1 / correction_threshold_length) - ) + df["length"] = df.length_haversine.where( + (df["length"] < 20) + | (ratio > correction_threshold_length) + | (ratio < 1 / correction_threshold_length) ) return df diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 5b4cd6b3..522f08cf 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -98,7 +98,7 @@ def calculate_line_rating(n, cutout): ------- xarray DataArray object with maximal power. """ - relevant_lines = n.lines[~n.lines["underground"]] + relevant_lines = n.lines[~n.lines["underground"]].copy() buses = relevant_lines[["bus0", "bus1"]].values x = n.buses.x y = n.buses.y diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e215e6c0..cb63c27e 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -83,7 +83,8 @@ if __name__ == "__main__": # correct for imprecision of Iinv*I pop_ct = nuts3.loc[nuts3.country == ct, "pop"].sum() - pop_cells_ct *= pop_ct / pop_cells_ct.sum() + if pop_cells_ct.sum() != 0: + pop_cells_ct *= pop_ct / pop_cells_ct.sum() # The first low density grid cells to reach rural fraction are rural asc_density_i = density_cells_ct.sort_values().index diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index bd285e18..60d74afa 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -297,8 +297,8 @@ def prepare_building_stock_data(): errors="ignore", ) - u_values.subsector.replace(rename_sectors, inplace=True) - u_values.btype.replace(rename_sectors, inplace=True) + u_values["subsector"] = u_values.subsector.replace(rename_sectors) + u_values["btype"] = u_values.btype.replace(rename_sectors) # for missing weighting of surfaces of building types assume MFH u_values["assumed_subsector"] = u_values.subsector @@ -306,8 +306,8 @@ def prepare_building_stock_data(): ~u_values.subsector.isin(rename_sectors.values()), "assumed_subsector" ] = "MFH" - u_values.country_code.replace({"UK": "GB"}, inplace=True) - u_values.bage.replace({"Berfore 1945": "Before 1945"}, inplace=True) + u_values["country_code"] = u_values.country_code.replace({"UK": "GB"}) + u_values["bage"] = u_values.bage.replace({"Berfore 1945": "Before 1945"}) u_values = u_values[~u_values.bage.isna()] u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 39b6ad96..34a138cd 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -488,7 +488,9 @@ if __name__ == "__main__": gens.efficiency, bins=[0, low, high, 1], labels=labels ).astype(str) carriers += [f"{c} {label} efficiency" for label in labels] - n.generators.carrier.update(gens.carrier + " " + suffix + " efficiency") + n.generators.update( + {"carrier": gens.carrier + " " + suffix + " efficiency"} + ) aggregate_carriers = carriers if n_clusters == len(n.buses): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 5652dc6e..e358c05e 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -269,8 +269,8 @@ def set_line_nom_max( hvdc = n.links.index[n.links.carrier == "DC"] n.links.loc[hvdc, "p_nom_max"] = n.links.loc[hvdc, "p_nom"] + p_nom_max_ext - n.lines.s_nom_max.clip(upper=s_nom_max_set, inplace=True) - n.links.p_nom_max.clip(upper=p_nom_max_set, inplace=True) + n.lines["s_nom_max"] = n.lines.s_nom_max.clip(upper=s_nom_max_set) + n.links["p_nom_max"] = n.links.p_nom_max.clip(upper=p_nom_max_set) if __name__ == "__main__": diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a1d790a7..551222c0 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -418,7 +418,7 @@ def add_CCL_constraints(n, config): Example ------- scenario: - opts: [Co2L-CCL-24H] + opts: [Co2L-CCL-24h] electricity: agg_p_nom_limits: data/agg_p_nom_minmax.csv """ @@ -463,7 +463,7 @@ def add_EQ_constraints(n, o, scaling=1e-1): Example ------- scenario: - opts: [Co2L-EQ0.7-24H] + opts: [Co2L-EQ0.7-24h] Require each country or node to on average produce a minimal share of its total electricity consumption itself. Example: EQ0.7c demands each country @@ -527,7 +527,7 @@ def add_BAU_constraints(n, config): Example ------- scenario: - opts: [Co2L-BAU-24H] + opts: [Co2L-BAU-24h] electricity: BAU_mincapacities: solar: 0 @@ -564,7 +564,7 @@ def add_SAFE_constraints(n, config): config.yaml requires to specify opts: scenario: - opts: [Co2L-SAFE-24H] + opts: [Co2L-SAFE-24h] electricity: SAFE_reservemargin: 0.1 Which sets a reserve margin of 10% above the peak demand. From c60e1c5b73ac23dc7d53ad88be3e30cab3924e1b Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 17:12:36 +0100 Subject: [PATCH 07/19] update release notes --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6fa88738..44ccdff5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,6 +64,9 @@ Upcoming Release * The ``highs`` solver was added to the default environment file. +* It is now possible to determine the directory for shared resources by setting `shared_resources` to a string. + +* A ``test.sh`` script was added to the repository to run the tests locally. PyPSA-Eur 0.9.0 (5th January 2024) From 4c0491341cd1242e0295e1e3ac47e403c934719b Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Wed, 31 Jan 2024 12:45:45 +0000 Subject: [PATCH 08/19] Move `build_existing_heating_distribution` rule --- rules/build_sector.smk | 34 ++++++++++++++++++++++++++++++++++ rules/solve_myopic.smk | 34 ---------------------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 0586ec01..a8f419c3 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -730,6 +730,40 @@ rule build_district_heat_share: "../scripts/build_district_heat_share.py" +rule build_existing_heating_distribution: + params: + baseyear=config["scenario"]["planning_horizons"][0], + sector=config["sector"], + existing_capacities=config["existing_capacities"], + input: + existing_heating="data/existing_infrastructure/existing_heating_raw.csv", + clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", + clustered_pop_energy_layout=RESOURCES + + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", + district_heat_share=RESOURCES + + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + output: + existing_heating_distribution=RESOURCES + + "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv", + wildcard_constraints: + planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear + threads: 1 + resources: + mem_mb=2000, + log: + LOGS + + "build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log", + benchmark: + ( + BENCHMARKS + + "build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}" + ) + conda: + "../envs/environment.yaml" + script: + "../scripts/build_existing_heating_distribution.py" + + rule prepare_sector_network: params: co2_budget=config["co2_budget"], diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 75334073..94e75a5d 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -3,40 +3,6 @@ # SPDX-License-Identifier: MIT -rule build_existing_heating_distribution: - params: - baseyear=config["scenario"]["planning_horizons"][0], - sector=config["sector"], - existing_capacities=config["existing_capacities"], - input: - existing_heating="data/existing_infrastructure/existing_heating_raw.csv", - clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - clustered_pop_energy_layout=RESOURCES - + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", - district_heat_share=RESOURCES - + "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - output: - existing_heating_distribution=RESOURCES - + "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv", - wildcard_constraints: - planning_horizons=config["scenario"]["planning_horizons"][0], #only applies to baseyear - threads: 1 - resources: - mem_mb=2000, - log: - LOGS - + "build_existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.log", - benchmark: - ( - BENCHMARKS - + "build_existing_heating_distribution/elec_s{simpl}_{clusters}_{planning_horizons}" - ) - conda: - "../envs/environment.yaml" - script: - "../scripts/build_existing_heating_distribution.py" - - rule add_existing_baseyear: params: baseyear=config["scenario"]["planning_horizons"][0], From 25a24f555225de2cef68dfb832e945fea06ca198 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 1 Feb 2024 14:18:27 +0000 Subject: [PATCH 09/19] Remove unnecessary `add_brownfield` rule from perfect foresight --- rules/solve_perfect.smk | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 76051976..9e164a16 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -45,38 +45,6 @@ rule add_existing_baseyear: "../scripts/add_existing_baseyear.py" -rule add_brownfield: - params: - H2_retrofit=config["sector"]["H2_retrofit"], - H2_retrofit_capacity_per_CH4=config["sector"]["H2_retrofit_capacity_per_CH4"], - threshold_capacity=config["existing_capacities"]["threshold_capacity"], - input: - network=RESULTS - + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - network_p=solved_previous_horizon, #solved network at previous time step - costs="data/costs_{planning_horizons}.csv", - cop_soil_total=RESOURCES + "cop_soil_total_elec_s{simpl}_{clusters}.nc", - cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", - output: - RESULTS - + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - threads: 4 - resources: - mem_mb=10000, - log: - LOGS - + "add_brownfield_elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log", - benchmark: - ( - BENCHMARKS - + "add_brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}" - ) - conda: - "../envs/environment.yaml" - script: - "../scripts/add_brownfield.py" - - rule prepare_perfect_foresight: input: **{ @@ -192,6 +160,3 @@ rule make_summary_perfect: "../envs/environment.yaml" script: "../scripts/make_summary_perfect.py" - - -ruleorder: add_existing_baseyear > add_brownfield From de384c7cf620c769df2ae91a588ac4e78ea77c3f Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 1 Feb 2024 14:46:13 +0000 Subject: [PATCH 10/19] Correctly carry over time-varying data between investment periods Previously, time-varying data in each "next" investment period was set to the default values, implying capacity factors of 1 for renewable generators for example. --- scripts/prepare_perfect_foresight.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index f5ae3919..8fd6bc1f 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -199,8 +199,13 @@ def concat_networks(years): pnl[k].loc[pnl_year.index, pnl_year.columns] = pnl_year else: - # this is to avoid adding multiple times assets with - # infinite lifetime as ror + # For components that aren't new, we just extend + # time-varying data from the previous investment + # period. + if i > 0: + pnl[k].loc[(year,)] = pnl[k].loc[(years[i - 1],)].values + + # Now, add time-varying data for new components. cols = pnl_year.columns.difference(pnl[k].columns) pnl[k] = pd.concat([pnl[k], pnl_year[cols]], axis=1) From b67e800da9c442079d20eb08cbf6eb242b9fa476 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Thu, 1 Feb 2024 14:52:16 +0000 Subject: [PATCH 11/19] Simplify concat of static components in `prepare_perfect_forsight` --- scripts/prepare_perfect_foresight.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 8fd6bc1f..278472f6 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -162,15 +162,17 @@ def concat_networks(years): add_build_year_to_new_assets(network, year) # static ---------------------------------- - # (1) add buses and carriers - for component in network.iterate_components(["Bus", "Carrier"]): - df_year = component.df - # get missing assets - missing = get_missing(df_year, n, component.list_name) - import_components_from_dataframe(n, missing, component.name) - # (2) add generators, links, stores and loads for component in network.iterate_components( - ["Generator", "Link", "Store", "Load", "Line", "StorageUnit"] + [ + "Bus", + "Carrier", + "Generator", + "Link", + "Store", + "Load", + "Line", + "StorageUnit", + ] ): df_year = component.df.copy() missing = get_missing(df_year, n, component.list_name) From 7f77c3690abd8f9f7c6d1251c5b5be5c540d2982 Mon Sep 17 00:00:00 2001 From: Koen van Greevenbroek Date: Fri, 2 Feb 2024 08:54:44 +0100 Subject: [PATCH 12/19] Add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6fa88738..1ffeb310 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,6 +64,8 @@ Upcoming Release * The ``highs`` solver was added to the default environment file. +* Various minor bugfixes to the perfect foresight workflow, though perfect foresight must still be considered experimental. + PyPSA-Eur 0.9.0 (5th January 2024) From 344989d76b4f81bea5e3c1555021a767d3b3bd9b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 5 Feb 2024 08:21:26 +0100 Subject: [PATCH 13/19] Apply suggestions from code review --- rules/postprocess.smk | 3 --- 1 file changed, 3 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index c1dd24c0..59f9ead7 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -31,7 +31,6 @@ if config["foresight"] != "perfect": rule plot_power_network: params: plotting=config["plotting"], - foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -55,7 +54,6 @@ if config["foresight"] != "perfect": rule plot_hydrogen_network: params: plotting=config["plotting"], - foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -79,7 +77,6 @@ if config["foresight"] != "perfect": rule plot_gas_network: params: plotting=config["plotting"], - foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From e76749ab31e543733a671c37968c548f3386037a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 5 Feb 2024 08:37:19 +0100 Subject: [PATCH 14/19] plot_hydrogen_network: add foresight params back in --- rules/postprocess.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 59f9ead7..3045d0da 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -54,6 +54,7 @@ if config["foresight"] != "perfect": rule plot_hydrogen_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From 76872fa0f1547bc0b7de9592ce5c77060959c687 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 5 Feb 2024 16:16:34 +0100 Subject: [PATCH 15/19] add sanitize_locations function and apply it address some deprecation warnings --- config/test/config.perfect.yaml | 2 +- scripts/add_electricity.py | 9 +++++++++ scripts/add_extra_components.py | 8 ++++---- scripts/make_summary.py | 12 +++++++++--- scripts/make_summary_perfect.py | 24 ++++++++++++++++-------- scripts/plot_hydrogen_network.py | 4 +++- scripts/prepare_perfect_foresight.py | 2 +- scripts/prepare_sector_network.py | 16 ++++++++++++++-- 8 files changed, 57 insertions(+), 20 deletions(-) diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index 6eb9113e..2e716066 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -18,7 +18,7 @@ scenario: clusters: - 5 sector_opts: - - 8760H-T-H-B-I-A-dist1 + - 8760h-T-H-B-I-A-dist1 planning_horizons: - 2030 - 2040 diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 6c63abda..d67e2fa2 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -178,6 +178,15 @@ def sanitize_carriers(n, config): n.carriers["color"] = n.carriers.color.where(n.carriers.color != "", colors) +def sanitize_locations(n): + n.buses["x"] = n.buses.x.where(n.buses.x != 0, n.buses.location.map(n.buses.x)) + n.buses["y"] = n.buses.y.where(n.buses.y != 0, n.buses.location.map(n.buses.y)) + n.buses["country"] = n.buses.country.where( + n.buses.country.ne("") & n.buses.country.notnull(), + n.buses.location.map(n.buses.country), + ) + + def add_co2_emissions(n, costs, carriers): """ Add CO2 emissions to the network's carriers attribute. diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index e00e1e5f..731e36be 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -56,7 +56,7 @@ import numpy as np import pandas as pd import pypsa from _helpers import configure_logging -from add_electricity import load_costs, sanitize_carriers +from add_electricity import load_costs, sanitize_carriers, sanitize_locations idx = pd.IndexSlice @@ -100,10 +100,9 @@ def attach_stores(n, costs, extendable_carriers): n.madd("Carrier", carriers) buses_i = n.buses.index - bus_sub_dict = {k: n.buses[k].values for k in ["x", "y", "country"]} if "H2" in carriers: - h2_buses_i = n.madd("Bus", buses_i + " H2", carrier="H2", **bus_sub_dict) + h2_buses_i = n.madd("Bus", buses_i + " H2", carrier="H2", location=buses_i) n.madd( "Store", @@ -143,7 +142,7 @@ def attach_stores(n, costs, extendable_carriers): if "battery" in carriers: b_buses_i = n.madd( - "Bus", buses_i + " battery", carrier="battery", **bus_sub_dict + "Bus", buses_i + " battery", carrier="battery", location=buses_i ) n.madd( @@ -246,6 +245,7 @@ if __name__ == "__main__": attach_hydrogen_pipelines(n, costs, extendable_carriers) sanitize_carriers(n, snakemake.config) + sanitize_locations(n) n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) n.export_to_netcdf(snakemake.output[0]) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 0fab5367..76d8099c 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -507,7 +507,7 @@ def calculate_weighted_prices(n, label, weighted_prices): if carrier in ["H2", "gas"]: load = pd.DataFrame(index=n.snapshots, columns=buses, data=0.0) else: - load = n.loads_t.p_set[buses] + load = n.loads_t.p_set[buses.intersection(n.loads.index)] for tech in value: names = n.links.index[n.links.index.to_series().str[-len(tech) :] == tech] @@ -560,7 +560,10 @@ def calculate_market_values(n, label, market_values): ) revenue = dispatch * n.buses_t.marginal_price[buses] - market_values.at[tech, label] = revenue.sum().sum() / dispatch.sum().sum() + if total_dispatch := dispatch.sum().sum(): + market_values.at[tech, label] = revenue.sum().sum() / total_dispatch + else: + market_values.at[tech, label] = np.nan ## Now do market value of links ## @@ -583,7 +586,10 @@ def calculate_market_values(n, label, market_values): revenue = dispatch * n.buses_t.marginal_price[buses] - market_values.at[tech, label] = revenue.sum().sum() / dispatch.sum().sum() + if total_dispatch := dispatch.sum().sum(): + market_values.at[tech, label] = revenue.sum().sum() / total_dispatch + else: + market_values.at[tech, label] = np.nan return market_values diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 555e5da5..d401b28e 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -265,7 +265,7 @@ def calculate_energy(n, label, energy): totals[no_bus] = float( n.component_attrs[c.name].loc["p" + port, "default"] ) - c_energies -= totals.groupby(c.df.carrier, axis=1).sum() + c_energies -= totals.T.groupby(c.df.carrier).sum().T c_energies = pd.concat([c_energies.T], keys=[c.list_name]) @@ -376,9 +376,8 @@ def calculate_supply_energy(n, label, supply_energy): .groupby(level=0) .sum() .multiply(c.df.loc[items, "sign"]) - .groupby(c.df.loc[items, "carrier"], axis=1) + .T.groupby(c.df.loc[items, "carrier"]) .sum() - .T ) s = pd.concat([s], keys=[c.list_name]) s = pd.concat([s], keys=[i]) @@ -525,9 +524,12 @@ def calculate_weighted_prices(n, label, weighted_prices): # stores[stores > 0.] = 0. # load += -stores - weighted_prices.loc[carrier, label] = ( - load * n.buses_t.marginal_price[buses] - ).sum().sum() / load.sum().sum() + if total_load := load.sum().sum(): + weighted_prices.loc[carrier, label] = ( + load * n.buses_t.marginal_price[buses] + ).sum().sum() / total_load + else: + weighted_prices.loc[carrier, label] = np.nan if carrier[:5] == "space": print(load * n.buses_t.marginal_price[buses]) @@ -562,7 +564,10 @@ def calculate_market_values(n, label, market_values): revenue = dispatch * n.buses_t.marginal_price[buses] - market_values.at[tech, label] = revenue.sum().sum() / dispatch.sum().sum() + if total_dispatch := dispatch.sum().sum(): + market_values.at[tech, label] = revenue.sum().sum() / total_dispatch + else: + market_values.at[tech, label] = np.nan ## Now do market value of links ## @@ -585,7 +590,10 @@ def calculate_market_values(n, label, market_values): revenue = dispatch * n.buses_t.marginal_price[buses] - market_values.at[tech, label] = revenue.sum().sum() / dispatch.sum().sum() + if total_dispatch := dispatch.sum().sum(): + market_values.at[tech, label] = revenue.sum().sum() / total_dispatch + else: + market_values.at[tech, label] = np.nan return market_values diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index a1183311..95741170 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -36,7 +36,9 @@ def group_pipes(df, drop_direction=False): lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", axis=1, ) - return df.groupby(level=0).agg({"p_nom_opt": sum, "bus0": "first", "bus1": "first"}) + return df.groupby(level=0).agg( + {"p_nom_opt": "sum", "bus0": "first", "bus1": "first"} + ) def plot_h2_map(n, regions): diff --git a/scripts/prepare_perfect_foresight.py b/scripts/prepare_perfect_foresight.py index 278472f6..3d72cd8c 100644 --- a/scripts/prepare_perfect_foresight.py +++ b/scripts/prepare_perfect_foresight.py @@ -221,7 +221,7 @@ def concat_networks(years): # set investment periods n.investment_periods = n.snapshots.levels[0] # weighting of the investment period -> assuming last period same weighting as the period before - time_w = n.investment_periods.to_series().diff().shift(-1).fillna(method="ffill") + time_w = n.investment_periods.to_series().diff().shift(-1).ffill() n.investment_period_weightings["years"] = time_w # set objective weightings objective_w = get_investment_weighting( diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f7e91605..5abd2ce6 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -19,7 +19,7 @@ import pandas as pd import pypsa import xarray as xr from _helpers import update_config_with_sector_opts -from add_electricity import calculate_annuity, sanitize_carriers +from add_electricity import calculate_annuity, sanitize_carriers, sanitize_locations from build_energy_totals import build_co2_totals, build_eea_co2, build_eurostat_co2 from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation @@ -546,6 +546,14 @@ def patch_electricity_network(n): n.loads_t.p_set.rename(lambda x: x.strip(), axis=1, inplace=True) +def add_eu_bus(n, x=-5.5, y=46): + """ + Add EU bus to the network. + """ + n.add("Bus", "EU", location="EU", x=x, y=y, carrier="none") + n.add("Carrier", "none") + + def add_co2_tracking(n, costs, options): # minus sign because opposite to how fossil fuels used: # CH4 burning puts CH4 down, atmosphere up @@ -1005,6 +1013,7 @@ def insert_electricity_distribution_grid(n, costs): "Store", nodes + " home battery", bus=nodes + " home battery", + location=nodes, e_cyclic=True, e_nom_extendable=True, carrier="home battery", @@ -3527,8 +3536,8 @@ def lossy_bidirectional_links(n, carrier, efficiencies={}): rev_links["reversed"] = True rev_links.index = rev_links.index.map(lambda x: x + "-reversed") + n.links["reversed"] = False n.links = pd.concat([n.links, rev_links], sort=False) - n.links["reversed"] = n.links["reversed"].fillna(False) n.links["length_original"] = n.links["length_original"].fillna(n.links.length) # do compression losses after concatenation to take electricity consumption at bus0 in either direction @@ -3594,6 +3603,8 @@ if __name__ == "__main__": for carrier in conventional: add_carrier_buses(n, carrier) + add_eu_bus(n) + add_co2_tracking(n, costs, options) add_generation(n, costs) @@ -3733,5 +3744,6 @@ if __name__ == "__main__": n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards))) sanitize_carriers(n, snakemake.config) + sanitize_locations(n) n.export_to_netcdf(snakemake.output[0]) From 32451f6ca8e45ca6f3c292f29c324104499e9276 Mon Sep 17 00:00:00 2001 From: Fabian Hofmann Date: Mon, 5 Feb 2024 19:13:42 +0100 Subject: [PATCH 16/19] Update scripts/prepare_sector_network.py Co-authored-by: Fabian Neumann --- scripts/prepare_sector_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 5abd2ce6..8eb38d69 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3536,8 +3536,8 @@ def lossy_bidirectional_links(n, carrier, efficiencies={}): rev_links["reversed"] = True rev_links.index = rev_links.index.map(lambda x: x + "-reversed") - n.links["reversed"] = False n.links = pd.concat([n.links, rev_links], sort=False) + n.links["reversed"] = n.links["reversed"].fillna(False) n.links["length_original"] = n.links["length_original"].fillna(n.links.length) # do compression losses after concatenation to take electricity consumption at bus0 in either direction From 2a01ed3d48f75b03e0ed0193304fa810e81b84a7 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 5 Feb 2024 19:15:55 +0100 Subject: [PATCH 17/19] prepare_sector: add description of eu bus --- scripts/prepare_sector_network.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 8eb38d69..4948d451 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -549,6 +549,9 @@ def patch_electricity_network(n): def add_eu_bus(n, x=-5.5, y=46): """ Add EU bus to the network. + + This cosmetic bus serves as a reference point for the location of + the EU buses in the plots and summaries. """ n.add("Bus", "EU", location="EU", x=x, y=y, carrier="none") n.add("Carrier", "none") From 0553108e3d2e722d13e91cac65b7f823b81dc1cd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:23:30 +0000 Subject: [PATCH 18/19] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/snakemake/snakefmt: v0.9.0 → v0.10.0](https://github.com/snakemake/snakefmt/compare/v0.9.0...v0.10.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d90f5cb9..5c41e781 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,7 +74,7 @@ repos: # Format Snakemake rule / workflow files - repo: https://github.com/snakemake/snakefmt - rev: v0.9.0 + rev: v0.10.0 hooks: - id: snakefmt From 39890630efc3bc1b2356ef274faf9fbebf92ce82 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:23:48 +0000 Subject: [PATCH 19/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 16 +++++++----- rules/build_sector.smk | 49 +++++++++++++++++++++---------------- rules/collect.smk | 14 +++++------ rules/postprocess.smk | 40 +++++++++++++++++------------- rules/retrieve.smk | 8 +++--- 5 files changed, 73 insertions(+), 54 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 6b092638..89f4f736 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -401,18 +401,22 @@ rule add_electricity: if str(fn).startswith("data/") }, base_network=RESOURCES + "networks/base.nc", - line_rating=RESOURCES + "networks/line_rating.nc" - if config["lines"]["dynamic_line_rating"]["activate"] - else RESOURCES + "networks/base.nc", + line_rating=( + RESOURCES + "networks/line_rating.nc" + if config["lines"]["dynamic_line_rating"]["activate"] + else RESOURCES + "networks/base.nc" + ), tech_costs=COSTS, regions=RESOURCES + "regions_onshore.geojson", powerplants=RESOURCES + "powerplants.csv", hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", unit_commitment="data/unit_commitment.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv" - if config["conventional"]["dynamic_fuel_price"] - else [], + fuel_price=( + RESOURCES + "monthly_fuel_price.csv" + if config["conventional"]["dynamic_fuel_price"] + else [] + ), load=RESOURCES + "load.csv", nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", ua_md_gdp="data/GDP_PPP_30arcsec_v3_mapped_default.csv", diff --git a/rules/build_sector.smk b/rules/build_sector.smk index a8f419c3..c25c8673 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -797,15 +797,19 @@ rule prepare_sector_network: dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv", co2_totals_name=RESOURCES + "co2_totals.csv", co2="data/bundle-sector/eea/UNFCCC_v23.csv", - biomass_potentials=RESOURCES - + "biomass_potentials_s{simpl}_{clusters}_" - + "{}.csv".format(config["biomass"]["year"]) - if config["foresight"] == "overnight" - else RESOURCES - + "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv", - costs="data/costs_{}.csv".format(config["costs"]["year"]) - if config["foresight"] == "overnight" - else "data/costs_{planning_horizons}.csv", + biomass_potentials=( + RESOURCES + + "biomass_potentials_s{simpl}_{clusters}_" + + "{}.csv".format(config["biomass"]["year"]) + if config["foresight"] == "overnight" + else RESOURCES + + "biomass_potentials_s{simpl}_{clusters}_{planning_horizons}.csv" + ), + costs=( + "data/costs_{}.csv".format(config["costs"]["year"]) + if config["foresight"] == "overnight" + else "data/costs_{planning_horizons}.csv" + ), profile_offwind_ac=RESOURCES + "profile_offwind-ac.nc", profile_offwind_dc=RESOURCES + "profile_offwind-dc.nc", h2_cavern=RESOURCES + "salt_cavern_potentials_s{simpl}_{clusters}.csv", @@ -831,18 +835,21 @@ rule prepare_sector_network: cop_air_total=RESOURCES + "cop_air_total_elec_s{simpl}_{clusters}.nc", cop_air_rural=RESOURCES + "cop_air_rural_elec_s{simpl}_{clusters}.nc", cop_air_urban=RESOURCES + "cop_air_urban_elec_s{simpl}_{clusters}.nc", - solar_thermal_total=RESOURCES - + "solar_thermal_total_elec_s{simpl}_{clusters}.nc" - if config["sector"]["solar_thermal"] - else [], - solar_thermal_urban=RESOURCES - + "solar_thermal_urban_elec_s{simpl}_{clusters}.nc" - if config["sector"]["solar_thermal"] - else [], - solar_thermal_rural=RESOURCES - + "solar_thermal_rural_elec_s{simpl}_{clusters}.nc" - if config["sector"]["solar_thermal"] - else [], + solar_thermal_total=( + RESOURCES + "solar_thermal_total_elec_s{simpl}_{clusters}.nc" + if config["sector"]["solar_thermal"] + else [] + ), + solar_thermal_urban=( + RESOURCES + "solar_thermal_urban_elec_s{simpl}_{clusters}.nc" + if config["sector"]["solar_thermal"] + else [] + ), + solar_thermal_rural=( + RESOURCES + "solar_thermal_rural_elec_s{simpl}_{clusters}.nc" + if config["sector"]["solar_thermal"] + else [] + ), output: RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/collect.smk b/rules/collect.smk index 1d977da1..dc0a94cc 100644 --- a/rules/collect.smk +++ b/rules/collect.smk @@ -29,7 +29,7 @@ rule prepare_elec_networks: input: expand( RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - **config["scenario"] + **config["scenario"], ), @@ -38,7 +38,7 @@ rule prepare_sector_networks: expand( RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"] + **config["scenario"], ), @@ -46,7 +46,7 @@ rule solve_elec_networks: input: expand( RESULTS + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", - **config["scenario"] + **config["scenario"], ), @@ -55,7 +55,7 @@ rule solve_sector_networks: expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"] + **config["scenario"], ), @@ -64,7 +64,7 @@ rule solve_sector_networks_perfect: expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_brownfield_all_years.nc", - **config["scenario"] + **config["scenario"], ), @@ -73,11 +73,11 @@ rule validate_elec_networks: expand( RESULTS + "figures/.statistics_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", - **config["scenario"] + **config["scenario"], ), expand( RESULTS + "figures/.validation_{kind}_plots_elec_s{simpl}_{clusters}_ec_l{ll}_{opts}", **config["scenario"], - kind=["production", "prices", "cross_border"] + kind=["production", "prices", "cross_border"], ), diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 3045d0da..2e90d233 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -153,38 +153,44 @@ rule make_summary: input: expand( RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", - **config["scenario"] + **config["scenario"], ), networks=expand( RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", - **config["scenario"] + **config["scenario"], + ), + costs=( + "data/costs_{}.csv".format(config["costs"]["year"]) + if config["foresight"] == "overnight" + else "data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]) ), - costs="data/costs_{}.csv".format(config["costs"]["year"]) - if config["foresight"] == "overnight" - else "data/costs_{}.csv".format(config["scenario"]["planning_horizons"][0]), ac_plot=expand( RESULTS + "maps/power-network-s{simpl}-{clusters}.pdf", - **config["scenario"] + **config["scenario"], ), costs_plot=expand( RESULTS + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-costs-all_{planning_horizons}.pdf", - **config["scenario"] + **config["scenario"], ), h2_plot=expand( - RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" - if config["sector"]["H2_network"] - else [], - **config["scenario"] + ( + RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-h2_network_{planning_horizons}.pdf" + if config["sector"]["H2_network"] + else [] + ), + **config["scenario"], ), ch4_plot=expand( - RESULTS - + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" - if config["sector"]["gas_network"] - else [], - **config["scenario"] + ( + RESULTS + + "maps/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}-ch4_network_{planning_horizons}.pdf" + if config["sector"]["gas_network"] + else [] + ), + **config["scenario"], ), output: nodal_costs=RESULTS + "csvs/nodal_costs.csv", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 05bbefd8..2980583f 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -191,9 +191,11 @@ if config["enable"]["retrieve"]: input: HTTP.remote( "data.open-power-system-data.org/time_series/{version}/time_series_60min_singleindex.csv".format( - version="2019-06-05" - if config["snapshots"]["end"] < "2019" - else "2020-10-06" + version=( + "2019-06-05" + if config["snapshots"]["end"] < "2019" + else "2020-10-06" + ) ), keep_local=True, static=True,