From d32789bd02bc29fafd2f0a411a06ff723b10d182 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 16:53:14 +0100 Subject: [PATCH 1/6] snakefile: allow to share resources in directory ci: use test.sh script which can be executed locally --- .github/workflows/ci.yaml | 6 +----- Snakefile | 7 ++++++- config/test/config.electricity.yaml | 4 ++-- config/test/config.myopic.yaml | 4 ++-- config/test/config.overnight.yaml | 4 ++-- config/test/config.perfect.yaml | 2 +- test.sh | 8 ++++++++ 7 files changed, 22 insertions(+), 13 deletions(-) create mode 100755 test.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4fa9f280..1af8e733 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,11 +81,7 @@ jobs: key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - name: Test snakemake workflow - run: | - snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime - snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime + run: ./test.sh - name: Upload artifacts uses: actions/upload-artifact@v4.3.0 diff --git a/Snakefile b/Snakefile index 14c9e821..8391156c 100644 --- a/Snakefile +++ b/Snakefile @@ -31,7 +31,12 @@ CDIR = RDIR if not run.get("shared_cutouts") else "" LOGS = "logs/" + RDIR BENCHMARKS = "benchmarks/" + RDIR -RESOURCES = "resources/" + RDIR if not run.get("shared_resources") else "resources/" +if not (shared_resources := run.get("shared_resources")): + RESOURCES = "resources/" + RDIR +elif isinstance(shared_resources, str): + RESOURCES = "resources/" + shared_resources + "/" +else: + RESOURCES = "resources/" RESULTS = "results/" + RDIR diff --git a/config/test/config.electricity.yaml b/config/test/config.electricity.yaml index b750bf62..22c8e8d3 100644 --- a/config/test/config.electricity.yaml +++ b/config/test/config.electricity.yaml @@ -8,14 +8,14 @@ tutorial: true run: name: "test-elec" # use this to keep track of runs with different settings disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true scenario: clusters: - 5 opts: - - Co2L-24H + - Co2L-24h countries: ['BE'] diff --git a/config/test/config.myopic.yaml b/config/test/config.myopic.yaml index 2dab7b04..2e7b3e6e 100644 --- a/config/test/config.myopic.yaml +++ b/config/test/config.myopic.yaml @@ -7,7 +7,7 @@ tutorial: true run: name: "test-sector-myopic" disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true foresight: myopic @@ -18,7 +18,7 @@ scenario: clusters: - 5 sector_opts: - - 24H-T-H-B-I-A-dist1 + - 24h-T-H-B-I-A-dist1 planning_horizons: - 2030 - 2040 diff --git a/config/test/config.overnight.yaml b/config/test/config.overnight.yaml index 6d1900cf..8b98fea9 100644 --- a/config/test/config.overnight.yaml +++ b/config/test/config.overnight.yaml @@ -7,7 +7,7 @@ tutorial: true run: name: "test-sector-overnight" disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true @@ -17,7 +17,7 @@ scenario: clusters: - 5 sector_opts: - - CO2L0-24H-T-H-B-I-A-dist1 + - CO2L0-24h-T-H-B-I-A-dist1 planning_horizons: - 2030 diff --git a/config/test/config.perfect.yaml b/config/test/config.perfect.yaml index f20a2c9f..6eb9113e 100644 --- a/config/test/config.perfect.yaml +++ b/config/test/config.perfect.yaml @@ -7,7 +7,7 @@ tutorial: true run: name: "test-sector-perfect" disable_progressbar: true - shared_resources: true + shared_resources: "test" shared_cutouts: true foresight: perfect diff --git a/test.sh b/test.sh new file mode 100755 index 00000000..a40276b8 --- /dev/null +++ b/test.sh @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: CC0-1.0 + +snakemake -call solve_elec_networks --configfile config/test/config.electricity.yaml --rerun-triggers=mtime && \ +snakemake -call all --configfile config/test/config.overnight.yaml --rerun-triggers=mtime && \ +snakemake -call all --configfile config/test/config.myopic.yaml --rerun-triggers=mtime && \ +snakemake -call all --configfile config/test/config.perfect.yaml --rerun-triggers=mtime From 0f80e2d089088e0c65f635516cac160954ee27e2 Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 17:07:01 +0100 Subject: [PATCH 2/6] plot: fix params and groupby deprecation warning --- rules/postprocess.smk | 3 +++ scripts/plot_power_network.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 59f9ead7..c1dd24c0 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -31,6 +31,7 @@ if config["foresight"] != "perfect": rule plot_power_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -54,6 +55,7 @@ if config["foresight"] != "perfect": rule plot_hydrogen_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -77,6 +79,7 @@ if config["foresight"] != "perfect": rule plot_gas_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/plot_power_network.py b/scripts/plot_power_network.py index 271e638d..0e13e497 100644 --- a/scripts/plot_power_network.py +++ b/scripts/plot_power_network.py @@ -98,7 +98,7 @@ def plot_map( logger.debug(f"{comp}, {costs}") - costs = costs.groupby(costs.columns, axis=1).sum() + costs = costs.T.groupby(costs.columns).sum().T costs.drop(list(costs.columns[(costs == 0.0).all()]), axis=1, inplace=True) From 46d8ce8f1f8293c127602e9c1a87eb58fcccf1ec Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 17:10:08 +0100 Subject: [PATCH 3/6] address deprecation warnings --- scripts/add_electricity.py | 12 +++++++----- scripts/add_existing_baseyear.py | 2 +- scripts/base_network.py | 8 +++++--- scripts/build_biomass_potentials.py | 6 +++--- scripts/build_gas_network.py | 20 ++++++++------------ scripts/build_line_rating.py | 2 +- scripts/build_population_layouts.py | 3 ++- scripts/build_retro_cost.py | 8 ++++---- scripts/cluster_network.py | 4 +++- scripts/prepare_network.py | 4 ++-- scripts/solve_network.py | 8 ++++---- 11 files changed, 40 insertions(+), 37 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index c9e5abca..a57edd11 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -327,7 +327,9 @@ def attach_load(n, regions, load, nuts3_shapes, ua_md_gdp, countries, scaling=1. axis=1, ) - n.madd("Load", substation_lv_i, bus=substation_lv_i, p_set=load) + n.madd( + "Load", substation_lv_i, bus=substation_lv_i, p_set=load + ) # carrier="electricity" def update_transmission_costs(n, costs, length_factor=1.0): @@ -504,8 +506,8 @@ def attach_conventional_generators( snakemake.input[f"conventional_{carrier}_{attr}"], index_col=0 ).iloc[:, 0] bus_values = n.buses.country.map(values) - n.generators[attr].update( - n.generators.loc[idx].bus.map(bus_values).dropna() + n.generators.update( + {attr: n.generators.loc[idx].bus.map(bus_values).dropna()} ) else: # Single value affecting all generators of technology k indiscriminantely of country @@ -749,8 +751,8 @@ def attach_OPSD_renewables(n: pypsa.Network, tech_map: Dict[str, List[str]]) -> caps = caps.groupby(["bus"]).Capacity.sum() caps = caps / gens_per_bus.reindex(caps.index, fill_value=1) - n.generators.p_nom.update(gens.bus.map(caps).dropna()) - n.generators.p_nom_min.update(gens.bus.map(caps).dropna()) + n.generators.update({"p_nom": gens.bus.map(caps).dropna()}) + n.generators.update({"p_nom_min": gens.bus.map(caps).dropna()}) def estimate_renewable_capacities( diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 46d4bc31..0235a4d7 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -48,7 +48,7 @@ def add_build_year_to_new_assets(n, baseyear): "series" ) & n.component_attrs[c.name].status.str.contains("Input") for attr in n.component_attrs[c.name].index[selection]: - c.pnl[attr].rename(columns=rename, inplace=True) + c.pnl[attr] = c.pnl[attr].rename(columns=rename) def add_existing_renewables(df_agg): diff --git a/scripts/base_network.py b/scripts/base_network.py index 2fa31f98..1de3ef96 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -138,7 +138,9 @@ def _load_buses_from_eg(eg_buses, europe_shape, config_elec): ) buses["carrier"] = buses.pop("dc").map({True: "DC", False: "AC"}) - buses["under_construction"] = buses["under_construction"].fillna(False).astype(bool) + buses["under_construction"] = buses.under_construction.where( + lambda s: s.notnull(), False + ).astype(bool) # remove all buses outside of all countries including exclusive economic zones (offshore) europe_shape = gpd.read_file(europe_shape).loc[0, "geometry"] @@ -525,9 +527,9 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes): gb = buses.loc[substation_b].groupby( ["x", "y"], as_index=False, group_keys=False, sort=False ) - bus_map_low = gb.apply(prefer_voltage, "min") + bus_map_low = gb.apply(prefer_voltage, "min", include_groups=False) lv_b = (bus_map_low == bus_map_low.index).reindex(buses.index, fill_value=False) - bus_map_high = gb.apply(prefer_voltage, "max") + bus_map_high = gb.apply(prefer_voltage, "max", include_groups=False) hv_b = (bus_map_high == bus_map_high.index).reindex(buses.index, fill_value=False) onshore_b = pd.Series(False, buses.index) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index b6cbbfbf..6b5cb147 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -132,14 +132,14 @@ def disaggregate_nuts0(bio): pop = build_nuts_population_data() # get population in nuts2 - pop_nuts2 = pop.loc[pop.index.str.len() == 4] + pop_nuts2 = pop.loc[pop.index.str.len() == 4].copy() by_country = pop_nuts2.total.groupby(pop_nuts2.ct).sum() - pop_nuts2.loc[:, "fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country) + pop_nuts2["fraction"] = pop_nuts2.total / pop_nuts2.ct.map(by_country) # distribute nuts0 data to nuts2 by population bio_nodal = bio.loc[pop_nuts2.ct] bio_nodal.index = pop_nuts2.index - bio_nodal = bio_nodal.mul(pop_nuts2.fraction, axis=0) + bio_nodal = bio_nodal.mul(pop_nuts2.fraction, axis=0).astype(float) # update inplace bio.update(bio_nodal) diff --git a/scripts/build_gas_network.py b/scripts/build_gas_network.py index 8df7744d..13cd75ba 100644 --- a/scripts/build_gas_network.py +++ b/scripts/build_gas_network.py @@ -114,12 +114,10 @@ def prepare_dataset( df["p_nom_diameter"] = df.diameter_mm.apply(diameter_to_capacity) ratio = df.p_nom / df.p_nom_diameter not_nordstream = df.max_pressure_bar < 220 - df.p_nom.update( - df.p_nom_diameter.where( - (df.p_nom <= 500) - | ((ratio > correction_threshold_p_nom) & not_nordstream) - | ((ratio < 1 / correction_threshold_p_nom) & not_nordstream) - ) + df["p_nom"] = df.p_nom_diameter.where( + (df.p_nom <= 500) + | ((ratio > correction_threshold_p_nom) & not_nordstream) + | ((ratio < 1 / correction_threshold_p_nom) & not_nordstream) ) # lines which have way too discrepant line lengths @@ -130,12 +128,10 @@ def prepare_dataset( axis=1, ) ratio = df.eval("length / length_haversine") - df["length"].update( - df.length_haversine.where( - (df["length"] < 20) - | (ratio > correction_threshold_length) - | (ratio < 1 / correction_threshold_length) - ) + df["length"] = df.length_haversine.where( + (df["length"] < 20) + | (ratio > correction_threshold_length) + | (ratio < 1 / correction_threshold_length) ) return df diff --git a/scripts/build_line_rating.py b/scripts/build_line_rating.py index 5b4cd6b3..522f08cf 100755 --- a/scripts/build_line_rating.py +++ b/scripts/build_line_rating.py @@ -98,7 +98,7 @@ def calculate_line_rating(n, cutout): ------- xarray DataArray object with maximal power. """ - relevant_lines = n.lines[~n.lines["underground"]] + relevant_lines = n.lines[~n.lines["underground"]].copy() buses = relevant_lines[["bus0", "bus1"]].values x = n.buses.x y = n.buses.y diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index e215e6c0..cb63c27e 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -83,7 +83,8 @@ if __name__ == "__main__": # correct for imprecision of Iinv*I pop_ct = nuts3.loc[nuts3.country == ct, "pop"].sum() - pop_cells_ct *= pop_ct / pop_cells_ct.sum() + if pop_cells_ct.sum() != 0: + pop_cells_ct *= pop_ct / pop_cells_ct.sum() # The first low density grid cells to reach rural fraction are rural asc_density_i = density_cells_ct.sort_values().index diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index bd285e18..60d74afa 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -297,8 +297,8 @@ def prepare_building_stock_data(): errors="ignore", ) - u_values.subsector.replace(rename_sectors, inplace=True) - u_values.btype.replace(rename_sectors, inplace=True) + u_values["subsector"] = u_values.subsector.replace(rename_sectors) + u_values["btype"] = u_values.btype.replace(rename_sectors) # for missing weighting of surfaces of building types assume MFH u_values["assumed_subsector"] = u_values.subsector @@ -306,8 +306,8 @@ def prepare_building_stock_data(): ~u_values.subsector.isin(rename_sectors.values()), "assumed_subsector" ] = "MFH" - u_values.country_code.replace({"UK": "GB"}, inplace=True) - u_values.bage.replace({"Berfore 1945": "Before 1945"}, inplace=True) + u_values["country_code"] = u_values.country_code.replace({"UK": "GB"}) + u_values["bage"] = u_values.bage.replace({"Berfore 1945": "Before 1945"}) u_values = u_values[~u_values.bage.isna()] u_values.set_index(["country_code", "subsector", "bage", "type"], inplace=True) diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 39b6ad96..34a138cd 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -488,7 +488,9 @@ if __name__ == "__main__": gens.efficiency, bins=[0, low, high, 1], labels=labels ).astype(str) carriers += [f"{c} {label} efficiency" for label in labels] - n.generators.carrier.update(gens.carrier + " " + suffix + " efficiency") + n.generators.update( + {"carrier": gens.carrier + " " + suffix + " efficiency"} + ) aggregate_carriers = carriers if n_clusters == len(n.buses): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 5652dc6e..e358c05e 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -269,8 +269,8 @@ def set_line_nom_max( hvdc = n.links.index[n.links.carrier == "DC"] n.links.loc[hvdc, "p_nom_max"] = n.links.loc[hvdc, "p_nom"] + p_nom_max_ext - n.lines.s_nom_max.clip(upper=s_nom_max_set, inplace=True) - n.links.p_nom_max.clip(upper=p_nom_max_set, inplace=True) + n.lines["s_nom_max"] = n.lines.s_nom_max.clip(upper=s_nom_max_set) + n.links["p_nom_max"] = n.links.p_nom_max.clip(upper=p_nom_max_set) if __name__ == "__main__": diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a1d790a7..551222c0 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -418,7 +418,7 @@ def add_CCL_constraints(n, config): Example ------- scenario: - opts: [Co2L-CCL-24H] + opts: [Co2L-CCL-24h] electricity: agg_p_nom_limits: data/agg_p_nom_minmax.csv """ @@ -463,7 +463,7 @@ def add_EQ_constraints(n, o, scaling=1e-1): Example ------- scenario: - opts: [Co2L-EQ0.7-24H] + opts: [Co2L-EQ0.7-24h] Require each country or node to on average produce a minimal share of its total electricity consumption itself. Example: EQ0.7c demands each country @@ -527,7 +527,7 @@ def add_BAU_constraints(n, config): Example ------- scenario: - opts: [Co2L-BAU-24H] + opts: [Co2L-BAU-24h] electricity: BAU_mincapacities: solar: 0 @@ -564,7 +564,7 @@ def add_SAFE_constraints(n, config): config.yaml requires to specify opts: scenario: - opts: [Co2L-SAFE-24H] + opts: [Co2L-SAFE-24h] electricity: SAFE_reservemargin: 0.1 Which sets a reserve margin of 10% above the peak demand. From c60e1c5b73ac23dc7d53ad88be3e30cab3924e1b Mon Sep 17 00:00:00 2001 From: Fabian Date: Wed, 31 Jan 2024 17:12:36 +0100 Subject: [PATCH 4/6] update release notes --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 6fa88738..44ccdff5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -64,6 +64,9 @@ Upcoming Release * The ``highs`` solver was added to the default environment file. +* It is now possible to determine the directory for shared resources by setting `shared_resources` to a string. + +* A ``test.sh`` script was added to the repository to run the tests locally. PyPSA-Eur 0.9.0 (5th January 2024) From 344989d76b4f81bea5e3c1555021a767d3b3bd9b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 5 Feb 2024 08:21:26 +0100 Subject: [PATCH 5/6] Apply suggestions from code review --- rules/postprocess.smk | 3 --- 1 file changed, 3 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index c1dd24c0..59f9ead7 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -31,7 +31,6 @@ if config["foresight"] != "perfect": rule plot_power_network: params: plotting=config["plotting"], - foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -55,7 +54,6 @@ if config["foresight"] != "perfect": rule plot_hydrogen_network: params: plotting=config["plotting"], - foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", @@ -79,7 +77,6 @@ if config["foresight"] != "perfect": rule plot_gas_network: params: plotting=config["plotting"], - foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From e76749ab31e543733a671c37968c548f3386037a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 5 Feb 2024 08:37:19 +0100 Subject: [PATCH 6/6] plot_hydrogen_network: add foresight params back in --- rules/postprocess.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 59f9ead7..3045d0da 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -54,6 +54,7 @@ if config["foresight"] != "perfect": rule plot_hydrogen_network: params: plotting=config["plotting"], + foresight=config["foresight"], input: network=RESULTS + "postnetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc",