From 407037a799817d8a99f47fc70a61a1deecdcb6bb Mon Sep 17 00:00:00 2001 From: "daniel.rdt" Date: Fri, 28 Jul 2023 15:09:13 +0200 Subject: [PATCH 01/18] fix in add_heating_capacities_installed_before_baseyear to account for case when there is no rural heating demand for some nodes in network --- scripts/add_existing_baseyear.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 52cb585e..009712d8 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -439,6 +439,11 @@ def add_heating_capacities_installed_before_baseyear( + n.loads_t.p_set.sum()[f"{node} services rural heat"] ) ) + # if rural heating demand for one of the nodes doesn't exist, + # then columns were dropped before and heating demand share should be 0.0 + if (f"{node} residential rural heat" in n.loads_t.p_set.sum().index) + & (f"{node} services rural heat" in n.loads_t.p_set.sum().index) + else 0.0 for node in nodal_df.index ], index=nodal_df.index, From 7c00f9d4517e2bf0179c4ab49e7582a1e17f6c17 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:14:40 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 009712d8..279b6644 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -442,7 +442,7 @@ def add_heating_capacities_installed_before_baseyear( # if rural heating demand for one of the nodes doesn't exist, # then columns were dropped before and heating demand share should be 0.0 if (f"{node} residential rural heat" in n.loads_t.p_set.sum().index) - & (f"{node} services rural heat" in n.loads_t.p_set.sum().index) + & (f"{node} services rural heat" in n.loads_t.p_set.sum().index) else 0.0 for node in nodal_df.index ], From e304efbf3c2e13da1423ccd107ab37a67655cb7b Mon Sep 17 00:00:00 2001 From: "daniel.rdt" Date: Wed, 9 Aug 2023 14:35:39 +0200 Subject: [PATCH 03/18] fix in co2_emissions_year function and correspondigly in plot_summary and snakemake rule. Also changes to historical_emissions in plot_summary since some countries are not in eea dataset. --- rules/postprocess.smk | 3 +++ scripts/plot_summary.py | 20 ++++++++++++++++++-- scripts/prepare_sector_network.py | 11 +++++------ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 2618680e..d0909c4e 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -106,6 +106,8 @@ rule plot_summary: countries=config["countries"], planning_horizons=config["scenario"]["planning_horizons"], sector_opts=config["scenario"]["sector_opts"], + emissions_scope=config["energy"]["emissions"], + eurostat_report_year=config["energy"]["eurostat_report_year"], plotting=config["plotting"], RDIR=RDIR, input: @@ -113,6 +115,7 @@ rule plot_summary: energy=RESULTS + "csvs/energy.csv", balances=RESULTS + "csvs/supply_energy.csv", eurostat=input_eurostat, + co2="data/eea/UNFCCC_v23.csv", output: costs=RESULTS + "graphs/costs.pdf", energy=RESULTS + "graphs/energy.pdf", diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index e7de5473..f77d01c8 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -387,6 +387,19 @@ def historical_emissions(countries): countries.remove("GB") countries.append("UK") + # Albania (AL) and Bosnia Herzegovina (BA), Montenegro (ME), Macedonia (MK) and Serbia (RS) + # not included in eea historical emission dataset + if "AL" in countries: + countries.remove("AL") + if "BA" in countries: + countries.remove("BA") + if "ME" in countries: + countries.remove("ME") + if "MK" in countries: + countries.remove("MK") + if "RS" in countries: + countries.remove("RS") + year = np.arange(1990, 2018).tolist() idx = pd.IndexSlice @@ -457,9 +470,12 @@ def plot_carbon_budget_distribution(input_eurostat): ax1.set_ylim([0, 5]) ax1.set_xlim([1990, snakemake.params.planning_horizons[-1] + 1]) - path_cb = "results/" + snakemake.params.RDIR + "/csvs/" + path_cb = "results/" + snakemake.params.RDIR + "csvs/" countries = snakemake.params.countries - e_1990 = co2_emissions_year(countries, input_eurostat, opts, year=1990) + emissions_scope = snakemake.params.emissions_scope + report_year = snakemake.params.eurostat_report_year + input_co2 = snakemake.input.co2 + e_1990 = co2_emissions_year(countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year=1990) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) ax1.plot(e_1990 * CO2_CAP[o], linewidth=3, color="dodgerblue", label=None) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 0c1faacc..03f0e33b 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -191,17 +191,15 @@ def get(item, investment_year=None): def co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year + countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year ): """ Calculate CO2 emissions in one specific year (e.g. 1990 or 2018). """ - emissions_scope = snakemake.params.energy["emissions"] - eea_co2 = build_eea_co2(snakemake.input.co2, year, emissions_scope) + eea_co2 = build_eea_co2(input_co2, year, emissions_scope) # TODO: read Eurostat data from year > 2014 # this only affects the estimation of CO2 emissions for BA, RS, AL, ME, MK - report_year = snakemake.params.energy["eurostat_report_year"] if year > 2014: eurostat_co2 = build_eurostat_co2( input_eurostat, countries, report_year, year=2014 @@ -240,12 +238,12 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries = snakemake.params.countries e_1990 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year=1990 + countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year=1990 ) # emissions at the beginning of the path (last year available 2018) e_0 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, year=2018 + countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year=2018 ) planning_horizons = snakemake.params.planning_horizons @@ -3396,6 +3394,7 @@ if __name__ == "__main__": if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope report_year = snakemake.params.eurostat_report_year + input_co2 = snakemake.input.co2 build_carbon_budget( o, snakemake.input.eurostat, fn, emissions_scope, report_year ) From eed52d04aeec6174fb18b25ca644e78cc8dda1ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 9 Aug 2023 13:04:35 +0000 Subject: [PATCH 04/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/plot_summary.py | 10 +++++++++- scripts/prepare_sector_network.py | 16 ++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index f77d01c8..0da0eb9d 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -475,7 +475,15 @@ def plot_carbon_budget_distribution(input_eurostat): emissions_scope = snakemake.params.emissions_scope report_year = snakemake.params.eurostat_report_year input_co2 = snakemake.input.co2 - e_1990 = co2_emissions_year(countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year=1990) + e_1990 = co2_emissions_year( + countries, + input_eurostat, + opts, + emissions_scope, + report_year, + input_co2, + year=1990, + ) CO2_CAP = pd.read_csv(path_cb + "carbon_budget_distribution.csv", index_col=0) ax1.plot(e_1990 * CO2_CAP[o], linewidth=3, color="dodgerblue", label=None) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 03f0e33b..a4e4e6b7 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -238,12 +238,24 @@ def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): countries = snakemake.params.countries e_1990 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year=1990 + countries, + input_eurostat, + opts, + emissions_scope, + report_year, + input_co2, + year=1990, ) # emissions at the beginning of the path (last year available 2018) e_0 = co2_emissions_year( - countries, input_eurostat, opts, emissions_scope, report_year, input_co2, year=2018 + countries, + input_eurostat, + opts, + emissions_scope, + report_year, + input_co2, + year=2018, ) planning_horizons = snakemake.params.planning_horizons From 6cdb1943322bf2fcc75f0f2a17f32a0c5bf83445 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Mon, 21 Aug 2023 11:19:16 +0200 Subject: [PATCH 05/18] sanitize_carriers: do not invent nice names, use carrier where not available --- scripts/add_electricity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 56375800..27793396 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -165,7 +165,7 @@ def sanitize_carriers(n, config): nice_names = ( pd.Series(config["plotting"]["nice_names"]) .reindex(carrier_i) - .fillna(carrier_i.to_series().str.title()) + .fillna(carrier_i.to_series()) ) n.carriers["nice_name"] = n.carriers.nice_name.where( n.carriers.nice_name != "", nice_names From 8d09e38627ffef4b9f5b6140d2eb7e0b6a35e845 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:16:53 +0200 Subject: [PATCH 06/18] drop duplicates in prepare_hotmaps_database --- scripts/build_industrial_distribution_key.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 979a1493..fe7f5d82 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -93,6 +93,20 @@ def prepare_hotmaps_database(regions): gdf.rename(columns={"index_right": "bus"}, inplace=True) gdf["country"] = gdf.bus.str[:2] + # the .sjoin can lead to duplicates if a geom is in two regions + if gdf.index.duplicated().any(): + import pycountry + # get all duplicated entries + duplicated_i = gdf.index[gdf.index.duplicated()] + # convert from raw data country name to iso-2-code + s = df.loc[duplicated_i, "Country"].apply(lambda x: pycountry.countries.lookup(x).alpha_2) + # Get a boolean mask where gdf's country column matches s's values for the same index + mask = gdf['country'] == gdf.index.map(s) + # Filter gdf using the mask + gdf_filtered = gdf[mask] + # concat not duplicated and filtered gdf + gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]).sort_index() + return gdf From dadc372ecd629cdeb708727e5abce3b7179eb307 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:17:19 +0000 Subject: [PATCH 07/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_industrial_distribution_key.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index fe7f5d82..3c62ca1b 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -96,12 +96,15 @@ def prepare_hotmaps_database(regions): # the .sjoin can lead to duplicates if a geom is in two regions if gdf.index.duplicated().any(): import pycountry + # get all duplicated entries duplicated_i = gdf.index[gdf.index.duplicated()] # convert from raw data country name to iso-2-code - s = df.loc[duplicated_i, "Country"].apply(lambda x: pycountry.countries.lookup(x).alpha_2) + s = df.loc[duplicated_i, "Country"].apply( + lambda x: pycountry.countries.lookup(x).alpha_2 + ) # Get a boolean mask where gdf's country column matches s's values for the same index - mask = gdf['country'] == gdf.index.map(s) + mask = gdf["country"] == gdf.index.map(s) # Filter gdf using the mask gdf_filtered = gdf[mask] # concat not duplicated and filtered gdf From 2391a5c4e90df3a5184a476514ddc6739d85b521 Mon Sep 17 00:00:00 2001 From: martavp Date: Tue, 22 Aug 2023 08:31:46 +0200 Subject: [PATCH 08/18] fix p_nom_min of renewable generators for myopic approach as described in https://github.com/PyPSA/pypsa-eur/issues/727 --- scripts/solve_network.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 8eccef19..a3a3f58c 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -51,6 +51,9 @@ def _add_land_use_constraint(n): # warning: this will miss existing offwind which is not classed AC-DC and has carrier 'offwind' for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: + extendable_i = (n.generators.carrier == carrier) & n.generators.p_nom_extendable + n.generators.loc[extendable_i, "p_nom_min"] = 0 + ext_i = (n.generators.carrier == carrier) & ~n.generators.p_nom_extendable existing = ( n.generators.loc[ext_i, "p_nom"] @@ -60,18 +63,18 @@ def _add_land_use_constraint(n): existing.index += " " + carrier + "-" + snakemake.wildcards.planning_horizons n.generators.loc[existing.index, "p_nom_max"] -= existing - # check if existing capacities are larger than technical potential - existing_large = n.generators[ - n.generators["p_nom_min"] > n.generators["p_nom_max"] - ].index - if len(existing_large): - logger.warning( - f"Existing capacities larger than technical potential for {existing_large},\ - adjust technical potential to existing capacities" - ) - n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[ - existing_large, "p_nom_min" - ] + # # check if existing capacities are larger than technical potential + # existing_large = n.generators[ + # n.generators["p_nom_min"] > n.generators["p_nom_max"] + # ].index + # if len(existing_large): + # logger.warning( + # f"Existing capacities larger than technical potential for {existing_large},\ + # adjust technical potential to existing capacities" + # ) + # n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[ + # existing_large, "p_nom_min" + # ] n.generators.p_nom_max.clip(lower=0, inplace=True) From b2e8efea19cff8ee197550eccc4bd717d830762d Mon Sep 17 00:00:00 2001 From: martavp Date: Tue, 22 Aug 2023 08:36:21 +0200 Subject: [PATCH 09/18] add small comment to release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b6db1b90..243e78ca 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -14,6 +14,8 @@ Upcoming Release * For industry distribution, use EPRTR as fallback if ETS data is not available. +* The minimum capacity for renewable generators when using the myopic option has been fixed. + PyPSA-Eur 0.8.1 (27th July 2023) ================================ From 69132bd6e24a72f2caa0a17647ab68ea02219858 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 06:41:41 +0000 Subject: [PATCH 10/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/solve_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a3a3f58c..c8e1957b 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -53,7 +53,7 @@ def _add_land_use_constraint(n): for carrier in ["solar", "onwind", "offwind-ac", "offwind-dc"]: extendable_i = (n.generators.carrier == carrier) & n.generators.p_nom_extendable n.generators.loc[extendable_i, "p_nom_min"] = 0 - + ext_i = (n.generators.carrier == carrier) & ~n.generators.p_nom_extendable existing = ( n.generators.loc[ext_i, "p_nom"] From 9cee4384bb946dc43e0acb91184978a90429dc28 Mon Sep 17 00:00:00 2001 From: martavp Date: Tue, 22 Aug 2023 11:08:51 +0200 Subject: [PATCH 11/18] revert commented lines --- scripts/solve_network.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a3a3f58c..7dbccb23 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -63,18 +63,18 @@ def _add_land_use_constraint(n): existing.index += " " + carrier + "-" + snakemake.wildcards.planning_horizons n.generators.loc[existing.index, "p_nom_max"] -= existing - # # check if existing capacities are larger than technical potential - # existing_large = n.generators[ - # n.generators["p_nom_min"] > n.generators["p_nom_max"] - # ].index - # if len(existing_large): - # logger.warning( - # f"Existing capacities larger than technical potential for {existing_large},\ - # adjust technical potential to existing capacities" - # ) - # n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[ - # existing_large, "p_nom_min" - # ] + # check if existing capacities are larger than technical potential + existing_large = n.generators[ + n.generators["p_nom_min"] > n.generators["p_nom_max"] + ].index + if len(existing_large): + logger.warning( + f"Existing capacities larger than technical potential for {existing_large},\ + adjust technical potential to existing capacities" + ) + n.generators.loc[existing_large, "p_nom_max"] = n.generators.loc[ + existing_large, "p_nom_min" + ] n.generators.p_nom_max.clip(lower=0, inplace=True) From f4d821ad2ad5709910988d05b2ff1867bc39dee3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 22 Aug 2023 14:22:25 +0200 Subject: [PATCH 12/18] use country_converter instead of pycountry --- scripts/build_industrial_distribution_key.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 3c62ca1b..25d0235a 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -93,22 +93,16 @@ def prepare_hotmaps_database(regions): gdf.rename(columns={"index_right": "bus"}, inplace=True) gdf["country"] = gdf.bus.str[:2] - # the .sjoin can lead to duplicates if a geom is in two regions + # the .sjoin can lead to duplicates if a geom is in two overlapping regions if gdf.index.duplicated().any(): - import pycountry - # get all duplicated entries duplicated_i = gdf.index[gdf.index.duplicated()] # convert from raw data country name to iso-2-code - s = df.loc[duplicated_i, "Country"].apply( - lambda x: pycountry.countries.lookup(x).alpha_2 - ) - # Get a boolean mask where gdf's country column matches s's values for the same index - mask = gdf["country"] == gdf.index.map(s) - # Filter gdf using the mask - gdf_filtered = gdf[mask] + code = cc.convert(gdf.loc[duplicated_i, "Country"], to="iso2") + # screen out malformed country allocation + gdf_filtered = gdf.loc[duplicated_i].query("country == @code") # concat not duplicated and filtered gdf - gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]).sort_index() + gdf = pd.concat([gdf.drop(duplicated_i), gdf_filtered]) return gdf From e4b220ff81cb4a4abc1b14f747953a1be7b089c5 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 22 Aug 2023 14:29:00 +0200 Subject: [PATCH 13/18] follow-up to #726: import country converter --- scripts/build_industrial_distribution_key.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 25d0235a..63299570 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -17,6 +17,8 @@ import geopandas as gpd import pandas as pd from packaging.version import Version, parse +import country_converter as coco +cc = coco.CountryConverter() def locate_missing_industrial_sites(df): """ From e12cad27892d10fe8bd556d3dcf9cfb2dd305efb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 22 Aug 2023 14:42:21 +0200 Subject: [PATCH 14/18] simplify if-else expression --- scripts/add_existing_baseyear.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 9c85a215..269705a7 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -435,20 +435,20 @@ def add_heating_capacities_installed_before_baseyear( # split existing capacities between residential and services # proportional to energy demand + p_set_sum = n.loads_t.p_set.sum() ratio_residential = pd.Series( [ ( - n.loads_t.p_set.sum()[f"{node} residential rural heat"] + p_set_sum[f"{node} residential rural heat"] / ( - n.loads_t.p_set.sum()[f"{node} residential rural heat"] - + n.loads_t.p_set.sum()[f"{node} services rural heat"] + p_set_sum[f"{node} residential rural heat"] + + p_set_sum[f"{node} services rural heat"] ) ) # if rural heating demand for one of the nodes doesn't exist, # then columns were dropped before and heating demand share should be 0.0 - if (f"{node} residential rural heat" in n.loads_t.p_set.sum().index) - & (f"{node} services rural heat" in n.loads_t.p_set.sum().index) - else 0.0 + if all(f"{node} {service} rural heat" in p_set_sum.index for service in ["residential", "services"]) + else 0. for node in nodal_df.index ], index=nodal_df.index, From 49b2cd382da8bf9416955a7e2557a134a3905576 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 22 Aug 2023 14:43:15 +0200 Subject: [PATCH 15/18] Update scripts/plot_summary.py Co-authored-by: Fabian Hofmann --- scripts/plot_summary.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 0da0eb9d..be2b4d86 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -387,19 +387,9 @@ def historical_emissions(countries): countries.remove("GB") countries.append("UK") - # Albania (AL) and Bosnia Herzegovina (BA), Montenegro (ME), Macedonia (MK) and Serbia (RS) - # not included in eea historical emission dataset - if "AL" in countries: - countries.remove("AL") - if "BA" in countries: - countries.remove("BA") - if "ME" in countries: - countries.remove("ME") - if "MK" in countries: - countries.remove("MK") - if "RS" in countries: - countries.remove("RS") - + # remove countries which are not included in eea historical emission dataset + countries_to_remove = {"AL", "BA", "ME", "MK", "RS"} + countries = list(set(countries) - countries_to_remove) year = np.arange(1990, 2018).tolist() idx = pd.IndexSlice From 3ccc63aae4f4865cc3b10234d0d7d5e506aa8428 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 22 Aug 2023 14:49:44 +0200 Subject: [PATCH 16/18] build_carbon_budget: parse input_co2 --- scripts/make_summary.py | 2 +- scripts/plot_summary.py | 2 +- scripts/prepare_sector_network.py | 11 ++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 56ee98c9..98a6a6d7 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -711,5 +711,5 @@ if __name__ == "__main__": if snakemake.params.foresight == "myopic": cumulative_cost = calculate_cumulative_cost() cumulative_cost.to_csv( - "results/" + snakemake.params.RDIR + "/csvs/cumulative_cost.csv" + "results/" + snakemake.params.RDIR + "csvs/cumulative_cost.csv" ) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index be2b4d86..072c7128 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -549,7 +549,7 @@ def plot_carbon_budget_distribution(input_eurostat): fancybox=True, fontsize=18, loc=(0.01, 0.01), facecolor="white", frameon=True ) - path_cb_plot = "results/" + snakemake.params.RDIR + "/graphs/" + path_cb_plot = "results/" + snakemake.params.RDIR + "graphs/" plt.savefig(path_cb_plot + "carbon_budget_plot.pdf", dpi=300) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a4e4e6b7..11406bff 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -220,7 +220,7 @@ def co2_emissions_year( # TODO: move to own rule with sector-opts wildcard? -def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year): +def build_carbon_budget(o, input_eurostat, fn, emissions_scope, report_year, input_co2): """ Distribute carbon budget following beta or exponential transition path. """ @@ -3402,13 +3402,18 @@ if __name__ == "__main__": if "cb" not in o: continue limit_type = "carbon budget" - fn = "results/" + snakemake.params.RDIR + "/csvs/carbon_budget_distribution.csv" + fn = "results/" + snakemake.params.RDIR + "csvs/carbon_budget_distribution.csv" if not os.path.exists(fn): emissions_scope = snakemake.params.emissions_scope report_year = snakemake.params.eurostat_report_year input_co2 = snakemake.input.co2 build_carbon_budget( - o, snakemake.input.eurostat, fn, emissions_scope, report_year + o, + snakemake.input.eurostat, + fn, + emissions_scope, + report_year, + input_co2, ) co2_cap = pd.read_csv(fn, index_col=0).squeeze() limit = co2_cap.loc[investment_year] From d0f2d52853b8c068909528bff04abfc5aedf4cf1 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Thu, 24 Aug 2023 11:11:33 +0200 Subject: [PATCH 17/18] apply protected() to zenodo downloads; only retrieve monthly fuel/CO2 prices when necessary --- .gitignore | 14 +++++++------- doc/release_notes.rst | 4 ++++ rules/build_electricity.smk | 4 ++-- rules/build_sector.smk | 28 ++++++++++++++-------------- rules/common.smk | 2 +- rules/postprocess.smk | 2 +- rules/retrieve.smk | 36 ++++++++++++++++++++---------------- scripts/plot_summary.py | 2 +- 8 files changed, 50 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 0adf0ae6..459ef860 100644 --- a/.gitignore +++ b/.gitignore @@ -28,18 +28,18 @@ dconf /data/links_p_nom.csv /data/*totals.csv /data/biomass* -/data/emobility/ -/data/eea* -/data/jrc* +/data/bundle-sector/emobility/ +/data/bundle-sector/eea* +/data/bundle-sector/jrc* /data/heating/ -/data/eurostat* +/data/bundle-sector/eurostat* /data/odyssee/ /data/transport_data.csv -/data/switzerland* +/data/bundle-sector/switzerland* /data/.nfs* -/data/Industrial_Database.csv +/data/bundle-sector/Industrial_Database.csv /data/retro/tabula-calculator-calcsetbuilding.csv -/data/nuts* +/data/bundle-sector/nuts* data/gas_network/scigrid-gas/ data/costs_*.csv diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 243e78ca..3a787104 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -16,6 +16,10 @@ Upcoming Release * The minimum capacity for renewable generators when using the myopic option has been fixed. +* Files downloaded from zenodo are now write-protected to prevent accidental re-download. + +* Files extracted from sector-coupled data bundle have been moved from ``data/`` to ``data/sector-bundle``. + PyPSA-Eur 0.8.1 (27th July 2023) ================================ diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index 383951bd..cf1732be 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -350,7 +350,7 @@ rule add_electricity: hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", unit_commitment="data/unit_commitment.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv", + fuel_price=RESOURCES + "monthly_fuel_price.csv" if config["conventional"]["dynamic_fuel_price"] else [], load=RESOURCES + "load.csv", nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: @@ -478,7 +478,7 @@ rule prepare_network: input: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec.nc", tech_costs=COSTS, - co2_price=RESOURCES + "co2_price.csv", + co2_price=lambda w: RESOURCES + "co2_price.csv" if "Ept" in w.opts else [], output: RESOURCES + "networks/elec_s{simpl}_{clusters}_ec_l{ll}_{opts}.nc", log: diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 356abdc5..10a5f821 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -242,9 +242,9 @@ rule build_energy_totals: energy=config["energy"], input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", - co2="data/eea/UNFCCC_v23.csv", - swiss="data/switzerland-sfoe/switzerland-new_format.csv", - idees="data/jrc-idees-2015", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", + swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv", + idees="data/bundle-sector/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", eurostat=input_eurostat, output: @@ -272,7 +272,7 @@ rule build_biomass_potentials: "https://cidportal.jrc.ec.europa.eu/ftp/jrc-opendata/ENSPRESO/ENSPRESO_BIOMASS.xlsx", keep_local=True, ), - nuts2="data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 + nuts2="data/bundle-sector/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", # https://gisco-services.ec.europa.eu/distribution/v2/nuts/download/#nuts21 regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", nuts3_population=ancient("data/bundle/nama_10r_3popgdp.tsv.gz"), swiss_cantons=ancient("data/bundle/ch_cantons.csv"), @@ -366,7 +366,7 @@ if not config["sector"]["regional_co2_sequestration_potential"]["enable"]: rule build_salt_cavern_potentials: input: - salt_caverns="data/h2_salt_caverns_GWh_per_sqkm.geojson", + salt_caverns="data/bundle-sector/h2_salt_caverns_GWh_per_sqkm.geojson", regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", regions_offshore=RESOURCES + "regions_offshore_elec_s{simpl}_{clusters}.geojson", output: @@ -388,7 +388,7 @@ rule build_ammonia_production: params: countries=config["countries"], input: - usgs="data/myb1-2017-nitro.xls", + usgs="data/bundle-sector/myb1-2017-nitro.xls", output: ammonia_production=RESOURCES + "ammonia_production.csv", threads: 1 @@ -410,7 +410,7 @@ rule build_industry_sector_ratios: ammonia=config["sector"].get("ammonia", False), input: ammonia_production=RESOURCES + "ammonia_production.csv", - idees="data/jrc-idees-2015", + idees="data/bundle-sector/jrc-idees-2015", output: industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", threads: 1 @@ -432,8 +432,8 @@ rule build_industrial_production_per_country: countries=config["countries"], input: ammonia_production=RESOURCES + "ammonia_production.csv", - jrc="data/jrc-idees-2015", - eurostat="data/eurostat-energy_balances-may_2018_edition", + jrc="data/bundle-sector/jrc-idees-2015", + eurostat="data/bundle-sector/eurostat-energy_balances-may_2018_edition", output: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -483,7 +483,7 @@ rule build_industrial_distribution_key: input: regions_onshore=RESOURCES + "regions_onshore_elec_s{simpl}_{clusters}.geojson", clustered_pop_layout=RESOURCES + "pop_layout_elec_s{simpl}_{clusters}.csv", - hotmaps_industrial_database="data/Industrial_Database.csv", + hotmaps_industrial_database="data/bundle-sector/Industrial_Database.csv", output: industrial_distribution_key=RESOURCES + "industrial_distribution_key_elec_s{simpl}_{clusters}.csv", @@ -558,7 +558,7 @@ rule build_industrial_energy_demand_per_country_today: countries=config["countries"], industry=config["industry"], input: - jrc="data/jrc-idees-2015", + jrc="data/bundle-sector/jrc-idees-2015", ammonia_production=RESOURCES + "ammonia_production.csv", industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", @@ -684,8 +684,8 @@ rule build_transport_demand: pop_weighted_energy_totals=RESOURCES + "pop_weighted_energy_totals_s{simpl}_{clusters}.csv", transport_data=RESOURCES + "transport_data.csv", - traffic_data_KFZ="data/emobility/KFZ__count", - traffic_data_Pkw="data/emobility/Pkw__count", + traffic_data_KFZ="data/bundle-sector/emobility/KFZ__count", + traffic_data_Pkw="data/bundle-sector/emobility/Pkw__count", temp_air_total=RESOURCES + "temp_air_total_elec_s{simpl}_{clusters}.nc", output: transport_demand=RESOURCES + "transport_demand_s{simpl}_{clusters}.csv", @@ -734,7 +734,7 @@ rule prepare_sector_network: avail_profile=RESOURCES + "avail_profile_s{simpl}_{clusters}.csv", dsm_profile=RESOURCES + "dsm_profile_s{simpl}_{clusters}.csv", co2_totals_name=RESOURCES + "co2_totals.csv", - co2="data/eea/UNFCCC_v23.csv", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", biomass_potentials=RESOURCES + "biomass_potentials_s{simpl}_{clusters}.csv", heat_profile="data/heat_load_profile_BDEW.csv", costs="data/costs_{}.csv".format(config["costs"]["year"]) diff --git a/rules/common.smk b/rules/common.smk index ec5be355..d3416050 100644 --- a/rules/common.smk +++ b/rules/common.smk @@ -42,7 +42,7 @@ def has_internet_access(url="www.zenodo.org") -> bool: def input_eurostat(w): # 2016 includes BA, 2017 does not report_year = config["energy"]["eurostat_report_year"] - return f"data/eurostat-energy_balances-june_{report_year}_edition" + return f"data/bundle-sector/eurostat-energy_balances-june_{report_year}_edition" def solved_previous_horizon(wildcards): diff --git a/rules/postprocess.smk b/rules/postprocess.smk index d0909c4e..cf0038a3 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -115,7 +115,7 @@ rule plot_summary: energy=RESULTS + "csvs/energy.csv", balances=RESULTS + "csvs/supply_energy.csv", eurostat=input_eurostat, - co2="data/eea/UNFCCC_v23.csv", + co2="data/bundle-sector/eea/UNFCCC_v23.csv", output: costs=RESULTS + "graphs/costs.pdf", energy=RESULTS + "graphs/energy.pdf", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 0b60ee2e..363058ad 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -27,7 +27,7 @@ if config["enable"]["retrieve"] and config["enable"].get("retrieve_databundle", rule retrieve_databundle: output: - expand("data/bundle/{file}", file=datafiles), + protected(expand("data/bundle/{file}", file=datafiles)), log: LOGS + "retrieve_databundle.log", resources: @@ -92,7 +92,7 @@ if config["enable"]["retrieve"] and config["enable"].get( static=True, ), output: - RESOURCES + "natura.tiff", + protected(RESOURCES + "natura.tiff"), log: LOGS + "retrieve_natura_raster.log", resources: @@ -106,22 +106,26 @@ if config["enable"]["retrieve"] and config["enable"].get( "retrieve_sector_databundle", True ): datafiles = [ - "data/eea/UNFCCC_v23.csv", - "data/switzerland-sfoe/switzerland-new_format.csv", - "data/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", - "data/myb1-2017-nitro.xls", - "data/Industrial_Database.csv", - "data/emobility/KFZ__count", - "data/emobility/Pkw__count", - "data/h2_salt_caverns_GWh_per_sqkm.geojson", - directory("data/eurostat-energy_balances-june_2016_edition"), - directory("data/eurostat-energy_balances-may_2018_edition"), - directory("data/jrc-idees-2015"), + "eea/UNFCCC_v23.csv", + "switzerland-sfoe/switzerland-new_format.csv", + "nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", + "myb1-2017-nitro.xls", + "Industrial_Database.csv", + "emobility/KFZ__count", + "emobility/Pkw__count", + "h2_salt_caverns_GWh_per_sqkm.geojson", + ] + + datafolders = [ + protected(directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")), + protected(directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")), + protected(directory("data/bundle-sector/jrc-idees-2015")), ] rule retrieve_sector_databundle: output: - *datafiles, + protected(expand("data/bundle-sector/{files}", files=datafiles)), + *datafolders log: LOGS + "retrieve_sector_databundle.log", retries: 2 @@ -143,7 +147,7 @@ if config["enable"]["retrieve"] and ( rule retrieve_gas_infrastructure_data: output: - expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles), + protected(expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)), log: LOGS + "retrieve_gas_infrastructure_data.log", retries: 2 @@ -187,7 +191,7 @@ if config["enable"]["retrieve"]: static=True, ), output: - "data/shipdensity_global.zip", + protected("data/shipdensity_global.zip"), log: LOGS + "retrieve_ship_raster.log", resources: diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 072c7128..a501dcf7 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -354,7 +354,7 @@ def historical_emissions(countries): """ # https://www.eea.europa.eu/data-and-maps/data/national-emissions-reported-to-the-unfccc-and-to-the-eu-greenhouse-gas-monitoring-mechanism-16 # downloaded 201228 (modified by EEA last on 201221) - fn = "data/eea/UNFCCC_v23.csv" + fn = "data/bundle-sector/eea/UNFCCC_v23.csv" df = pd.read_csv(fn, encoding="latin-1") df.loc[df["Year"] == "1985-1987", "Year"] = 1986 df["Year"] = df["Year"].astype(int) From b8dee184a9a9cb2ac16f24b18dd14cbfac6b3a35 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 24 Aug 2023 09:12:05 +0000 Subject: [PATCH 18/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_electricity.smk | 4 +++- rules/retrieve.smk | 14 ++++++++++---- scripts/add_existing_baseyear.py | 7 +++++-- scripts/build_industrial_distribution_key.py | 3 ++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/rules/build_electricity.smk b/rules/build_electricity.smk index cf1732be..f9fdc3ac 100644 --- a/rules/build_electricity.smk +++ b/rules/build_electricity.smk @@ -350,7 +350,9 @@ rule add_electricity: hydro_capacities=ancient("data/bundle/hydro_capacities.csv"), geth_hydro_capacities="data/geth2015_hydro_capacities.csv", unit_commitment="data/unit_commitment.csv", - fuel_price=RESOURCES + "monthly_fuel_price.csv" if config["conventional"]["dynamic_fuel_price"] else [], + fuel_price=RESOURCES + "monthly_fuel_price.csv" + if config["conventional"]["dynamic_fuel_price"] + else [], load=RESOURCES + "load.csv", nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", output: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 363058ad..66ce76df 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -117,15 +117,19 @@ if config["enable"]["retrieve"] and config["enable"].get( ] datafolders = [ - protected(directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition")), - protected(directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition")), + protected( + directory("data/bundle-sector/eurostat-energy_balances-june_2016_edition") + ), + protected( + directory("data/bundle-sector/eurostat-energy_balances-may_2018_edition") + ), protected(directory("data/bundle-sector/jrc-idees-2015")), ] rule retrieve_sector_databundle: output: protected(expand("data/bundle-sector/{files}", files=datafiles)), - *datafolders + *datafolders, log: LOGS + "retrieve_sector_databundle.log", retries: 2 @@ -147,7 +151,9 @@ if config["enable"]["retrieve"] and ( rule retrieve_gas_infrastructure_data: output: - protected(expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles)), + protected( + expand("data/gas_network/scigrid-gas/data/{files}", files=datafiles) + ), log: LOGS + "retrieve_gas_infrastructure_data.log", retries: 2 diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 269705a7..08810470 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -447,8 +447,11 @@ def add_heating_capacities_installed_before_baseyear( ) # if rural heating demand for one of the nodes doesn't exist, # then columns were dropped before and heating demand share should be 0.0 - if all(f"{node} {service} rural heat" in p_set_sum.index for service in ["residential", "services"]) - else 0. + if all( + f"{node} {service} rural heat" in p_set_sum.index + for service in ["residential", "services"] + ) + else 0.0 for node in nodal_df.index ], index=nodal_df.index, diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 63299570..bc4a26bc 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -13,13 +13,14 @@ logger = logging.getLogger(__name__) import uuid from itertools import product +import country_converter as coco import geopandas as gpd import pandas as pd from packaging.version import Version, parse -import country_converter as coco cc = coco.CountryConverter() + def locate_missing_industrial_sites(df): """ Locate industrial sites without valid locations based on city and