From 0bcb21559704accdc5900f536c4899489c132676 Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Mon, 2 Sep 2024 19:32:13 +0200 Subject: [PATCH 01/24] use JRC-IDEES thermal energy service instead of FE for buildings This change consists of: - reading the final energy (FE) to thermal energy service (TES) efficiency for each each country, seperately for gas and oil (this efficiency represents e.g. the losses in older non-condensing boilers) - using TES instead of FE for the n.loads, so that it is pure energy service instead of including losses in legacy equipment - using the stored efficiencies for baseyear equipment in add_existing_baseyear In the baseyear (e.g. 2020) this should have no effect on FE, since the reduction to TES is conpensated by the lower efficiencies of existing equipment. In the future (e.g. 2050) this leads to a reduction in heating demand, since new equipment is more efficient than existing. --- rules/build_sector.smk | 2 + rules/solve_myopic.smk | 1 + scripts/add_existing_baseyear.py | 25 +++++++- scripts/build_energy_totals.py | 101 +++++++++++++++++++++++++++++- scripts/prepare_sector_network.py | 11 +++- 5 files changed, 135 insertions(+), 5 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index f95731b7..ec7fcd0c 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -305,6 +305,7 @@ rule build_energy_totals: co2_name=resources("co2_totals.csv"), transport_name=resources("transport_data.csv"), district_heat_share=resources("district_heat_share.csv"), + heating_efficiencies=resources("heating_efficiencies.csv"), threads: 16 resources: mem_mb=10000, @@ -1037,6 +1038,7 @@ rule prepare_sector_network: district_heat_share=resources( "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), + heating_efficiencies=resources("heating_efficiencies.csv"), temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), cop_profiles=resources("cop_profiles_elec_s{simpl}_{clusters}.nc"), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8d7fa284..f868a997 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -26,6 +26,7 @@ rule add_existing_baseyear: existing_heating_distribution=resources( "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), + heating_efficiencies=resources("heating_efficiencies.csv"), output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 212ae8af..7e9dd082 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -546,6 +546,14 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) + if "residential" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["gas residential space efficiency"]) + elif "services" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["gas services space efficiency"]) + else: + #default used for urban central, since no info on district heating boilers + efficiency = costs.at[heat_system.gas_boiler_costs_name, "efficiency"] + n.madd( "Link", nodes, @@ -554,7 +562,7 @@ def add_heating_capacities_installed_before_baseyear( bus1=nodes + " " + heat_system.value + " heat", bus2="co2 atmosphere", carrier=heat_system.value + " gas boiler", - efficiency=costs.at[heat_system.gas_boiler_costs_name, "efficiency"], + efficiency=efficiency, efficiency2=costs.at["gas", "CO2 intensity"], capital_cost=( costs.at[heat_system.gas_boiler_costs_name, "efficiency"] @@ -569,6 +577,14 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) + if "residential" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["oil residential space efficiency"]) + elif "services" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["oil services space efficiency"]) + else: + #default used for urban central, since no info on district heating boilers + efficiency = costs.at[heat_system.oil_boiler_costs_name, "efficiency"] + n.madd( "Link", nodes, @@ -577,7 +593,7 @@ def add_heating_capacities_installed_before_baseyear( bus1=nodes + " " + heat_system.value + " heat", bus2="co2 atmosphere", carrier=heat_system.value + " oil boiler", - efficiency=costs.at[heat_system.oil_boiler_costs_name, "efficiency"], + efficiency=efficiency, efficiency2=costs.at["oil", "CO2 intensity"], capital_cost=costs.at[heat_system.oil_boiler_costs_name, "efficiency"] * costs.at[heat_system.oil_boiler_costs_name, "fixed"], @@ -660,6 +676,11 @@ if __name__ == "__main__": if options["heating"]: + #one could use baseyear here instead (but dangerous if no data) + heating_efficiencies = ( + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) + ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + add_heating_capacities_installed_before_baseyear( n=n, baseyear=baseyear, diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 24ba86bb..73c3d1a0 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -367,6 +367,24 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[43] == "Thermal uses" ct_totals["thermal uses residential"] = df.iloc[43] + df = pd.read_excel(fn_residential, "RES_hh_eff", index_col=0) + + ct_totals["total residential space efficiency"] = df.loc["Space heating"] + + assert df.index[5] == "Diesel oil" + ct_totals["oil residential space efficiency"] = df.iloc[5] + + assert df.index[6] == "Natural gas" + ct_totals["gas residential space efficiency"] = df.iloc[6] + + ct_totals["total residential water efficiency"] = df.loc["Water heating"] + + assert df.index[18] == "Diesel oil" + ct_totals["oil residential water efficiency"] = df.iloc[18] + + assert df.index[19] == "Natural gas" + ct_totals["gas residential water efficiency"] = df.iloc[19] + # services df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0) @@ -400,6 +418,24 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[46] == "Thermal uses" ct_totals["thermal uses services"] = df.iloc[46] + df = pd.read_excel(fn_tertiary, "SER_hh_eff", index_col=0) + + ct_totals["total services space efficiency"] = df.loc["Space heating"] + + assert df.index[5] == "Diesel oil" + ct_totals["oil services space efficiency"] = df.iloc[5] + + assert df.index[7] == "Conventional gas heaters" + ct_totals["gas services space efficiency"] = df.iloc[7] + + ct_totals["total services water efficiency"] = df.loc["Hot water"] + + assert df.index[20] == "Diesel oil" + ct_totals["oil services water efficiency"] = df.iloc[20] + + assert df.index[21] == "Natural gas" + ct_totals["gas services water efficiency"] = df.iloc[21] + # agriculture, forestry and fishing start = "Detailed split of energy consumption (ktoe)" @@ -576,7 +612,9 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh - exclude = totals.columns.str.fullmatch("passenger cars") + exclude = totals.columns.str.fullmatch("passenger cars") \ + ^ totals.columns.str.fullmatch(".*space efficiency") \ + ^ totals.columns.str.fullmatch(".*water efficiency") totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 @@ -654,11 +692,16 @@ def build_energy_totals( eurostat_countries = eurostat.index.unique(0) eurostat_years = eurostat.index.unique(1) - to_drop = ["passenger cars", "passenger car efficiency"] new_index = pd.MultiIndex.from_product( [countries, eurostat_years], names=["country", "year"] ) + to_drop = idees.columns[idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency")] + to_drop = to_drop.append(pd.Index( + ["passenger cars", "passenger car efficiency"] + )) + df = idees.reindex(new_index).drop(to_drop, axis=1) in_eurostat = df.index.levels[0].intersection(eurostat_countries) @@ -1500,6 +1543,57 @@ def build_transformation_output_coke(eurostat, fn): df = eurostat.loc[slicer, :].droplevel(level=[2, 3, 4, 5]) df.to_csv(fn) +def build_heating_efficiencies( + countries: List[str], idees: pd.DataFrame +) -> pd.DataFrame: + """ + Build heating efficiencies for a set of countries based on IDEES data. + + Parameters + ---------- + countries : List[str] + List of country codes. + idees : pd.DataFrame + DataFrame with IDEES data. + + Returns + ------- + pd.DataFrame + DataFrame with heating efficiencies. + + + Notes + ----- + - It fills missing data with average data. + """ + + years = np.arange(2000, 2022) + + cols = idees.columns[idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency")] + + logger.info(cols) + + heating_efficiencies = pd.DataFrame(idees[cols]) + + new_index = pd.MultiIndex.from_product( + [countries, heating_efficiencies.index.unique(1)], + names=["country", "year"], + ) + + heating_efficiencies = heating_efficiencies.reindex(index=new_index) + + for col in cols: + unstacked = heating_efficiencies[col].unstack() + + fillvalue = unstacked.mean() + + for ct in unstacked.index: + mask = unstacked.loc[ct].isna() + unstacked.loc[ct, mask] = fillvalue[mask] + heating_efficiencies[col] = unstacked.stack() + + return heating_efficiencies # %% if __name__ == "__main__": @@ -1556,3 +1650,6 @@ if __name__ == "__main__": transport = build_transport_data(countries, population, idees) transport.to_csv(snakemake.output.transport_name) + + heating_efficiencies = build_heating_efficiencies(countries, idees) + heating_efficiencies.to_csv(snakemake.output.heating_efficiencies) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 87aa90fa..f303c10c 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1806,9 +1806,14 @@ def build_heat_demand(n): for sector, use in product(sectors, uses): name = f"{sector} {use}" + #efficiency for final energy to thermal energy service + eff = pop_weighted_energy_totals.index.str[:2].map( + heating_efficiencies[f"total {sector} {use} efficiency"] + ) + heat_demand[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() - ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]) * 1e6 + ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]*eff) * 1e6 electric_heat_supply[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() ).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 @@ -4282,6 +4287,10 @@ if __name__ == "__main__": ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) + heating_efficiencies = ( + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) + ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + patch_electricity_network(n) spatial = define_spatial(pop_layout.index, options) From ff943d87ada4584e7dc3fe22fa8edb8f7d63c6bb Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 3 Sep 2024 13:34:27 +0200 Subject: [PATCH 02/24] add missing snakemake input --- rules/solve_perfect.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index a06c6dfa..4db4e31a 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -25,6 +25,7 @@ rule add_existing_baseyear: "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), existing_heating="data/existing_infrastructure/existing_heating_raw.csv", + heating_efficiencies=resources("heating_efficiencies.csv"), output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From 93ec44de77d09f328f71d57545f680bd708f1370 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 9 Sep 2024 18:10:59 +0200 Subject: [PATCH 03/24] use heat_system.sector --- scripts/add_existing_baseyear.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 7e9dd082..34114a66 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -546,9 +546,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) - if "residential" in heat_system.value: + if "residential"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["gas residential space efficiency"]) - elif "services" in heat_system.value: + elif "services"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["gas services space efficiency"]) else: #default used for urban central, since no info on district heating boilers @@ -577,9 +577,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) - if "residential" in heat_system.value: + if "residential"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["oil residential space efficiency"]) - elif "services" in heat_system.value: + elif "services"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["oil services space efficiency"]) else: #default used for urban central, since no info on district heating boilers @@ -639,11 +639,11 @@ if __name__ == "__main__": "add_existing_baseyear", configfiles="config/config.yaml", simpl="", - clusters="20", - ll="v1.5", + clusters="38", + ll="vopt", opts="", - sector_opts="none", - planning_horizons=2030, + sector_opts="", + planning_horizons=2020, ) configure_logging(snakemake) @@ -676,7 +676,7 @@ if __name__ == "__main__": if options["heating"]: - #one could use baseyear here instead (but dangerous if no data) + # one could use baseyear here instead (but dangerous if no data) heating_efficiencies = ( pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] From 59984d06003f120de9efba68e0639586749a6db9 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 9 Sep 2024 18:19:18 +0200 Subject: [PATCH 04/24] add function for efficiency --- scripts/add_existing_baseyear.py | 58 ++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 34114a66..10277ad5 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -418,6 +418,46 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas ] +def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): + """ + Computes the heating system efficiency based on the sector and carrier type. + + Parameters: + ----------- + heat_system : object + carrier : str + The type of fuel or energy carrier (e.g., 'gas', 'oil'). + nodes : pandas.Series + A pandas Series containing node information used to match the heating efficiency data. + heating_efficiencies : dict + A dictionary containing efficiency values for different carriers and sectors. + costs : pandas.DataFrame + A DataFrame containing boiler cost and efficiency data for different heating systems. + + Returns: + -------- + efficiency : pandas.Series or float + A pandas Series mapping the efficiencies based on nodes for residential and services sectors, or a single + efficiency value for other heating systems (e.g., urban central). + + Notes: + ------ + - For residential and services sectors, efficiency is mapped based on the nodes. + - For other sectors, the default boiler efficiency is retrieved from the `costs` database. + """ + + if heat_system.sector.value == "residential": + key = f"{carrier} residential space efficiency" + efficiency = nodes.str[:2].map(heating_efficiencies[key]) + elif heat_system.sector.value == "services": + key = f"{carrier} services space efficiency" + efficiency = nodes.str[:2].map(heating_efficiencies[key]) + else: + boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") + efficiency = costs.at[boiler_costs_name, "efficiency"] + + return efficiency + def add_heating_capacities_installed_before_baseyear( n: pypsa.Network, baseyear: int, @@ -546,13 +586,8 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) - if "residential"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["gas residential space efficiency"]) - elif "services"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["gas services space efficiency"]) - else: - #default used for urban central, since no info on district heating boilers - efficiency = costs.at[heat_system.gas_boiler_costs_name, "efficiency"] + efficiency = get_efficiency(heat_system, "gas", nodes, + heating_efficiencies, costs) n.madd( "Link", @@ -577,13 +612,8 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) - if "residential"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["oil residential space efficiency"]) - elif "services"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["oil services space efficiency"]) - else: - #default used for urban central, since no info on district heating boilers - efficiency = costs.at[heat_system.oil_boiler_costs_name, "efficiency"] + efficiency = get_efficiency(heat_system, "oil", nodes, + heating_efficiencies, costs) n.madd( "Link", From 861054961ac4e126b9762a031249c457458e2af5 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 9 Sep 2024 18:20:30 +0200 Subject: [PATCH 05/24] simplify expression --- scripts/add_existing_baseyear.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 10277ad5..c34de362 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -708,8 +708,8 @@ if __name__ == "__main__": # one could use baseyear here instead (but dangerous if no data) heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) - ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1,0]) + ).loc[int(snakemake.config["energy"]["energy_totals_year"])] add_heating_capacities_installed_before_baseyear( n=n, From 5465d8cc75455b3a220d5e7526853a968b173be2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:28:07 +0000 Subject: [PATCH 06/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 18 +++++++++++------- scripts/build_energy_totals.py | 24 +++++++++++++++--------- scripts/prepare_sector_network.py | 12 +++++++----- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c34de362..1635fab7 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -420,7 +420,8 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): """ - Computes the heating system efficiency based on the sector and carrier type. + Computes the heating system efficiency based on the sector and carrier + type. Parameters: ----------- @@ -445,7 +446,7 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): - For residential and services sectors, efficiency is mapped based on the nodes. - For other sectors, the default boiler efficiency is retrieved from the `costs` database. """ - + if heat_system.sector.value == "residential": key = f"{carrier} residential space efficiency" efficiency = nodes.str[:2].map(heating_efficiencies[key]) @@ -458,6 +459,7 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): return efficiency + def add_heating_capacities_installed_before_baseyear( n: pypsa.Network, baseyear: int, @@ -586,8 +588,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) - efficiency = get_efficiency(heat_system, "gas", nodes, - heating_efficiencies, costs) + efficiency = get_efficiency( + heat_system, "gas", nodes, heating_efficiencies, costs + ) n.madd( "Link", @@ -612,8 +615,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) - efficiency = get_efficiency(heat_system, "oil", nodes, - heating_efficiencies, costs) + efficiency = get_efficiency( + heat_system, "oil", nodes, heating_efficiencies, costs + ) n.madd( "Link", @@ -708,7 +712,7 @@ if __name__ == "__main__": # one could use baseyear here instead (but dangerous if no data) heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1,0]) + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1, 0]) ).loc[int(snakemake.config["energy"]["energy_totals_year"])] add_heating_capacities_installed_before_baseyear( diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 73c3d1a0..a4064af6 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -612,9 +612,11 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh - exclude = totals.columns.str.fullmatch("passenger cars") \ - ^ totals.columns.str.fullmatch(".*space efficiency") \ + exclude = ( + totals.columns.str.fullmatch("passenger cars") + ^ totals.columns.str.fullmatch(".*space efficiency") ^ totals.columns.str.fullmatch(".*water efficiency") + ) totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 @@ -696,11 +698,11 @@ def build_energy_totals( [countries, eurostat_years], names=["country", "year"] ) - to_drop = idees.columns[idees.columns.str.contains("space efficiency") - ^ idees.columns.str.contains("water efficiency")] - to_drop = to_drop.append(pd.Index( - ["passenger cars", "passenger car efficiency"] - )) + to_drop = idees.columns[ + idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency") + ] + to_drop = to_drop.append(pd.Index(["passenger cars", "passenger car efficiency"])) df = idees.reindex(new_index).drop(to_drop, axis=1) @@ -1543,6 +1545,7 @@ def build_transformation_output_coke(eurostat, fn): df = eurostat.loc[slicer, :].droplevel(level=[2, 3, 4, 5]) df.to_csv(fn) + def build_heating_efficiencies( countries: List[str], idees: pd.DataFrame ) -> pd.DataFrame: @@ -1569,8 +1572,10 @@ def build_heating_efficiencies( years = np.arange(2000, 2022) - cols = idees.columns[idees.columns.str.contains("space efficiency") - ^ idees.columns.str.contains("water efficiency")] + cols = idees.columns[ + idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency") + ] logger.info(cols) @@ -1595,6 +1600,7 @@ def build_heating_efficiencies( return heating_efficiencies + # %% if __name__ == "__main__": if "snakemake" not in globals(): diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f303c10c..097f9bb5 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1806,14 +1806,14 @@ def build_heat_demand(n): for sector, use in product(sectors, uses): name = f"{sector} {use}" - #efficiency for final energy to thermal energy service + # efficiency for final energy to thermal energy service eff = pop_weighted_energy_totals.index.str[:2].map( heating_efficiencies[f"total {sector} {use} efficiency"] - ) + ) heat_demand[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() - ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]*eff) * 1e6 + ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"] * eff) * 1e6 electric_heat_supply[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() ).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 @@ -4288,8 +4288,10 @@ if __name__ == "__main__": pop_weighted_energy_totals.update(pop_weighted_heat_totals) heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) - ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + (pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0, 1])) + .swaplevel() + .loc[int(snakemake.config["energy"]["energy_totals_year"])] + ) patch_electricity_network(n) From 879c896bea45ca6908a96ddb7d6cdb8111220e77 Mon Sep 17 00:00:00 2001 From: "daniel.rdt" Date: Tue, 10 Sep 2024 13:51:34 +0200 Subject: [PATCH 07/24] fix bug for plotting of hydrogen network with myopic foresight. --- scripts/plot_hydrogen_network.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index b4585fb2..d7e42ff8 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -115,7 +115,7 @@ def plot_h2_map(n, regions): retro_wo_new_i = h2_retro.index.difference(h2_new.index) h2_retro_wo_new = h2_retro.loc[retro_wo_new_i] - h2_retro_wo_new.index = h2_retro_wo_new.index_orig + h2_retro_wo_new.index = h2_retro_wo_new.index_orig.apply(lambda x: x.split('-2')[0]) to_concat = [h2_new, h2_retro_w_new, h2_retro_wo_new] h2_total = pd.concat(to_concat).p_nom_opt.groupby(level=0).sum() @@ -126,7 +126,10 @@ def plot_h2_map(n, regions): link_widths_total = h2_total / linewidth_factor n.links.rename(index=lambda x: x.split("-2")[0], inplace=True) - n.links = n.links.groupby(level=0).first() + # group links by summing up p_nom values and taking the first value of the rest of the columns + other_cols = dict.fromkeys(n.links.columns.drop(["p_nom_opt", "p_nom"]), "first") + n.links = n.links.groupby(level=0).agg({"p_nom_opt": "sum", "p_nom": "sum", **other_cols}) + link_widths_total = link_widths_total.reindex(n.links.index).fillna(0.0) link_widths_total[n.links.p_nom_opt < line_lower_threshold] = 0.0 From e89027e9466c70ca47600fb543cd592d73848dec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 12:04:16 +0000 Subject: [PATCH 08/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/plot_hydrogen_network.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index d7e42ff8..6d666acd 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -115,7 +115,9 @@ def plot_h2_map(n, regions): retro_wo_new_i = h2_retro.index.difference(h2_new.index) h2_retro_wo_new = h2_retro.loc[retro_wo_new_i] - h2_retro_wo_new.index = h2_retro_wo_new.index_orig.apply(lambda x: x.split('-2')[0]) + h2_retro_wo_new.index = h2_retro_wo_new.index_orig.apply( + lambda x: x.split("-2")[0] + ) to_concat = [h2_new, h2_retro_w_new, h2_retro_wo_new] h2_total = pd.concat(to_concat).p_nom_opt.groupby(level=0).sum() @@ -128,8 +130,10 @@ def plot_h2_map(n, regions): n.links.rename(index=lambda x: x.split("-2")[0], inplace=True) # group links by summing up p_nom values and taking the first value of the rest of the columns other_cols = dict.fromkeys(n.links.columns.drop(["p_nom_opt", "p_nom"]), "first") - n.links = n.links.groupby(level=0).agg({"p_nom_opt": "sum", "p_nom": "sum", **other_cols}) - + n.links = n.links.groupby(level=0).agg( + {"p_nom_opt": "sum", "p_nom": "sum", **other_cols} + ) + link_widths_total = link_widths_total.reindex(n.links.index).fillna(0.0) link_widths_total[n.links.p_nom_opt < line_lower_threshold] = 0.0 From ca9556597ed7b1ca1e4287caace4f8c192c4f53d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Sep 2024 20:40:31 +0200 Subject: [PATCH 09/24] limit powerplantmatching version to below v0.6 due to upcoming breaking changes --- envs/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment.yaml b/envs/environment.yaml index 98cf858d..e40d07d8 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -24,7 +24,7 @@ dependencies: - yaml - pytables - lxml -- powerplantmatching>=0.5.15 +- powerplantmatching>=0.5.15,<0.6 - numpy - pandas>=2.1 - geopandas>=1 From ac606c1f67db517b13f8e6fb2b7735266ad0b899 Mon Sep 17 00:00:00 2001 From: Bobby Xiong <36541459+bobbyxng@users.noreply.github.com> Date: Tue, 10 Sep 2024 20:51:28 +0200 Subject: [PATCH 10/24] Build biomass transport costs from pre-extracted .csv instead of pdf (#1272) * Removed tabula dependency for building biomass transport costs. Instead use pre-extracted .csvs. Updated rule and script accordingly. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated script to optionally allow running the script with tabula-py dependency to recreate csv, if needed. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- data/biomass_transport_costs_supplychain1.csv | 41 ++++++++++++ data/biomass_transport_costs_supplychain2.csv | 41 ++++++++++++ envs/environment.yaml | 1 - rules/build_sector.smk | 6 +- scripts/build_biomass_transport_costs.py | 66 +++++++++++++------ 5 files changed, 129 insertions(+), 26 deletions(-) create mode 100644 data/biomass_transport_costs_supplychain1.csv create mode 100644 data/biomass_transport_costs_supplychain2.csv diff --git a/data/biomass_transport_costs_supplychain1.csv b/data/biomass_transport_costs_supplychain1.csv new file mode 100644 index 00000000..b5f5a0e3 --- /dev/null +++ b/data/biomass_transport_costs_supplychain1.csv @@ -0,0 +1,41 @@ +country,Loading,Loading,Loading,Transport to plant,Transport to plant,Unloading,TOTAL,TOTAL,TOTAL +country,Time,Machinery costs,Waiting time truck driver,60 average speed km/h,Trailer,Waiting time truck driver,Fix,per km,per km/ton +country,h,EUR,EUR.1,EUR/km,EUR.2,EUR.3,EUR.4,EUR/km.1,EUR/km/ton +BE,0.08,3.56,1.31,1.45,6.42,1.31,12.61,0.02,0.47 +BG,0.08,2.06,0.56,1.06,2.61,0.56,5.79,0.02,0.22 +CZ,0.08,2.34,0.62,1.1,4.08,0.62,7.65,0.02,0.29 +DK,0.08,5.14,2.54,1.85,7.91,2.54,18.13,0.03,0.68 +DE,0.08,3.41,1.31,1.37,5.99,1.31,12.01,0.02,0.45 +EE,0.08,2.26,0.5,1.1,4.14,0.5,7.38,0.02,0.28 +IE,0.08,3.29,1.03,1.4,6.29,1.03,11.64,0.02,0.44 +EL,0.08,2.83,0.85,1.25,5.17,0.85,9.7,0.02,0.37 +ES,0.08,3.19,1.22,1.32,5.28,1.22,10.91,0.02,0.41 +FR,0.08,3.52,1.35,1.38,6.49,1.35,12.71,0.02,0.48 +IT,0.08,3.59,1.26,1.56,5.82,1.26,11.94,0.03,0.45 +CY,0.08,2.82,0.85,1.25,5.09,0.85,9.6,0.02,0.36 +LV,0.08,2.11,0.43,1.05,3.87,0.43,6.84,0.02,0.26 +LT,0.08,1.99,0.39,1.02,3.49,0.39,6.25,0.02,0.24 +LU,0.08,3.38,1.21,1.31,6.97,1.21,12.78,0.02,0.48 +HU,0.08,2.12,0.42,1.12,3.33,0.42,6.28,0.02,0.24 +MT,0.08,2.44,0.62,1.16,4.32,0.62,7.99,0.02,0.3 +NL,0.08,3.83,1.57,1.51,6.37,1.57,13.33,0.03,0.5 +AT,0.08,3.33,1.18,1.35,6.37,1.18,12.06,0.02,0.45 +PL,0.08,2.27,0.68,1.08,3.35,0.68,6.98,0.02,0.27 +PT,0.08,2.3,0.37,1.16,4.67,0.37,7.71,0.02,0.29 +RO,0.08,1.88,0.4,0.99,2.8,0.4,5.49,0.02,0.21 +SI,0.08,2.47,0.58,1.18,4.65,0.58,8.29,0.02,0.32 +SK,0.08,2.24,0.48,1.12,3.93,0.48,7.13,0.02,0.27 +FI,0.08,3.59,1.2,1.48,7.0,1.2,12.98,0.02,0.49 +SE,0.08,3.84,1.26,1.57,7.71,1.26,14.07,0.03,0.53 +UK,0.08,3.65,1.21,1.56,6.56,1.21,12.64,0.03,0.48 +HR,0.08,2.12,0.46,1.05,3.79,0.46,6.84,0.02,0.26 +AL,0.08,1.67,0.19,0.96,2.55,0.19,4.61,0.02,0.18 +BA,0.08,1.66,0.23,0.91,2.81,0.23,4.92,0.02,0.19 +MK,0.08,1.5,0.22,0.84,2.32,0.22,4.27,0.01,0.17 +ME,0.08,1.71,0.24,0.94,2.83,0.24,5.02,0.02,0.2 +RS,0.08,1.69,0.22,0.96,2.64,0.22,4.77,0.02,0.19 +XK,0.08,1.62,0.21,0.89,2.81,0.21,4.85,0.01,0.19 +UA,0.08,1.25,0.13,0.65,2.8,0.13,4.32,0.01,0.16 +TR,0.08,2.35,0.52,1.24,3.44,0.52,6.82,0.02,0.26 +MD,0.08,1.33,0.09,0.74,2.8,0.09,4.31,0.01,0.17 +CH,0.08,3.79,1.18,1.47,8.9,1.18,15.06,0.02,0.56 diff --git a/data/biomass_transport_costs_supplychain2.csv b/data/biomass_transport_costs_supplychain2.csv new file mode 100644 index 00000000..530ff7d9 --- /dev/null +++ b/data/biomass_transport_costs_supplychain2.csv @@ -0,0 +1,41 @@ +country,Loading,Loading,Loading,Transport to plant,Transport to plant,Unloading,TOTAL,TOTAL,TOTAL +country,Time,Machinery costs,Waiting time truck driver,60 average speed km/h,Trailer,Waiting time truck driver,Fix,per km,per km/ton +country,h,EUR,EUR.1,EUR/km,EUR.2,EUR.3,EUR.4,EUR/km.1,EUR/km/ton +BE,0.05,3.06,0.88,1.49,16.57,0.88,21.39,0.02,0.88 +BG,0.05,1.77,0.37,1.07,6.74,0.37,9.26,0.02,0.39 +CZ,0.05,2.05,0.41,1.13,10.52,0.41,13.4,0.02,0.55 +DK,0.05,4.24,1.7,1.89,20.4,1.7,28.04,0.03,1.15 +DE,0.05,2.91,0.87,1.41,15.45,0.87,20.11,0.02,0.83 +EE,0.05,2.01,0.33,1.13,10.67,0.33,13.34,0.02,0.55 +IE,0.05,2.87,0.69,1.44,16.23,0.69,20.48,0.02,0.84 +EL,0.05,2.47,0.57,1.28,13.34,0.57,16.95,0.02,0.7 +ES,0.05,2.71,0.82,1.35,13.63,0.82,17.98,0.02,0.74 +FR,0.05,3.02,0.91,1.42,16.74,0.91,21.56,0.02,0.89 +IT,0.05,3.07,0.85,1.59,15.0,0.85,19.77,0.03,0.82 +CY,0.05,2.46,0.57,1.28,13.12,0.57,16.71,0.02,0.69 +LV,0.05,1.89,0.29,1.08,9.98,0.29,12.44,0.02,0.52 +LT,0.05,1.78,0.26,1.04,9.01,0.26,11.3,0.02,0.47 +LU,0.05,2.94,0.81,1.35,17.98,0.81,22.54,0.02,0.92 +HU,0.05,1.88,0.28,1.14,8.59,0.28,11.03,0.02,0.46 +MT,0.05,2.15,0.41,1.19,11.15,0.41,14.12,0.02,0.58 +NL,0.05,3.23,1.05,1.55,16.42,1.05,21.76,0.03,0.9 +AT,0.05,2.88,0.79,1.38,16.42,0.79,20.88,0.02,0.86 +PL,0.05,1.96,0.46,1.1,8.64,0.46,11.51,0.02,0.48 +PT,0.05,2.09,0.25,1.18,12.04,0.25,14.63,0.02,0.6 +RO,0.05,1.66,0.27,1.01,7.23,0.27,9.43,0.02,0.39 +SI,0.05,2.2,0.39,1.2,12.0,0.39,14.98,0.02,0.62 +SK,0.05,1.99,0.32,1.15,10.13,0.32,12.77,0.02,0.53 +FI,0.05,3.12,0.8,1.52,18.05,0.8,22.78,0.03,0.94 +SE,0.05,3.35,0.85,1.62,19.89,0.85,24.93,0.03,1.02 +UK,0.05,3.16,0.81,1.6,16.93,0.81,21.71,0.03,0.9 +HR,0.05,1.89,0.31,1.07,9.77,0.31,12.28,0.02,0.51 +AL,0.05,1.5,0.13,0.98,6.59,0.13,8.35,0.02,0.35 +BA,0.05,1.5,0.15,0.93,7.25,0.15,9.05,0.02,0.38 +MK,0.05,1.35,0.15,0.85,5.98,0.15,7.62,0.01,0.32 +ME,0.05,1.54,0.16,0.96,7.31,0.16,9.17,0.02,0.38 +RS,0.05,1.52,0.15,0.97,6.8,0.15,8.62,0.02,0.36 +XK,0.05,1.46,0.14,0.91,7.25,0.14,8.99,0.02,0.37 +UA,0.05,1.16,0.09,0.66,7.23,0.09,8.57,0.01,0.35 +TR,0.05,2.07,0.35,1.26,8.88,0.35,11.63,0.02,0.49 +MD,0.05,1.24,0.06,0.75,7.23,0.06,8.59,0.01,0.36 +CH,0.05,3.37,0.79,1.52,22.97,0.79,27.92,0.03,1.14 diff --git a/envs/environment.yaml b/envs/environment.yaml index e40d07d8..0b56c1ef 100644 --- a/envs/environment.yaml +++ b/envs/environment.yaml @@ -64,4 +64,3 @@ dependencies: - snakemake-executor-plugin-slurm - snakemake-executor-plugin-cluster-generic - highspy - - tabula-py diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 9f94dbbd..c719c338 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -435,10 +435,8 @@ rule build_biomass_potentials: rule build_biomass_transport_costs: input: - transport_cost_data=storage( - "https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass potentials in europe_web rev.pdf", - keep_local=True, - ), + sc1="data/biomass_transport_costs_supplychain1.csv", + sc2="data/biomass_transport_costs_supplychain2.csv", output: biomass_transport_costs=resources("biomass_transport_costs.csv"), threads: 1 diff --git a/scripts/build_biomass_transport_costs.py b/scripts/build_biomass_transport_costs.py index db369542..d1836230 100644 --- a/scripts/build_biomass_transport_costs.py +++ b/scripts/build_biomass_transport_costs.py @@ -16,33 +16,51 @@ assuming as an approximation energy content of wood pellets @author: bw0928 """ - -import platform - import pandas as pd -import tabula as tbl ENERGY_CONTENT = 4.8 # unit MWh/t (wood pellets) -system = platform.system() -encoding = "cp1252" if system == "Windows" else "utf-8" -def get_countries(): - pandas_options = dict( +def get_cost_per_tkm(pdf, datapage, countrypage): + """ + Extracts the cost tables from the JRC report PDF. + + https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass%20potentials%20in%20europe_web%20rev.pdf + - pdf (str): The filepath of the PDF file containing the data. + - datapage (int): The page number of the data table in the PDF. + - countrypage (int): The page number of the table containing the country indices in the PDF. + + Returns: + - pandas.DataFrame: The data table with the cost per tkm for biomass transport, indexed by country. + + Raises: + - ImportError: If tabula-py and platform are not installed. + """ + try: + import platform + + import tabula as tbl + except: + ImportError("Please install tabula-py and platform") + + system = platform.system() + encoding = "cp1252" if system == "Windows" else "utf-8" + + # Obtain countries: + pandas_options_country = dict( skiprows=range(6), header=None, index_col=0, encoding=encoding ) - return tbl.read_pdf( - str(snakemake.input.transport_cost_data), - pages="145", + countries = tbl.read_pdf( + pdf, + pages=countrypage, multiple_tables=False, - pandas_options=pandas_options, + pandas_options=pandas_options_country, encoding=encoding, )[0].index - -def get_cost_per_tkm(page, countries): - pandas_options = dict( + # Obtain data tables + pandas_options_data = dict( skiprows=range(6), header=0, sep=" |,", @@ -52,10 +70,10 @@ def get_cost_per_tkm(page, countries): ) sc = tbl.read_pdf( - str(snakemake.input.transport_cost_data), - pages=page, + pdf, + pages=datapage, multiple_tables=False, - pandas_options=pandas_options, + pandas_options=pandas_options_data, encoding=encoding, )[0] sc.index = countries @@ -65,10 +83,16 @@ def get_cost_per_tkm(page, countries): def build_biomass_transport_costs(): - countries = get_countries() + # Optional build from JRC report pdf, requires tabula and java dependencies. + # Update `pdf` path to the JRC report if needed. + # sc1 = get_cost_per_tkm(pdf = "report.pdf", datapage=146, countrypage=145) + # sc2 = get_cost_per_tkm(pdf = "report.pdf", datapage=147, countrypage=145) - sc1 = get_cost_per_tkm(146, countries) - sc2 = get_cost_per_tkm(147, countries) + # Use extracted csv from JRC report + # https://publications.jrc.ec.europa.eu/repository/bitstream/JRC98626/biomass%20potentials%20in%20europe_web%20rev.pdf + # Pages 146 (144) for supply chain 1 and 147 (145) for supply chain 2 + sc1 = pd.read_csv(snakemake.input.sc1, index_col=0, skiprows=2) + sc2 = pd.read_csv(snakemake.input.sc2, index_col=0, skiprows=2) # take mean of both supply chains to_concat = [sc1["EUR/km/ton"], sc2["EUR/km/ton"]] From 8c57a8037952dbf0bec0d6731cdd8f7de2c1d90a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Sep 2024 21:07:13 +0200 Subject: [PATCH 11/24] smk: use storage() only in combination with retrieve rules (#1274) * smk: use storage() only in combination with retrieve rules * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- doc/release_notes.rst | 4 +++ rules/build_sector.smk | 19 +++----------- rules/retrieve.smk | 59 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 97db4baf..e75d3344 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -17,6 +17,10 @@ Release Notes * bugfix: The oil generator was incorrectly dropped when the config `oil_refining_emissions` was greater than zero. This was the default behaviour in 0.12.0. +* Uses of Snakemake's ``storage()`` function are integrated into retrieval + rules. This simplifies the use of ``mock_snakemake`` and places downloaded + data more transparently into the ``data`` directory. + PyPSA-Eur 0.12.0 (30th August 2024) =================================== diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c719c338..1c80cd80 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -402,10 +402,7 @@ rule build_biomass_potentials: params: biomass=config_provider("biomass"), input: - enspreso_biomass=storage( - "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx", - keep_local=True, - ), + enspreso_biomass="data/ENSPRESO_BIOMASS.xlsx", eurostat="data/eurostat/Balances-April2023", nuts2="data/bundle/nuts/NUTS_RG_10M_2013_4326_LEVL_2.geojson", regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), @@ -458,10 +455,7 @@ rule build_sequestration_potentials: "sector", "regional_co2_sequestration_potential" ), input: - sequestration_potential=storage( - "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", - keep_local=True, - ), + sequestration_potential="data/omplete_map_2020_unit_Mt.geojson", regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: @@ -503,9 +497,7 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: input: - usgs=storage( - "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx" - ), + usgs="data/myb1-2022-nitro-ert.xlsx", output: ammonia_production=resources("ammonia_production.csv"), threads: 1 @@ -634,10 +626,7 @@ rule build_industrial_distribution_key: input: regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), clustered_pop_layout=resources("pop_layout_elec_s{simpl}_{clusters}.csv"), - hotmaps=storage( - "https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/raw/master/data/Industrial_Database.csv", - keep_local=True, - ), + hotmaps="data/Industrial_Database.csv", gem_gspt="data/gem/Global-Steel-Plant-Tracker-April-2024-Standard-Copy-V1.xlsx", ammonia="data/ammonia_plants.csv", cement_supplement="data/cement-plants-noneu.csv", diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 67b91b99..844618e0 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -191,6 +191,65 @@ if config["enable"]["retrieve"]: validate_checksum(output[0], input[0]) +if config["enable"]["retrieve"]: + + rule retrieve_jrc_enspreso_biomass: + input: + storage( + "https://zenodo.org/records/10356004/files/ENSPRESO_BIOMASS.xlsx", + keep_local=True, + ), + output: + "data/ENSPRESO_BIOMASS.xlsx", + retries: 1 + run: + move(input[0], output[0]) + + +if config["enable"]["retrieve"]: + + rule retrieve_hotmaps_industrial_sites: + input: + storage( + "https://gitlab.com/hotmaps/industrial_sites/industrial_sites_Industrial_Database/-/raw/master/data/Industrial_Database.csv", + keep_local=True, + ), + output: + "data/Industrial_Database.csv", + retries: 1 + run: + move(input[0], output[0]) + + +if config["enable"]["retrieve"]: + + rule retrieve_usgs_ammonia_production: + input: + storage( + "https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/media/files/myb1-2022-nitro-ert.xlsx" + ), + output: + "data/myb1-2022-nitro-ert.xlsx", + retries: 1 + run: + move(input[0], output[0]) + + +if config["enable"]["retrieve"]: + + rule retrieve_geological_co2_storage_potential: + input: + storage( + "https://raw.githubusercontent.com/ericzhou571/Co2Storage/main/resources/complete_map_2020_unit_Mt.geojson", + keep_local=True, + ), + output: + "data/complete_map_2020_unit_Mt.geojson", + retries: 1 + run: + move(input[0], output[0]) + + if config["enable"]["retrieve"]: # Downloading Copernicus Global Land Cover for land cover and land use: From f118d15a4bf9a886e5c90735c78b5249191666af Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Tue, 10 Sep 2024 21:54:24 +0200 Subject: [PATCH 12/24] fix typo --- rules/build_sector.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index 1c80cd80..ccb56aaa 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -455,7 +455,7 @@ rule build_sequestration_potentials: "sector", "regional_co2_sequestration_potential" ), input: - sequestration_potential="data/omplete_map_2020_unit_Mt.geojson", + sequestration_potential="data/complete_map_2020_unit_Mt.geojson", regions_onshore=resources("regions_onshore_elec_s{simpl}_{clusters}.geojson"), regions_offshore=resources("regions_offshore_elec_s{simpl}_{clusters}.geojson"), output: From b90f83a659105ee49316027be4dc6c1d09e27c17 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:35:31 +0200 Subject: [PATCH 13/24] simplify drop expression Co-authored-by: Fabian Neumann --- scripts/build_energy_totals.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3d37a8ee..4f096262 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -698,11 +698,9 @@ def build_energy_totals( [countries, eurostat_years], names=["country", "year"] ) - to_drop = idees.columns[ - idees.columns.str.contains("space efficiency") - ^ idees.columns.str.contains("water efficiency") - ] - to_drop = to_drop.append(pd.Index(["passenger cars", "passenger car efficiency"])) + efficiency_keywords = ["space efficiency", "water efficiency"] + to_drop = idees.columns[~idees.columns.str.contains('|'.join(efficiency_keywords))] + to_drop = to_drop.append(["passenger cars", "passenger car efficiency"]) df = idees.reindex(new_index).drop(to_drop, axis=1) From 163f8344817d43344d69e7b02a64f9fdcb1a13fc Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:35:50 +0200 Subject: [PATCH 14/24] remove snakemake.config Co-authored-by: Fabian Neumann --- scripts/add_existing_baseyear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 1635fab7..3757d4ba 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -711,9 +711,9 @@ if __name__ == "__main__": if options["heating"]: # one could use baseyear here instead (but dangerous if no data) - heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1, 0]) - ).loc[int(snakemake.config["energy"]["energy_totals_year"])] + fn = snakemake.input.heating_efficiencies + year = int(snakemake.params["energy_totals_year"]) + heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] add_heating_capacities_installed_before_baseyear( n=n, From 649e48e1a9805388ed7f92f8b97f5d8fd2c907fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 07:35:58 +0000 Subject: [PATCH 15/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 4f096262..72bc01b5 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -699,7 +699,7 @@ def build_energy_totals( ) efficiency_keywords = ["space efficiency", "water efficiency"] - to_drop = idees.columns[~idees.columns.str.contains('|'.join(efficiency_keywords))] + to_drop = idees.columns[~idees.columns.str.contains("|".join(efficiency_keywords))] to_drop = to_drop.append(["passenger cars", "passenger car efficiency"]) df = idees.reindex(new_index).drop(to_drop, axis=1) From 0ac6a6a1f2c07a5ee0ff8696e29de5e615fbbd35 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:38:07 +0200 Subject: [PATCH 16/24] simplify exclude statement Co-authored-by: Fabian Neumann --- scripts/build_energy_totals.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 72bc01b5..5fe405a6 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -612,11 +612,8 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh - exclude = ( - totals.columns.str.fullmatch("passenger cars") - ^ totals.columns.str.fullmatch(".*space efficiency") - ^ totals.columns.str.fullmatch(".*water efficiency") - ) + patterns = ["passenger cars", ".*space efficiency", ".*water efficiency"] + exclude = totals.columns.str.fullmatch("|".join(patterns)) totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 From 82a669635b7b12ad65d8dd9c3d91ad1ea42745e5 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:38:21 +0200 Subject: [PATCH 17/24] removing logging Co-authored-by: Fabian Neumann --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5fe405a6..ce5bf7b0 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1572,7 +1572,6 @@ def build_heating_efficiencies( ^ idees.columns.str.contains("water efficiency") ] - logger.info(cols) heating_efficiencies = pd.DataFrame(idees[cols]) From 0bdf90280bc65a805b20013e08d4b75c7b0eea6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 07:38:40 +0000 Subject: [PATCH 18/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index ce5bf7b0..b09ba080 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1572,7 +1572,6 @@ def build_heating_efficiencies( ^ idees.columns.str.contains("water efficiency") ] - heating_efficiencies = pd.DataFrame(idees[cols]) new_index = pd.MultiIndex.from_product( From c0172117004c4d7bf2d6eb195387219af5dcdd61 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 10:16:09 +0200 Subject: [PATCH 19/24] use snakemake.params --- rules/build_sector.smk | 1 + scripts/prepare_sector_network.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index db7084a9..cb3db79c 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1027,6 +1027,7 @@ rule prepare_sector_network: RDIR=RDIR, heat_pump_sources=config_provider("sector", "heat_pump_sources"), heat_systems=config_provider("sector", "heat_systems"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: unpack(input_profile_offwind), **rules.cluster_gas_network.output, diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 4aac5801..4ddcac32 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -4378,11 +4378,9 @@ if __name__ == "__main__": ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) - heating_efficiencies = ( - (pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0, 1])) - .swaplevel() - .loc[int(snakemake.config["energy"]["energy_totals_year"])] - ) + fn = snakemake.input.heating_efficiencies + year = int(snakemake.params["energy_totals_year"]) + heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] patch_electricity_network(n) From 4315b3a18ac98f50f16610de0ae8656900b96a6a Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 14:09:23 +0200 Subject: [PATCH 20/24] fix dropping bug --- scripts/build_energy_totals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index b09ba080..6afa6b33 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -696,8 +696,8 @@ def build_energy_totals( ) efficiency_keywords = ["space efficiency", "water efficiency"] - to_drop = idees.columns[~idees.columns.str.contains("|".join(efficiency_keywords))] - to_drop = to_drop.append(["passenger cars", "passenger car efficiency"]) + to_drop = idees.columns[idees.columns.str.contains("|".join(efficiency_keywords))] + to_drop = to_drop.append(pd.Index(["passenger cars", "passenger car efficiency"])) df = idees.reindex(new_index).drop(to_drop, axis=1) From 6a1438cffdf819a15c8191c1d5fcc7277cb64419 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 15:06:17 +0200 Subject: [PATCH 21/24] fix heating_system.sector bug --- scripts/add_existing_baseyear.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 3757d4ba..53d8e390 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -446,16 +446,18 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): - For residential and services sectors, efficiency is mapped based on the nodes. - For other sectors, the default boiler efficiency is retrieved from the `costs` database. """ - - if heat_system.sector.value == "residential": + + if heat_system.value == "urban central": + boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") + efficiency = costs.at[boiler_costs_name, "efficiency"] + elif heat_system.sector.value == "residential": key = f"{carrier} residential space efficiency" efficiency = nodes.str[:2].map(heating_efficiencies[key]) elif heat_system.sector.value == "services": key = f"{carrier} services space efficiency" efficiency = nodes.str[:2].map(heating_efficiencies[key]) else: - boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") - efficiency = costs.at[boiler_costs_name, "efficiency"] + logger.warning(f"{heat_system} not defined.") return efficiency @@ -671,13 +673,13 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_existing_baseyear", - configfiles="config/config.yaml", + configfiles="config/test/config.myopic.yaml", simpl="", - clusters="38", - ll="vopt", + clusters="5", + ll="v1.5", opts="", sector_opts="", - planning_horizons=2020, + planning_horizons=2030, ) configure_logging(snakemake) From b7bcf3d30204b9cc4c4518c283bc4542de4d9cca Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 15:06:32 +0200 Subject: [PATCH 22/24] add energy totals year to params --- rules/solve_myopic.smk | 1 + rules/solve_perfect.smk | 1 + 2 files changed, 2 insertions(+) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f868a997..6340787a 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -10,6 +10,7 @@ rule add_existing_baseyear: existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 4db4e31a..f1ec9966 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -8,6 +8,7 @@ rule add_existing_baseyear: existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From a57e39b3e716a787785b088d647c9cbec2c27ce1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:09:51 +0000 Subject: [PATCH 23/24] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 53d8e390..84d20b72 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -446,7 +446,7 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): - For residential and services sectors, efficiency is mapped based on the nodes. - For other sectors, the default boiler efficiency is retrieved from the `costs` database. """ - + if heat_system.value == "urban central": boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") efficiency = costs.at[boiler_costs_name, "efficiency"] From d9ab4940804c180d70442e5281012c53eaa441b8 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 15:15:22 +0200 Subject: [PATCH 24/24] add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e75d3344..7340fe31 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,8 @@ Release Notes .. Upcoming Release +* Change the heating demand from final energy which includes losses in legacy equipment to thermal energy service based on JRC-IDEES. Efficiencies of existing heating capacities are lowered according to the conversion of final energy to thermal energy service. For overnight scenarios or future planning horizon this change leads to a reduction in heat supply. + * Updated district heating supply temperatures based on `Euroheat's DHC Market Outlook 2024`__ and `AGFW-Hauptbericht 2022 `__. `min_forward_temperature` and `return_temperature` (not given by Euroheat) are extrapolated based on German values. * Made the overdimensioning factor for heating systems specific for central/decentral heating, defaults to no overdimensionining for central heating and no changes to decentral heating compared to previous version.