From 0bcb21559704accdc5900f536c4899489c132676 Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Mon, 2 Sep 2024 19:32:13 +0200 Subject: [PATCH 01/18] use JRC-IDEES thermal energy service instead of FE for buildings This change consists of: - reading the final energy (FE) to thermal energy service (TES) efficiency for each each country, seperately for gas and oil (this efficiency represents e.g. the losses in older non-condensing boilers) - using TES instead of FE for the n.loads, so that it is pure energy service instead of including losses in legacy equipment - using the stored efficiencies for baseyear equipment in add_existing_baseyear In the baseyear (e.g. 2020) this should have no effect on FE, since the reduction to TES is conpensated by the lower efficiencies of existing equipment. In the future (e.g. 2050) this leads to a reduction in heating demand, since new equipment is more efficient than existing. --- rules/build_sector.smk | 2 + rules/solve_myopic.smk | 1 + scripts/add_existing_baseyear.py | 25 +++++++- scripts/build_energy_totals.py | 101 +++++++++++++++++++++++++++++- scripts/prepare_sector_network.py | 11 +++- 5 files changed, 135 insertions(+), 5 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index f95731b7..ec7fcd0c 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -305,6 +305,7 @@ rule build_energy_totals: co2_name=resources("co2_totals.csv"), transport_name=resources("transport_data.csv"), district_heat_share=resources("district_heat_share.csv"), + heating_efficiencies=resources("heating_efficiencies.csv"), threads: 16 resources: mem_mb=10000, @@ -1037,6 +1038,7 @@ rule prepare_sector_network: district_heat_share=resources( "district_heat_share_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), + heating_efficiencies=resources("heating_efficiencies.csv"), temp_soil_total=resources("temp_soil_total_elec_s{simpl}_{clusters}.nc"), temp_air_total=resources("temp_air_total_elec_s{simpl}_{clusters}.nc"), cop_profiles=resources("cop_profiles_elec_s{simpl}_{clusters}.nc"), diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index 8d7fa284..f868a997 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -26,6 +26,7 @@ rule add_existing_baseyear: existing_heating_distribution=resources( "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), + heating_efficiencies=resources("heating_efficiencies.csv"), output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 212ae8af..7e9dd082 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -546,6 +546,14 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) + if "residential" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["gas residential space efficiency"]) + elif "services" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["gas services space efficiency"]) + else: + #default used for urban central, since no info on district heating boilers + efficiency = costs.at[heat_system.gas_boiler_costs_name, "efficiency"] + n.madd( "Link", nodes, @@ -554,7 +562,7 @@ def add_heating_capacities_installed_before_baseyear( bus1=nodes + " " + heat_system.value + " heat", bus2="co2 atmosphere", carrier=heat_system.value + " gas boiler", - efficiency=costs.at[heat_system.gas_boiler_costs_name, "efficiency"], + efficiency=efficiency, efficiency2=costs.at["gas", "CO2 intensity"], capital_cost=( costs.at[heat_system.gas_boiler_costs_name, "efficiency"] @@ -569,6 +577,14 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) + if "residential" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["oil residential space efficiency"]) + elif "services" in heat_system.value: + efficiency = nodes.str[:2].map(heating_efficiencies["oil services space efficiency"]) + else: + #default used for urban central, since no info on district heating boilers + efficiency = costs.at[heat_system.oil_boiler_costs_name, "efficiency"] + n.madd( "Link", nodes, @@ -577,7 +593,7 @@ def add_heating_capacities_installed_before_baseyear( bus1=nodes + " " + heat_system.value + " heat", bus2="co2 atmosphere", carrier=heat_system.value + " oil boiler", - efficiency=costs.at[heat_system.oil_boiler_costs_name, "efficiency"], + efficiency=efficiency, efficiency2=costs.at["oil", "CO2 intensity"], capital_cost=costs.at[heat_system.oil_boiler_costs_name, "efficiency"] * costs.at[heat_system.oil_boiler_costs_name, "fixed"], @@ -660,6 +676,11 @@ if __name__ == "__main__": if options["heating"]: + #one could use baseyear here instead (but dangerous if no data) + heating_efficiencies = ( + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) + ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + add_heating_capacities_installed_before_baseyear( n=n, baseyear=baseyear, diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 24ba86bb..73c3d1a0 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -367,6 +367,24 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[43] == "Thermal uses" ct_totals["thermal uses residential"] = df.iloc[43] + df = pd.read_excel(fn_residential, "RES_hh_eff", index_col=0) + + ct_totals["total residential space efficiency"] = df.loc["Space heating"] + + assert df.index[5] == "Diesel oil" + ct_totals["oil residential space efficiency"] = df.iloc[5] + + assert df.index[6] == "Natural gas" + ct_totals["gas residential space efficiency"] = df.iloc[6] + + ct_totals["total residential water efficiency"] = df.loc["Water heating"] + + assert df.index[18] == "Diesel oil" + ct_totals["oil residential water efficiency"] = df.iloc[18] + + assert df.index[19] == "Natural gas" + ct_totals["gas residential water efficiency"] = df.iloc[19] + # services df = pd.read_excel(fn_tertiary, "SER_hh_fec", index_col=0) @@ -400,6 +418,24 @@ def idees_per_country(ct: str, base_dir: str) -> pd.DataFrame: assert df.index[46] == "Thermal uses" ct_totals["thermal uses services"] = df.iloc[46] + df = pd.read_excel(fn_tertiary, "SER_hh_eff", index_col=0) + + ct_totals["total services space efficiency"] = df.loc["Space heating"] + + assert df.index[5] == "Diesel oil" + ct_totals["oil services space efficiency"] = df.iloc[5] + + assert df.index[7] == "Conventional gas heaters" + ct_totals["gas services space efficiency"] = df.iloc[7] + + ct_totals["total services water efficiency"] = df.loc["Hot water"] + + assert df.index[20] == "Diesel oil" + ct_totals["oil services water efficiency"] = df.iloc[20] + + assert df.index[21] == "Natural gas" + ct_totals["gas services water efficiency"] = df.iloc[21] + # agriculture, forestry and fishing start = "Detailed split of energy consumption (ktoe)" @@ -576,7 +612,9 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh - exclude = totals.columns.str.fullmatch("passenger cars") + exclude = totals.columns.str.fullmatch("passenger cars") \ + ^ totals.columns.str.fullmatch(".*space efficiency") \ + ^ totals.columns.str.fullmatch(".*water efficiency") totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 @@ -654,11 +692,16 @@ def build_energy_totals( eurostat_countries = eurostat.index.unique(0) eurostat_years = eurostat.index.unique(1) - to_drop = ["passenger cars", "passenger car efficiency"] new_index = pd.MultiIndex.from_product( [countries, eurostat_years], names=["country", "year"] ) + to_drop = idees.columns[idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency")] + to_drop = to_drop.append(pd.Index( + ["passenger cars", "passenger car efficiency"] + )) + df = idees.reindex(new_index).drop(to_drop, axis=1) in_eurostat = df.index.levels[0].intersection(eurostat_countries) @@ -1500,6 +1543,57 @@ def build_transformation_output_coke(eurostat, fn): df = eurostat.loc[slicer, :].droplevel(level=[2, 3, 4, 5]) df.to_csv(fn) +def build_heating_efficiencies( + countries: List[str], idees: pd.DataFrame +) -> pd.DataFrame: + """ + Build heating efficiencies for a set of countries based on IDEES data. + + Parameters + ---------- + countries : List[str] + List of country codes. + idees : pd.DataFrame + DataFrame with IDEES data. + + Returns + ------- + pd.DataFrame + DataFrame with heating efficiencies. + + + Notes + ----- + - It fills missing data with average data. + """ + + years = np.arange(2000, 2022) + + cols = idees.columns[idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency")] + + logger.info(cols) + + heating_efficiencies = pd.DataFrame(idees[cols]) + + new_index = pd.MultiIndex.from_product( + [countries, heating_efficiencies.index.unique(1)], + names=["country", "year"], + ) + + heating_efficiencies = heating_efficiencies.reindex(index=new_index) + + for col in cols: + unstacked = heating_efficiencies[col].unstack() + + fillvalue = unstacked.mean() + + for ct in unstacked.index: + mask = unstacked.loc[ct].isna() + unstacked.loc[ct, mask] = fillvalue[mask] + heating_efficiencies[col] = unstacked.stack() + + return heating_efficiencies # %% if __name__ == "__main__": @@ -1556,3 +1650,6 @@ if __name__ == "__main__": transport = build_transport_data(countries, population, idees) transport.to_csv(snakemake.output.transport_name) + + heating_efficiencies = build_heating_efficiencies(countries, idees) + heating_efficiencies.to_csv(snakemake.output.heating_efficiencies) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 87aa90fa..f303c10c 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1806,9 +1806,14 @@ def build_heat_demand(n): for sector, use in product(sectors, uses): name = f"{sector} {use}" + #efficiency for final energy to thermal energy service + eff = pop_weighted_energy_totals.index.str[:2].map( + heating_efficiencies[f"total {sector} {use} efficiency"] + ) + heat_demand[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() - ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]) * 1e6 + ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]*eff) * 1e6 electric_heat_supply[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() ).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 @@ -4282,6 +4287,10 @@ if __name__ == "__main__": ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) + heating_efficiencies = ( + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) + ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + patch_electricity_network(n) spatial = define_spatial(pop_layout.index, options) From ff943d87ada4584e7dc3fe22fa8edb8f7d63c6bb Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Tue, 3 Sep 2024 13:34:27 +0200 Subject: [PATCH 02/18] add missing snakemake input --- rules/solve_perfect.smk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index a06c6dfa..4db4e31a 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -25,6 +25,7 @@ rule add_existing_baseyear: "existing_heating_distribution_elec_s{simpl}_{clusters}_{planning_horizons}.csv" ), existing_heating="data/existing_infrastructure/existing_heating_raw.csv", + heating_efficiencies=resources("heating_efficiencies.csv"), output: RESULTS + "prenetworks-brownfield/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From 93ec44de77d09f328f71d57545f680bd708f1370 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 9 Sep 2024 18:10:59 +0200 Subject: [PATCH 03/18] use heat_system.sector --- scripts/add_existing_baseyear.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 7e9dd082..34114a66 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -546,9 +546,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) - if "residential" in heat_system.value: + if "residential"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["gas residential space efficiency"]) - elif "services" in heat_system.value: + elif "services"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["gas services space efficiency"]) else: #default used for urban central, since no info on district heating boilers @@ -577,9 +577,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) - if "residential" in heat_system.value: + if "residential"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["oil residential space efficiency"]) - elif "services" in heat_system.value: + elif "services"==heat_system.sector.value: efficiency = nodes.str[:2].map(heating_efficiencies["oil services space efficiency"]) else: #default used for urban central, since no info on district heating boilers @@ -639,11 +639,11 @@ if __name__ == "__main__": "add_existing_baseyear", configfiles="config/config.yaml", simpl="", - clusters="20", - ll="v1.5", + clusters="38", + ll="vopt", opts="", - sector_opts="none", - planning_horizons=2030, + sector_opts="", + planning_horizons=2020, ) configure_logging(snakemake) @@ -676,7 +676,7 @@ if __name__ == "__main__": if options["heating"]: - #one could use baseyear here instead (but dangerous if no data) + # one could use baseyear here instead (but dangerous if no data) heating_efficiencies = ( pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] From 59984d06003f120de9efba68e0639586749a6db9 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 9 Sep 2024 18:19:18 +0200 Subject: [PATCH 04/18] add function for efficiency --- scripts/add_existing_baseyear.py | 58 ++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 34114a66..10277ad5 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -418,6 +418,46 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas ] +def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): + """ + Computes the heating system efficiency based on the sector and carrier type. + + Parameters: + ----------- + heat_system : object + carrier : str + The type of fuel or energy carrier (e.g., 'gas', 'oil'). + nodes : pandas.Series + A pandas Series containing node information used to match the heating efficiency data. + heating_efficiencies : dict + A dictionary containing efficiency values for different carriers and sectors. + costs : pandas.DataFrame + A DataFrame containing boiler cost and efficiency data for different heating systems. + + Returns: + -------- + efficiency : pandas.Series or float + A pandas Series mapping the efficiencies based on nodes for residential and services sectors, or a single + efficiency value for other heating systems (e.g., urban central). + + Notes: + ------ + - For residential and services sectors, efficiency is mapped based on the nodes. + - For other sectors, the default boiler efficiency is retrieved from the `costs` database. + """ + + if heat_system.sector.value == "residential": + key = f"{carrier} residential space efficiency" + efficiency = nodes.str[:2].map(heating_efficiencies[key]) + elif heat_system.sector.value == "services": + key = f"{carrier} services space efficiency" + efficiency = nodes.str[:2].map(heating_efficiencies[key]) + else: + boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") + efficiency = costs.at[boiler_costs_name, "efficiency"] + + return efficiency + def add_heating_capacities_installed_before_baseyear( n: pypsa.Network, baseyear: int, @@ -546,13 +586,8 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) - if "residential"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["gas residential space efficiency"]) - elif "services"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["gas services space efficiency"]) - else: - #default used for urban central, since no info on district heating boilers - efficiency = costs.at[heat_system.gas_boiler_costs_name, "efficiency"] + efficiency = get_efficiency(heat_system, "gas", nodes, + heating_efficiencies, costs) n.madd( "Link", @@ -577,13 +612,8 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) - if "residential"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["oil residential space efficiency"]) - elif "services"==heat_system.sector.value: - efficiency = nodes.str[:2].map(heating_efficiencies["oil services space efficiency"]) - else: - #default used for urban central, since no info on district heating boilers - efficiency = costs.at[heat_system.oil_boiler_costs_name, "efficiency"] + efficiency = get_efficiency(heat_system, "oil", nodes, + heating_efficiencies, costs) n.madd( "Link", From 861054961ac4e126b9762a031249c457458e2af5 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Mon, 9 Sep 2024 18:20:30 +0200 Subject: [PATCH 05/18] simplify expression --- scripts/add_existing_baseyear.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 10277ad5..c34de362 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -708,8 +708,8 @@ if __name__ == "__main__": # one could use baseyear here instead (but dangerous if no data) heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) - ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1,0]) + ).loc[int(snakemake.config["energy"]["energy_totals_year"])] add_heating_capacities_installed_before_baseyear( n=n, From 5465d8cc75455b3a220d5e7526853a968b173be2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:28:07 +0000 Subject: [PATCH 06/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 18 +++++++++++------- scripts/build_energy_totals.py | 24 +++++++++++++++--------- scripts/prepare_sector_network.py | 12 +++++++----- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index c34de362..1635fab7 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -420,7 +420,8 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): """ - Computes the heating system efficiency based on the sector and carrier type. + Computes the heating system efficiency based on the sector and carrier + type. Parameters: ----------- @@ -445,7 +446,7 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): - For residential and services sectors, efficiency is mapped based on the nodes. - For other sectors, the default boiler efficiency is retrieved from the `costs` database. """ - + if heat_system.sector.value == "residential": key = f"{carrier} residential space efficiency" efficiency = nodes.str[:2].map(heating_efficiencies[key]) @@ -458,6 +459,7 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): return efficiency + def add_heating_capacities_installed_before_baseyear( n: pypsa.Network, baseyear: int, @@ -586,8 +588,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.resistive_heater_costs_name, "lifetime"], ) - efficiency = get_efficiency(heat_system, "gas", nodes, - heating_efficiencies, costs) + efficiency = get_efficiency( + heat_system, "gas", nodes, heating_efficiencies, costs + ) n.madd( "Link", @@ -612,8 +615,9 @@ def add_heating_capacities_installed_before_baseyear( lifetime=costs.at[heat_system.gas_boiler_costs_name, "lifetime"], ) - efficiency = get_efficiency(heat_system, "oil", nodes, - heating_efficiencies, costs) + efficiency = get_efficiency( + heat_system, "oil", nodes, heating_efficiencies, costs + ) n.madd( "Link", @@ -708,7 +712,7 @@ if __name__ == "__main__": # one could use baseyear here instead (but dangerous if no data) heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1,0]) + pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1, 0]) ).loc[int(snakemake.config["energy"]["energy_totals_year"])] add_heating_capacities_installed_before_baseyear( diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 73c3d1a0..a4064af6 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -612,9 +612,11 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh - exclude = totals.columns.str.fullmatch("passenger cars") \ - ^ totals.columns.str.fullmatch(".*space efficiency") \ + exclude = ( + totals.columns.str.fullmatch("passenger cars") + ^ totals.columns.str.fullmatch(".*space efficiency") ^ totals.columns.str.fullmatch(".*water efficiency") + ) totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 @@ -696,11 +698,11 @@ def build_energy_totals( [countries, eurostat_years], names=["country", "year"] ) - to_drop = idees.columns[idees.columns.str.contains("space efficiency") - ^ idees.columns.str.contains("water efficiency")] - to_drop = to_drop.append(pd.Index( - ["passenger cars", "passenger car efficiency"] - )) + to_drop = idees.columns[ + idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency") + ] + to_drop = to_drop.append(pd.Index(["passenger cars", "passenger car efficiency"])) df = idees.reindex(new_index).drop(to_drop, axis=1) @@ -1543,6 +1545,7 @@ def build_transformation_output_coke(eurostat, fn): df = eurostat.loc[slicer, :].droplevel(level=[2, 3, 4, 5]) df.to_csv(fn) + def build_heating_efficiencies( countries: List[str], idees: pd.DataFrame ) -> pd.DataFrame: @@ -1569,8 +1572,10 @@ def build_heating_efficiencies( years = np.arange(2000, 2022) - cols = idees.columns[idees.columns.str.contains("space efficiency") - ^ idees.columns.str.contains("water efficiency")] + cols = idees.columns[ + idees.columns.str.contains("space efficiency") + ^ idees.columns.str.contains("water efficiency") + ] logger.info(cols) @@ -1595,6 +1600,7 @@ def build_heating_efficiencies( return heating_efficiencies + # %% if __name__ == "__main__": if "snakemake" not in globals(): diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index f303c10c..097f9bb5 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1806,14 +1806,14 @@ def build_heat_demand(n): for sector, use in product(sectors, uses): name = f"{sector} {use}" - #efficiency for final energy to thermal energy service + # efficiency for final energy to thermal energy service eff = pop_weighted_energy_totals.index.str[:2].map( heating_efficiencies[f"total {sector} {use} efficiency"] - ) + ) heat_demand[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() - ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"]*eff) * 1e6 + ).multiply(pop_weighted_energy_totals[f"total {sector} {use}"] * eff) * 1e6 electric_heat_supply[name] = ( heat_demand_shape[name] / heat_demand_shape[name].sum() ).multiply(pop_weighted_energy_totals[f"electricity {sector} {use}"]) * 1e6 @@ -4288,8 +4288,10 @@ if __name__ == "__main__": pop_weighted_energy_totals.update(pop_weighted_heat_totals) heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0,1]) - ).swaplevel().loc[int(snakemake.config["energy"]["energy_totals_year"])] + (pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0, 1])) + .swaplevel() + .loc[int(snakemake.config["energy"]["energy_totals_year"])] + ) patch_electricity_network(n) From b90f83a659105ee49316027be4dc6c1d09e27c17 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:35:31 +0200 Subject: [PATCH 07/18] simplify drop expression Co-authored-by: Fabian Neumann --- scripts/build_energy_totals.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 3d37a8ee..4f096262 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -698,11 +698,9 @@ def build_energy_totals( [countries, eurostat_years], names=["country", "year"] ) - to_drop = idees.columns[ - idees.columns.str.contains("space efficiency") - ^ idees.columns.str.contains("water efficiency") - ] - to_drop = to_drop.append(pd.Index(["passenger cars", "passenger car efficiency"])) + efficiency_keywords = ["space efficiency", "water efficiency"] + to_drop = idees.columns[~idees.columns.str.contains('|'.join(efficiency_keywords))] + to_drop = to_drop.append(["passenger cars", "passenger car efficiency"]) df = idees.reindex(new_index).drop(to_drop, axis=1) From 163f8344817d43344d69e7b02a64f9fdcb1a13fc Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:35:50 +0200 Subject: [PATCH 08/18] remove snakemake.config Co-authored-by: Fabian Neumann --- scripts/add_existing_baseyear.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 1635fab7..3757d4ba 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -711,9 +711,9 @@ if __name__ == "__main__": if options["heating"]: # one could use baseyear here instead (but dangerous if no data) - heating_efficiencies = ( - pd.read_csv(snakemake.input.heating_efficiencies, index_col=[1, 0]) - ).loc[int(snakemake.config["energy"]["energy_totals_year"])] + fn = snakemake.input.heating_efficiencies + year = int(snakemake.params["energy_totals_year"]) + heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] add_heating_capacities_installed_before_baseyear( n=n, From 649e48e1a9805388ed7f92f8b97f5d8fd2c907fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 07:35:58 +0000 Subject: [PATCH 09/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 4f096262..72bc01b5 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -699,7 +699,7 @@ def build_energy_totals( ) efficiency_keywords = ["space efficiency", "water efficiency"] - to_drop = idees.columns[~idees.columns.str.contains('|'.join(efficiency_keywords))] + to_drop = idees.columns[~idees.columns.str.contains("|".join(efficiency_keywords))] to_drop = to_drop.append(["passenger cars", "passenger car efficiency"]) df = idees.reindex(new_index).drop(to_drop, axis=1) From 0ac6a6a1f2c07a5ee0ff8696e29de5e615fbbd35 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:38:07 +0200 Subject: [PATCH 10/18] simplify exclude statement Co-authored-by: Fabian Neumann --- scripts/build_energy_totals.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 72bc01b5..5fe405a6 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -612,11 +612,8 @@ def build_idees(countries: List[str]) -> pd.DataFrame: # efficiency kgoe/100km -> ktoe/100km so that after conversion TWh/100km totals.loc[:, "passenger car efficiency"] /= 1e6 # convert ktoe to TWh - exclude = ( - totals.columns.str.fullmatch("passenger cars") - ^ totals.columns.str.fullmatch(".*space efficiency") - ^ totals.columns.str.fullmatch(".*water efficiency") - ) + patterns = ["passenger cars", ".*space efficiency", ".*water efficiency"] + exclude = totals.columns.str.fullmatch("|".join(patterns)) totals = totals.copy() totals.loc[:, ~exclude] *= 11.63 / 1e3 From 82a669635b7b12ad65d8dd9c3d91ad1ea42745e5 Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:38:21 +0200 Subject: [PATCH 11/18] removing logging Co-authored-by: Fabian Neumann --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 5fe405a6..ce5bf7b0 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1572,7 +1572,6 @@ def build_heating_efficiencies( ^ idees.columns.str.contains("water efficiency") ] - logger.info(cols) heating_efficiencies = pd.DataFrame(idees[cols]) From 0bdf90280bc65a805b20013e08d4b75c7b0eea6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 07:38:40 +0000 Subject: [PATCH 12/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_energy_totals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index ce5bf7b0..b09ba080 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1572,7 +1572,6 @@ def build_heating_efficiencies( ^ idees.columns.str.contains("water efficiency") ] - heating_efficiencies = pd.DataFrame(idees[cols]) new_index = pd.MultiIndex.from_product( From c0172117004c4d7bf2d6eb195387219af5dcdd61 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 10:16:09 +0200 Subject: [PATCH 13/18] use snakemake.params --- rules/build_sector.smk | 1 + scripts/prepare_sector_network.py | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index db7084a9..cb3db79c 100755 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -1027,6 +1027,7 @@ rule prepare_sector_network: RDIR=RDIR, heat_pump_sources=config_provider("sector", "heat_pump_sources"), heat_systems=config_provider("sector", "heat_systems"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: unpack(input_profile_offwind), **rules.cluster_gas_network.output, diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 4aac5801..4ddcac32 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -4378,11 +4378,9 @@ if __name__ == "__main__": ) pop_weighted_energy_totals.update(pop_weighted_heat_totals) - heating_efficiencies = ( - (pd.read_csv(snakemake.input.heating_efficiencies, index_col=[0, 1])) - .swaplevel() - .loc[int(snakemake.config["energy"]["energy_totals_year"])] - ) + fn = snakemake.input.heating_efficiencies + year = int(snakemake.params["energy_totals_year"]) + heating_efficiencies = pd.read_csv(fn, index_col=[1, 0]).loc[year] patch_electricity_network(n) From 4315b3a18ac98f50f16610de0ae8656900b96a6a Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 14:09:23 +0200 Subject: [PATCH 14/18] fix dropping bug --- scripts/build_energy_totals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index b09ba080..6afa6b33 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -696,8 +696,8 @@ def build_energy_totals( ) efficiency_keywords = ["space efficiency", "water efficiency"] - to_drop = idees.columns[~idees.columns.str.contains("|".join(efficiency_keywords))] - to_drop = to_drop.append(["passenger cars", "passenger car efficiency"]) + to_drop = idees.columns[idees.columns.str.contains("|".join(efficiency_keywords))] + to_drop = to_drop.append(pd.Index(["passenger cars", "passenger car efficiency"])) df = idees.reindex(new_index).drop(to_drop, axis=1) From 6a1438cffdf819a15c8191c1d5fcc7277cb64419 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 15:06:17 +0200 Subject: [PATCH 15/18] fix heating_system.sector bug --- scripts/add_existing_baseyear.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 3757d4ba..53d8e390 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -446,16 +446,18 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): - For residential and services sectors, efficiency is mapped based on the nodes. - For other sectors, the default boiler efficiency is retrieved from the `costs` database. """ - - if heat_system.sector.value == "residential": + + if heat_system.value == "urban central": + boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") + efficiency = costs.at[boiler_costs_name, "efficiency"] + elif heat_system.sector.value == "residential": key = f"{carrier} residential space efficiency" efficiency = nodes.str[:2].map(heating_efficiencies[key]) elif heat_system.sector.value == "services": key = f"{carrier} services space efficiency" efficiency = nodes.str[:2].map(heating_efficiencies[key]) else: - boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") - efficiency = costs.at[boiler_costs_name, "efficiency"] + logger.warning(f"{heat_system} not defined.") return efficiency @@ -671,13 +673,13 @@ if __name__ == "__main__": snakemake = mock_snakemake( "add_existing_baseyear", - configfiles="config/config.yaml", + configfiles="config/test/config.myopic.yaml", simpl="", - clusters="38", - ll="vopt", + clusters="5", + ll="v1.5", opts="", sector_opts="", - planning_horizons=2020, + planning_horizons=2030, ) configure_logging(snakemake) From b7bcf3d30204b9cc4c4518c283bc4542de4d9cca Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 15:06:32 +0200 Subject: [PATCH 16/18] add energy totals year to params --- rules/solve_myopic.smk | 1 + rules/solve_perfect.smk | 1 + 2 files changed, 2 insertions(+) diff --git a/rules/solve_myopic.smk b/rules/solve_myopic.smk index f868a997..6340787a 100644 --- a/rules/solve_myopic.smk +++ b/rules/solve_myopic.smk @@ -10,6 +10,7 @@ rule add_existing_baseyear: existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", diff --git a/rules/solve_perfect.smk b/rules/solve_perfect.smk index 4db4e31a..f1ec9966 100644 --- a/rules/solve_perfect.smk +++ b/rules/solve_perfect.smk @@ -8,6 +8,7 @@ rule add_existing_baseyear: existing_capacities=config_provider("existing_capacities"), costs=config_provider("costs"), heat_pump_sources=config_provider("sector", "heat_pump_sources"), + energy_totals_year=config_provider("energy", "energy_totals_year"), input: network=RESULTS + "prenetworks/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.nc", From a57e39b3e716a787785b088d647c9cbec2c27ce1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:09:51 +0000 Subject: [PATCH 17/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_existing_baseyear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 53d8e390..84d20b72 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -446,7 +446,7 @@ def get_efficiency(heat_system, carrier, nodes, heating_efficiencies, costs): - For residential and services sectors, efficiency is mapped based on the nodes. - For other sectors, the default boiler efficiency is retrieved from the `costs` database. """ - + if heat_system.value == "urban central": boiler_costs_name = getattr(heat_system, f"{carrier}_boiler_costs_name") efficiency = costs.at[boiler_costs_name, "efficiency"] From d9ab4940804c180d70442e5281012c53eaa441b8 Mon Sep 17 00:00:00 2001 From: lisazeyen Date: Wed, 11 Sep 2024 15:15:22 +0200 Subject: [PATCH 18/18] add release note --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index e75d3344..7340fe31 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,8 @@ Release Notes .. Upcoming Release +* Change the heating demand from final energy which includes losses in legacy equipment to thermal energy service based on JRC-IDEES. Efficiencies of existing heating capacities are lowered according to the conversion of final energy to thermal energy service. For overnight scenarios or future planning horizon this change leads to a reduction in heat supply. + * Updated district heating supply temperatures based on `Euroheat's DHC Market Outlook 2024`__ and `AGFW-Hauptbericht 2022 `__. `min_forward_temperature` and `return_temperature` (not given by Euroheat) are extrapolated based on German values. * Made the overdimensioning factor for heating systems specific for central/decentral heating, defaults to no overdimensionining for central heating and no changes to decentral heating compared to previous version.