From 4b6199de42d8547c35b4b35dec482ab23ffb55b3 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 9 Feb 2024 15:43:27 +0100 Subject: [PATCH 01/28] create a bus for every unique coordinate, not only substations (closes #699) --- rules/postprocess.smk | 6 +++--- scripts/base_network.py | 1 + scripts/build_bus_regions.py | 8 +++++++- scripts/simplify_network.py | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/rules/postprocess.smk b/rules/postprocess.smk index 18d65c16..98b90fd1 100644 --- a/rules/postprocess.smk +++ b/rules/postprocess.smk @@ -45,7 +45,7 @@ if config["foresight"] != "perfect": ( LOGS + "plot_power_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ) + ), benchmark: ( BENCHMARKS @@ -74,7 +74,7 @@ if config["foresight"] != "perfect": ( LOGS + "plot_hydrogen_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ) + ), benchmark: ( BENCHMARKS @@ -102,7 +102,7 @@ if config["foresight"] != "perfect": ( LOGS + "plot_gas_network/elec_s{simpl}_{clusters}_l{ll}_{opts}_{sector_opts}_{planning_horizons}.log" - ) + ), benchmark: ( BENCHMARKS diff --git a/scripts/base_network.py b/scripts/base_network.py index 1de3ef96..684ca613 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -555,6 +555,7 @@ def _set_countries_and_substations(n, config, country_shapes, offshore_shapes): for b, df in product(("bus0", "bus1"), (n.lines, n.links)): has_connections_b |= ~df.groupby(b).under_construction.min() + buses["onshore_bus"] = onshore_b buses["substation_lv"] = ( lv_b & onshore_b & (~buses["under_construction"]) & has_connections_b ) diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index a6500bb0..f9bf287e 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -135,7 +135,13 @@ if __name__ == "__main__": c_b = n.buses.country == country onshore_shape = country_shapes[country] - onshore_locs = n.buses.loc[c_b & n.buses.substation_lv, ["x", "y"]] + onshore_locs = ( + n.buses.loc[c_b & n.buses.onshore_bus] + .sort_values( + by="substation_lv", ascending=False + ) # preference for substations + .drop_duplicates(subset=["x", "y"], keep="first")[["x", "y"]] + ) onshore_regions.append( gpd.GeoDataFrame( { diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 31bc8f4e..cc4ff4f6 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -611,6 +611,7 @@ if __name__ == "__main__": "symbol", "tags", "under_construction", + "onshore_bus", "substation_lv", "substation_off", "geometry", From 8781e690661b064432689511f215eaa48affa7bf Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Fri, 9 Feb 2024 18:36:16 +0100 Subject: [PATCH 02/28] bugfix: coal emissions for industry weren't tracked Also allow industrial coal demand to be regional (so we can include them in regional CO2 constraints). --- config/config.default.yaml | 1 + scripts/prepare_sector_network.py | 39 +++++++++++++++++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 1033d49d..44b54148 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -501,6 +501,7 @@ sector: SMR_cc: true regional_methanol_demand: false regional_oil_demand: false + regional_coal_demand: false regional_co2_sequestration_potential: enable: false attribute: 'conservative estimate Mt' diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index be8aea53..18d1feb0 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -172,6 +172,13 @@ def define_spatial(nodes, options): spatial.coal.nodes = ["EU coal"] spatial.coal.locations = ["EU"] + if options["regional_coal_demand"]: + spatial.coal.demand_locations = nodes + spatial.coal.industry = nodes + " coal for industry" + else: + spatial.coal.demand_locations = ["EU"] + spatial.coal.industry = ["EU coal for industry"] + # lignite spatial.lignite = SimpleNamespace() spatial.lignite.nodes = ["EU lignite"] @@ -3048,19 +3055,41 @@ def add_industry(n, costs): mwh_coal_per_mwh_coke = 1.366 # from eurostat energy balance p_set = ( - industrial_demand["coal"].sum() - + mwh_coal_per_mwh_coke * industrial_demand["coke"].sum() + industrial_demand["coal"] + + mwh_coal_per_mwh_coke * industrial_demand["coke"] ) / nhours + if not options["regional_coal_demand"]: + p_set = p_set.sum() + + n.madd( + "Bus", + spatial.coal.industry, + location=spatial.coal.demand_locations, + carrier="coal for industry", + unit="MWh_LHV", + ) + n.madd( "Load", - spatial.coal.nodes, - suffix=" for industry", - bus=spatial.coal.nodes, + spatial.coal.industry, + bus=spatial.coal.industry, carrier="coal for industry", p_set=p_set, ) + n.madd( + "Link", + spatial.coal.industry, + bus0=spatial.coal.nodes, + bus1=spatial.coal.industry, + bus2="co2 atmosphere", + carrier="coal for industry", + p_nom_extendable=True, + efficiency2=costs.at["coal", "CO2 intensity"], + ) + + def add_waste_heat(n): # TODO options? From 17105b81256a5364739a4a2f4c3e865a3ba3fc95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 10 Feb 2024 18:17:57 +0000 Subject: [PATCH 03/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/prepare_sector_network.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 18d1feb0..5d5e271b 100755 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -3090,7 +3090,6 @@ def add_industry(n, costs): ) - def add_waste_heat(n): # TODO options? From 9c592d9f73630d350447ba45c9c408f06456c7f8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 14 Feb 2024 10:49:40 +0100 Subject: [PATCH 04/28] add config/config.yaml to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c9d2e171..6454616c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ gurobi.log doc/_build config.yaml +config/config.yaml dconf /data/links_p_nom.csv From ecedea02d630d49ff14ad54c97c11b919211ea77 Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Mon, 12 Feb 2024 18:17:53 +0100 Subject: [PATCH 05/28] bugfix: include all countries in ammonia production resource This is so that the full EU28 ammonia demand can be correctly subtracted in the build_industry_sector_ratios.py script. No other downstream scripts are affected by this change. --- scripts/build_ammonia_production.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 1bcdf9ae..bfcfaf02 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -8,6 +8,7 @@ Build historical annual ammonia production per country in ktonNH3/a. import country_converter as coco import pandas as pd +import numpy as np cc = coco.CountryConverter() @@ -30,8 +31,12 @@ if __name__ == "__main__": ammonia.index = cc.convert(ammonia.index, to="iso2") years = [str(i) for i in range(2013, 2018)] - countries = ammonia.index.intersection(snakemake.params.countries) - ammonia = ammonia.loc[countries, years].astype(float) + + ammonia = ammonia[years] + ammonia.replace("--", + np.nan, + inplace=True) + ammonia = ammonia.astype(float) # convert from ktonN to ktonNH3 ammonia *= 17 / 14 From c54626cc1a847ffded57e50cba2330f1f994a4ec Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Mon, 12 Feb 2024 18:19:25 +0100 Subject: [PATCH 06/28] bugfix: correct units of subtracted chlorine and methanol in build_industry_sector_ratios.py. In the config the units are Mt/a, they are multiplied by MWh/t, but what is desired is GWh/a. --- scripts/build_industry_sector_ratios.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 45705002..805cf3e7 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -408,15 +408,15 @@ def chemicals_industry(): df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"] df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"] - # subtract chlorine demand + # subtract chlorine demand (in MtCl/a) chlorine_total = params["chlorine_production_today"] - df.loc["hydrogen", sector] -= chlorine_total * params["MWh_H2_per_tCl"] - df.loc["elec", sector] -= chlorine_total * params["MWh_elec_per_tCl"] + df.loc["hydrogen", sector] -= chlorine_total * params["MWh_H2_per_tCl"] * 1e3 + df.loc["elec", sector] -= chlorine_total * params["MWh_elec_per_tCl"] * 1e3 - # subtract methanol demand + # subtract methanol demand (in MtMeOH/a) methanol_total = params["methanol_production_today"] - df.loc["methane", sector] -= methanol_total * params["MWh_CH4_per_tMeOH"] - df.loc["elec", sector] -= methanol_total * params["MWh_elec_per_tMeOH"] + df.loc["methane", sector] -= methanol_total * params["MWh_CH4_per_tMeOH"] * 1e3 + df.loc["elec", sector] -= methanol_total * params["MWh_elec_per_tMeOH"] * 1e3 # MWh/t material df.loc[sources, sector] = df.loc[sources, sector] / s_out From cbf7ed0d38d059c60b8e220040be5be2e7ffa9ce Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Wed, 14 Feb 2024 10:09:13 +0100 Subject: [PATCH 07/28] for today's industry energy demand, separate MeOH, Cl and HVC I.e. split basic chemicals (without ammonia) into MeOH, Cl and HVC. This now agrees with scheme for industrial sectors tomorrow. --- config/config.default.yaml | 1 + ...ustrial_energy_demand_per_country_today.py | 50 +++++++++---------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 1033d49d..40dfb330 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -636,6 +636,7 @@ industry: 2040: 0.12 2045: 0.16 2050: 0.20 + basic_chemicals_without_NH3_energy_demand_today: 1138. #TWh/a HVC_production_today: 52. MWh_elec_per_tHVC_mechanical_recycling: 0.547 MWh_elec_per_tHVC_chemical_recycling: 6.9 diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index d1c672f1..696921de 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -73,7 +73,7 @@ def industrial_energy_demand_per_country(country, year, jrc_dir): def get_subsector_data(sheet): df = df_dict[sheet][year].groupby(fuels).sum() - df["ammonia"] = 0.0 + df["hydrogen"] = 0.0 df["other"] = df["all"] - df.loc[df.index != "all"].sum() @@ -94,36 +94,41 @@ def industrial_energy_demand_per_country(country, year, jrc_dir): return df -def add_ammonia_energy_demand(demand): +def separate_basic_chemicals(demand): # MtNH3/a fn = snakemake.input.ammonia_production ammonia = pd.read_csv(fn, index_col=0)[str(year)] / 1e3 - def get_ammonia_by_fuel(x): - fuels = { - "gas": params["MWh_CH4_per_tNH3_SMR"], - "electricity": params["MWh_elec_per_tNH3_SMR"], - } - - return pd.Series({k: x * v for k, v in fuels.items()}) - - ammonia_by_fuel = ammonia.apply(get_ammonia_by_fuel).T - ammonia_by_fuel = ammonia_by_fuel.unstack().reindex( - index=demand.index, fill_value=0.0 - ) - - ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T + ammonia = pd.DataFrame({"gas": ammonia * params["MWh_CH4_per_tNH3_SMR"], + "electricity" : ammonia * params["MWh_elec_per_tNH3_SMR"]}).T demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0) demand["Basic chemicals (without ammonia)"] = ( - demand["Basic chemicals"] - ammonia_by_fuel + demand["Basic chemicals"] - demand["Ammonia"] ) - demand["Basic chemicals (without ammonia)"].clip(lower=0, inplace=True) - demand.drop(columns="Basic chemicals", inplace=True) + distribution = demand["Basic chemicals (without ammonia)"].groupby(level=0).sum()/params["basic_chemicals_without_NH3_energy_demand_today"] + + chlorine = pd.DataFrame({"hydrogen": distribution * params["chlorine_production_today"] * params["MWh_H2_per_tCl"], + "electricity" : distribution * params["chlorine_production_today"] * params["MWh_elec_per_tCl"]}).T + + methanol = pd.DataFrame({"gas": distribution * params["methanol_production_today"] * params["MWh_CH4_per_tMeOH"], + "electricity" : distribution * params["methanol_production_today"] * params["MWh_elec_per_tMeOH"]}).T + + demand["Chlorine"] = chlorine.unstack().reindex(index=demand.index, fill_value=0.0) + demand["Methanol"] = methanol.unstack().reindex(index=demand.index, fill_value=0.0) + + demand["HVC"] = ( + demand["Basic chemicals (without ammonia)"] -demand["Methanol"] - demand["Chlorine"] + ) + + demand.drop(columns="Basic chemicals (without ammonia)", inplace=True) + + demand["HVC"].clip(lower=0, inplace=True) + return demand @@ -135,11 +140,6 @@ def add_non_eu28_industrial_energy_demand(countries, demand): fn = snakemake.input.industrial_production_per_country production = pd.read_csv(fn, index_col=0) / 1e3 - # recombine HVC, Chlorine and Methanol to Basic chemicals (without ammonia) - chemicals = ["HVC", "Chlorine", "Methanol"] - production["Basic chemicals (without ammonia)"] = production[chemicals].sum(axis=1) - production.drop(columns=chemicals, inplace=True) - eu28_production = production.loc[countries.intersection(eu28)].sum() eu28_energy = demand.groupby(level=1).sum() eu28_averages = eu28_energy / eu28_production @@ -182,7 +182,7 @@ if __name__ == "__main__": demand = industrial_energy_demand(countries.intersection(eu28), year) - demand = add_ammonia_energy_demand(demand) + demand = separate_basic_chemicals(demand) demand = add_non_eu28_industrial_energy_demand(countries, demand) From cc57952402a80a785d16c4c1887d16d7c5aaed2b Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Wed, 14 Feb 2024 10:33:50 +0100 Subject: [PATCH 08/28] industrial prod: use EU28 total for denominator for distribution key This makes sure the distribution key is correct when only subsets of countries are used. This is then consistent with the HVC, MeOH and Cl totals being EU28 totals. Without this change, industry production is overestimated when using subsets of countries. Or the user has to adjust the totals for industrial production themselves. --- config/config.default.yaml | 1 + scripts/build_industrial_production_per_country.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 40dfb330..4245d926 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -637,6 +637,7 @@ industry: 2045: 0.16 2050: 0.20 basic_chemicals_without_NH3_energy_demand_today: 1138. #TWh/a + basic_chemicals_without_NH3_production_today: 69. #Mt/a HVC_production_today: 52. MWh_elec_per_tHVC_mechanical_recycling: 0.547 MWh_elec_per_tHVC_chemical_recycling: 6.9 diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index 0aea4f15..afbe4de7 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -261,7 +261,7 @@ def separate_basic_chemicals(demand, year): demand["Basic chemicals"].clip(lower=0.0, inplace=True) # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals - distribution_key = demand["Basic chemicals"] / demand["Basic chemicals"].sum() + distribution_key = demand["Basic chemicals"] / params["basic_chemicals_without_NH3_production_today"] / 1e3 demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key From 7f3ad792a9a7e36f09ce85c819834d29b247b558 Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Wed, 14 Feb 2024 12:24:58 +0100 Subject: [PATCH 09/28] use production to determine today's energy demand for basic chemicals This uniformises how demand for basic chemicals is calculated. We also avoid unnecessary use of ammonia production separately. --- config/config.default.yaml | 3 +- rules/build_sector.smk | 1 - ...ustrial_energy_demand_per_country_today.py | 45 +++++++------------ 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 4245d926..79ca890d 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -636,8 +636,7 @@ industry: 2040: 0.12 2045: 0.16 2050: 0.20 - basic_chemicals_without_NH3_energy_demand_today: 1138. #TWh/a - basic_chemicals_without_NH3_production_today: 69. #Mt/a + basic_chemicals_without_NH3_production_today: 69. #Mt/a, = 86 Mtethylene-equiv - 17 MtNH3 HVC_production_today: 52. MWh_elec_per_tHVC_mechanical_recycling: 0.547 MWh_elec_per_tHVC_chemical_recycling: 6.9 diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c25c8673..f50432d6 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -566,7 +566,6 @@ rule build_industrial_energy_demand_per_country_today: industry=config["industry"], input: jrc="data/bundle-sector/jrc-idees-2015", - ammonia_production=RESOURCES + "ammonia_production.csv", industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", output: diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 696921de..65569b55 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -94,51 +94,34 @@ def industrial_energy_demand_per_country(country, year, jrc_dir): return df -def separate_basic_chemicals(demand): - # MtNH3/a - fn = snakemake.input.ammonia_production - ammonia = pd.read_csv(fn, index_col=0)[str(year)] / 1e3 +def separate_basic_chemicals(demand, production): - ammonia = pd.DataFrame({"gas": ammonia * params["MWh_CH4_per_tNH3_SMR"], - "electricity" : ammonia * params["MWh_elec_per_tNH3_SMR"]}).T + ammonia = pd.DataFrame({"hydrogen": production["Ammonia"] * params["MWh_H2_per_tNH3_electrolysis"], + "electricity" : production["Ammonia"] * params["MWh_elec_per_tNH3_electrolysis"]}).T + chlorine = pd.DataFrame({"hydrogen": production["Chlorine"] * params["MWh_H2_per_tCl"], + "electricity" : production["Chlorine"] * params["MWh_elec_per_tCl"]}).T + methanol = pd.DataFrame({"gas": production["Methanol"] * params["MWh_CH4_per_tMeOH"], + "electricity" : production["Methanol"] * params["MWh_elec_per_tMeOH"]}).T demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0) - - demand["Basic chemicals (without ammonia)"] = ( - demand["Basic chemicals"] - demand["Ammonia"] - ) - - demand.drop(columns="Basic chemicals", inplace=True) - - distribution = demand["Basic chemicals (without ammonia)"].groupby(level=0).sum()/params["basic_chemicals_without_NH3_energy_demand_today"] - - chlorine = pd.DataFrame({"hydrogen": distribution * params["chlorine_production_today"] * params["MWh_H2_per_tCl"], - "electricity" : distribution * params["chlorine_production_today"] * params["MWh_elec_per_tCl"]}).T - - methanol = pd.DataFrame({"gas": distribution * params["methanol_production_today"] * params["MWh_CH4_per_tMeOH"], - "electricity" : distribution * params["methanol_production_today"] * params["MWh_elec_per_tMeOH"]}).T - demand["Chlorine"] = chlorine.unstack().reindex(index=demand.index, fill_value=0.0) demand["Methanol"] = methanol.unstack().reindex(index=demand.index, fill_value=0.0) demand["HVC"] = ( - demand["Basic chemicals (without ammonia)"] -demand["Methanol"] - demand["Chlorine"] + demand["Basic chemicals"] - demand["Ammonia"] - demand["Methanol"] - demand["Chlorine"] ) - demand.drop(columns="Basic chemicals (without ammonia)", inplace=True) + demand.drop(columns="Basic chemicals", inplace=True) demand["HVC"].clip(lower=0, inplace=True) return demand -def add_non_eu28_industrial_energy_demand(countries, demand): +def add_non_eu28_industrial_energy_demand(countries, demand, production): non_eu28 = countries.difference(eu28) if non_eu28.empty: return demand - # output in MtMaterial/a - fn = snakemake.input.industrial_production_per_country - production = pd.read_csv(fn, index_col=0) / 1e3 eu28_production = production.loc[countries.intersection(eu28)].sum() eu28_energy = demand.groupby(level=1).sum() @@ -182,9 +165,13 @@ if __name__ == "__main__": demand = industrial_energy_demand(countries.intersection(eu28), year) - demand = separate_basic_chemicals(demand) + # output in MtMaterial/a + production = pd.read_csv(snakemake.input.industrial_production_per_country, + index_col=0) / 1e3 - demand = add_non_eu28_industrial_energy_demand(countries, demand) + demand = separate_basic_chemicals(demand, production) + + demand = add_non_eu28_industrial_energy_demand(countries, demand, production) # for format compatibility demand = demand.stack(dropna=False).unstack(level=[0, 2]) From e14bae345b5009cf65c49d573517176bda7d69f4 Mon Sep 17 00:00:00 2001 From: Tom Brown Date: Wed, 14 Feb 2024 18:15:18 +0100 Subject: [PATCH 10/28] new script to interpolate industry sector ratios today to tomorrow For each country we gradually switch industry processes from today's specific energy carrier usage per ton material output to the best-in-class energy consumption of tomorrow in the industry_sector_ratios.csv. This is done on a per-country basis. The ratio of today to tomorrow's energy consumption is set with the config["industry"]["sector_ratios_fraction_future"] parameter. --- config/config.default.yaml | 8 ++ rules/build_sector.smk | 26 ++++++- ...build_industrial_energy_demand_per_node.py | 21 +++-- ...ild_industry_sector_ratios_intermediate.py | 77 +++++++++++++++++++ 4 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 scripts/build_industry_sector_ratios_intermediate.py diff --git a/config/config.default.yaml b/config/config.default.yaml index 79ca890d..0f28ee93 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -636,6 +636,14 @@ industry: 2040: 0.12 2045: 0.16 2050: 0.20 + sector_ratios_fraction_future: + 2020: 0.0 + 2025: 0.1 + 2030: 0.3 + 2035: 0.5 + 2040: 0.7 + 2045: 0.9 + 2050: 1.0 basic_chemicals_without_NH3_production_today: 69. #Mt/a, = 86 Mtethylene-equiv - 17 MtNH3 HVC_production_today: 52. MWh_elec_per_tHVC_mechanical_recycling: 0.547 diff --git a/rules/build_sector.smk b/rules/build_sector.smk index f50432d6..bec9aa7a 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -433,6 +433,30 @@ rule build_industry_sector_ratios: "../scripts/build_industry_sector_ratios.py" +rule build_industry_sector_ratios_intermediate: + params: + industry=config["industry"], + input: + industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", + industrial_energy_demand_per_country_today=RESOURCES + + "industrial_energy_demand_per_country_today.csv", + industrial_production_per_country=RESOURCES + + "industrial_production_per_country.csv", + output: + industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv", + threads: 1 + resources: + mem_mb=1000, + log: + LOGS + "build_industry_sector_ratios_{planning_horizons}.log", + benchmark: + BENCHMARKS + "build_industry_sector_ratios_{planning_horizons}" + conda: + "../envs/environment.yaml" + script: + "../scripts/build_industry_sector_ratios_intermediate.py" + + rule build_industrial_production_per_country: params: industry=config["industry"], @@ -535,7 +559,7 @@ rule build_industrial_production_per_node: rule build_industrial_energy_demand_per_node: input: - industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv", + industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv", industrial_production_per_node=RESOURCES + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_energy_demand_per_node_today=RESOURCES diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py index 55c10c5d..84f8679a 100644 --- a/scripts/build_industrial_energy_demand_per_node.py +++ b/scripts/build_industrial_energy_demand_per_node.py @@ -19,23 +19,28 @@ if __name__ == "__main__": planning_horizons=2030, ) - # import EU ratios df as csv + # import ratios fn = snakemake.input.industry_sector_ratios - industry_sector_ratios = pd.read_csv(fn, index_col=0) + sector_ratios = pd.read_csv(fn, + header=[0,1], + index_col=0) - # material demand per node and industry (kton/a) + # material demand per node and industry (Mton/a) fn = snakemake.input.industrial_production_per_node - nodal_production = pd.read_csv(fn, index_col=0) + nodal_production = pd.read_csv(fn, index_col=0) / 1e3 # energy demand today to get current electricity fn = snakemake.input.industrial_energy_demand_per_node_today nodal_today = pd.read_csv(fn, index_col=0) - # final energy consumption per node and industry (TWh/a) - nodal_df = nodal_production.dot(industry_sector_ratios.T) + nodal_sector_ratios = pd.concat({node: sector_ratios[node[:2]] for node in nodal_production.index}, + axis=1) - # convert GWh to TWh and ktCO2 to MtCO2 - nodal_df *= 0.001 + nodal_production_stacked = nodal_production.stack() + nodal_production_stacked.index.names = [None,None] + + # final energy consumption per node and industry (TWh/a) + nodal_df = (nodal_sector_ratios.multiply(nodal_production_stacked)).T.groupby(level=0).sum() rename_sectors = { "elec": "electricity", diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py new file mode 100644 index 00000000..86f88218 --- /dev/null +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors +# +# SPDX-License-Identifier: MIT +""" +Build specific energy consumption by carrier and industries and by country, +that interpolates between the current average energy consumption (from 2015-2020) +and the ideal future best-in-class consumption. +""" + +import pandas as pd + +from prepare_sector_network import get + +def build_industry_sector_ratios_intermediate(): + + # in TWh/a + demand = pd.read_csv(snakemake.input.industrial_energy_demand_per_country_today, + header=[0,1], + index_col=0) + + # in Mt/a + production = pd.read_csv(snakemake.input.industrial_production_per_country, + index_col=0) / 1e3 + production = production.unstack().swaplevel() + + # in MWh/t + future_sector_ratios = pd.read_csv(snakemake.input.industry_sector_ratios, + index_col=0) + + production.index.names = [None,None] + + today_sector_ratios = demand.div(production, axis=1) + + today_sector_ratios.drop(columns=today_sector_ratios.columns[today_sector_ratios.isna().all()], + inplace=True) + + rename = pd.Series(today_sector_ratios.index, + today_sector_ratios.index) + rename["waste"] = "biomass" + rename["electricity"] = "elec" + rename["solid"] = "coke" + rename["gas"] = "methane" + rename["other"] = "biomass" + rename["liquid"] = "naphtha" + + today_sector_ratios.rename(rename, + inplace=True) + + + fraction_future = get(params["sector_ratios_fraction_future"], year) + + intermediate_sector_ratios = {} + + for ct in today_sector_ratios.columns.unique(level=0): + + intermediate_sector_ratio = future_sector_ratios.copy() + + intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns] = (fraction_future*intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns] + + (1 - fraction_future)*today_sector_ratios[ct]) + intermediate_sector_ratios[ct] = intermediate_sector_ratio + + intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) + + intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios) + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("build_industry_sector_ratios_intermediate") + + year = int(snakemake.wildcards.planning_horizons[-4:]) + + params = snakemake.params.industry + + build_industry_sector_ratios_intermediate() From b45df1724bda1611c5792f0775b6496e36d88638 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:31:48 +0000 Subject: [PATCH 11/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rules/build_sector.smk | 6 ++- ...ustrial_energy_demand_per_country_today.py | 36 +++++++++---- ...build_industrial_energy_demand_per_node.py | 17 +++--- ...build_industrial_production_per_country.py | 6 ++- ...ild_industry_sector_ratios_intermediate.py | 52 ++++++++++++------- 5 files changed, 78 insertions(+), 39 deletions(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index bec9aa7a..0c53fdd1 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -443,7 +443,8 @@ rule build_industry_sector_ratios_intermediate: industrial_production_per_country=RESOURCES + "industrial_production_per_country.csv", output: - industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv", + industry_sector_ratios=RESOURCES + + "industry_sector_ratios_{planning_horizons}.csv", threads: 1 resources: mem_mb=1000, @@ -559,7 +560,8 @@ rule build_industrial_production_per_node: rule build_industrial_energy_demand_per_node: input: - industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv", + industry_sector_ratios=RESOURCES + + "industry_sector_ratios_{planning_horizons}.csv", industrial_production_per_node=RESOURCES + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv", industrial_energy_demand_per_node_today=RESOURCES diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py index 65569b55..9105a790 100644 --- a/scripts/build_industrial_energy_demand_per_country_today.py +++ b/scripts/build_industrial_energy_demand_per_country_today.py @@ -96,19 +96,35 @@ def industrial_energy_demand_per_country(country, year, jrc_dir): def separate_basic_chemicals(demand, production): - ammonia = pd.DataFrame({"hydrogen": production["Ammonia"] * params["MWh_H2_per_tNH3_electrolysis"], - "electricity" : production["Ammonia"] * params["MWh_elec_per_tNH3_electrolysis"]}).T - chlorine = pd.DataFrame({"hydrogen": production["Chlorine"] * params["MWh_H2_per_tCl"], - "electricity" : production["Chlorine"] * params["MWh_elec_per_tCl"]}).T - methanol = pd.DataFrame({"gas": production["Methanol"] * params["MWh_CH4_per_tMeOH"], - "electricity" : production["Methanol"] * params["MWh_elec_per_tMeOH"]}).T + ammonia = pd.DataFrame( + { + "hydrogen": production["Ammonia"] * params["MWh_H2_per_tNH3_electrolysis"], + "electricity": production["Ammonia"] + * params["MWh_elec_per_tNH3_electrolysis"], + } + ).T + chlorine = pd.DataFrame( + { + "hydrogen": production["Chlorine"] * params["MWh_H2_per_tCl"], + "electricity": production["Chlorine"] * params["MWh_elec_per_tCl"], + } + ).T + methanol = pd.DataFrame( + { + "gas": production["Methanol"] * params["MWh_CH4_per_tMeOH"], + "electricity": production["Methanol"] * params["MWh_elec_per_tMeOH"], + } + ).T demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0) demand["Chlorine"] = chlorine.unstack().reindex(index=demand.index, fill_value=0.0) demand["Methanol"] = methanol.unstack().reindex(index=demand.index, fill_value=0.0) demand["HVC"] = ( - demand["Basic chemicals"] - demand["Ammonia"] - demand["Methanol"] - demand["Chlorine"] + demand["Basic chemicals"] + - demand["Ammonia"] + - demand["Methanol"] + - demand["Chlorine"] ) demand.drop(columns="Basic chemicals", inplace=True) @@ -166,8 +182,10 @@ if __name__ == "__main__": demand = industrial_energy_demand(countries.intersection(eu28), year) # output in MtMaterial/a - production = pd.read_csv(snakemake.input.industrial_production_per_country, - index_col=0) / 1e3 + production = ( + pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0) + / 1e3 + ) demand = separate_basic_chemicals(demand, production) diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py index 84f8679a..ec571c71 100644 --- a/scripts/build_industrial_energy_demand_per_node.py +++ b/scripts/build_industrial_energy_demand_per_node.py @@ -21,9 +21,7 @@ if __name__ == "__main__": # import ratios fn = snakemake.input.industry_sector_ratios - sector_ratios = pd.read_csv(fn, - header=[0,1], - index_col=0) + sector_ratios = pd.read_csv(fn, header=[0, 1], index_col=0) # material demand per node and industry (Mton/a) fn = snakemake.input.industrial_production_per_node @@ -33,14 +31,19 @@ if __name__ == "__main__": fn = snakemake.input.industrial_energy_demand_per_node_today nodal_today = pd.read_csv(fn, index_col=0) - nodal_sector_ratios = pd.concat({node: sector_ratios[node[:2]] for node in nodal_production.index}, - axis=1) + nodal_sector_ratios = pd.concat( + {node: sector_ratios[node[:2]] for node in nodal_production.index}, axis=1 + ) nodal_production_stacked = nodal_production.stack() - nodal_production_stacked.index.names = [None,None] + nodal_production_stacked.index.names = [None, None] # final energy consumption per node and industry (TWh/a) - nodal_df = (nodal_sector_ratios.multiply(nodal_production_stacked)).T.groupby(level=0).sum() + nodal_df = ( + (nodal_sector_ratios.multiply(nodal_production_stacked)) + .T.groupby(level=0) + .sum() + ) rename_sectors = { "elec": "electricity", diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index afbe4de7..afab1403 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -261,7 +261,11 @@ def separate_basic_chemicals(demand, year): demand["Basic chemicals"].clip(lower=0.0, inplace=True) # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals - distribution_key = demand["Basic chemicals"] / params["basic_chemicals_without_NH3_production_today"] / 1e3 + distribution_key = ( + demand["Basic chemicals"] + / params["basic_chemicals_without_NH3_production_today"] + / 1e3 + ) demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py index 86f88218..0cbdfa06 100644 --- a/scripts/build_industry_sector_ratios_intermediate.py +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -4,39 +4,45 @@ # SPDX-License-Identifier: MIT """ Build specific energy consumption by carrier and industries and by country, -that interpolates between the current average energy consumption (from 2015-2020) -and the ideal future best-in-class consumption. +that interpolates between the current average energy consumption (from +2015-2020) and the ideal future best-in-class consumption. """ import pandas as pd - from prepare_sector_network import get + def build_industry_sector_ratios_intermediate(): # in TWh/a - demand = pd.read_csv(snakemake.input.industrial_energy_demand_per_country_today, - header=[0,1], - index_col=0) + demand = pd.read_csv( + snakemake.input.industrial_energy_demand_per_country_today, + header=[0, 1], + index_col=0, + ) # in Mt/a - production = pd.read_csv(snakemake.input.industrial_production_per_country, - index_col=0) / 1e3 + production = ( + pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0) + / 1e3 + ) production = production.unstack().swaplevel() # in MWh/t - future_sector_ratios = pd.read_csv(snakemake.input.industry_sector_ratios, - index_col=0) + future_sector_ratios = pd.read_csv( + snakemake.input.industry_sector_ratios, index_col=0 + ) - production.index.names = [None,None] + production.index.names = [None, None] today_sector_ratios = demand.div(production, axis=1) - today_sector_ratios.drop(columns=today_sector_ratios.columns[today_sector_ratios.isna().all()], - inplace=True) + today_sector_ratios.drop( + columns=today_sector_ratios.columns[today_sector_ratios.isna().all()], + inplace=True, + ) - rename = pd.Series(today_sector_ratios.index, - today_sector_ratios.index) + rename = pd.Series(today_sector_ratios.index, today_sector_ratios.index) rename["waste"] = "biomass" rename["electricity"] = "elec" rename["solid"] = "coke" @@ -44,9 +50,7 @@ def build_industry_sector_ratios_intermediate(): rename["other"] = "biomass" rename["liquid"] = "naphtha" - today_sector_ratios.rename(rename, - inplace=True) - + today_sector_ratios.rename(rename, inplace=True) fraction_future = get(params["sector_ratios_fraction_future"], year) @@ -56,14 +60,22 @@ def build_industry_sector_ratios_intermediate(): intermediate_sector_ratio = future_sector_ratios.copy() - intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns] = (fraction_future*intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns] - + (1 - fraction_future)*today_sector_ratios[ct]) + intermediate_sector_ratio.loc[ + today_sector_ratios[ct].index, today_sector_ratios[ct].columns + ] = ( + fraction_future + * intermediate_sector_ratio.loc[ + today_sector_ratios[ct].index, today_sector_ratios[ct].columns + ] + + (1 - fraction_future) * today_sector_ratios[ct] + ) intermediate_sector_ratios[ct] = intermediate_sector_ratio intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios) + if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake From 4fa504b0dbd60aa0b410d2bffba4c68e525159ff Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 17:32:20 +0000 Subject: [PATCH 12/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_ammonia_production.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index bfcfaf02..77b33075 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -7,8 +7,8 @@ Build historical annual ammonia production per country in ktonNH3/a. """ import country_converter as coco -import pandas as pd import numpy as np +import pandas as pd cc = coco.CountryConverter() @@ -33,9 +33,7 @@ if __name__ == "__main__": years = [str(i) for i in range(2013, 2018)] ammonia = ammonia[years] - ammonia.replace("--", - np.nan, - inplace=True) + ammonia.replace("--", np.nan, inplace=True) ammonia = ammonia.astype(float) # convert from ktonN to ktonNH3 From 87ab0a55759507aa5eb4115a9b7fb98634fd40ea Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 15 Feb 2024 14:58:53 +0100 Subject: [PATCH 13/28] update h2 pipe losses --- config/config.default.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 1033d49d..cfda5224 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -538,8 +538,8 @@ sector: efficiency_static: 0.98 efficiency_per_1000km: 0.977 H2 pipeline: - efficiency_per_1000km: 1 # 0.979 - compression_per_1000km: 0.019 + efficiency_per_1000km: 0.996 + compression_per_1000km: 0.018 gas pipeline: efficiency_per_1000km: 1 #0.977 compression_per_1000km: 0.01 From 8464415b356171c633e53fb0268660f9b9ee994d Mon Sep 17 00:00:00 2001 From: lisazeyen <35347358+lisazeyen@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:14:32 +0100 Subject: [PATCH 14/28] update for 70 bars instead of 140 --- config/config.default.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index cfda5224..97b0edb3 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -538,8 +538,8 @@ sector: efficiency_static: 0.98 efficiency_per_1000km: 0.977 H2 pipeline: - efficiency_per_1000km: 0.996 - compression_per_1000km: 0.018 + efficiency_per_1000km: 0.9905 + compression_per_1000km: 0.008 gas pipeline: efficiency_per_1000km: 1 #0.977 compression_per_1000km: 0.01 From 16955d24abec236e757c5f19cf0af003939defbb Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 15 Feb 2024 16:27:44 +0100 Subject: [PATCH 15/28] data: upload swiss energy totals data for all years --- data/switzerland-new_format-all_years.csv | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 data/switzerland-new_format-all_years.csv diff --git a/data/switzerland-new_format-all_years.csv b/data/switzerland-new_format-all_years.csv new file mode 100644 index 00000000..93123009 --- /dev/null +++ b/data/switzerland-new_format-all_years.csv @@ -0,0 +1,25 @@ +country,item,2010,2011,2012,2013,2014,2015 +CH,total residential,268.2,223.4,243.4,261.3,214.2,229.1 +CH,total residential space,192.2,149.0,168.1,185.5,139.7,154.4 +CH,total residential water,32.2,31.6,31.9,32.2,31.7,31.9 +CH,total residential cooking,9.3,9.3,9.3,9.4,9.5,9.6 +CH,electricity residential,67.9,63.7,65.7,67.6,63.0,64.4 +CH,electricity residential space,15.9,12.8,14.3,15.8,12.3,13.5 +CH,electricity residential water,8.8,8.5,8.5,8.6,8.5,8.6 +CH,electricity residential cooking,4.9,4.9,4.9,4.9,5.0,5.0 +CH,total services,145.9,127.4,136.7,144.0,124.5,132.5 +CH,total services space,80.0,62.2,70.8,77.4,58.3,64.3 +CH,total services water,10.1,10.0,10.1,10.1,10.0,10.0 +CH,total services cooking,2.5,2.4,2.3,2.3,2.4,2.3 +CH,electricity services,60.5,59.2,60.3,61.4,60.3,62.6 +CH,electricity services space,4.0,3.2,3.8,4.2,3.3,3.6 +CH,electricity services water,0.7,0.7,0.7,0.7,0.7,0.7 +CH,electricity services cooking,2.5,2.4,2.3,2.3,2.4,2.3 +CH,total rail,11.5,11.1,11.2,11.4,11.1,11.4 +CH,total road,199.4,200.4,200.4,201.2,202.0,203.1 +CH,electricity road,0.,0.,0.,0.,0.,0. +CH,electricity rail,11.5,11.1,11.2,11.4,11.1,11.4 +CH,total domestic aviation,3.3,3.2,3.4,3.4,3.5,3.5 +CH,total international aviation,58.0,62.0,63.5,64.2,64.5,66.8 +CH,total domestic navigation,1.6,1.6,1.6,1.6,1.6,1.6 +CH,total international navigation,0.,0.,0.,0.,0.,0. From a22b19dcc49639ce927d72ac1bb640c29059fcdf Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 15 Feb 2024 16:28:24 +0100 Subject: [PATCH 16/28] use new swiss data for building energy totals --- rules/build_sector.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c25c8673..d99c39ae 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -248,7 +248,7 @@ rule build_energy_totals: input: nuts3_shapes=RESOURCES + "nuts3_shapes.geojson", co2="data/bundle-sector/eea/UNFCCC_v23.csv", - swiss="data/bundle-sector/switzerland-sfoe/switzerland-new_format.csv", + swiss="data/switzerland-new_format-all_years.csv", idees="data/bundle-sector/jrc-idees-2015", district_heat_share="data/district_heat_share.csv", eurostat=input_eurostat, From 6196fd689283e25c0be0660bfc313a2fb4f07430 Mon Sep 17 00:00:00 2001 From: martacki Date: Thu, 15 Feb 2024 16:29:29 +0100 Subject: [PATCH 17/28] change default energy totals year to 2013 to comply with default weather year --- config/config.default.yaml | 2 +- doc/release_notes.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 1033d49d..ba99e3ba 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -314,7 +314,7 @@ pypsa_eur: # docs in https://pypsa-eur.readthedocs.io/en/latest/configuration.html#energy energy: - energy_totals_year: 2011 + energy_totals_year: 2013 base_emissions_year: 1990 eurostat_report_year: 2016 emissions: CO2 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index cab8229a..fde06f97 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -113,6 +113,8 @@ Upcoming Release workflows with foresight "myopic" and still needs to be added foresight option "perfect". +* Swiched the energy totals year from 2011 to 2013 to comply with the assumed default weather year. + PyPSA-Eur 0.9.0 (5th January 2024) ================================== From ccbf835976e2cf776f421a5f868158dbe9d42848 Mon Sep 17 00:00:00 2001 From: martacki Date: Fri, 16 Feb 2024 09:49:00 +0100 Subject: [PATCH 18/28] fix typo to make pre-commit.ci happy --- doc/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index fde06f97..e017b07d 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -113,7 +113,7 @@ Upcoming Release workflows with foresight "myopic" and still needs to be added foresight option "perfect". -* Swiched the energy totals year from 2011 to 2013 to comply with the assumed default weather year. +* Switched the energy totals year from 2011 to 2013 to comply with the assumed default weather year. PyPSA-Eur 0.9.0 (5th January 2024) From c22e47cd58e46b98fb591e5d951652518257e61a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 12:07:37 +0100 Subject: [PATCH 19/28] add release note --- doc/release_notes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index cab8229a..08ec9dd5 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,11 @@ Release Notes Upcoming Release ================ +* Regions are assigned to all buses with unique coordinates in the network with + a preference given to substations. Previously, only substations had assigned + regions, but this could lead to issues when a high spatial resolution was + applied. + * The default configuration ``config/config.default.yaml`` is now automatically used as a base configuration file and no longer copied to ``config/config.yaml`` on first use. The file ``config/config.yaml`` should be From 67484f9cf0428bf8d8c8e463730db9c0001e064d Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 12:13:35 +0100 Subject: [PATCH 20/28] remove numpy import, add release ntoe --- doc/release_notes.rst | 4 ++++ rules/build_sector.smk | 2 -- scripts/build_ammonia_production.py | 4 +--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index cab8229a..fc58c899 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,10 @@ Release Notes Upcoming Release ================ +* Include all countries in ammonia production resource. This is so that the full + EU28 ammonia demand can be correctly subtracted in the rule + :mod:`build_industry_sector_ratios`. + * The default configuration ``config/config.default.yaml`` is now automatically used as a base configuration file and no longer copied to ``config/config.yaml`` on first use. The file ``config/config.yaml`` should be diff --git a/rules/build_sector.smk b/rules/build_sector.smk index c25c8673..35a59008 100644 --- a/rules/build_sector.smk +++ b/rules/build_sector.smk @@ -392,8 +392,6 @@ rule build_salt_cavern_potentials: rule build_ammonia_production: - params: - countries=config["countries"], input: usgs="data/bundle-sector/myb1-2017-nitro.xls", output: diff --git a/scripts/build_ammonia_production.py b/scripts/build_ammonia_production.py index 77b33075..e2cf6a7b 100644 --- a/scripts/build_ammonia_production.py +++ b/scripts/build_ammonia_production.py @@ -7,7 +7,6 @@ Build historical annual ammonia production per country in ktonNH3/a. """ import country_converter as coco -import numpy as np import pandas as pd cc = coco.CountryConverter() @@ -26,6 +25,7 @@ if __name__ == "__main__": header=0, index_col=0, skipfooter=19, + na_values=["--"], ) ammonia.index = cc.convert(ammonia.index, to="iso2") @@ -33,8 +33,6 @@ if __name__ == "__main__": years = [str(i) for i in range(2013, 2018)] ammonia = ammonia[years] - ammonia.replace("--", np.nan, inplace=True) - ammonia = ammonia.astype(float) # convert from ktonN to ktonNH3 ammonia *= 17 / 14 From 7e66c1428d5a538efd0917ed2ff8002e1e9fcfec Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 12:40:05 +0100 Subject: [PATCH 21/28] add release note --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index cab8229a..ffe1a794 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,9 @@ Release Notes Upcoming Release ================ +* Bugfix: Correct units of subtracted chlorine and methanol demand in + :mod:`build_industry_sector_ratios`. + * The default configuration ``config/config.default.yaml`` is now automatically used as a base configuration file and no longer copied to ``config/config.yaml`` on first use. The file ``config/config.yaml`` should be From 6d80b332e606c7569e180f8519204a3be30724a4 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 12:47:01 +0100 Subject: [PATCH 22/28] add release note --- doc/release_notes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index ee7bd64b..ccb886e6 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,11 @@ Release Notes Upcoming Release ================ +* Bugfix: The industry coal emissions for industry were not properly tracked. + +* Allow industrial coal demand to be regional so its emissions can be included + in regional emission limits. + * Add new default to overdimension heating in individual buildings. This allows them to cover heat demand peaks e.g. 10% higher than those in the data. The disadvantage of manipulating the costs is that the capacity is then not quite From 472a09988101d280b434d5d26947a5408f54cceb Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 13:30:52 +0100 Subject: [PATCH 23/28] add_electricity: set locations and units of electricity buses --- scripts/add_electricity.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 614e3330..9ac3638d 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -903,5 +903,9 @@ if __name__ == "__main__": sanitize_carriers(n, snakemake.config) + # set locations and units of electricity buses + n.buses["location"] = n.buses.index + n.buses["unit"] = "MWh_el" + n.meta = snakemake.config n.export_to_netcdf(snakemake.output[0]) From 9e8ed7394d65fcf24d0f631f22656a2dc544d61e Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 13:32:24 +0100 Subject: [PATCH 24/28] Update config/config.default.yaml --- config/config.default.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.default.yaml b/config/config.default.yaml index 97b0edb3..48f2a78d 100644 --- a/config/config.default.yaml +++ b/config/config.default.yaml @@ -538,8 +538,8 @@ sector: efficiency_static: 0.98 efficiency_per_1000km: 0.977 H2 pipeline: - efficiency_per_1000km: 0.9905 - compression_per_1000km: 0.008 + efficiency_per_1000km: 1 # 0.982 + compression_per_1000km: 0.018 gas pipeline: efficiency_per_1000km: 1 #0.977 compression_per_1000km: 0.01 From 93bb4e5f54c09ae9f3e571e07e83444150c6f59b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 15:14:18 +0100 Subject: [PATCH 25/28] add release note, simplify build_industry_sector_ratios_intermediate script --- doc/configtables/industry.csv | 2 + doc/release_notes.rst | 9 +++ ...ild_industry_sector_ratios_intermediate.py | 60 ++++++++----------- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/doc/configtables/industry.csv b/doc/configtables/industry.csv index fc1b3f0f..d1b560ed 100644 --- a/doc/configtables/industry.csv +++ b/doc/configtables/industry.csv @@ -17,6 +17,8 @@ HVC_primary_fraction,--,float,The fraction of high value chemicals (HVC) produce HVC_mechanical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling HVC_chemical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling ,,, +sector_ratios_fraction_future,--,Dictionary with planning horizons as keys.,The fraction of total progress in fuel and process switching achieved in the industry sector. +basic_chemicals_without_NH3_production_today,Mt/a,float,"The amount of basic chemicals produced without ammonia (= 86 Mtethylene-equiv - 17 MtNH3)." HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) `_, Figure 16, page 107" Mwh_elec_per_tHVC _mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) `_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756." Mwh_elec_per_tHVC _chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) `_, page 125" diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1a7564d5..b4425308 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -10,6 +10,15 @@ Release Notes Upcoming Release ================ +* Improved representation of industry transition pathways. A new script was + added to interpolate industry sector ratios from today's status quo to future + systems (i.e. specific emissions and demands for energy and feedstocks). For + each country we gradually switch industry processes from today's specific + energy carrier usage per ton material output to the best-in-class energy + consumption of tomorrow. This is done on a per-country basis. The ratio of + today to tomorrow's energy consumption is set with the ``industry: + sector_ratios_fraction_future:`` parameter. + * Bugfix: Correct units of subtracted chlorine and methanol demand in :mod:`build_industry_sector_ratios`. diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py index 0cbdfa06..4b1a2d34 100644 --- a/scripts/build_industry_sector_ratios_intermediate.py +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -25,62 +25,52 @@ def build_industry_sector_ratios_intermediate(): production = ( pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0) / 1e3 - ) - production = production.unstack().swaplevel() + ).stack() + production.index.names = [None, None] # in MWh/t future_sector_ratios = pd.read_csv( snakemake.input.industry_sector_ratios, index_col=0 ) - production.index.names = [None, None] - today_sector_ratios = demand.div(production, axis=1) - today_sector_ratios.drop( - columns=today_sector_ratios.columns[today_sector_ratios.isna().all()], - inplace=True, - ) + today_sector_ratios.dropna(how="all", axis=1, inplace=True) - rename = pd.Series(today_sector_ratios.index, today_sector_ratios.index) - rename["waste"] = "biomass" - rename["electricity"] = "elec" - rename["solid"] = "coke" - rename["gas"] = "methane" - rename["other"] = "biomass" - rename["liquid"] = "naphtha" - - today_sector_ratios.rename(rename, inplace=True) + rename = { + "waste": "biomass", + "electricity": "elec", + "solid": "coke", + "gas": "methane", + "other": "biomass", + "liquid": "naphtha", + } + today_sector_ratios = today_sector_ratios.rename(rename).groupby(level=0).sum() fraction_future = get(params["sector_ratios_fraction_future"], year) intermediate_sector_ratios = {} - - for ct in today_sector_ratios.columns.unique(level=0): - - intermediate_sector_ratio = future_sector_ratios.copy() - - intermediate_sector_ratio.loc[ - today_sector_ratios[ct].index, today_sector_ratios[ct].columns - ] = ( - fraction_future - * intermediate_sector_ratio.loc[ - today_sector_ratios[ct].index, today_sector_ratios[ct].columns - ] - + (1 - fraction_future) * today_sector_ratios[ct] + for ct, group in today_sector_ratios.T.groupby(level=0): + today_sector_ratios_ct = ( + group.droplevel(0) + .T.reindex_like(future_sector_ratios) + .fillna(future_sector_ratios) + ) + intermediate_sector_ratios[ct] = ( + today_sector_ratios_ct * (1 - fraction_future) + + future_sector_ratios * fraction_future ) - intermediate_sector_ratios[ct] = intermediate_sector_ratio - intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) - intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios) - if __name__ == "__main__": if "snakemake" not in globals(): from _helpers import mock_snakemake - snakemake = mock_snakemake("build_industry_sector_ratios_intermediate") + snakemake = mock_snakemake( + "build_industry_sector_ratios_intermediate", + planning_horizons="2030", + ) year = int(snakemake.wildcards.planning_horizons[-4:]) From 8ae0669b333f0c9387f1a1a8a8bb38f4fa01b82b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 15:31:17 +0100 Subject: [PATCH 26/28] Revert "add_electricity: set locations and units of electricity buses" This reverts commit 472a09988101d280b434d5d26947a5408f54cceb. --- scripts/add_electricity.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 9ac3638d..614e3330 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -903,9 +903,5 @@ if __name__ == "__main__": sanitize_carriers(n, snakemake.config) - # set locations and units of electricity buses - n.buses["location"] = n.buses.index - n.buses["unit"] = "MWh_el" - n.meta = snakemake.config n.export_to_netcdf(snakemake.output[0]) From 27f9fb8cbaf7b377f182fb126246bf8a08772ba6 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 16:18:31 +0100 Subject: [PATCH 27/28] add back in to_csv export (accidentally deleted) --- scripts/build_industry_sector_ratios_intermediate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py index 4b1a2d34..14e09505 100644 --- a/scripts/build_industry_sector_ratios_intermediate.py +++ b/scripts/build_industry_sector_ratios_intermediate.py @@ -62,6 +62,8 @@ def build_industry_sector_ratios_intermediate(): ) intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1) + intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios) + if __name__ == "__main__": if "snakemake" not in globals(): From 7f1ff0c3240b545fc5d93eeb8fdef28ef328975a Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Fri, 16 Feb 2024 16:41:21 +0100 Subject: [PATCH 28/28] Fix plotting of retrofitted hydrogen pipelines with pathway optimisation. --- doc/release_notes.rst | 2 ++ scripts/plot_hydrogen_network.py | 25 ++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b4425308..7feb4d72 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -19,6 +19,8 @@ Upcoming Release today to tomorrow's energy consumption is set with the ``industry: sector_ratios_fraction_future:`` parameter. +* Fix plotting of retrofitted hydrogen pipelines with pathway optimisation. + * Bugfix: Correct units of subtracted chlorine and methanol demand in :mod:`build_industry_sector_ratios`. diff --git a/scripts/plot_hydrogen_network.py b/scripts/plot_hydrogen_network.py index 95741170..c6b90ceb 100644 --- a/scripts/plot_hydrogen_network.py +++ b/scripts/plot_hydrogen_network.py @@ -24,6 +24,7 @@ def group_pipes(df, drop_direction=False): """ Group pipes which connect same buses and return overall capacity. """ + df = df.copy() if drop_direction: positive_order = df.bus0 < df.bus1 df_p = df[positive_order] @@ -32,12 +33,13 @@ def group_pipes(df, drop_direction=False): df = pd.concat([df_p, df_n]) # there are pipes for each investment period rename to AC buses name for plotting + df["index_orig"] = df.index df.index = df.apply( lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", axis=1, ) return df.groupby(level=0).agg( - {"p_nom_opt": "sum", "bus0": "first", "bus1": "first"} + {"p_nom_opt": "sum", "bus0": "first", "bus1": "first", "index_orig": "first"} ) @@ -95,17 +97,18 @@ def plot_h2_map(n, regions): ) if not h2_retro.empty: - positive_order = h2_retro.bus0 < h2_retro.bus1 - h2_retro_p = h2_retro[positive_order] - swap_buses = {"bus0": "bus1", "bus1": "bus0"} - h2_retro_n = h2_retro[~positive_order].rename(columns=swap_buses) - h2_retro = pd.concat([h2_retro_p, h2_retro_n]) + if snakemake.params.foresight != "myopic": + positive_order = h2_retro.bus0 < h2_retro.bus1 + h2_retro_p = h2_retro[positive_order] + swap_buses = {"bus0": "bus1", "bus1": "bus0"} + h2_retro_n = h2_retro[~positive_order].rename(columns=swap_buses) + h2_retro = pd.concat([h2_retro_p, h2_retro_n]) - h2_retro["index_orig"] = h2_retro.index - h2_retro.index = h2_retro.apply( - lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", - axis=1, - ) + h2_retro["index_orig"] = h2_retro.index + h2_retro.index = h2_retro.apply( + lambda x: f"H2 pipeline {x.bus0.replace(' H2', '')} -> {x.bus1.replace(' H2', '')}", + axis=1, + ) retro_w_new_i = h2_retro.index.intersection(h2_new.index) h2_retro_w_new = h2_retro.loc[retro_w_new_i]