From cbf7ed0d38d059c60b8e220040be5be2e7ffa9ce Mon Sep 17 00:00:00 2001
From: Tom Brown <tom@nworbmot.org>
Date: Wed, 14 Feb 2024 10:09:13 +0100
Subject: [PATCH 1/7] for today's industry energy demand, separate MeOH, Cl and
 HVC

I.e. split basic chemicals (without ammonia) into MeOH, Cl and HVC.

This now agrees with scheme for industrial sectors tomorrow.
---
 config/config.default.yaml                    |  1 +
 ...ustrial_energy_demand_per_country_today.py | 50 +++++++++----------
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 1033d49d..40dfb330 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -636,6 +636,7 @@ industry:
     2040: 0.12
     2045: 0.16
     2050: 0.20
+  basic_chemicals_without_NH3_energy_demand_today: 1138. #TWh/a
   HVC_production_today: 52.
   MWh_elec_per_tHVC_mechanical_recycling: 0.547
   MWh_elec_per_tHVC_chemical_recycling: 6.9
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index d1c672f1..696921de 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -73,7 +73,7 @@ def industrial_energy_demand_per_country(country, year, jrc_dir):
     def get_subsector_data(sheet):
         df = df_dict[sheet][year].groupby(fuels).sum()
 
-        df["ammonia"] = 0.0
+        df["hydrogen"] = 0.0
 
         df["other"] = df["all"] - df.loc[df.index != "all"].sum()
 
@@ -94,36 +94,41 @@ def industrial_energy_demand_per_country(country, year, jrc_dir):
     return df
 
 
-def add_ammonia_energy_demand(demand):
+def separate_basic_chemicals(demand):
     # MtNH3/a
     fn = snakemake.input.ammonia_production
     ammonia = pd.read_csv(fn, index_col=0)[str(year)] / 1e3
 
-    def get_ammonia_by_fuel(x):
-        fuels = {
-            "gas": params["MWh_CH4_per_tNH3_SMR"],
-            "electricity": params["MWh_elec_per_tNH3_SMR"],
-        }
-
-        return pd.Series({k: x * v for k, v in fuels.items()})
-
-    ammonia_by_fuel = ammonia.apply(get_ammonia_by_fuel).T
-    ammonia_by_fuel = ammonia_by_fuel.unstack().reindex(
-        index=demand.index, fill_value=0.0
-    )
-
-    ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T
+    ammonia = pd.DataFrame({"gas": ammonia * params["MWh_CH4_per_tNH3_SMR"],
+                            "electricity" : ammonia * params["MWh_elec_per_tNH3_SMR"]}).T
 
     demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0)
 
     demand["Basic chemicals (without ammonia)"] = (
-        demand["Basic chemicals"] - ammonia_by_fuel
+        demand["Basic chemicals"] - demand["Ammonia"]
     )
 
-    demand["Basic chemicals (without ammonia)"].clip(lower=0, inplace=True)
-
     demand.drop(columns="Basic chemicals", inplace=True)
 
+    distribution = demand["Basic chemicals (without ammonia)"].groupby(level=0).sum()/params["basic_chemicals_without_NH3_energy_demand_today"]
+
+    chlorine = pd.DataFrame({"hydrogen": distribution * params["chlorine_production_today"] * params["MWh_H2_per_tCl"],
+                             "electricity" : distribution * params["chlorine_production_today"] * params["MWh_elec_per_tCl"]}).T
+
+    methanol = pd.DataFrame({"gas": distribution * params["methanol_production_today"] * params["MWh_CH4_per_tMeOH"],
+                             "electricity" : distribution * params["methanol_production_today"] * params["MWh_elec_per_tMeOH"]}).T
+
+    demand["Chlorine"] = chlorine.unstack().reindex(index=demand.index, fill_value=0.0)
+    demand["Methanol"] = methanol.unstack().reindex(index=demand.index, fill_value=0.0)
+
+    demand["HVC"] = (
+        demand["Basic chemicals (without ammonia)"] -demand["Methanol"] - demand["Chlorine"]
+    )
+
+    demand.drop(columns="Basic chemicals (without ammonia)", inplace=True)
+
+    demand["HVC"].clip(lower=0, inplace=True)
+
     return demand
 
 
@@ -135,11 +140,6 @@ def add_non_eu28_industrial_energy_demand(countries, demand):
     fn = snakemake.input.industrial_production_per_country
     production = pd.read_csv(fn, index_col=0) / 1e3
 
-    # recombine HVC, Chlorine and Methanol to Basic chemicals (without ammonia)
-    chemicals = ["HVC", "Chlorine", "Methanol"]
-    production["Basic chemicals (without ammonia)"] = production[chemicals].sum(axis=1)
-    production.drop(columns=chemicals, inplace=True)
-
     eu28_production = production.loc[countries.intersection(eu28)].sum()
     eu28_energy = demand.groupby(level=1).sum()
     eu28_averages = eu28_energy / eu28_production
@@ -182,7 +182,7 @@ if __name__ == "__main__":
 
     demand = industrial_energy_demand(countries.intersection(eu28), year)
 
-    demand = add_ammonia_energy_demand(demand)
+    demand = separate_basic_chemicals(demand)
 
     demand = add_non_eu28_industrial_energy_demand(countries, demand)
 

From cc57952402a80a785d16c4c1887d16d7c5aaed2b Mon Sep 17 00:00:00 2001
From: Tom Brown <tom@nworbmot.org>
Date: Wed, 14 Feb 2024 10:33:50 +0100
Subject: [PATCH 2/7] industrial prod: use EU28 total for denominator for
 distribution key

This makes sure the distribution key is correct when only subsets of
countries are used. This is then consistent with the HVC, MeOH and Cl
totals being EU28 totals.

Without this change, industry production is overestimated when using
subsets of countries.

Or the user has to adjust the totals for industrial production
themselves.
---
 config/config.default.yaml                         | 1 +
 scripts/build_industrial_production_per_country.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 40dfb330..4245d926 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -637,6 +637,7 @@ industry:
     2045: 0.16
     2050: 0.20
   basic_chemicals_without_NH3_energy_demand_today: 1138. #TWh/a
+  basic_chemicals_without_NH3_production_today: 69. #Mt/a
   HVC_production_today: 52.
   MWh_elec_per_tHVC_mechanical_recycling: 0.547
   MWh_elec_per_tHVC_chemical_recycling: 6.9
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index 0aea4f15..afbe4de7 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -261,7 +261,7 @@ def separate_basic_chemicals(demand, year):
     demand["Basic chemicals"].clip(lower=0.0, inplace=True)
 
     # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals
-    distribution_key = demand["Basic chemicals"] / demand["Basic chemicals"].sum()
+    distribution_key = demand["Basic chemicals"] / params["basic_chemicals_without_NH3_production_today"] / 1e3
     demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key
     demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key
     demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key

From 7f3ad792a9a7e36f09ce85c819834d29b247b558 Mon Sep 17 00:00:00 2001
From: Tom Brown <tom@nworbmot.org>
Date: Wed, 14 Feb 2024 12:24:58 +0100
Subject: [PATCH 3/7] use production to determine today's energy demand for
 basic chemicals

This uniformises how demand for basic chemicals is calculated.

We also avoid unnecessary use of ammonia production separately.
---
 config/config.default.yaml                    |  3 +-
 rules/build_sector.smk                        |  1 -
 ...ustrial_energy_demand_per_country_today.py | 45 +++++++------------
 3 files changed, 17 insertions(+), 32 deletions(-)

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 4245d926..79ca890d 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -636,8 +636,7 @@ industry:
     2040: 0.12
     2045: 0.16
     2050: 0.20
-  basic_chemicals_without_NH3_energy_demand_today: 1138. #TWh/a
-  basic_chemicals_without_NH3_production_today: 69. #Mt/a
+  basic_chemicals_without_NH3_production_today: 69. #Mt/a, = 86 Mtethylene-equiv - 17 MtNH3
   HVC_production_today: 52.
   MWh_elec_per_tHVC_mechanical_recycling: 0.547
   MWh_elec_per_tHVC_chemical_recycling: 6.9
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index c25c8673..f50432d6 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -566,7 +566,6 @@ rule build_industrial_energy_demand_per_country_today:
         industry=config["industry"],
     input:
         jrc="data/bundle-sector/jrc-idees-2015",
-        ammonia_production=RESOURCES + "ammonia_production.csv",
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
     output:
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index 696921de..65569b55 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -94,51 +94,34 @@ def industrial_energy_demand_per_country(country, year, jrc_dir):
     return df
 
 
-def separate_basic_chemicals(demand):
-    # MtNH3/a
-    fn = snakemake.input.ammonia_production
-    ammonia = pd.read_csv(fn, index_col=0)[str(year)] / 1e3
+def separate_basic_chemicals(demand, production):
 
-    ammonia = pd.DataFrame({"gas": ammonia * params["MWh_CH4_per_tNH3_SMR"],
-                            "electricity" : ammonia * params["MWh_elec_per_tNH3_SMR"]}).T
+    ammonia = pd.DataFrame({"hydrogen": production["Ammonia"] * params["MWh_H2_per_tNH3_electrolysis"],
+                            "electricity" : production["Ammonia"] * params["MWh_elec_per_tNH3_electrolysis"]}).T
+    chlorine = pd.DataFrame({"hydrogen": production["Chlorine"] * params["MWh_H2_per_tCl"],
+                             "electricity" : production["Chlorine"] * params["MWh_elec_per_tCl"]}).T
+    methanol = pd.DataFrame({"gas": production["Methanol"] * params["MWh_CH4_per_tMeOH"],
+                             "electricity" : production["Methanol"] * params["MWh_elec_per_tMeOH"]}).T
 
     demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0)
-
-    demand["Basic chemicals (without ammonia)"] = (
-        demand["Basic chemicals"] - demand["Ammonia"]
-    )
-
-    demand.drop(columns="Basic chemicals", inplace=True)
-
-    distribution = demand["Basic chemicals (without ammonia)"].groupby(level=0).sum()/params["basic_chemicals_without_NH3_energy_demand_today"]
-
-    chlorine = pd.DataFrame({"hydrogen": distribution * params["chlorine_production_today"] * params["MWh_H2_per_tCl"],
-                             "electricity" : distribution * params["chlorine_production_today"] * params["MWh_elec_per_tCl"]}).T
-
-    methanol = pd.DataFrame({"gas": distribution * params["methanol_production_today"] * params["MWh_CH4_per_tMeOH"],
-                             "electricity" : distribution * params["methanol_production_today"] * params["MWh_elec_per_tMeOH"]}).T
-
     demand["Chlorine"] = chlorine.unstack().reindex(index=demand.index, fill_value=0.0)
     demand["Methanol"] = methanol.unstack().reindex(index=demand.index, fill_value=0.0)
 
     demand["HVC"] = (
-        demand["Basic chemicals (without ammonia)"] -demand["Methanol"] - demand["Chlorine"]
+        demand["Basic chemicals"] - demand["Ammonia"] - demand["Methanol"] - demand["Chlorine"]
     )
 
-    demand.drop(columns="Basic chemicals (without ammonia)", inplace=True)
+    demand.drop(columns="Basic chemicals", inplace=True)
 
     demand["HVC"].clip(lower=0, inplace=True)
 
     return demand
 
 
-def add_non_eu28_industrial_energy_demand(countries, demand):
+def add_non_eu28_industrial_energy_demand(countries, demand, production):
     non_eu28 = countries.difference(eu28)
     if non_eu28.empty:
         return demand
-    # output in MtMaterial/a
-    fn = snakemake.input.industrial_production_per_country
-    production = pd.read_csv(fn, index_col=0) / 1e3
 
     eu28_production = production.loc[countries.intersection(eu28)].sum()
     eu28_energy = demand.groupby(level=1).sum()
@@ -182,9 +165,13 @@ if __name__ == "__main__":
 
     demand = industrial_energy_demand(countries.intersection(eu28), year)
 
-    demand = separate_basic_chemicals(demand)
+    # output in MtMaterial/a
+    production = pd.read_csv(snakemake.input.industrial_production_per_country,
+                             index_col=0) / 1e3
 
-    demand = add_non_eu28_industrial_energy_demand(countries, demand)
+    demand = separate_basic_chemicals(demand, production)
+
+    demand = add_non_eu28_industrial_energy_demand(countries, demand, production)
 
     # for format compatibility
     demand = demand.stack(dropna=False).unstack(level=[0, 2])

From e14bae345b5009cf65c49d573517176bda7d69f4 Mon Sep 17 00:00:00 2001
From: Tom Brown <tom@nworbmot.org>
Date: Wed, 14 Feb 2024 18:15:18 +0100
Subject: [PATCH 4/7] new script to interpolate industry sector ratios today to
 tomorrow

For each country we gradually switch industry processes from today's
specific energy carrier usage per ton material output to the
best-in-class energy consumption of tomorrow in the
industry_sector_ratios.csv.

This is done on a per-country basis. The ratio of today to tomorrow's
energy consumption is set with the
config["industry"]["sector_ratios_fraction_future"] parameter.
---
 config/config.default.yaml                    |  8 ++
 rules/build_sector.smk                        | 26 ++++++-
 ...build_industrial_energy_demand_per_node.py | 21 +++--
 ...ild_industry_sector_ratios_intermediate.py | 77 +++++++++++++++++++
 4 files changed, 123 insertions(+), 9 deletions(-)
 create mode 100644 scripts/build_industry_sector_ratios_intermediate.py

diff --git a/config/config.default.yaml b/config/config.default.yaml
index 79ca890d..0f28ee93 100644
--- a/config/config.default.yaml
+++ b/config/config.default.yaml
@@ -636,6 +636,14 @@ industry:
     2040: 0.12
     2045: 0.16
     2050: 0.20
+  sector_ratios_fraction_future:
+    2020: 0.0
+    2025: 0.1
+    2030: 0.3
+    2035: 0.5
+    2040: 0.7
+    2045: 0.9
+    2050: 1.0
   basic_chemicals_without_NH3_production_today: 69. #Mt/a, = 86 Mtethylene-equiv - 17 MtNH3
   HVC_production_today: 52.
   MWh_elec_per_tHVC_mechanical_recycling: 0.547
diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index f50432d6..bec9aa7a 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -433,6 +433,30 @@ rule build_industry_sector_ratios:
         "../scripts/build_industry_sector_ratios.py"
 
 
+rule build_industry_sector_ratios_intermediate:
+    params:
+        industry=config["industry"],
+    input:
+        industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv",
+        industrial_energy_demand_per_country_today=RESOURCES
+        + "industrial_energy_demand_per_country_today.csv",
+        industrial_production_per_country=RESOURCES
+        + "industrial_production_per_country.csv",
+    output:
+        industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv",
+    threads: 1
+    resources:
+        mem_mb=1000,
+    log:
+        LOGS + "build_industry_sector_ratios_{planning_horizons}.log",
+    benchmark:
+        BENCHMARKS + "build_industry_sector_ratios_{planning_horizons}"
+    conda:
+        "../envs/environment.yaml"
+    script:
+        "../scripts/build_industry_sector_ratios_intermediate.py"
+
+
 rule build_industrial_production_per_country:
     params:
         industry=config["industry"],
@@ -535,7 +559,7 @@ rule build_industrial_production_per_node:
 
 rule build_industrial_energy_demand_per_node:
     input:
-        industry_sector_ratios=RESOURCES + "industry_sector_ratios.csv",
+        industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv",
         industrial_production_per_node=RESOURCES
         + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
         industrial_energy_demand_per_node_today=RESOURCES
diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py
index 55c10c5d..84f8679a 100644
--- a/scripts/build_industrial_energy_demand_per_node.py
+++ b/scripts/build_industrial_energy_demand_per_node.py
@@ -19,23 +19,28 @@ if __name__ == "__main__":
             planning_horizons=2030,
         )
 
-    # import EU ratios df as csv
+    # import ratios
     fn = snakemake.input.industry_sector_ratios
-    industry_sector_ratios = pd.read_csv(fn, index_col=0)
+    sector_ratios = pd.read_csv(fn,
+                                header=[0,1],
+                                index_col=0)
 
-    # material demand per node and industry (kton/a)
+    # material demand per node and industry (Mton/a)
     fn = snakemake.input.industrial_production_per_node
-    nodal_production = pd.read_csv(fn, index_col=0)
+    nodal_production = pd.read_csv(fn, index_col=0) / 1e3
 
     # energy demand today to get current electricity
     fn = snakemake.input.industrial_energy_demand_per_node_today
     nodal_today = pd.read_csv(fn, index_col=0)
 
-    # final energy consumption per node and industry (TWh/a)
-    nodal_df = nodal_production.dot(industry_sector_ratios.T)
+    nodal_sector_ratios = pd.concat({node: sector_ratios[node[:2]] for node in nodal_production.index},
+                                    axis=1)
 
-    # convert GWh to TWh and ktCO2 to MtCO2
-    nodal_df *= 0.001
+    nodal_production_stacked = nodal_production.stack()
+    nodal_production_stacked.index.names = [None,None]
+
+    # final energy consumption per node and industry (TWh/a)
+    nodal_df = (nodal_sector_ratios.multiply(nodal_production_stacked)).T.groupby(level=0).sum()
 
     rename_sectors = {
         "elec": "electricity",
diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py
new file mode 100644
index 00000000..86f88218
--- /dev/null
+++ b/scripts/build_industry_sector_ratios_intermediate.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: : 2020-2024 The PyPSA-Eur Authors
+#
+# SPDX-License-Identifier: MIT
+"""
+Build specific energy consumption by carrier and industries and by country,
+that interpolates between the current average energy consumption (from 2015-2020)
+and the ideal future best-in-class consumption.
+"""
+
+import pandas as pd
+
+from prepare_sector_network import get
+
+def build_industry_sector_ratios_intermediate():
+
+    # in TWh/a
+    demand = pd.read_csv(snakemake.input.industrial_energy_demand_per_country_today,
+                         header=[0,1],
+                         index_col=0)
+
+    # in Mt/a
+    production = pd.read_csv(snakemake.input.industrial_production_per_country,
+                             index_col=0) / 1e3
+    production = production.unstack().swaplevel()
+
+    # in MWh/t
+    future_sector_ratios = pd.read_csv(snakemake.input.industry_sector_ratios,
+                                       index_col=0)
+
+    production.index.names = [None,None]
+
+    today_sector_ratios = demand.div(production, axis=1)
+
+    today_sector_ratios.drop(columns=today_sector_ratios.columns[today_sector_ratios.isna().all()],
+                             inplace=True)
+
+    rename = pd.Series(today_sector_ratios.index,
+                       today_sector_ratios.index)
+    rename["waste"] = "biomass"
+    rename["electricity"] = "elec"
+    rename["solid"] = "coke"
+    rename["gas"] = "methane"
+    rename["other"] = "biomass"
+    rename["liquid"] = "naphtha"
+
+    today_sector_ratios.rename(rename,
+                               inplace=True)
+
+
+    fraction_future = get(params["sector_ratios_fraction_future"], year)
+
+    intermediate_sector_ratios = {}
+
+    for ct in today_sector_ratios.columns.unique(level=0):
+
+        intermediate_sector_ratio = future_sector_ratios.copy()
+
+        intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns] = (fraction_future*intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns]
+                                                                                                        + (1 - fraction_future)*today_sector_ratios[ct])
+        intermediate_sector_ratios[ct] = intermediate_sector_ratio
+
+    intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1)
+
+    intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios)
+
+if __name__ == "__main__":
+    if "snakemake" not in globals():
+        from _helpers import mock_snakemake
+
+        snakemake = mock_snakemake("build_industry_sector_ratios_intermediate")
+
+    year = int(snakemake.wildcards.planning_horizons[-4:])
+
+    params = snakemake.params.industry
+
+    build_industry_sector_ratios_intermediate()

From b45df1724bda1611c5792f0775b6496e36d88638 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 14 Feb 2024 17:31:48 +0000
Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 rules/build_sector.smk                        |  6 ++-
 ...ustrial_energy_demand_per_country_today.py | 36 +++++++++----
 ...build_industrial_energy_demand_per_node.py | 17 +++---
 ...build_industrial_production_per_country.py |  6 ++-
 ...ild_industry_sector_ratios_intermediate.py | 52 ++++++++++++-------
 5 files changed, 78 insertions(+), 39 deletions(-)

diff --git a/rules/build_sector.smk b/rules/build_sector.smk
index bec9aa7a..0c53fdd1 100644
--- a/rules/build_sector.smk
+++ b/rules/build_sector.smk
@@ -443,7 +443,8 @@ rule build_industry_sector_ratios_intermediate:
         industrial_production_per_country=RESOURCES
         + "industrial_production_per_country.csv",
     output:
-        industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv",
+        industry_sector_ratios=RESOURCES
+        + "industry_sector_ratios_{planning_horizons}.csv",
     threads: 1
     resources:
         mem_mb=1000,
@@ -559,7 +560,8 @@ rule build_industrial_production_per_node:
 
 rule build_industrial_energy_demand_per_node:
     input:
-        industry_sector_ratios=RESOURCES + "industry_sector_ratios_{planning_horizons}.csv",
+        industry_sector_ratios=RESOURCES
+        + "industry_sector_ratios_{planning_horizons}.csv",
         industrial_production_per_node=RESOURCES
         + "industrial_production_elec_s{simpl}_{clusters}_{planning_horizons}.csv",
         industrial_energy_demand_per_node_today=RESOURCES
diff --git a/scripts/build_industrial_energy_demand_per_country_today.py b/scripts/build_industrial_energy_demand_per_country_today.py
index 65569b55..9105a790 100644
--- a/scripts/build_industrial_energy_demand_per_country_today.py
+++ b/scripts/build_industrial_energy_demand_per_country_today.py
@@ -96,19 +96,35 @@ def industrial_energy_demand_per_country(country, year, jrc_dir):
 
 def separate_basic_chemicals(demand, production):
 
-    ammonia = pd.DataFrame({"hydrogen": production["Ammonia"] * params["MWh_H2_per_tNH3_electrolysis"],
-                            "electricity" : production["Ammonia"] * params["MWh_elec_per_tNH3_electrolysis"]}).T
-    chlorine = pd.DataFrame({"hydrogen": production["Chlorine"] * params["MWh_H2_per_tCl"],
-                             "electricity" : production["Chlorine"] * params["MWh_elec_per_tCl"]}).T
-    methanol = pd.DataFrame({"gas": production["Methanol"] * params["MWh_CH4_per_tMeOH"],
-                             "electricity" : production["Methanol"] * params["MWh_elec_per_tMeOH"]}).T
+    ammonia = pd.DataFrame(
+        {
+            "hydrogen": production["Ammonia"] * params["MWh_H2_per_tNH3_electrolysis"],
+            "electricity": production["Ammonia"]
+            * params["MWh_elec_per_tNH3_electrolysis"],
+        }
+    ).T
+    chlorine = pd.DataFrame(
+        {
+            "hydrogen": production["Chlorine"] * params["MWh_H2_per_tCl"],
+            "electricity": production["Chlorine"] * params["MWh_elec_per_tCl"],
+        }
+    ).T
+    methanol = pd.DataFrame(
+        {
+            "gas": production["Methanol"] * params["MWh_CH4_per_tMeOH"],
+            "electricity": production["Methanol"] * params["MWh_elec_per_tMeOH"],
+        }
+    ).T
 
     demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0)
     demand["Chlorine"] = chlorine.unstack().reindex(index=demand.index, fill_value=0.0)
     demand["Methanol"] = methanol.unstack().reindex(index=demand.index, fill_value=0.0)
 
     demand["HVC"] = (
-        demand["Basic chemicals"] - demand["Ammonia"] - demand["Methanol"] - demand["Chlorine"]
+        demand["Basic chemicals"]
+        - demand["Ammonia"]
+        - demand["Methanol"]
+        - demand["Chlorine"]
     )
 
     demand.drop(columns="Basic chemicals", inplace=True)
@@ -166,8 +182,10 @@ if __name__ == "__main__":
     demand = industrial_energy_demand(countries.intersection(eu28), year)
 
     # output in MtMaterial/a
-    production = pd.read_csv(snakemake.input.industrial_production_per_country,
-                             index_col=0) / 1e3
+    production = (
+        pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0)
+        / 1e3
+    )
 
     demand = separate_basic_chemicals(demand, production)
 
diff --git a/scripts/build_industrial_energy_demand_per_node.py b/scripts/build_industrial_energy_demand_per_node.py
index 84f8679a..ec571c71 100644
--- a/scripts/build_industrial_energy_demand_per_node.py
+++ b/scripts/build_industrial_energy_demand_per_node.py
@@ -21,9 +21,7 @@ if __name__ == "__main__":
 
     # import ratios
     fn = snakemake.input.industry_sector_ratios
-    sector_ratios = pd.read_csv(fn,
-                                header=[0,1],
-                                index_col=0)
+    sector_ratios = pd.read_csv(fn, header=[0, 1], index_col=0)
 
     # material demand per node and industry (Mton/a)
     fn = snakemake.input.industrial_production_per_node
@@ -33,14 +31,19 @@ if __name__ == "__main__":
     fn = snakemake.input.industrial_energy_demand_per_node_today
     nodal_today = pd.read_csv(fn, index_col=0)
 
-    nodal_sector_ratios = pd.concat({node: sector_ratios[node[:2]] for node in nodal_production.index},
-                                    axis=1)
+    nodal_sector_ratios = pd.concat(
+        {node: sector_ratios[node[:2]] for node in nodal_production.index}, axis=1
+    )
 
     nodal_production_stacked = nodal_production.stack()
-    nodal_production_stacked.index.names = [None,None]
+    nodal_production_stacked.index.names = [None, None]
 
     # final energy consumption per node and industry (TWh/a)
-    nodal_df = (nodal_sector_ratios.multiply(nodal_production_stacked)).T.groupby(level=0).sum()
+    nodal_df = (
+        (nodal_sector_ratios.multiply(nodal_production_stacked))
+        .T.groupby(level=0)
+        .sum()
+    )
 
     rename_sectors = {
         "elec": "electricity",
diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py
index afbe4de7..afab1403 100644
--- a/scripts/build_industrial_production_per_country.py
+++ b/scripts/build_industrial_production_per_country.py
@@ -261,7 +261,11 @@ def separate_basic_chemicals(demand, year):
     demand["Basic chemicals"].clip(lower=0.0, inplace=True)
 
     # assume HVC, methanol, chlorine production proportional to non-ammonia basic chemicals
-    distribution_key = demand["Basic chemicals"] / params["basic_chemicals_without_NH3_production_today"] / 1e3
+    distribution_key = (
+        demand["Basic chemicals"]
+        / params["basic_chemicals_without_NH3_production_today"]
+        / 1e3
+    )
     demand["HVC"] = params["HVC_production_today"] * 1e3 * distribution_key
     demand["Chlorine"] = params["chlorine_production_today"] * 1e3 * distribution_key
     demand["Methanol"] = params["methanol_production_today"] * 1e3 * distribution_key
diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py
index 86f88218..0cbdfa06 100644
--- a/scripts/build_industry_sector_ratios_intermediate.py
+++ b/scripts/build_industry_sector_ratios_intermediate.py
@@ -4,39 +4,45 @@
 # SPDX-License-Identifier: MIT
 """
 Build specific energy consumption by carrier and industries and by country,
-that interpolates between the current average energy consumption (from 2015-2020)
-and the ideal future best-in-class consumption.
+that interpolates between the current average energy consumption (from
+2015-2020) and the ideal future best-in-class consumption.
 """
 
 import pandas as pd
-
 from prepare_sector_network import get
 
+
 def build_industry_sector_ratios_intermediate():
 
     # in TWh/a
-    demand = pd.read_csv(snakemake.input.industrial_energy_demand_per_country_today,
-                         header=[0,1],
-                         index_col=0)
+    demand = pd.read_csv(
+        snakemake.input.industrial_energy_demand_per_country_today,
+        header=[0, 1],
+        index_col=0,
+    )
 
     # in Mt/a
-    production = pd.read_csv(snakemake.input.industrial_production_per_country,
-                             index_col=0) / 1e3
+    production = (
+        pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0)
+        / 1e3
+    )
     production = production.unstack().swaplevel()
 
     # in MWh/t
-    future_sector_ratios = pd.read_csv(snakemake.input.industry_sector_ratios,
-                                       index_col=0)
+    future_sector_ratios = pd.read_csv(
+        snakemake.input.industry_sector_ratios, index_col=0
+    )
 
-    production.index.names = [None,None]
+    production.index.names = [None, None]
 
     today_sector_ratios = demand.div(production, axis=1)
 
-    today_sector_ratios.drop(columns=today_sector_ratios.columns[today_sector_ratios.isna().all()],
-                             inplace=True)
+    today_sector_ratios.drop(
+        columns=today_sector_ratios.columns[today_sector_ratios.isna().all()],
+        inplace=True,
+    )
 
-    rename = pd.Series(today_sector_ratios.index,
-                       today_sector_ratios.index)
+    rename = pd.Series(today_sector_ratios.index, today_sector_ratios.index)
     rename["waste"] = "biomass"
     rename["electricity"] = "elec"
     rename["solid"] = "coke"
@@ -44,9 +50,7 @@ def build_industry_sector_ratios_intermediate():
     rename["other"] = "biomass"
     rename["liquid"] = "naphtha"
 
-    today_sector_ratios.rename(rename,
-                               inplace=True)
-
+    today_sector_ratios.rename(rename, inplace=True)
 
     fraction_future = get(params["sector_ratios_fraction_future"], year)
 
@@ -56,14 +60,22 @@ def build_industry_sector_ratios_intermediate():
 
         intermediate_sector_ratio = future_sector_ratios.copy()
 
-        intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns] = (fraction_future*intermediate_sector_ratio.loc[today_sector_ratios[ct].index,today_sector_ratios[ct].columns]
-                                                                                                        + (1 - fraction_future)*today_sector_ratios[ct])
+        intermediate_sector_ratio.loc[
+            today_sector_ratios[ct].index, today_sector_ratios[ct].columns
+        ] = (
+            fraction_future
+            * intermediate_sector_ratio.loc[
+                today_sector_ratios[ct].index, today_sector_ratios[ct].columns
+            ]
+            + (1 - fraction_future) * today_sector_ratios[ct]
+        )
         intermediate_sector_ratios[ct] = intermediate_sector_ratio
 
     intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1)
 
     intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios)
 
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake

From 93bb4e5f54c09ae9f3e571e07e83444150c6f59b Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 16 Feb 2024 15:14:18 +0100
Subject: [PATCH 6/7] add release note, simplify
 build_industry_sector_ratios_intermediate script

---
 doc/configtables/industry.csv                 |  2 +
 doc/release_notes.rst                         |  9 +++
 ...ild_industry_sector_ratios_intermediate.py | 60 ++++++++-----------
 3 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/doc/configtables/industry.csv b/doc/configtables/industry.csv
index fc1b3f0f..d1b560ed 100644
--- a/doc/configtables/industry.csv
+++ b/doc/configtables/industry.csv
@@ -17,6 +17,8 @@ HVC_primary_fraction,--,float,The fraction of high value chemicals (HVC) produce
 HVC_mechanical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using mechanical recycling
 HVC_chemical_recycling _fraction,--,float,The fraction of high value chemicals (HVC) produced using chemical recycling
 ,,,
+sector_ratios_fraction_future,--,Dictionary with planning horizons as keys.,The fraction of total progress in fuel and process switching achieved in the industry sector.
+basic_chemicals_without_NH3_production_today,Mt/a,float,"The amount of basic chemicals produced without ammonia (= 86 Mtethylene-equiv - 17 MtNH3)."
 HVC_production_today,MtHVC/a,float,"The amount of high value chemicals (HVC) produced. This includes ethylene, propylene and BTX. From `DECHEMA (2017) <https://dechema.de/dechema_media/Downloads/Positionspapiere/Technology_study_Low_carbon_energy_and_feedstock_for_the_European_chemical_industry-p-20002750.pdf>`_, Figure 16, page 107"
 Mwh_elec_per_tHVC _mechanical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using mechanical recycling. From SI of `Meys et al (2020) <https://doi.org/10.1016/j.resconrec.2020.105010>`_, Table S5, for HDPE, PP, PS, PET. LDPE would be 0.756."
 Mwh_elec_per_tHVC _chemical_recycling,MWh/tHVC,float,"The energy amount of electricity needed to produce a ton of high value chemical (HVC) using chemical recycling. The default value is based on pyrolysis and electric steam cracking. From `Material Economics (2019) <https://materialeconomics.com/latest-updates/industrial-transformation-2050>`_, page 125"
diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 1a7564d5..b4425308 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -10,6 +10,15 @@ Release Notes
 Upcoming Release
 ================
 
+* Improved representation of industry transition pathways. A new script was
+  added to interpolate industry sector ratios from today's status quo to future
+  systems (i.e. specific emissions and demands for energy and feedstocks). For
+  each country we gradually switch industry processes from today's specific
+  energy carrier usage per ton material output to the best-in-class energy
+  consumption of tomorrow. This is done on a per-country basis. The ratio of
+  today to tomorrow's energy consumption is set with the ``industry:
+  sector_ratios_fraction_future:`` parameter.
+
 * Bugfix: Correct units of subtracted chlorine and methanol demand in
   :mod:`build_industry_sector_ratios`.
 
diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py
index 0cbdfa06..4b1a2d34 100644
--- a/scripts/build_industry_sector_ratios_intermediate.py
+++ b/scripts/build_industry_sector_ratios_intermediate.py
@@ -25,62 +25,52 @@ def build_industry_sector_ratios_intermediate():
     production = (
         pd.read_csv(snakemake.input.industrial_production_per_country, index_col=0)
         / 1e3
-    )
-    production = production.unstack().swaplevel()
+    ).stack()
+    production.index.names = [None, None]
 
     # in MWh/t
     future_sector_ratios = pd.read_csv(
         snakemake.input.industry_sector_ratios, index_col=0
     )
 
-    production.index.names = [None, None]
-
     today_sector_ratios = demand.div(production, axis=1)
 
-    today_sector_ratios.drop(
-        columns=today_sector_ratios.columns[today_sector_ratios.isna().all()],
-        inplace=True,
-    )
+    today_sector_ratios.dropna(how="all", axis=1, inplace=True)
 
-    rename = pd.Series(today_sector_ratios.index, today_sector_ratios.index)
-    rename["waste"] = "biomass"
-    rename["electricity"] = "elec"
-    rename["solid"] = "coke"
-    rename["gas"] = "methane"
-    rename["other"] = "biomass"
-    rename["liquid"] = "naphtha"
-
-    today_sector_ratios.rename(rename, inplace=True)
+    rename = {
+        "waste": "biomass",
+        "electricity": "elec",
+        "solid": "coke",
+        "gas": "methane",
+        "other": "biomass",
+        "liquid": "naphtha",
+    }
+    today_sector_ratios = today_sector_ratios.rename(rename).groupby(level=0).sum()
 
     fraction_future = get(params["sector_ratios_fraction_future"], year)
 
     intermediate_sector_ratios = {}
-
-    for ct in today_sector_ratios.columns.unique(level=0):
-
-        intermediate_sector_ratio = future_sector_ratios.copy()
-
-        intermediate_sector_ratio.loc[
-            today_sector_ratios[ct].index, today_sector_ratios[ct].columns
-        ] = (
-            fraction_future
-            * intermediate_sector_ratio.loc[
-                today_sector_ratios[ct].index, today_sector_ratios[ct].columns
-            ]
-            + (1 - fraction_future) * today_sector_ratios[ct]
+    for ct, group in today_sector_ratios.T.groupby(level=0):
+        today_sector_ratios_ct = (
+            group.droplevel(0)
+            .T.reindex_like(future_sector_ratios)
+            .fillna(future_sector_ratios)
+        )
+        intermediate_sector_ratios[ct] = (
+            today_sector_ratios_ct * (1 - fraction_future)
+            + future_sector_ratios * fraction_future
         )
-        intermediate_sector_ratios[ct] = intermediate_sector_ratio
-
     intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1)
 
-    intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios)
-
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
-        snakemake = mock_snakemake("build_industry_sector_ratios_intermediate")
+        snakemake = mock_snakemake(
+            "build_industry_sector_ratios_intermediate",
+            planning_horizons="2030",
+        )
 
     year = int(snakemake.wildcards.planning_horizons[-4:])
 

From 27f9fb8cbaf7b377f182fb126246bf8a08772ba6 Mon Sep 17 00:00:00 2001
From: Fabian Neumann <fabian.neumann@outlook.de>
Date: Fri, 16 Feb 2024 16:18:31 +0100
Subject: [PATCH 7/7] add back in to_csv export (accidentally deleted)

---
 scripts/build_industry_sector_ratios_intermediate.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/build_industry_sector_ratios_intermediate.py b/scripts/build_industry_sector_ratios_intermediate.py
index 4b1a2d34..14e09505 100644
--- a/scripts/build_industry_sector_ratios_intermediate.py
+++ b/scripts/build_industry_sector_ratios_intermediate.py
@@ -62,6 +62,8 @@ def build_industry_sector_ratios_intermediate():
         )
     intermediate_sector_ratios = pd.concat(intermediate_sector_ratios, axis=1)
 
+    intermediate_sector_ratios.to_csv(snakemake.output.industry_sector_ratios)
+
 
 if __name__ == "__main__":
     if "snakemake" not in globals():