update energy totals from Eurostat residential data

rules/build_sector.smk

@@ -293,6 +293,7 @@ rule build_energy_totals:
         idees="data/bundle-sector/jrc-idees-2015",
         district_heat_share="data/district_heat_share.csv",
         eurostat="data/eurostat/eurostat-energy_balances-april_2023_edition",
+        eurostat_households="data/eurostat/eurostat-household_energy_balances-february_2024.csv",
     output:
         energy_name=resources("energy_totals.csv"),
         co2_name=resources("co2_totals.csv"),

scripts/build_energy_totals.py

@@ -943,6 +943,44 @@ def rescale_idees_from_eurostat(
     return energy
 
 
+def update_residential_from_eurostat(energy):
+    """
+        Updates energy balances for residential from disaggregated data from Eurostat
+    """
+    # Read disaggregated Eurostat's data
+    fn = snakemake.input.eurostat_households
+    eurostat_data = pd.read_csv(fn)
+
+    # Column mapping for energy type 
+    nrg_type = {"total residential":"FC_OTH_HH_E",
+                "total residential space":"FC_OTH_HH_E_SH",
+                "total residential water":"FC_OTH_HH_E_WH",
+                "total residential cooking":"FC_OTH_HH_E_CK"}
+    
+    # Make temporary copy of energy_totals
+    energy_totals = energy.copy().reset_index()
+
+    for nrg_name, code in nrg_type.items():
+        # Select energy balance type
+        nrg_data = eurostat_data.query("nrg_bal in @code").copy()
+        # Rename columns
+        nrg_data.rename(columns={"geo":"country", "TIME_PERIOD":"year", "OBS_VALUE":nrg_name}, inplace=True)
+        # Convert TJ to TWh
+        nrg_data[nrg_name] = nrg_data[nrg_name] / 3.6e3
+        # Select value, country, year columns 
+        nrg_data = nrg_data[["country","year", nrg_name]]
+        # To update energy data with Eurostat households data 
+        # 1) Merge the two DataFrames on 'year' and 'country'
+        merged_df = energy_totals.merge(nrg_data, on=['year', 'country'], suffixes=('_energy_totals', '_nrg_data'), how='left')
+        # 2) Update the 'nrg_name' column in energy with the values from nrg_data
+        energy_totals[nrg_name] = merged_df[f'{nrg_name}_nrg_data'].combine_first(merged_df[f'{nrg_name}_energy_totals'])
+        
+    # Set indexes back
+    energy_totals.set_index(['country', 'year'], inplace=True)
+
+    return energy_totals
+
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -976,6 +1014,8 @@ if __name__ == "__main__":
     logger.info("Extrapolate IDEES data based on eurostat for years 2015-2021.")
     energy = rescale_idees_from_eurostat(idees_countries, energy, eurostat)
 
+    energy = update_residential_from_eurostat(energy)
+
     energy.to_csv(snakemake.output.energy_name)
 
     # use rescaled idees data to calculate district heat share