Merge pull request #1225 from PyPSA/fix-transport-data-year-selection

Make year selection more secure in transport data creation

scripts/base_network.py

@@ -95,7 +95,7 @@ logger = logging.getLogger(__name__)
 
 def _get_oid(df):
     if "tags" in df.columns:
-        return df.tags.str.extract('"oid"=>"(\d+)"', expand=False)
+        return df.tags.str.extract(r'"oid"=>"(\d+)"', expand=False)
     else:
         return pd.Series(np.nan, df.index)
 

scripts/build_biomass_potentials.py

@@ -311,7 +311,7 @@ def add_unsustainable_potentials(df):
     # Phase out unsustainable biomass potentials linearly from 2020 to 2035 while phasing in sustainable potentials
     share_unsus = params.get("share_unsustainable_use_retained").get(investment_year)
 
-    df_wo_ch = df.drop(df.filter(regex="CH\d", axis=0).index)
+    df_wo_ch = df.drop(df.filter(regex=r"CH\d", axis=0).index)
 
     # Calculate unsustainable solid biomass
     df_wo_ch["unsustainable solid biomass"] = _calc_unsustainable_potential(

scripts/build_energy_totals.py

@@ -242,7 +242,7 @@ def build_eurostat(
     temp.index = pd.MultiIndex.from_frame(
         temp.index.to_frame().fillna("International aviation")
     )
-    df = pd.concat([temp, df.loc[~int_avia]])
+    df = pd.concat([temp, df.loc[~int_avia]]).sort_index()
 
     # Fill in missing data on "Domestic aviation" for each country.
     for country in countries:
@@ -651,8 +651,8 @@ def build_energy_totals(
     """
 
     eurostat_fuels = {"electricity": "Electricity", "total": "Total all products"}
-    eurostat_countries = eurostat.index.levels[0]
+    eurostat_countries = eurostat.index.unique(0)
-    eurostat_years = eurostat.index.levels[1]
+    eurostat_years = eurostat.index.unique(1)
 
     to_drop = ["passenger cars", "passenger car efficiency"]
     new_index = pd.MultiIndex.from_product(
@@ -1153,13 +1153,14 @@ def build_transport_data(
     ----------
     - Swiss transport data: `BFS <https://www.bfs.admin.ch/bfs/en/home/statistics/mobility-transport/transport-infrastructure-vehicles/vehicles/road-vehicles-stock-level-motorisation.html>`_
     """
+    years = np.arange(2000, 2022)
 
     # first collect number of cars
     transport_data = pd.DataFrame(idees["passenger cars"])
 
     countries_without_ch = set(countries) - {"CH"}
     new_index = pd.MultiIndex.from_product(
-        [countries_without_ch, transport_data.index.levels[1]],
+        [countries_without_ch, transport_data.index.unique(1)],
         names=["country", "year"],
     )
 
@@ -1167,7 +1168,7 @@ def build_transport_data(
 
     if "CH" in countries:
         fn = snakemake.input.swiss_transport
-        swiss_cars = pd.read_csv(fn, index_col=0).loc[2000:2023, ["passenger cars"]]
+        swiss_cars = pd.read_csv(fn, index_col=0).loc[years, ["passenger cars"]]
 
         swiss_cars.index = pd.MultiIndex.from_product(
             [["CH"], swiss_cars.index], names=["country", "year"]
@@ -1175,10 +1176,8 @@ def build_transport_data(
 
         transport_data = pd.concat([transport_data, swiss_cars]).sort_index()
 
-    transport_data.rename(columns={"passenger cars": "number cars"}, inplace=True)
+    transport_data = transport_data.rename(columns={"passenger cars": "number cars"})
-
     # clean up dataframe
-    years = np.arange(2000, 2022)
     transport_data = transport_data[
         transport_data.index.get_level_values(1).isin(years)
     ]
@@ -1188,11 +1187,10 @@ def build_transport_data(
         logger.info(
             f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data."
         )
-
         cars_pp = transport_data["number cars"] / population
 
         fill_values = {
-            year: cars_pp.mean() * population for year in transport_data.index.levels[1]
+            year: cars_pp.mean() * population for year in transport_data.index.unique(1)
         }
         fill_values = pd.DataFrame(fill_values).stack()
         fill_values = pd.DataFrame(fill_values, columns=["number cars"])

scripts/build_industrial_energy_demand_per_node_today.py

@@ -66,7 +66,7 @@ def build_nodal_industrial_energy_demand():
     )
 
     countries = keys.country.unique()
-    sectors = industrial_demand.columns.levels[1]
+    sectors = industrial_demand.columns.unique(1)
 
     for country, sector in product(countries, sectors):
         buses = keys.index[keys.country == country]

scripts/build_industry_sector_ratios.py

@@ -445,7 +445,9 @@ def chemicals_industry():
 
     # subtract ammonia energy demand (in ktNH3/a)
     ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0)
-    ammonia_total = ammonia.loc[ammonia.index.intersection(eu27), str(max(2018, year))].sum()
+    ammonia_total = ammonia.loc[
+        ammonia.index.intersection(eu27), str(max(2018, year))
+    ].sum()
     df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"]
     df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"]
 

scripts/build_retro_cost.py

@@ -254,7 +254,7 @@ def prepare_building_stock_data():
         index = pd.MultiIndex.from_product([[ct], averaged_data.index.to_list()])
         averaged_data.index = index
         averaged_data["estimated"] = 1
-        if ct not in area_tot.index.levels[0]:
+        if ct not in area_tot.index.unique(0):
             area_tot = pd.concat([area_tot, averaged_data], sort=True)
         else:
             area_tot.loc[averaged_data.index] = averaged_data

scripts/make_summary_perfect.py

@@ -40,9 +40,9 @@ def calculate_costs(n, label, costs):
     investments = n.investment_periods
     cols = pd.MultiIndex.from_product(
         [
-            costs.columns.levels[0],
+            costs.columns.unique(0),
-            costs.columns.levels[1],
+            costs.columns.unique(1),
-            costs.columns.levels[2],
+            costs.columns.unique(2),
             investments,
         ],
         names=costs.columns.names[:3] + ["year"],
@@ -339,9 +339,9 @@ def calculate_supply_energy(n, label, supply_energy):
     investments = n.investment_periods
     cols = pd.MultiIndex.from_product(
         [
-            supply_energy.columns.levels[0],
+            supply_energy.columns.unique(0),
-            supply_energy.columns.levels[1],
+            supply_energy.columns.unique(1),
-            supply_energy.columns.levels[2],
+            supply_energy.columns.unique(2),
             investments,
         ],
         names=supply_energy.columns.names[:3] + ["year"],