From ca47becc873431da30f4770d1cf3e0e330ad50f1 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 19 Aug 2024 13:33:28 +0200 Subject: [PATCH 1/2] build_transport_data: select swiss years according to data selection use index.unique instead of index.levels where needed --- scripts/base_network.py | 2 +- scripts/build_biomass_potentials.py | 2 +- scripts/build_energy_totals.py | 18 ++++++++---------- ..._industrial_energy_demand_per_node_today.py | 2 +- scripts/build_retro_cost.py | 2 +- scripts/make_summary_perfect.py | 12 ++++++------ 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/scripts/base_network.py b/scripts/base_network.py index 78cf6984..2b0fee63 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -95,7 +95,7 @@ logger = logging.getLogger(__name__) def _get_oid(df): if "tags" in df.columns: - return df.tags.str.extract('"oid"=>"(\d+)"', expand=False) + return df.tags.str.extract(r'"oid"=>"(\d+)"', expand=False) else: return pd.Series(np.nan, df.index) diff --git a/scripts/build_biomass_potentials.py b/scripts/build_biomass_potentials.py index 0a2692e8..b69fcfbe 100644 --- a/scripts/build_biomass_potentials.py +++ b/scripts/build_biomass_potentials.py @@ -311,7 +311,7 @@ def add_unsustainable_potentials(df): # Phase out unsustainable biomass potentials linearly from 2020 to 2035 while phasing in sustainable potentials share_unsus = params.get("share_unsustainable_use_retained").get(investment_year) - df_wo_ch = df.drop(df.filter(regex="CH\d", axis=0).index) + df_wo_ch = df.drop(df.filter(regex=r"CH\d", axis=0).index) # Calculate unsustainable solid biomass df_wo_ch["unsustainable solid biomass"] = _calc_unsustainable_potential( diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index a4f22220..5aefb26b 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -242,7 +242,7 @@ def build_eurostat( temp.index = pd.MultiIndex.from_frame( temp.index.to_frame().fillna("International aviation") ) - df = pd.concat([temp, df.loc[~int_avia]]) + df = pd.concat([temp, df.loc[~int_avia]]).sort_index() # Fill in missing data on "Domestic aviation" for each country. for country in countries: @@ -651,8 +651,8 @@ def build_energy_totals( """ eurostat_fuels = {"electricity": "Electricity", "total": "Total all products"} - eurostat_countries = eurostat.index.levels[0] - eurostat_years = eurostat.index.levels[1] + eurostat_countries = eurostat.index.unique(0) + eurostat_years = eurostat.index.unique(1) to_drop = ["passenger cars", "passenger car efficiency"] new_index = pd.MultiIndex.from_product( @@ -1153,13 +1153,14 @@ def build_transport_data( ---------- - Swiss transport data: `BFS `_ """ + years = np.arange(2000, 2022) # first collect number of cars transport_data = pd.DataFrame(idees["passenger cars"]) countries_without_ch = set(countries) - {"CH"} new_index = pd.MultiIndex.from_product( - [countries_without_ch, transport_data.index.levels[1]], + [countries_without_ch, transport_data.index.unique(1)], names=["country", "year"], ) @@ -1167,7 +1168,7 @@ def build_transport_data( if "CH" in countries: fn = snakemake.input.swiss_transport - swiss_cars = pd.read_csv(fn, index_col=0).loc[2000:2023, ["passenger cars"]] + swiss_cars = pd.read_csv(fn, index_col=0).loc[years, ["passenger cars"]] swiss_cars.index = pd.MultiIndex.from_product( [["CH"], swiss_cars.index], names=["country", "year"] @@ -1175,10 +1176,8 @@ def build_transport_data( transport_data = pd.concat([transport_data, swiss_cars]).sort_index() - transport_data.rename(columns={"passenger cars": "number cars"}, inplace=True) - + transport_data = transport_data.rename(columns={"passenger cars": "number cars"}) # clean up dataframe - years = np.arange(2000, 2022) transport_data = transport_data[ transport_data.index.get_level_values(1).isin(years) ] @@ -1188,11 +1187,10 @@ def build_transport_data( logger.info( f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data." ) - cars_pp = transport_data["number cars"] / population fill_values = { - year: cars_pp.mean() * population for year in transport_data.index.levels[1] + year: cars_pp.mean() * population for year in transport_data.index.unique(1) } fill_values = pd.DataFrame(fill_values).stack() fill_values = pd.DataFrame(fill_values, columns=["number cars"]) diff --git a/scripts/build_industrial_energy_demand_per_node_today.py b/scripts/build_industrial_energy_demand_per_node_today.py index 7a1ee7ac..aa9e9dff 100644 --- a/scripts/build_industrial_energy_demand_per_node_today.py +++ b/scripts/build_industrial_energy_demand_per_node_today.py @@ -66,7 +66,7 @@ def build_nodal_industrial_energy_demand(): ) countries = keys.country.unique() - sectors = industrial_demand.columns.levels[1] + sectors = industrial_demand.columns.unique(1) for country, sector in product(countries, sectors): buses = keys.index[keys.country == country] diff --git a/scripts/build_retro_cost.py b/scripts/build_retro_cost.py index 44f4a738..c6dd31cf 100755 --- a/scripts/build_retro_cost.py +++ b/scripts/build_retro_cost.py @@ -254,7 +254,7 @@ def prepare_building_stock_data(): index = pd.MultiIndex.from_product([[ct], averaged_data.index.to_list()]) averaged_data.index = index averaged_data["estimated"] = 1 - if ct not in area_tot.index.levels[0]: + if ct not in area_tot.index.unique(0): area_tot = pd.concat([area_tot, averaged_data], sort=True) else: area_tot.loc[averaged_data.index] = averaged_data diff --git a/scripts/make_summary_perfect.py b/scripts/make_summary_perfect.py index 8e56e5d4..59aa6e74 100644 --- a/scripts/make_summary_perfect.py +++ b/scripts/make_summary_perfect.py @@ -40,9 +40,9 @@ def calculate_costs(n, label, costs): investments = n.investment_periods cols = pd.MultiIndex.from_product( [ - costs.columns.levels[0], - costs.columns.levels[1], - costs.columns.levels[2], + costs.columns.unique(0), + costs.columns.unique(1), + costs.columns.unique(2), investments, ], names=costs.columns.names[:3] + ["year"], @@ -339,9 +339,9 @@ def calculate_supply_energy(n, label, supply_energy): investments = n.investment_periods cols = pd.MultiIndex.from_product( [ - supply_energy.columns.levels[0], - supply_energy.columns.levels[1], - supply_energy.columns.levels[2], + supply_energy.columns.unique(0), + supply_energy.columns.unique(1), + supply_energy.columns.unique(2), investments, ], names=supply_energy.columns.names[:3] + ["year"], From 54a2d24517c140fdd9026d4b1e882865f1302d2e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:39:34 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_industry_sector_ratios.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index 530ac910..c2438f91 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -445,7 +445,9 @@ def chemicals_industry(): # subtract ammonia energy demand (in ktNH3/a) ammonia = pd.read_csv(snakemake.input.ammonia_production, index_col=0) - ammonia_total = ammonia.loc[ammonia.index.intersection(eu27), str(max(2018, year))].sum() + ammonia_total = ammonia.loc[ + ammonia.index.intersection(eu27), str(max(2018, year)) + ].sum() df.loc["methane", sector] -= ammonia_total * params["MWh_CH4_per_tNH3_SMR"] df.loc["elec", sector] -= ammonia_total * params["MWh_elec_per_tNH3_SMR"]