From d237e60b2555a9f110a2c2212b591ee5de744e3c Mon Sep 17 00:00:00 2001 From: virio-andreyana Date: Thu, 23 Feb 2023 10:30:32 +0100 Subject: [PATCH] Replacing print statement with logging --- scripts/add_brownfield.py | 7 ++-- scripts/add_existing_baseyear.py | 4 +-- scripts/build_energy_totals.py | 13 +++++--- scripts/build_industrial_distribution_key.py | 10 ++++-- ...build_industrial_production_per_country.py | 7 +++- scripts/build_population_layouts.py | 7 +++- scripts/make_summary.py | 11 ++++--- scripts/plot_network.py | 13 +++++--- scripts/plot_summary.py | 32 ++++++++++--------- 9 files changed, 67 insertions(+), 37 deletions(-) diff --git a/scripts/add_brownfield.py b/scripts/add_brownfield.py index 8a8f0af3..b68f5326 100644 --- a/scripts/add_brownfield.py +++ b/scripts/add_brownfield.py @@ -16,7 +16,7 @@ from helper import override_component_attrs, update_config_with_sector_opts def add_brownfield(n, n_p, year): - print("adding brownfield") + logger.info(f"preparing brownfield for the year {year}") # electric transmission grid set optimised capacities of previous as minimum n.lines.s_nom_min = n_p.lines.s_nom_opt @@ -122,10 +122,11 @@ if __name__ == "__main__": planning_horizons=2030, ) + logging.basicConfig(level=snakemake.config['logging_level']) + update_config_with_sector_opts(snakemake.config, snakemake.wildcards.sector_opts) - print(snakemake.input.network_p) - logging.basicConfig(level=snakemake.config['logging_level']) + logger.info(f"preparing brownfield from the file {snakemake.input.network_p}") year = int(snakemake.wildcards.planning_horizons) diff --git a/scripts/add_existing_baseyear.py b/scripts/add_existing_baseyear.py index 74b26211..7f21e8d1 100644 --- a/scripts/add_existing_baseyear.py +++ b/scripts/add_existing_baseyear.py @@ -120,7 +120,7 @@ def add_power_capacities_installed_before_baseyear(n, grouping_years, costs, bas to read lifetime to estimate YearDecomissioning baseyear : int """ - print("adding power capacities installed before baseyear from powerplants.csv") + logger.debug(f"adding power capacities installed before {baseyear} from powerplants.csv") df_agg = pd.read_csv(snakemake.input.powerplants, index_col=0) @@ -352,7 +352,7 @@ def add_heating_capacities_installed_before_baseyear(n, baseyear, grouping_years services proportional to heating load in both 50% capacities in rural busess 50% in urban buses """ - print("adding heating capacities installed before baseyear") + logger.debug(f"adding heating capacities installed before {baseyear}") # Add existing heating capacities, data comes from the study # "Mapping and analyses of the current and future (2020 - 2030) diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 7ff5d216..48852728 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1,3 +1,6 @@ +import logging +logger = logging.getLogger(__name__) + from functools import partial from tqdm import tqdm from helper import mute_print @@ -454,7 +457,7 @@ def build_energy_totals(countries, eurostat, swiss, idees): fuel_use = df[f"electricity {sector} {use}"] fuel = df[f"electricity {sector}"] avg = fuel_use.div(fuel).mean() - print(f"{sector}: average fraction of electricity for {use} is {avg:.3f}") + logger.debug(f"{sector}: average fraction of electricity for {use} is {avg:.3f}") df.loc[to_fill, f"electricity {sector} {use}"] = avg * df.loc[to_fill, f"electricity {sector}"] # non-electric use @@ -463,7 +466,7 @@ def build_energy_totals(countries, eurostat, swiss, idees): nonelectric_use = df[f"total {sector} {use}"] - df[f"electricity {sector} {use}"] nonelectric = df[f"total {sector}"] - df[f"electricity {sector}"] avg = nonelectric_use.div(nonelectric).mean() - print(f"{sector}: average fraction of non-electric for {use} is {avg:.3f}") + logger.debug(f"{sector}: average fraction of non-electric for {use} is {avg:.3f}") electric_use = df.loc[to_fill, f"electricity {sector} {use}"] nonelectric = df.loc[to_fill, f"total {sector}"] - df.loc[to_fill, f"electricity {sector}"] df.loc[to_fill, f"total {sector} {use}"] = electric_use + avg * nonelectric @@ -673,7 +676,7 @@ def build_transport_data(countries, population, idees): transport_data.at["CH", "number cars"] = 4.136e6 missing = transport_data.index[transport_data["number cars"].isna()] - print(f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data.") + logger.info(f"Missing data on cars from:\n{list(missing)}\nFilling gaps with averaged data.") cars_pp = transport_data["number cars"] / population transport_data.loc[missing, "number cars"] = cars_pp.mean() * population @@ -683,7 +686,7 @@ def build_transport_data(countries, population, idees): transport_data["average fuel efficiency"] = idees["passenger car efficiency"] missing = transport_data.index[transport_data["average fuel efficiency"].isna()] - print(f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data.") + logger.info(f"Missing data on fuel efficiency from:\n{list(missing)}\nFilling gapswith averaged data.") fill_values = transport_data["average fuel efficiency"].mean() transport_data.loc[missing, "average fuel efficiency"] = fill_values @@ -696,6 +699,8 @@ if __name__ == "__main__": from helper import mock_snakemake snakemake = mock_snakemake('build_energy_totals') + logging.basicConfig(level=snakemake.config['logging_level']) + config = snakemake.config["energy"] nuts3 = gpd.read_file(snakemake.input.nuts3_shapes).set_index("index") diff --git a/scripts/build_industrial_distribution_key.py b/scripts/build_industrial_distribution_key.py index 835445da..b888e24f 100644 --- a/scripts/build_industrial_distribution_key.py +++ b/scripts/build_industrial_distribution_key.py @@ -1,5 +1,8 @@ """Build industrial distribution keys from hotmaps database.""" +import logging +logger = logging.getLogger(__name__) + import uuid import pandas as pd import geopandas as gpd @@ -33,7 +36,7 @@ def locate_missing_industrial_sites(df): loc = geocode([s.City, s.Country], geometry='wkt') if loc is not None: - print(f"Found:\t{loc}\nFor:\t{s['City']}, {s['Country']}\n") + logger.debug(f"Found:\t{loc}\nFor:\t{s['City']}, {s['Country']}\n") return f"POINT({loc.longitude} {loc.latitude})" else: return None @@ -46,8 +49,7 @@ def locate_missing_industrial_sites(df): num_found = len(missing) - num_still_missing share_missing = len(missing) / len(df) * 100 share_still_missing = num_still_missing / len(df) * 100 - print(f"Found {num_found} missing locations.", - f"Share of missing locations reduced from {share_missing:.2f}% to {share_still_missing:.2f}%.") + logger.warning(f"Found {num_found} missing locations. \nShare of missing locations reduced from {share_missing:.2f}% to {share_still_missing:.2f}%.") return df @@ -125,6 +127,8 @@ if __name__ == "__main__": clusters=48, ) + logging.basicConfig(level=snakemake.config['logging_level']) + regions = gpd.read_file(snakemake.input.regions_onshore).set_index('name') hotmaps = prepare_hotmaps_database(regions) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index a00e41ab..120d4373 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -1,5 +1,8 @@ """Build industrial production per country.""" +import logging +logger = logging.getLogger(__name__) + import pandas as pd import numpy as np import multiprocessing as mp @@ -189,7 +192,7 @@ def separate_basic_chemicals(demand): there = ammonia.index.intersection(demand.index) missing = demand.index.symmetric_difference(there) - print("Following countries have no ammonia demand:", missing) + logger.info(f"Following countries have no ammonia demand: {missing.tolist()}") demand["Ammonia"] = 0. @@ -213,6 +216,8 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake('build_industrial_production_per_country') + logging.basicConfig(level=snakemake.config['logging_level']) + countries = non_EU + eu28 year = snakemake.config['industry']['reference_year'] diff --git a/scripts/build_population_layouts.py b/scripts/build_population_layouts.py index 7936d831..e414f135 100644 --- a/scripts/build_population_layouts.py +++ b/scripts/build_population_layouts.py @@ -1,5 +1,8 @@ """Build mapping between grid cells and population (total, urban, rural)""" +import logging +logger = logging.getLogger(__name__) + import multiprocessing as mp import atlite import numpy as np @@ -14,6 +17,8 @@ if __name__ == '__main__': from helper import mock_snakemake snakemake = mock_snakemake('build_population_layouts') + logging.basicConfig(level=snakemake.config['logging_level']) + cutout = atlite.Cutout(snakemake.config['atlite']['cutout']) grid_cells = cutout.grid.geometry @@ -57,7 +62,7 @@ if __name__ == '__main__': pop_urban = pd.Series(0., density_cells.index) for ct in countries: - print(ct, urban_fraction[ct]) + logger.debug(f"The urbanization rate for {ct} is {round(urban_fraction[ct]*100)}%") indicator_nuts3_ct = nuts3.country.apply(lambda x: 1. if x == ct else 0.) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 06680cb4..599c5902 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -1,3 +1,5 @@ +import logging +logger = logging.getLogger(__name__) import sys import yaml @@ -431,8 +433,9 @@ def calculate_weighted_prices(n, label, weighted_prices): weighted_prices.loc[carrier,label] = (load * n.buses_t.marginal_price[buses]).sum().sum() / load.sum().sum() + # still have no idea what this is for, only for debug reasons. if carrier[:5] == "space": - print(load * n.buses_t.marginal_price[buses]) + logger.debug(load * n.buses_t.marginal_price[buses]) return weighted_prices @@ -537,7 +540,7 @@ def make_summaries(networks_dict): df[output] = pd.DataFrame(columns=columns, dtype=float) for label, filename in networks_dict.items(): - print(label, filename) + logger.info(f"make summary for scenario {label}, using {filename}") overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(filename, override_component_attrs=overrides) @@ -561,6 +564,8 @@ if __name__ == "__main__": from helper import mock_snakemake snakemake = mock_snakemake('make_summary') + logging.basicConfig(level=snakemake.config['logging_level']) + networks_dict = { (cluster, lv, opt+sector_opt, planning_horizon) : snakemake.config['results_dir'] + snakemake.config['run'] + f'/postnetworks/elec_s{simpl}_{cluster}_lv{lv}_{opt}_{sector_opt}_{planning_horizon}.nc' \ @@ -572,8 +577,6 @@ if __name__ == "__main__": for planning_horizon in snakemake.config['scenario']['planning_horizons'] } - print(networks_dict) - Nyears = 1 costs_db = prepare_costs( diff --git a/scripts/plot_network.py b/scripts/plot_network.py index df72bce1..567cd9ba 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -1,3 +1,6 @@ +import logging +logger = logging.getLogger(__name__) + import pypsa import pandas as pd @@ -75,7 +78,7 @@ def plot_map(network, components=["links", "stores", "storage_units", "generator .unstack().fillna(0.)) costs = pd.concat([costs, costs_c], axis=1) - print(comp, costs) + logger.debug(f"{comp}, {costs}") costs = costs.groupby(costs.columns, axis=1).sum() @@ -87,7 +90,7 @@ def plot_map(network, components=["links", "stores", "storage_units", "generator for item in new_columns: if item not in tech_colors: - print("Warning!",item,"not in config/plotting/tech_colors") + logger.warning(f"{item} not in config/plotting/tech_colors") costs = costs.stack() # .sort_index() @@ -102,7 +105,7 @@ def plot_map(network, components=["links", "stores", "storage_units", "generator # drop non-bus to_drop = costs.index.levels[0].symmetric_difference(n.buses.index) if len(to_drop) != 0: - print("dropping non-buses", to_drop) + logger.info(f"dropping non-buses {to_drop.tolist()}") costs.drop(to_drop, level=0, inplace=True, axis=0, errors="ignore") # make sure they are removed from index @@ -751,7 +754,7 @@ def plot_series(network, carrier="AC", name="test"): to_drop = supply.columns[(abs(supply) < threshold).all()] if len(to_drop) != 0: - print("dropping", to_drop) + logger.info(f"dropping {to_drop.tolist()} from supply") supply.drop(columns=to_drop, inplace=True) supply.index.name = None @@ -841,6 +844,8 @@ if __name__ == "__main__": planning_horizons="2050", ) + logging.basicConfig(level=snakemake.config['logging_level']) + overrides = override_component_attrs(snakemake.input.overrides) n = pypsa.Network(snakemake.input.network, override_component_attrs=overrides) diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index be08a3af..6f107e03 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -1,4 +1,5 @@ - +import logging +logger = logging.getLogger(__name__) import numpy as np import pandas as pd @@ -135,13 +136,12 @@ def plot_costs(): to_drop = df.index[df.max(axis=1) < snakemake.config['plotting']['costs_threshold']] - print("dropping") - - print(df.loc[to_drop]) + logger.info(f"dropping technology with costs below {snakemake.config['plotting']['costs_threshold']} EUR billion per year") + logger.debug(df.loc[to_drop]) df = df.drop(to_drop) - print(df.sum()) + logger.info(f"Total system cost of {round(df.sum()[0])} EUR billion per year") new_index = preferred_order.intersection(df.index).append(df.index.difference(preferred_order)) @@ -191,15 +191,12 @@ def plot_energy(): to_drop = df.index[df.abs().max(axis=1) < snakemake.config['plotting']['energy_threshold']] - print("dropping") - - print(df.loc[to_drop]) + logger.info(f"dropping all technology with energy consumption or production below {snakemake.config['plotting']['energy_threshold']} TWh/a") + logger.debug(df.loc[to_drop]) df = df.drop(to_drop) - print(df.sum()) - - print(df) + logger.info(f"Total energy of {round(df.sum()[0])} TWh/a") new_index = preferred_order.intersection(df.index).append(df.index.difference(preferred_order)) @@ -207,7 +204,7 @@ def plot_energy(): fig, ax = plt.subplots(figsize=(12,8)) - print(df.loc[new_index, new_columns]) + logger.debug(df.loc[new_index, new_columns]) df.loc[new_index, new_columns].T.plot( kind="bar", @@ -263,13 +260,17 @@ def plot_balances(): to_drop = df.index[df.abs().max(axis=1) < snakemake.config['plotting']['energy_threshold']/10] - print("dropping") + if v[0] in co2_carriers: + units = "MtCO2/a" + else: + units = "TWh/a" - print(df.loc[to_drop]) + logger.info(f"dropping technology energy balance smaller than {snakemake.config['plotting']['energy_threshold']/10} {units}") + logger.debug(df.loc[to_drop]) df = df.drop(to_drop) - print(df.sum()) + logger.info(f"Total energy balance for {v} of {round(df.sum()[0],2)} {units}") if df.empty: continue @@ -441,6 +442,7 @@ if __name__ == "__main__": from helper import mock_snakemake snakemake = mock_snakemake('plot_summary') + logging.basicConfig(level=snakemake.config['logging_level']) n_header = 4