diff --git a/scripts/build_energy_totals.py b/scripts/build_energy_totals.py index 575070ad..38a9895c 100644 --- a/scripts/build_energy_totals.py +++ b/scripts/build_energy_totals.py @@ -1,5 +1,6 @@ from functools import partial from tqdm import tqdm +from helper import mute import multiprocessing as mp import pandas as pd @@ -8,6 +9,8 @@ import numpy as np idx = pd.IndexSlice +mute() + def cartesian(s1, s2): """Cartesian product of two pd.Series""" return pd.DataFrame(np.outer(s1, s2), index=s1.index, columns=s2.index) @@ -126,7 +129,6 @@ to_ipcc = { "total woL": "Total (without LULUCF)", } - def build_eurostat(input_eurostat, countries, report_year, year): """Return multi-index for all countries' energy data in TWh/a.""" @@ -380,7 +382,7 @@ def build_idees(countries, year): func = partial(idees_per_country, year=year) tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), desc='Build from IDEES database') - with mp.Pool(processes=nprocesses) as pool: + with mp.Pool(processes=nprocesses, initializer=mute) as pool: totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) diff --git a/scripts/build_industrial_production_per_country.py b/scripts/build_industrial_production_per_country.py index f21b711a..c88919ab 100644 --- a/scripts/build_industrial_production_per_country.py +++ b/scripts/build_industrial_production_per_country.py @@ -4,7 +4,7 @@ import pandas as pd import numpy as np import multiprocessing as mp from tqdm import tqdm - +from helper import mute tj_to_ktoe = 0.0238845 ktoe_to_twh = 0.01163 @@ -99,7 +99,6 @@ e_switzerland = pd.Series({'Iron and steel': 7889., 'Other Industrial Sectors': 10825., 'current electricity': 53760.}) - def find_physical_output(df): start = np.where(df.index.str.contains('Physical output', na=''))[0][0] empty_row = np.where(df.index.isnull())[0] @@ -169,7 +168,7 @@ def industry_production(countries): func = industry_production_per_country tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries), desc="Build industry production") - with mp.Pool(processes=nprocesses) as pool: + with mp.Pool(processes=nprocesses, initializer=mute) as pool: demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs)) demand = pd.concat(demand_l, axis=1).T diff --git a/scripts/build_industry_sector_ratios.py b/scripts/build_industry_sector_ratios.py index d2a9ca48..a8c00941 100644 --- a/scripts/build_industry_sector_ratios.py +++ b/scripts/build_industry_sector_ratios.py @@ -1,6 +1,9 @@ """Build industry sector ratios.""" import pandas as pd +from helper import mute + +mute() # GWh/ktoe OR MWh/toe toe_to_MWh = 11.630 diff --git a/scripts/helper.py b/scripts/helper.py index 62ae33c0..b099061c 100644 --- a/scripts/helper.py +++ b/scripts/helper.py @@ -1,4 +1,5 @@ import os +import sys import yaml import pytz import pandas as pd @@ -10,6 +11,10 @@ from pypsa.components import components, component_attrs import logging logger = logging.getLogger(__name__) +def mute(): + """hide irrelevant outputs of subprocess in multiprocessing pools. + also hide irrelevant outputs caused by pd.read_excel""" + sys.stdout = open(os.devnull, 'w') def override_component_attrs(directory): """Tell PyPSA that links can have multiple outputs by