Merge pull request #306 from PyPSA/hide-subprocess-output

Hide uninformative text caused by subprocess outputs

scripts/build_energy_totals.py

@@ -1,5 +1,6 @@
 from functools import partial
 from tqdm import tqdm
+from helper import mute
 
 import multiprocessing as mp
 import pandas as pd
@@ -8,6 +9,8 @@ import numpy as np
 
 idx = pd.IndexSlice
 
+mute()
+
 def cartesian(s1, s2):
     """Cartesian product of two pd.Series"""
     return pd.DataFrame(np.outer(s1, s2), index=s1.index, columns=s2.index)
@@ -126,7 +129,6 @@ to_ipcc = {
     "total woL": "Total (without LULUCF)",
 }
 
-
 def build_eurostat(input_eurostat, countries, report_year,  year):
     """Return multi-index for all countries' energy data in TWh/a."""
 
@@ -380,7 +382,7 @@ def build_idees(countries, year):
     func = partial(idees_per_country, year=year)
     tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries),
                        desc='Build from IDEES database')
-    with mp.Pool(processes=nprocesses) as pool:
+    with mp.Pool(processes=nprocesses, initializer=mute) as pool:
         totals_list = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))
 
 

scripts/build_industrial_production_per_country.py

@@ -4,7 +4,7 @@ import pandas as pd
 import numpy as np
 import multiprocessing as mp
 from tqdm import tqdm
-
+from helper import mute
 
 tj_to_ktoe = 0.0238845
 ktoe_to_twh = 0.01163
@@ -99,7 +99,6 @@ e_switzerland = pd.Series({'Iron and steel': 7889.,
                    'Other Industrial Sectors': 10825.,
                    'current electricity': 53760.})
 
-
 def find_physical_output(df):
     start = np.where(df.index.str.contains('Physical output', na=''))[0][0]
     empty_row = np.where(df.index.isnull())[0]
@@ -169,7 +168,7 @@ def industry_production(countries):
     func = industry_production_per_country
     tqdm_kwargs = dict(ascii=False, unit=' country', total=len(countries),
                        desc="Build industry production")
-    with mp.Pool(processes=nprocesses) as pool:
+    with mp.Pool(processes=nprocesses, initializer=mute) as pool:
         demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))
 
     demand = pd.concat(demand_l, axis=1).T

scripts/build_industry_sector_ratios.py

@@ -1,6 +1,9 @@
 """Build industry sector ratios."""
 
 import pandas as pd
+from helper import mute
+
+mute()
 
 # GWh/ktoe OR MWh/toe
 toe_to_MWh = 11.630

scripts/helper.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import yaml
 import pytz
 import pandas as pd
@@ -10,6 +11,10 @@ from pypsa.components import components, component_attrs
 import logging
 logger = logging.getLogger(__name__)
 
+def mute():
+    """hide irrelevant outputs of subprocess in multiprocessing pools.
+    also hide irrelevant outputs caused by pd.read_excel"""
+    sys.stdout = open(os.devnull, 'w')
 
 def override_component_attrs(directory):
     """Tell PyPSA that links can have multiple outputs by