From 64cbfd673c0a6022d711fb0d4e10bca6f2683b94 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:33:23 +0000 Subject: [PATCH] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_benchmark.py | 77 ++++++++++++++++++++++++++-------------- scripts/solve_network.py | 2 +- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/scripts/_benchmark.py b/scripts/_benchmark.py index 777ad7ac..ba49ddc5 100644 --- a/scripts/_benchmark.py +++ b/scripts/_benchmark.py @@ -14,35 +14,38 @@ ## You should have received a copy of the GNU General Public License ## along with this program. If not, see . - -""" """ -from __future__ import absolute_import -from __future__ import print_function +""" -import time -import sys, os +from __future__ import absolute_import, print_function import logging +import os +import sys +import time + logger = logging.getLogger(__name__) # TODO: provide alternative when multiprocessing is not available try: - from multiprocessing import Process, Pipe + from multiprocessing import Pipe, Process except ImportError: from multiprocessing.dummy import Process, Pipe from memory_profiler import _get_memory, choose_backend + # The memory logging facilities have been adapted from memory_profiler class MemTimer(Process): """ Write memory consumption over a time interval to file until signaled to - stop on the pipe + stop on the pipe. """ - def __init__(self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw): + def __init__( + self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw + ): self.monitor_pid = monitor_pid self.interval = interval self.pipe = pipe @@ -57,17 +60,18 @@ class MemTimer(Process): def run(self): # get baseline memory usage - cur_mem = ( - _get_memory(self.monitor_pid, self.backend, - timestamps=self.timestamps, - include_children=self.include_children) + cur_mem = _get_memory( + self.monitor_pid, + self.backend, + timestamps=self.timestamps, + include_children=self.include_children, ) n_measurements = 1 mem_usage = cur_mem if self.max_usage else [cur_mem] if self.filename is not None: - stream = open(self.filename, 'w') + stream = open(self.filename, "w") stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem)) stream.flush() else: @@ -76,10 +80,11 @@ class MemTimer(Process): self.pipe.send(0) # we're ready stop = False while True: - cur_mem = ( - _get_memory(self.monitor_pid, self.backend, - timestamps=self.timestamps, - include_children=self.include_children) + cur_mem = _get_memory( + self.monitor_pid, + self.backend, + timestamps=self.timestamps, + include_children=self.include_children, ) if stream is not None: @@ -103,6 +108,7 @@ class MemTimer(Process): self.pipe.send(mem_usage) self.pipe.send(n_measurements) + class memory_logger(object): """ Context manager for taking and reporting memory measurements at fixed @@ -144,8 +150,15 @@ class memory_logger(object): max_mem, timestamp = mem.mem_usage """ - def __init__(self, filename=None, interval=1., max_usage=True, - timestamps=True, include_children=True): + + def __init__( + self, + filename=None, + interval=1.0, + max_usage=True, + timestamps=True, + include_children=True, + ): if filename is not None: timestamps = True @@ -159,9 +172,16 @@ class memory_logger(object): backend = choose_backend() self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results - self.p = MemTimer(os.getpid(), self.interval, self.child_conn, self.filename, - backend=backend, timestamps=self.timestamps, max_usage=self.max_usage, - include_children=self.include_children) + self.p = MemTimer( + os.getpid(), + self.interval, + self.child_conn, + self.filename, + backend=backend, + timestamps=self.timestamps, + max_usage=self.max_usage, + include_children=self.include_children, + ) self.p.start() self.parent_conn.recv() # wait until memory logging in subprocess is ready @@ -169,7 +189,7 @@ class memory_logger(object): def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is None: - self.parent_conn.send(0) # finish timing + self.parent_conn.send(0) # finish timing self.mem_usage = self.parent_conn.recv() self.n_measurements = self.parent_conn.recv() @@ -178,6 +198,7 @@ class memory_logger(object): return False + class timer(object): level = 0 opened = False @@ -213,7 +234,6 @@ class timer(object): sec = msec / 1000 print("%.1f sec" % sec) - def __exit__(self, exc_type, exc_val, exc_tb): if not self.opened and self.verbose: sys.stdout.write(".. " * self.level) @@ -221,15 +241,18 @@ class timer(object): if exc_type is None: stop = time.time() self.usec = usec = (stop - self.start) * 1e6 - if self.verbose: self.print_usec(usec) + if self.verbose: + self.print_usec(usec) elif self.verbose: print("failed") sys.stdout.flush() self.__class__.level -= 1 - if self.verbose: self.__class__.opened = False + if self.verbose: + self.__class__.opened = False return False + class optional(object): def __init__(self, variable, contextman): self.variable = variable diff --git a/scripts/solve_network.py b/scripts/solve_network.py index e1b243c8..2e4c3305 100644 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -33,9 +33,9 @@ import numpy as np import pandas as pd import pypsa import xarray as xr +from _benchmark import memory_logger from _helpers import configure_logging, update_config_with_sector_opts from pypsa.descriptors import get_activity_mask -from _benchmark import memory_logger logger = logging.getLogger(__name__) pypsa.pf.logger.setLevel(logging.WARNING)