[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2023-08-30 12:33:23 +00:00
parent c278a3159f
commit 64cbfd673c
2 changed files with 51 additions and 28 deletions

View File

@ -14,35 +14,38 @@
## You should have received a copy of the GNU General Public License ## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>. ## along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
""" """
from __future__ import absolute_import """
from __future__ import print_function
import time from __future__ import absolute_import, print_function
import sys, os
import logging import logging
import os
import sys
import time
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# TODO: provide alternative when multiprocessing is not available # TODO: provide alternative when multiprocessing is not available
try: try:
from multiprocessing import Process, Pipe from multiprocessing import Pipe, Process
except ImportError: except ImportError:
from multiprocessing.dummy import Process, Pipe from multiprocessing.dummy import Process, Pipe
from memory_profiler import _get_memory, choose_backend from memory_profiler import _get_memory, choose_backend
# The memory logging facilities have been adapted from memory_profiler # The memory logging facilities have been adapted from memory_profiler
class MemTimer(Process): class MemTimer(Process):
""" """
Write memory consumption over a time interval to file until signaled to Write memory consumption over a time interval to file until signaled to
stop on the pipe stop on the pipe.
""" """
def __init__(self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw): def __init__(
self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw
):
self.monitor_pid = monitor_pid self.monitor_pid = monitor_pid
self.interval = interval self.interval = interval
self.pipe = pipe self.pipe = pipe
@ -57,17 +60,18 @@ class MemTimer(Process):
def run(self): def run(self):
# get baseline memory usage # get baseline memory usage
cur_mem = ( cur_mem = _get_memory(
_get_memory(self.monitor_pid, self.backend, self.monitor_pid,
timestamps=self.timestamps, self.backend,
include_children=self.include_children) timestamps=self.timestamps,
include_children=self.include_children,
) )
n_measurements = 1 n_measurements = 1
mem_usage = cur_mem if self.max_usage else [cur_mem] mem_usage = cur_mem if self.max_usage else [cur_mem]
if self.filename is not None: if self.filename is not None:
stream = open(self.filename, 'w') stream = open(self.filename, "w")
stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem)) stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem))
stream.flush() stream.flush()
else: else:
@ -76,10 +80,11 @@ class MemTimer(Process):
self.pipe.send(0) # we're ready self.pipe.send(0) # we're ready
stop = False stop = False
while True: while True:
cur_mem = ( cur_mem = _get_memory(
_get_memory(self.monitor_pid, self.backend, self.monitor_pid,
timestamps=self.timestamps, self.backend,
include_children=self.include_children) timestamps=self.timestamps,
include_children=self.include_children,
) )
if stream is not None: if stream is not None:
@ -103,6 +108,7 @@ class MemTimer(Process):
self.pipe.send(mem_usage) self.pipe.send(mem_usage)
self.pipe.send(n_measurements) self.pipe.send(n_measurements)
class memory_logger(object): class memory_logger(object):
""" """
Context manager for taking and reporting memory measurements at fixed Context manager for taking and reporting memory measurements at fixed
@ -144,8 +150,15 @@ class memory_logger(object):
max_mem, timestamp = mem.mem_usage max_mem, timestamp = mem.mem_usage
""" """
def __init__(self, filename=None, interval=1., max_usage=True,
timestamps=True, include_children=True): def __init__(
self,
filename=None,
interval=1.0,
max_usage=True,
timestamps=True,
include_children=True,
):
if filename is not None: if filename is not None:
timestamps = True timestamps = True
@ -159,9 +172,16 @@ class memory_logger(object):
backend = choose_backend() backend = choose_backend()
self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results
self.p = MemTimer(os.getpid(), self.interval, self.child_conn, self.filename, self.p = MemTimer(
backend=backend, timestamps=self.timestamps, max_usage=self.max_usage, os.getpid(),
include_children=self.include_children) self.interval,
self.child_conn,
self.filename,
backend=backend,
timestamps=self.timestamps,
max_usage=self.max_usage,
include_children=self.include_children,
)
self.p.start() self.p.start()
self.parent_conn.recv() # wait until memory logging in subprocess is ready self.parent_conn.recv() # wait until memory logging in subprocess is ready
@ -169,7 +189,7 @@ class memory_logger(object):
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None: if exc_type is None:
self.parent_conn.send(0) # finish timing self.parent_conn.send(0) # finish timing
self.mem_usage = self.parent_conn.recv() self.mem_usage = self.parent_conn.recv()
self.n_measurements = self.parent_conn.recv() self.n_measurements = self.parent_conn.recv()
@ -178,6 +198,7 @@ class memory_logger(object):
return False return False
class timer(object): class timer(object):
level = 0 level = 0
opened = False opened = False
@ -213,7 +234,6 @@ class timer(object):
sec = msec / 1000 sec = msec / 1000
print("%.1f sec" % sec) print("%.1f sec" % sec)
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
if not self.opened and self.verbose: if not self.opened and self.verbose:
sys.stdout.write(".. " * self.level) sys.stdout.write(".. " * self.level)
@ -221,15 +241,18 @@ class timer(object):
if exc_type is None: if exc_type is None:
stop = time.time() stop = time.time()
self.usec = usec = (stop - self.start) * 1e6 self.usec = usec = (stop - self.start) * 1e6
if self.verbose: self.print_usec(usec) if self.verbose:
self.print_usec(usec)
elif self.verbose: elif self.verbose:
print("failed") print("failed")
sys.stdout.flush() sys.stdout.flush()
self.__class__.level -= 1 self.__class__.level -= 1
if self.verbose: self.__class__.opened = False if self.verbose:
self.__class__.opened = False
return False return False
class optional(object): class optional(object):
def __init__(self, variable, contextman): def __init__(self, variable, contextman):
self.variable = variable self.variable = variable

View File

@ -33,9 +33,9 @@ import numpy as np
import pandas as pd import pandas as pd
import pypsa import pypsa
import xarray as xr import xarray as xr
from _benchmark import memory_logger
from _helpers import configure_logging, update_config_with_sector_opts from _helpers import configure_logging, update_config_with_sector_opts
from pypsa.descriptors import get_activity_mask from pypsa.descriptors import get_activity_mask
from _benchmark import memory_logger
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
pypsa.pf.logger.setLevel(logging.WARNING) pypsa.pf.logger.setLevel(logging.WARNING)