[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2023-08-30 12:33:23 +00:00
parent c278a3159f
commit 64cbfd673c
2 changed files with 51 additions and 28 deletions

View File

@ -14,35 +14,38 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
"""
from __future__ import absolute_import
from __future__ import print_function
"""
import time
import sys, os
from __future__ import absolute_import, print_function
import logging
import os
import sys
import time
logger = logging.getLogger(__name__)
# TODO: provide alternative when multiprocessing is not available
try:
from multiprocessing import Process, Pipe
from multiprocessing import Pipe, Process
except ImportError:
from multiprocessing.dummy import Process, Pipe
from memory_profiler import _get_memory, choose_backend
# The memory logging facilities have been adapted from memory_profiler
class MemTimer(Process):
"""
Write memory consumption over a time interval to file until signaled to
stop on the pipe
stop on the pipe.
"""
def __init__(self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw):
def __init__(
self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw
):
self.monitor_pid = monitor_pid
self.interval = interval
self.pipe = pipe
@ -57,17 +60,18 @@ class MemTimer(Process):
def run(self):
# get baseline memory usage
cur_mem = (
_get_memory(self.monitor_pid, self.backend,
cur_mem = _get_memory(
self.monitor_pid,
self.backend,
timestamps=self.timestamps,
include_children=self.include_children)
include_children=self.include_children,
)
n_measurements = 1
mem_usage = cur_mem if self.max_usage else [cur_mem]
if self.filename is not None:
stream = open(self.filename, 'w')
stream = open(self.filename, "w")
stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem))
stream.flush()
else:
@ -76,10 +80,11 @@ class MemTimer(Process):
self.pipe.send(0) # we're ready
stop = False
while True:
cur_mem = (
_get_memory(self.monitor_pid, self.backend,
cur_mem = _get_memory(
self.monitor_pid,
self.backend,
timestamps=self.timestamps,
include_children=self.include_children)
include_children=self.include_children,
)
if stream is not None:
@ -103,6 +108,7 @@ class MemTimer(Process):
self.pipe.send(mem_usage)
self.pipe.send(n_measurements)
class memory_logger(object):
"""
Context manager for taking and reporting memory measurements at fixed
@ -144,8 +150,15 @@ class memory_logger(object):
max_mem, timestamp = mem.mem_usage
"""
def __init__(self, filename=None, interval=1., max_usage=True,
timestamps=True, include_children=True):
def __init__(
self,
filename=None,
interval=1.0,
max_usage=True,
timestamps=True,
include_children=True,
):
if filename is not None:
timestamps = True
@ -159,9 +172,16 @@ class memory_logger(object):
backend = choose_backend()
self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results
self.p = MemTimer(os.getpid(), self.interval, self.child_conn, self.filename,
backend=backend, timestamps=self.timestamps, max_usage=self.max_usage,
include_children=self.include_children)
self.p = MemTimer(
os.getpid(),
self.interval,
self.child_conn,
self.filename,
backend=backend,
timestamps=self.timestamps,
max_usage=self.max_usage,
include_children=self.include_children,
)
self.p.start()
self.parent_conn.recv() # wait until memory logging in subprocess is ready
@ -178,6 +198,7 @@ class memory_logger(object):
return False
class timer(object):
level = 0
opened = False
@ -213,7 +234,6 @@ class timer(object):
sec = msec / 1000
print("%.1f sec" % sec)
def __exit__(self, exc_type, exc_val, exc_tb):
if not self.opened and self.verbose:
sys.stdout.write(".. " * self.level)
@ -221,15 +241,18 @@ class timer(object):
if exc_type is None:
stop = time.time()
self.usec = usec = (stop - self.start) * 1e6
if self.verbose: self.print_usec(usec)
if self.verbose:
self.print_usec(usec)
elif self.verbose:
print("failed")
sys.stdout.flush()
self.__class__.level -= 1
if self.verbose: self.__class__.opened = False
if self.verbose:
self.__class__.opened = False
return False
class optional(object):
def __init__(self, variable, contextman):
self.variable = variable

View File

@ -33,9 +33,9 @@ import numpy as np
import pandas as pd
import pypsa
import xarray as xr
from _benchmark import memory_logger
from _helpers import configure_logging, update_config_with_sector_opts
from pypsa.descriptors import get_activity_mask
from _benchmark import memory_logger
logger = logging.getLogger(__name__)
pypsa.pf.logger.setLevel(logging.WARNING)