[pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
This commit is contained in:
parent
c278a3159f
commit
64cbfd673c
@ -14,35 +14,38 @@
|
|||||||
|
|
||||||
## You should have received a copy of the GNU General Public License
|
## You should have received a copy of the GNU General Public License
|
||||||
## along with this program. If not, see <http://www.gnu.org/licenses/>.
|
## along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
"""
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import absolute_import
|
"""
|
||||||
from __future__ import print_function
|
|
||||||
|
|
||||||
import time
|
from __future__ import absolute_import, print_function
|
||||||
import sys, os
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
# TODO: provide alternative when multiprocessing is not available
|
# TODO: provide alternative when multiprocessing is not available
|
||||||
try:
|
try:
|
||||||
from multiprocessing import Process, Pipe
|
from multiprocessing import Pipe, Process
|
||||||
except ImportError:
|
except ImportError:
|
||||||
from multiprocessing.dummy import Process, Pipe
|
from multiprocessing.dummy import Process, Pipe
|
||||||
|
|
||||||
from memory_profiler import _get_memory, choose_backend
|
from memory_profiler import _get_memory, choose_backend
|
||||||
|
|
||||||
|
|
||||||
# The memory logging facilities have been adapted from memory_profiler
|
# The memory logging facilities have been adapted from memory_profiler
|
||||||
class MemTimer(Process):
|
class MemTimer(Process):
|
||||||
"""
|
"""
|
||||||
Write memory consumption over a time interval to file until signaled to
|
Write memory consumption over a time interval to file until signaled to
|
||||||
stop on the pipe
|
stop on the pipe.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw):
|
def __init__(
|
||||||
|
self, monitor_pid, interval, pipe, filename, max_usage, backend, *args, **kw
|
||||||
|
):
|
||||||
self.monitor_pid = monitor_pid
|
self.monitor_pid = monitor_pid
|
||||||
self.interval = interval
|
self.interval = interval
|
||||||
self.pipe = pipe
|
self.pipe = pipe
|
||||||
@ -57,17 +60,18 @@ class MemTimer(Process):
|
|||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
# get baseline memory usage
|
# get baseline memory usage
|
||||||
cur_mem = (
|
cur_mem = _get_memory(
|
||||||
_get_memory(self.monitor_pid, self.backend,
|
self.monitor_pid,
|
||||||
timestamps=self.timestamps,
|
self.backend,
|
||||||
include_children=self.include_children)
|
timestamps=self.timestamps,
|
||||||
|
include_children=self.include_children,
|
||||||
)
|
)
|
||||||
|
|
||||||
n_measurements = 1
|
n_measurements = 1
|
||||||
mem_usage = cur_mem if self.max_usage else [cur_mem]
|
mem_usage = cur_mem if self.max_usage else [cur_mem]
|
||||||
|
|
||||||
if self.filename is not None:
|
if self.filename is not None:
|
||||||
stream = open(self.filename, 'w')
|
stream = open(self.filename, "w")
|
||||||
stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem))
|
stream.write("MEM {0:.6f} {1:.4f}\n".format(*cur_mem))
|
||||||
stream.flush()
|
stream.flush()
|
||||||
else:
|
else:
|
||||||
@ -76,10 +80,11 @@ class MemTimer(Process):
|
|||||||
self.pipe.send(0) # we're ready
|
self.pipe.send(0) # we're ready
|
||||||
stop = False
|
stop = False
|
||||||
while True:
|
while True:
|
||||||
cur_mem = (
|
cur_mem = _get_memory(
|
||||||
_get_memory(self.monitor_pid, self.backend,
|
self.monitor_pid,
|
||||||
timestamps=self.timestamps,
|
self.backend,
|
||||||
include_children=self.include_children)
|
timestamps=self.timestamps,
|
||||||
|
include_children=self.include_children,
|
||||||
)
|
)
|
||||||
|
|
||||||
if stream is not None:
|
if stream is not None:
|
||||||
@ -103,6 +108,7 @@ class MemTimer(Process):
|
|||||||
self.pipe.send(mem_usage)
|
self.pipe.send(mem_usage)
|
||||||
self.pipe.send(n_measurements)
|
self.pipe.send(n_measurements)
|
||||||
|
|
||||||
|
|
||||||
class memory_logger(object):
|
class memory_logger(object):
|
||||||
"""
|
"""
|
||||||
Context manager for taking and reporting memory measurements at fixed
|
Context manager for taking and reporting memory measurements at fixed
|
||||||
@ -144,8 +150,15 @@ class memory_logger(object):
|
|||||||
|
|
||||||
max_mem, timestamp = mem.mem_usage
|
max_mem, timestamp = mem.mem_usage
|
||||||
"""
|
"""
|
||||||
def __init__(self, filename=None, interval=1., max_usage=True,
|
|
||||||
timestamps=True, include_children=True):
|
def __init__(
|
||||||
|
self,
|
||||||
|
filename=None,
|
||||||
|
interval=1.0,
|
||||||
|
max_usage=True,
|
||||||
|
timestamps=True,
|
||||||
|
include_children=True,
|
||||||
|
):
|
||||||
if filename is not None:
|
if filename is not None:
|
||||||
timestamps = True
|
timestamps = True
|
||||||
|
|
||||||
@ -159,9 +172,16 @@ class memory_logger(object):
|
|||||||
backend = choose_backend()
|
backend = choose_backend()
|
||||||
|
|
||||||
self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results
|
self.child_conn, self.parent_conn = Pipe() # this will store MemTimer's results
|
||||||
self.p = MemTimer(os.getpid(), self.interval, self.child_conn, self.filename,
|
self.p = MemTimer(
|
||||||
backend=backend, timestamps=self.timestamps, max_usage=self.max_usage,
|
os.getpid(),
|
||||||
include_children=self.include_children)
|
self.interval,
|
||||||
|
self.child_conn,
|
||||||
|
self.filename,
|
||||||
|
backend=backend,
|
||||||
|
timestamps=self.timestamps,
|
||||||
|
max_usage=self.max_usage,
|
||||||
|
include_children=self.include_children,
|
||||||
|
)
|
||||||
self.p.start()
|
self.p.start()
|
||||||
self.parent_conn.recv() # wait until memory logging in subprocess is ready
|
self.parent_conn.recv() # wait until memory logging in subprocess is ready
|
||||||
|
|
||||||
@ -169,7 +189,7 @@ class memory_logger(object):
|
|||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
if exc_type is None:
|
if exc_type is None:
|
||||||
self.parent_conn.send(0) # finish timing
|
self.parent_conn.send(0) # finish timing
|
||||||
|
|
||||||
self.mem_usage = self.parent_conn.recv()
|
self.mem_usage = self.parent_conn.recv()
|
||||||
self.n_measurements = self.parent_conn.recv()
|
self.n_measurements = self.parent_conn.recv()
|
||||||
@ -178,6 +198,7 @@ class memory_logger(object):
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
class timer(object):
|
class timer(object):
|
||||||
level = 0
|
level = 0
|
||||||
opened = False
|
opened = False
|
||||||
@ -213,7 +234,6 @@ class timer(object):
|
|||||||
sec = msec / 1000
|
sec = msec / 1000
|
||||||
print("%.1f sec" % sec)
|
print("%.1f sec" % sec)
|
||||||
|
|
||||||
|
|
||||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
if not self.opened and self.verbose:
|
if not self.opened and self.verbose:
|
||||||
sys.stdout.write(".. " * self.level)
|
sys.stdout.write(".. " * self.level)
|
||||||
@ -221,15 +241,18 @@ class timer(object):
|
|||||||
if exc_type is None:
|
if exc_type is None:
|
||||||
stop = time.time()
|
stop = time.time()
|
||||||
self.usec = usec = (stop - self.start) * 1e6
|
self.usec = usec = (stop - self.start) * 1e6
|
||||||
if self.verbose: self.print_usec(usec)
|
if self.verbose:
|
||||||
|
self.print_usec(usec)
|
||||||
elif self.verbose:
|
elif self.verbose:
|
||||||
print("failed")
|
print("failed")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
self.__class__.level -= 1
|
self.__class__.level -= 1
|
||||||
if self.verbose: self.__class__.opened = False
|
if self.verbose:
|
||||||
|
self.__class__.opened = False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
class optional(object):
|
class optional(object):
|
||||||
def __init__(self, variable, contextman):
|
def __init__(self, variable, contextman):
|
||||||
self.variable = variable
|
self.variable = variable
|
||||||
|
@ -33,9 +33,9 @@ import numpy as np
|
|||||||
import pandas as pd
|
import pandas as pd
|
||||||
import pypsa
|
import pypsa
|
||||||
import xarray as xr
|
import xarray as xr
|
||||||
|
from _benchmark import memory_logger
|
||||||
from _helpers import configure_logging, update_config_with_sector_opts
|
from _helpers import configure_logging, update_config_with_sector_opts
|
||||||
from pypsa.descriptors import get_activity_mask
|
from pypsa.descriptors import get_activity_mask
|
||||||
from _benchmark import memory_logger
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
pypsa.pf.logger.setLevel(logging.WARNING)
|
pypsa.pf.logger.setLevel(logging.WARNING)
|
||||||
|
Loading…
Reference in New Issue
Block a user