Commit 287946b3 authored by S. Lackner's avatar S. Lackner

[sched] Simplify the scheduler by extracting the logging logic

parent 2831f5ce
"""
batsim.sched.logging
~~~~~~~~~~~~~~~~~~~~
This module provides logging and exporting utilities for data in the scheduler.
"""
import logging
import os
class LoggingEvent:
"""Class for storing data about events triggered by the scheduler.
:param time: the simulation time when the event occurred
:param level: the importance level of the event
:param msg: the actual message of the event
:param type: the type of the event (`str`)
:param data: additional data attached to the event (`dict`)
"""
def __init__(self, time, level, msg, type, data):
self.time = time
self.level = level
self.msg = msg
self.type = type
self.data = data
def __str__(self):
data = ";".join(
["{}={}".format(
str(k).replace(";", ","),
str(v).replace(";", ",")) for k, v in self.data.items()])
return "{:.6f};{};{};{};{}".format(
self.time, self.level, self.type, self.msg, data)
class Logger:
"""Base logger class which handles various cases needed for the loggers.
:param obj: either a string with the name of the logger or an object or a type
(in which case the logger will be based on the type).
:param logger_suffix: a suffix appended to the generated logger name
:param debug: whether the output of debug messages should be enabled
:param streamhandler: whether or not the logger should output to stdout/stderr
:param to_file: the file to log to if set (will be overwritten each time)
:param append_to_file: the file to log to if set (new lines will be appended)
"""
def __init__(
self,
obj,
logger_suffix=None,
debug=False,
streamhandler=True,
to_file=None,
append_to_file=None):
if isinstance(obj, type):
obj = obj.__name__
elif not isinstance(obj, str):
obj = obj.__class__.__name__
if logger_suffix:
obj = obj + "_" + logger_suffix
self._logger = logger = logging.getLogger(obj)
self._debug = debug = debug or str(debug).lower() in [
"y", "yes", "true", "1"]
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
if streamhandler:
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(self.formatter)
logger.addHandler(handler)
if to_file:
try:
os.remove(to_file)
except OSError:
pass
handler = logging.FileHandler(to_file)
handler.setLevel(logging.DEBUG)
handler.setFormatter(self.file_formatter)
logger.addHandler(handler)
if append_to_file:
handler = logging.FileHandler(append_to_file)
handler.setLevel(logging.DEBUG)
handler.setFormatter(self.file_formatter)
logger.addHandler(handler)
@property
def formatter(self):
return logging.Formatter('[%(name)s::%(levelname)s] %(message)s')
@property
def file_formatter(self):
return self.formatter
@property
def has_debug(self):
return self._debug
def debug(self, *args, **kwargs):
"""Writes a debug message to the logger."""
self._logger.debug(*args, **kwargs)
def info(self, *args, **kwargs):
"""Writes a info message to the logger."""
self._logger.info(*args, **kwargs)
def warn(self, *args, **kwargs):
"""Writes a warn message to the logger."""
self._logger.warn(*args, **kwargs)
def error(self, *args, **kwargs):
"""Writes a error message to the logger."""
self._logger.error(*args, **kwargs)
class EventLogger(Logger):
"""Logger for events which will only log to files and will write the log messages
without any additional formatting.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, streamhandler=False, **kwargs)
@property
def file_formatter(self):
return logging.Formatter('%(message)s')
......@@ -7,11 +7,9 @@
interact with the basic scheduler to provide a richer interface.
"""
import logging
import os
from abc import ABCMeta, abstractmethod
from batsim.batsim import BatsimScheduler
from batsim.batsim import BatsimScheduler, Batsim
from .resource import Resources, Resource
from .job import Job, Jobs
......@@ -19,6 +17,7 @@ from .reply import ConsumedEnergyReply
from .utils import DictWrapper
from .messages import Message
from .utils import ListView
from .logging import LoggingEvent, Logger, EventLogger
class BaseBatsimScheduler(BatsimScheduler):
......@@ -185,39 +184,26 @@ class Scheduler(metaclass=ABCMeta):
"""
class Event:
"""Class for storing data about events triggered by the scheduler.
:param time: the simulation time when the event occurred
:param level: the importance level of the event
:param msg: the actual message of the event
:param type: the type of the event (`str`)
def __init__(self, options={}):
self._options = options
debug = self.options.get("debug", False)
export_prefix = self.options.get("export_prefix", "out")
:param data: additional data attached to the event (`dict`)
"""
# Create the logger
self._logger = Logger(self, debug=debug)
def __init__(self, time, level, msg, type, data):
self.time = time
self.level = level
self.msg = msg
self.type = type
self.data = data
def __str__(self):
data = ";".join(
["{}={}".format(
str(k).replace(";", ","),
str(v).replace(";", ",")) for k, v in self.data.items()])
return "{:.6f};{};{};{};{}".format(
self.time, self.level, self.type, self.msg, data)
self._event_logger = EventLogger(
self, "Events", debug=debug, to_file="{}_last_events.csv".format(
export_prefix),
append_to_file="{}_events.csv".format(export_prefix))
def __init__(self, options={}):
self._options = options
self._sched_jobs_logger = EventLogger(
self,
"SchedJobs",
debug=debug,
to_file="{}_sched_jobs.csv".format(export_prefix))
self._log_job_header()
self._init_logger()
self._events = []
# Use the basic Pybatsim scheduler to wrap the Batsim API
......@@ -237,71 +223,6 @@ class Scheduler(metaclass=ABCMeta):
self.debug("Scheduler initialised", type="scheduler_initialised")
def _init_logger(self):
debug = self.options.get("debug", False)
if isinstance(debug, str):
debug = debug.lower() in ["y", "yes", "true", "1"]
self._logger = logging.getLogger(self.__class__.__name__)
if debug:
self._logger.setLevel(logging.DEBUG)
else:
self._logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'[%(name)s::%(levelname)s] %(message)s')
# Add the stream handler (stdout)
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
self._logger.addHandler(handler)
self._event_logger = logging.getLogger(
self.__class__.__name__ + "Events")
self._event_logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
export_prefix = self.options.get("export_prefix", "out")
# Add the persistent event logging handler (not purged between runs)
handler = logging.FileHandler(
"{}_events.csv".format(export_prefix))
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
self._event_logger.addHandler(handler)
# Add the event logging handler for the last run (purged when the next
# run starts)
filename_lastschedule = "{}_last_events.csv".format(
export_prefix)
try:
os.remove(filename_lastschedule)
except OSError:
pass
handler = logging.FileHandler(filename_lastschedule)
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
self._event_logger.addHandler(handler)
# Add the logger with information about scheduled jobs.
self._sched_jobs_logger = logging.getLogger(
self.__class__.__name__ + "SchedJobs")
self._sched_jobs_logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
filename_sched_jobs = "{}_sched_jobs.csv".format(
export_prefix)
try:
os.remove(filename_sched_jobs)
except OSError:
pass
handler = logging.FileHandler(filename_sched_jobs)
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
self._sched_jobs_logger.addHandler(handler)
self._log_job_header()
@property
def events(self):
"""The events happened in the scheduler."""
......@@ -373,7 +294,7 @@ class Scheduler(metaclass=ABCMeta):
def _format_event_msg(self, level, msg, type="msg", **kwargs):
msg = msg.format(**kwargs)
event = Scheduler.Event(self.time, level, msg, type, kwargs)
event = LoggingEvent(self.time, level, msg, type, kwargs)
self._events.append(event)
self.on_event(event)
......@@ -436,6 +357,7 @@ class Scheduler(metaclass=ABCMeta):
type_of_completion,
reason_for_completion
]
msg = ["" if s is None else s for s in msg]
self._sched_jobs_logger.info(";".join([str(i) for i in msg]))
def debug(self, msg, **kwargs):
......@@ -627,7 +549,7 @@ class Scheduler(metaclass=ABCMeta):
def on_event(self, event):
"""Hook called on each event triggered by the scheduler.
:param event: the triggered event (class: `Scheduler.Event`)
:param event: the triggered event (class: `LoggingEvent`)
"""
pass
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment