From b62f7ed6fbd50dba4d0813e35e9a801d21a8adbb Mon Sep 17 00:00:00 2001
From: Paul Andrey <paul.andrey@inria.fr>
Date: Fri, 31 Mar 2023 15:42:17 +0200
Subject: [PATCH] Add a custom logging level to filter multiprocessing outputs.
---
declearn/main/utils/_training.py | 5 +++--
declearn/quickrun/run.py | 11 +++++++++--
declearn/utils/__init__.py | 12 ++++++++++--
declearn/utils/_logging.py | 12 ++++++++++++
4 files changed, 34 insertions(+), 6 deletions(-)
diff --git a/declearn/main/utils/_training.py b/declearn/main/utils/_training.py
index 76e46bcc..901d9538 100644
--- a/declearn/main/utils/_training.py
+++ b/declearn/main/utils/_training.py
@@ -33,7 +33,7 @@ from declearn.metrics import MeanMetric, Metric, MetricInputType, MetricSet
from declearn.model.api import Model
from declearn.optimizer import Optimizer
from declearn.typing import Batch
-from declearn.utils import get_logger
+from declearn.utils import LOGGING_LEVEL_MAJOR, get_logger
__all__ = [
"TrainingManager",
@@ -344,7 +344,8 @@ class TrainingManager:
effort = constraints.get_values()
result = self.metrics.get_result()
states = self.metrics.get_states()
- self.logger.info(
+ self.logger.log(
+ LOGGING_LEVEL_MAJOR,
"Local scalar evaluation metrics: %s",
{k: v for k, v in result.items() if isinstance(v, float)},
)
diff --git a/declearn/quickrun/run.py b/declearn/quickrun/run.py
index bb9969e3..6d653a63 100644
--- a/declearn/quickrun/run.py
+++ b/declearn/quickrun/run.py
@@ -33,6 +33,7 @@ using privided model and data, and stores its result in the same folder.
import argparse
import importlib
+import logging
import os
import re
import textwrap
@@ -51,7 +52,7 @@ from declearn.quickrun._config import (
from declearn.quickrun._parser import parse_data_folder
from declearn.quickrun._split_data import split_data
from declearn.test_utils import make_importable
-from declearn.utils import run_as_processes
+from declearn.utils import LOGGING_LEVEL_MAJOR, get_logger, run_as_processes
__all__ = ["quickrun"]
@@ -85,8 +86,9 @@ def run_server(
else:
checkpoint = os.path.join(folder, "result")
checkpoint = os.path.join(checkpoint, "server")
+ logger = get_logger("Server", fpath=os.path.join(checkpoint, "logger.txt"))
server = FederatedServer(
- model, network, optim, expe_config.metrics, checkpoint
+ model, network, optim, expe_config.metrics, checkpoint, logger
)
server.run(config)
@@ -110,6 +112,11 @@ def run_client(
else:
checkpoint = os.path.join(folder, "result")
checkpoint = os.path.join(checkpoint, name)
+ # Set up a logger: write everything to file, but filter console outputs.
+ logger = get_logger(name, fpath=os.path.join(checkpoint, "logs.txt"))
+ for handler in logger.handlers:
+ if isinstance(handler, logging.StreamHandler):
+ handler.setLevel(LOGGING_LEVEL_MAJOR)
# Wrap train and validation data as Dataset objects.
train = InMemoryDataset(
paths.get("train_data"),
diff --git a/declearn/utils/__init__.py b/declearn/utils/__init__.py
index ba1084fb..451517d3 100644
--- a/declearn/utils/__init__.py
+++ b/declearn/utils/__init__.py
@@ -80,6 +80,15 @@ Utils to access or update parameters defining a global device-selection policy.
* [set_device_policy][declearn.utils.set_device_policy]:
Update the current global device policy.
+Logging utils
+-------------
+Utils to set up and configure loggers:
+
+* [get_logger][declearn.utils.get_logger]:
+ Access or create a logger, automating basic handlers' configuration.
+* [LOGGING_LEVEL_MAJOR][declearn.utils.LOGGING_LEVEL_MAJOR]:
+ Custom "MAJOR" severity level, between stdlib "INFO" and "WARNING".
+
Miscellaneous
-------------
@@ -89,8 +98,6 @@ Miscellaneous
Automatically build a dataclass matching a function's signature.
* [dataclass_from_init][declearn.utils.dataclass_from_init]:
Automatically build a dataclass matching a class's init signature.
-* [get_logger][declearn.utils.get_logger]:
- Access or create a logger, automating basic handlers' configuration.
* [run_as_processes][declearn.utils.run_as_processes]:
Run coroutines concurrently within individual processes.
"""
@@ -112,6 +119,7 @@ from ._json import (
json_unpack,
)
from ._logging import (
+ LOGGING_LEVEL_MAJOR,
get_logger,
)
from ._multiprocess import run_as_processes
diff --git a/declearn/utils/_logging.py b/declearn/utils/_logging.py
index b1c92691..07f47627 100644
--- a/declearn/utils/_logging.py
+++ b/declearn/utils/_logging.py
@@ -22,6 +22,18 @@ import os
from typing import Optional
+__all__ = [
+ "get_logger",
+ "LOGGING_LEVEL_MAJOR",
+]
+
+
+# Add a logging level between INFO and WARNING.
+LOGGING_LEVEL_MAJOR = (logging.WARNING + logging.INFO) // 2
+"""Custom "MAJOR" severity level, between stdlib "INFO" and "WARNING"."""
+logging.addLevelName(level=LOGGING_LEVEL_MAJOR, levelName="MAJOR")
+
+
DEFAULT_FORMAT = "%(asctime)s:%(name)s:%(levelname)s: %(message)s"
--
GitLab