From 61bd0972e74e14e89df5c988a207b8933c4d429f Mon Sep 17 00:00:00 2001
From: BIGAUD Nathan <nathan.bigaud@inria.fr>
Date: Fri, 4 Aug 2023 16:06:03 +0200
Subject: [PATCH] Updating doc, not yet tested or linted
---
deploy_mock/scripts/build_compose.py | 28 ++++++++--
deploy_mock/scripts/client.py | 7 +++
deploy_mock/scripts/client.sh | 2 +-
deploy_mock/scripts/server.py | 21 ++++++--
deploy_mock/scripts/setup.py | 76 +++++++++++++++++++---------
5 files changed, 104 insertions(+), 30 deletions(-)
diff --git a/deploy_mock/scripts/build_compose.py b/deploy_mock/scripts/build_compose.py
index 38fd0e02..80329a8e 100644
--- a/deploy_mock/scripts/build_compose.py
+++ b/deploy_mock/scripts/build_compose.py
@@ -1,4 +1,7 @@
+"""Utility generating a compose.yaml file"""
+
import os
+from io import TextIOWrapper
from pathlib import Path
from typing import Optional
@@ -8,16 +11,35 @@ FILEDIR = os.path.dirname(__file__)
CPUS = os.cpu_count()
-def get_text_file(filename: str):
+def get_text_file(filename: str) -> TextIOWrapper:
+ """Utility opening a file in the same directory"""
return open(os.path.join(FILEDIR, filename), "r", encoding="utf-8")
def build_compose_file(
clients: int = 2,
port: int = 8000,
- mem_limit: int = 100,
+ mem_limit: int = 30,
cpu_limit: Optional[float] = None,
-):
+) -> None:
+ """Utility recursively generating a compose.yaml file
+
+ Parameters
+ ----------
+ clients: int, default=2
+ The number of clients
+ port: int, default=8000
+ The host port on which to hook the server container.
+ Clients will hook sequentially on the next ports,
+ e.g., 8001, 8002, ...
+ mem_limit: int, default=30
+ Maximum number of MB available to each client
+ cpu_limit: optional float
+ Maximim percentage of the CPU to be used by an
+ individual client. If None, default to
+ 0.7* <number_of_cpus> / <number of clients>
+ """
+
# Getting templates
with get_text_file("compose_server.txt") as file:
server_string = file.read().replace("$HOSTPORT$", str(port))
diff --git a/deploy_mock/scripts/client.py b/deploy_mock/scripts/client.py
index cd5547b9..97c86bb2 100644
--- a/deploy_mock/scripts/client.py
+++ b/deploy_mock/scripts/client.py
@@ -1,3 +1,5 @@
+"Mock client python script"
+
import sys
import fire
@@ -9,6 +11,11 @@ from comms import message_pb2, message_pb2_grpc
def run(server: str = "server", port: str = "8000"):
+
+ """Read data.csv, peform a simple calcultaion and send
+ its result to the server, awaiting its response to close
+ """
+
print("Calculating result")
df = pd.read_csv("data.csv")
out = int(df.to_numpy().sum() ** 2)
diff --git a/deploy_mock/scripts/client.sh b/deploy_mock/scripts/client.sh
index 8502baec..3f321b19 100644
--- a/deploy_mock/scripts/client.sh
+++ b/deploy_mock/scripts/client.sh
@@ -1,2 +1,2 @@
-# Run the client python script with the the server address as input
+# Run the client python script
python -u run.py
diff --git a/deploy_mock/scripts/server.py b/deploy_mock/scripts/server.py
index e420fb54..9654a2c2 100644
--- a/deploy_mock/scripts/server.py
+++ b/deploy_mock/scripts/server.py
@@ -14,15 +14,30 @@ FILEDIR = os.path.dirname(__file__)
class Servicer(message_pb2_grpc.BasicMessageServicer):
+ """Servicer for the GRPC server"""
+
def __init__(self):
self.responses = []
def SendMessage(self, request, context):
+ """Python counterpart of the SendMessage method
+ defined in the .proto file"""
self.responses.append(request.res)
return message_pb2.BasicReply(ok=f"{request.res}")
-def serve(clients: int = 5, port: str = "8000"):
+def serve(clients: int = 2, port: str = "8000") -> None:
+ """Mian server script. Open a communication line, wait for all
+ clients to send their results, close once all rsuslts were received.
+
+ Parameters
+ ----------
+ clients: int, default=2
+ The number of clients
+ port: int, default=8000
+ The host port on which to hook the server container.
+ """
+
# Set up
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
servicer = Servicer()
@@ -32,7 +47,7 @@ def serve(clients: int = 5, port: str = "8000"):
server.start()
print("Server started, listening on " + port)
# Monitoring results
- target_sum = sum((i**2 for i in range(1,clients+1)))
+ target_sum = sum((i**2 for i in range(1, clients + 1)))
while sum(servicer.responses) < target_sum:
print(f"Current sum:{sum(servicer.responses)}, goal:{target_sum}")
time.sleep(2)
@@ -44,7 +59,7 @@ def serve(clients: int = 5, port: str = "8000"):
print("Target reached, exchange closed")
server.stop(None)
else:
- raise (ValueError("Unexpected results from clients"))
+ raise ValueError("Unexpected results from clients")
if __name__ == "__main__":
diff --git a/deploy_mock/scripts/setup.py b/deploy_mock/scripts/setup.py
index 2f569aad..3763d2b1 100644
--- a/deploy_mock/scripts/setup.py
+++ b/deploy_mock/scripts/setup.py
@@ -11,10 +11,61 @@ import fire
FILEDIR = os.path.dirname(__file__)
+def write_dir(
+ dest_dir: str,
+ grpc_dir: str,
+ dir_type: str,
+ client_nb: Optional[int] = None,
+ dotenv: bool = False,
+) -> str:
+
+ """Utility copying over selected files into a destination directory.
+
+ Parameters
+ ----------
+ dest_dir: str
+ The target directory
+ grpc_dir: str
+ The source directory for grpc files
+ dir_type: str
+ The type of directory created, intented to be
+ "server" or "client"
+ client_nb: optional int
+ The id of the client, ignored if dir_type is not
+ "client"
+ dotenv: bool, default=False
+ Whether or not to copy over the .env file
+
+ """
+
+ opj = os.path.join # local alias to shorten code
+ name = (
+ f"{dir_type}_{client_nb}"
+ if client_nb is not None and dir_type == "client"
+ else dir_type
+ )
+ out_dir = opj(dest_dir, name)
+ os.makedirs(out_dir, exist_ok=True)
+ shutil.copyfile(opj(FILEDIR, f"{dir_type}.py"), opj(out_dir, "run.py"))
+ shutil.copyfile(opj(FILEDIR, f"{dir_type}.sh"), opj(out_dir, "run.sh"))
+ shutil.copytree(grpc_dir, opj(out_dir, "comms"))
+ if dotenv:
+ shutil.copyfile(opj(FILEDIR, ".env"), opj(out_dir, ".env"))
+
+ return out_dir
+
+
def generate_folders(clients: int):
- """Generate the server folder and the appropriate number of client folders,
- with appropriate scripts and simulated, unique data"""
+ """
+ Generate the server folder and the appropriate number of client folders,
+ with appropriate scripts and simulated, unique data
+
+ Parameters
+ ----------
+ clients: int
+ The number of clients
+ """
# Clear existing files
dest_dir = os.path.join(Path(FILEDIR).resolve().parents[0], "nodes")
@@ -36,26 +87,5 @@ def generate_folders(clients: int):
writer.writerow([clt / 2, clt / 2])
-def write_dir(
- dest_dir: str,
- grpc_dir: str,
- dir_type: str,
- client_nb: Optional[int] = None,
- dotenv: bool = False,
-) -> str:
-
- opj = os.path.join # local alias to shorten code
- name = f"{dir_type}_{client_nb}" if client_nb is not None else dir_type
- out_dir = opj(dest_dir, name)
- os.makedirs(out_dir, exist_ok=True)
- shutil.copyfile(opj(FILEDIR, f"{dir_type}.py"), opj(out_dir, "run.py"))
- shutil.copyfile(opj(FILEDIR, f"{dir_type}.sh"), opj(out_dir, "run.sh"))
- shutil.copytree(grpc_dir, opj(out_dir, "comms"))
- if dotenv:
- shutil.copyfile(opj(FILEDIR, ".env"), opj(out_dir, ".env"))
-
- return out_dir
-
-
if __name__ == "__main__":
fire.Fire(generate_folders)
--
GitLab