Mentions légales du service

Skip to content
Snippets Groups Projects
Commit 61bd0972 authored by BIGAUD Nathan's avatar BIGAUD Nathan
Browse files

Updating doc, not yet tested or linted

parent ebbbaad7
No related branches found
No related tags found
1 merge request!46Draft: Docker deploy
Pipeline #842124 waiting for manual action
"""Utility generating a compose.yaml file"""
import os import os
from io import TextIOWrapper
from pathlib import Path from pathlib import Path
from typing import Optional from typing import Optional
...@@ -8,16 +11,35 @@ FILEDIR = os.path.dirname(__file__) ...@@ -8,16 +11,35 @@ FILEDIR = os.path.dirname(__file__)
CPUS = os.cpu_count() CPUS = os.cpu_count()
def get_text_file(filename: str): def get_text_file(filename: str) -> TextIOWrapper:
"""Utility opening a file in the same directory"""
return open(os.path.join(FILEDIR, filename), "r", encoding="utf-8") return open(os.path.join(FILEDIR, filename), "r", encoding="utf-8")
def build_compose_file( def build_compose_file(
clients: int = 2, clients: int = 2,
port: int = 8000, port: int = 8000,
mem_limit: int = 100, mem_limit: int = 30,
cpu_limit: Optional[float] = None, cpu_limit: Optional[float] = None,
): ) -> None:
"""Utility recursively generating a compose.yaml file
Parameters
----------
clients: int, default=2
The number of clients
port: int, default=8000
The host port on which to hook the server container.
Clients will hook sequentially on the next ports,
e.g., 8001, 8002, ...
mem_limit: int, default=30
Maximum number of MB available to each client
cpu_limit: optional float
Maximim percentage of the CPU to be used by an
individual client. If None, default to
0.7* <number_of_cpus> / <number of clients>
"""
# Getting templates # Getting templates
with get_text_file("compose_server.txt") as file: with get_text_file("compose_server.txt") as file:
server_string = file.read().replace("$HOSTPORT$", str(port)) server_string = file.read().replace("$HOSTPORT$", str(port))
......
"Mock client python script"
import sys import sys
import fire import fire
...@@ -9,6 +11,11 @@ from comms import message_pb2, message_pb2_grpc ...@@ -9,6 +11,11 @@ from comms import message_pb2, message_pb2_grpc
def run(server: str = "server", port: str = "8000"): def run(server: str = "server", port: str = "8000"):
"""Read data.csv, peform a simple calcultaion and send
its result to the server, awaiting its response to close
"""
print("Calculating result") print("Calculating result")
df = pd.read_csv("data.csv") df = pd.read_csv("data.csv")
out = int(df.to_numpy().sum() ** 2) out = int(df.to_numpy().sum() ** 2)
......
# Run the client python script with the the server address as input # Run the client python script
python -u run.py python -u run.py
...@@ -14,15 +14,30 @@ FILEDIR = os.path.dirname(__file__) ...@@ -14,15 +14,30 @@ FILEDIR = os.path.dirname(__file__)
class Servicer(message_pb2_grpc.BasicMessageServicer): class Servicer(message_pb2_grpc.BasicMessageServicer):
"""Servicer for the GRPC server"""
def __init__(self): def __init__(self):
self.responses = [] self.responses = []
def SendMessage(self, request, context): def SendMessage(self, request, context):
"""Python counterpart of the SendMessage method
defined in the .proto file"""
self.responses.append(request.res) self.responses.append(request.res)
return message_pb2.BasicReply(ok=f"{request.res}") return message_pb2.BasicReply(ok=f"{request.res}")
def serve(clients: int = 5, port: str = "8000"): def serve(clients: int = 2, port: str = "8000") -> None:
"""Mian server script. Open a communication line, wait for all
clients to send their results, close once all rsuslts were received.
Parameters
----------
clients: int, default=2
The number of clients
port: int, default=8000
The host port on which to hook the server container.
"""
# Set up # Set up
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
servicer = Servicer() servicer = Servicer()
...@@ -32,7 +47,7 @@ def serve(clients: int = 5, port: str = "8000"): ...@@ -32,7 +47,7 @@ def serve(clients: int = 5, port: str = "8000"):
server.start() server.start()
print("Server started, listening on " + port) print("Server started, listening on " + port)
# Monitoring results # Monitoring results
target_sum = sum((i**2 for i in range(1,clients+1))) target_sum = sum((i**2 for i in range(1, clients + 1)))
while sum(servicer.responses) < target_sum: while sum(servicer.responses) < target_sum:
print(f"Current sum:{sum(servicer.responses)}, goal:{target_sum}") print(f"Current sum:{sum(servicer.responses)}, goal:{target_sum}")
time.sleep(2) time.sleep(2)
...@@ -44,7 +59,7 @@ def serve(clients: int = 5, port: str = "8000"): ...@@ -44,7 +59,7 @@ def serve(clients: int = 5, port: str = "8000"):
print("Target reached, exchange closed") print("Target reached, exchange closed")
server.stop(None) server.stop(None)
else: else:
raise (ValueError("Unexpected results from clients")) raise ValueError("Unexpected results from clients")
if __name__ == "__main__": if __name__ == "__main__":
......
...@@ -11,10 +11,61 @@ import fire ...@@ -11,10 +11,61 @@ import fire
FILEDIR = os.path.dirname(__file__) FILEDIR = os.path.dirname(__file__)
def write_dir(
dest_dir: str,
grpc_dir: str,
dir_type: str,
client_nb: Optional[int] = None,
dotenv: bool = False,
) -> str:
"""Utility copying over selected files into a destination directory.
Parameters
----------
dest_dir: str
The target directory
grpc_dir: str
The source directory for grpc files
dir_type: str
The type of directory created, intented to be
"server" or "client"
client_nb: optional int
The id of the client, ignored if dir_type is not
"client"
dotenv: bool, default=False
Whether or not to copy over the .env file
"""
opj = os.path.join # local alias to shorten code
name = (
f"{dir_type}_{client_nb}"
if client_nb is not None and dir_type == "client"
else dir_type
)
out_dir = opj(dest_dir, name)
os.makedirs(out_dir, exist_ok=True)
shutil.copyfile(opj(FILEDIR, f"{dir_type}.py"), opj(out_dir, "run.py"))
shutil.copyfile(opj(FILEDIR, f"{dir_type}.sh"), opj(out_dir, "run.sh"))
shutil.copytree(grpc_dir, opj(out_dir, "comms"))
if dotenv:
shutil.copyfile(opj(FILEDIR, ".env"), opj(out_dir, ".env"))
return out_dir
def generate_folders(clients: int): def generate_folders(clients: int):
"""Generate the server folder and the appropriate number of client folders, """
with appropriate scripts and simulated, unique data""" Generate the server folder and the appropriate number of client folders,
with appropriate scripts and simulated, unique data
Parameters
----------
clients: int
The number of clients
"""
# Clear existing files # Clear existing files
dest_dir = os.path.join(Path(FILEDIR).resolve().parents[0], "nodes") dest_dir = os.path.join(Path(FILEDIR).resolve().parents[0], "nodes")
...@@ -36,26 +87,5 @@ def generate_folders(clients: int): ...@@ -36,26 +87,5 @@ def generate_folders(clients: int):
writer.writerow([clt / 2, clt / 2]) writer.writerow([clt / 2, clt / 2])
def write_dir(
dest_dir: str,
grpc_dir: str,
dir_type: str,
client_nb: Optional[int] = None,
dotenv: bool = False,
) -> str:
opj = os.path.join # local alias to shorten code
name = f"{dir_type}_{client_nb}" if client_nb is not None else dir_type
out_dir = opj(dest_dir, name)
os.makedirs(out_dir, exist_ok=True)
shutil.copyfile(opj(FILEDIR, f"{dir_type}.py"), opj(out_dir, "run.py"))
shutil.copyfile(opj(FILEDIR, f"{dir_type}.sh"), opj(out_dir, "run.sh"))
shutil.copytree(grpc_dir, opj(out_dir, "comms"))
if dotenv:
shutil.copyfile(opj(FILEDIR, ".env"), opj(out_dir, ".env"))
return out_dir
if __name__ == "__main__": if __name__ == "__main__":
fire.Fire(generate_folders) fire.Fire(generate_folders)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment