Commit d3d3ddb3 authored by David Glesser's avatar David Glesser
Browse files

Add PyBatsim, a python module that helps you make your own scheduler.

As today, only one scheduler is written here. Soon, more toy schedulers will be added!
parent a288a391
PyBatsim helps you developping your own scheduler in python! -> This is the main entry point to launch things
Ex: python filler_sched ../../workload_profiles/test_workload_profile.json (a server should be started)
batsim/ -> this class helps you communicate with the batsim server
schedulers/ -> contains all the schedulers
Schedulers name should follow this convention: contains the FooBar classname which has as an ancestor BatsimScheduler -> A kind of first fit without topology scheduler
import json
import struct
import socket
class Batsim(object):
def __init__(self, json_file, scheduler, server_address = '/tmp/bat_socket', verbose=0):
self.server_address = server_address
self.verbose = verbose
self.scheduler = scheduler
#load json file
#open connection
self._connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
print("[BATSIM]: connecting to %r" % server_address)
if self.verbose > 1: print('[BATSIM]: connected')
except socket.error:
print("[BATSIM]: socket error")
#initialize some public attributes
self.last_msg_recv_time = -1 = self
def time(self):
return self._current_time
def consume_time(self, t):
self._current_time += t
return self._current_time
def start_job(self, jobid, res):
self._msgs_to_send.append( ( self.time(), "J:"+str(jobid)+"="+ ",".join([str(i) for i in res]) ) )
def start_jobs(self, jobids, res):
msg = "J:"
for jid in jobids:
msg += str(jid) + "="
for r in res[jid]:
msg += str(r) + ","
msg = msg[:-1] + ";" # replace last comma by semicolon separtor between jobs
msg = msg[:-1] # remove last semicolon
self._msgs_to_send.append( ( self.time(), msg ) )
def do_next_event(self):
def start(self):
while True:
def _read_bat_msg(self):
lg_str = self._connection.recv(4)
if not lg_str:
print("[BATSIM]: connection is closed by batsim core")
lg = struct.unpack("i",lg_str)[0]
msg = self._connection.recv(lg).decode()
if self.verbose > 0: print('[BATSIM]: from batsim : %r' % msg)
sub_msgs = msg.split('|')
data = sub_msgs[0].split(":")
version = int(data[0])
self.last_msg_recv_time = float(data[1])
self._current_time = float(data[1])
if self.verbose > 1: print("[BATSIM]: version: %r now: %r" % (version, self.time()))
# [ (timestamp, txtDATA), ...]
self._msgs_to_send = []
for i in range(1, len(sub_msgs)):
data = sub_msgs[i].split(':')
if data[1] == 'R':
if data[1] == 'N':
if data[1] == 'S':
elif data[1] == 'C':
elif data[1] == 'p':
opts = data[2].split('=')
self.scheduler.onMachinePStateChanged(int(opts[0]), int(opts[1]))
elif data[1] == 'J' or data[1] == 'P':
raise "Only the server can receive this kind of message"
raise Exception("Unknow submessage type " + data[1] )
msg = "0:" + str(self.last_msg_recv_time) + "|"
if len(self._msgs_to_send) > 0:
#sort msgs by timestamp
self._msgs_to_send = sorted(self._msgs_to_send, key=lambda m: m[0])
for m in self._msgs_to_send:
msg += str(m[0])+":"+m[1]
msg += str(self.time()) +":N"
if self.verbose > 0: print("[BATSIM]: to batsim : %r" % msg)
lg = struct.pack("i",int(len(msg)))
def _load_json_workload_profile(self, filename):
wkp_file = open(filename)
wkp = json.load(wkp_file)
self.nb_res = wkp["nb_res"] = {j["id"]: Job(j["id"], j["subtime"], j["walltime"], j["res"], j["profile"]) for j in wkp["jobs"]}
#TODO: profiles
class Job(object):
def __init__(self, id, subtime, walltime, res, profile): = id
self.submit_time = subtime
self.requested_time = walltime
self.requested_resources = res
self.profile = profile
class BatsimScheduler(object):
def onAfterBatsimInit(self):
#You now have access to and all other functions
def onJobRejection(self):
raise "not implemented"
def onNOP(self):
raise "not implemented"
def onJobSubmission(self, job):
raise "not implemented"
def onJobCompletion(self, job):
raise "not implemented"
def onMachinePStateChanged(self, nodeid, pstate):
raise "not implemented" ../../workload_profiles/test_workload_profile.json
import sys
from batsim.batsim import Batsim
scheduler_filename = sys.argv[1]
json_filename = sys.argv[2]
def module_to_class(module):
transform to FooBar
return ''.join(w.title() for w in str.split(module, "_"))
def filename_to_module(fn):
return str(fn).split(".")[0]
def instanciate_scheduler(name):
my_module = name#filename_to_module(my_filename)
my_class = module_to_class(my_module)
#load module(or file)
package = __import__ ('schedulers', fromlist=[my_module])
if my_module not in package.__dict__:
print "No such scheduler (module file not found)."
if my_class not in package.__dict__[my_module].__dict__:
print "No such scheduler (class within the module file not found)."
#load the class
scheduler_non_instancied = package.__dict__[my_module].__dict__[my_class]
scheduler = scheduler_non_instancied()
return scheduler
scheduler = instanciate_scheduler(scheduler_filename)
bs = Batsim(json_filename, scheduler, verbose=999)
\ No newline at end of file
from batsim.batsim import BatsimScheduler, Batsim
import sys
import os
from random import sample
from sortedcontainers import SortedSet
class FillerSched(BatsimScheduler):
def __init__(self):
def onAfterBatsimInit(self):
self.nb_completed_jobs = 0
self.jobs_res = {}
self.jobs_completed = []
self.jobs_waiting = []
self.sched_delay = 5.0
self.openJobs = set()
self.availableResources = SortedSet(range(
self.previousAllocations = dict()
def scheduleJobs(self):
scheduledJobs = []
print('openJobs = ', self.openJobs)
print('available = ', self.availableResources)
print('previous = ', self.previousAllocations)
# Iterating over a copy to be able to remove jobs from openJobs at traversal
for job in set(self.openJobs):
nb_res_req =[job].requested_resources
if nb_res_req <= len(self.availableResources):
res = self.availableResources[:nb_res_req]
self.jobs_res[job] = res
self.previousAllocations[job] = res
for r in res:
# update time
# send to uds
if len(scheduledJobs) > 0:, self.jobs_res)
print('openJobs = ', self.openJobs)
print('available = ', self.availableResources)
print('previous = ', self.previousAllocations)
def onJobSubmission(self, job):
def onJobCompletion(self, job):
for res in self.previousAllocations[job]:
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment