Commit bb00a40e authored by Maverick Chardet's avatar Maverick Chardet
Browse files

Refactoring for Gantt charts: split GanttRecord (used to store Concerto

events) from GanttChart (figure). GanttChart now uses Plotly as backend.
A few fixes and TODO added in Component.
Added ability to export Gantt charts from ReconfAnalysis
Added TODOs
Various little changes
parent 01f98139
......@@ -214,7 +214,7 @@ Example (modified extract from from `examples/server_clients/test_server_client.
from server_client_assembly import ServerClient
sca = ServerClient()
sca.set_use_gantt_chart(True)
sca.set_record_gantt(True)
sca.set_verbosity(-1)
sca.set_print_time(False)
......@@ -228,7 +228,7 @@ sca.restart()
sca.terminate()
gc : GanttChart = sca.get_gantt_chart()
gc : GanttRecord = sca.get_gantt_record()
gc.export_gnuplot("results_server.gpl", "Gantt chart with the Server component only")
gc.export_json("results_server.json")
```
* TODOs
#TODOs
** Urgent
##Urgent
** Hacks temporaires
- Partie "comportements" de la sortie JSON des Gantt charts étrange : à vérifier
##Hacks temporaires
- Switch: pour l'instant, un switch est une place spéciale qui surcharge la fonction qui renvoie sa liste des docks de sortie. L'utilisateur le déclare dans self.switches comme un tuple (nom,f) où f prend en entrée une référence vers le composant et le nom du comportement actuel et retourne une liste d'indices de docks de sortie (ordre = ordre de déclaration des transitions qui sortent du switch). Si le switch utilise un port data, il doit être bindé aux transitions qui arrivent sur le switch.
** Vérifications
##Vérifications
- Vérifier que le groupes se comportent comme prévu
- Vérifier que la transition _init a résolu les problèmes de ports sur l'état initial et de groupes contenant l'état initial
- Vérifier la suppression correcte de DataProvider
** Fonctionalités
##Fonctionalités
- Discuter de la sémantique du composant : faire les quatre étapes d'un coup ou une seule ? Modifié actuallement aux quatre étapes d'un coup en commençant par idocks to place pour ne pas fournir un provide si l'état n'est pas stable
- S'assurer que la sémantique "on ne peut pas quitter une place deux fois dans le même comportement" correspond à ce qui est attendu (objectif : éviter les boucles)
......@@ -21,15 +23,15 @@
- Gérer les ports optionnels
- Plusieurs comportements pour les transitions
** Corrections par rapport à la sémantique formelle
##Corrections par rapport à la sémantique formelle
- Groupes : gérer le cas où plusieurs groupes sont connectés à un port : ET au lieu de OU
- Vrai "switch"
** Code
##Code
- Documentation
** Modèle
##Modèle
- Conflits
......@@ -15,7 +15,7 @@ from concerto.component import Component
from concerto.connection import Connection
from concerto.internal_instruction import InternalInstruction
from concerto.reconfiguration import Reconfiguration
from concerto.gantt_chart import GanttChart
from concerto.gantt_record import GanttRecord
from concerto.utility import Messages, COLORS, Printer
......@@ -65,7 +65,7 @@ class Assembly (object):
self.verbosity : int = 0
self.print_time : bool = False
self.dryrun : bool = False
self.gantt : GanttChart = None
self.gantt : GanttRecord = None
self.name : str = None
self.dump_program : bool = False
......@@ -87,18 +87,18 @@ class Assembly (object):
for c in self._components:
self._components[c].set_dryrun(value)
def set_use_gantt_chart(self, value : bool):
def set_record_gantt(self, value : bool):
if value:
if self.gantt is None:
self.gantt = GanttChart()
self.gantt = GanttRecord()
for c in self._components:
self._components[c].set_gantt_chart(self.gantt)
self._components[c].set_gantt_record(self.gantt)
else:
self.gantt = None
for c in self._components:
self._components[c].set_gantt_chart(None)
self._components[c].set_gantt_record(None)
def get_gantt_chart(self):
def get_gantt_record(self) -> GanttRecord:
return self.gantt
def set_dump_program(self, value : bool):
......@@ -170,7 +170,7 @@ class Assembly (object):
if not self.semantics_thread.is_alive():
self.semantics_thread.start()
def run_reconfiguration(self, reconfiguration : Reconfiguration):
def run_reconfiguration(self, reconfiguration: Reconfiguration):
for instr in reconfiguration._get_instructions():
self.add_instruction(instr)
......@@ -190,7 +190,7 @@ class Assembly (object):
comp.set_verbosity(self.verbosity)
comp.set_print_time(self.print_time)
comp.set_dryrun(self.dryrun)
comp.set_gantt_chart(self.gantt)
comp.set_gantt_record(self.gantt)
self._components[name]=comp
self.component_connections[name] = set()
self.act_components.add(name) # _init
......
......@@ -14,7 +14,7 @@ from typing import Dict, Tuple, List, Set, Callable
from concerto.place import Dock, Place
from concerto.dependency import DepType, Dependency
from concerto.transition import Transition
from concerto.gantt_chart import GanttChart
from concerto.gantt_record import GanttRecord
from concerto.utility import Messages, Printer
class Group(object):
......@@ -118,7 +118,7 @@ class Component (object, metaclass=ABCMeta):
self.forced_verobisty : int = None
self.print_time : bool = False
self.dryrun : bool = False
self.gantt : GanttChart = None
self.gantt : GanttRecord = None
self.hidden_from_gantt_chart = False
self.places : List[str] = []
......@@ -146,7 +146,7 @@ class Component (object, metaclass=ABCMeta):
self.act_idocks : Set[Dock] = set()
self.act_behavior : str = "_init"
self.queued_behaviors : Queue = Queue()
self.visited_places : List[Place] = set()
self.visited_places : Set[Place] = set()
self.initialized : bool = False
......@@ -167,7 +167,7 @@ class Component (object, metaclass=ABCMeta):
def set_dryrun(self, value : bool):
self.dryrun = value
def set_gantt_chart(self, gc : GanttChart):
def set_gantt_record(self, gc : GanttRecord):
if not self.hidden_from_gantt_chart:
self.gantt = gc
......@@ -184,7 +184,6 @@ class Component (object, metaclass=ABCMeta):
else:
return self.forced_verobisty
def add_places(self, places : List[str], initial=None):
"""
This method add all places declared in the user component class as a
......@@ -217,8 +216,7 @@ class Component (object, metaclass=ABCMeta):
self.place_groups[name] = []
if initial:
self.set_initial_place(initial)
self.set_initial_place(name)
def add_switches(self, switches : List[Tuple[str,Callable[[Place,str],List[int]]]], initial=None):
for key in switches:
......@@ -247,9 +245,8 @@ class Component (object, metaclass=ABCMeta):
self.st_switches.add(name)
if initial:
self.set_initial_place(initial)
self.set_initial_place(name)
def add_groups(self, groups : Dict[str,List[str]]):
for name in groups:
self.add_group(name, groups[name])
......@@ -274,7 +271,6 @@ class Component (object, metaclass=ABCMeta):
for place_name in places:
self.place_groups[place_name].append(self.st_groups[name])
def add_transitions(self, transitions : Dict[str,Tuple]):
"""
This method add all transitions declared in the user component class
......@@ -332,7 +328,6 @@ class Component (object, metaclass=ABCMeta):
raise Exception("Trying to add transition '%s' going to unexisting place '%s'"%(name, dst_name))
self._force_add_transition(name,src_name,dst_name,bhv,idset,func,args)
def add_dependencies(self, dep : Dict[str,Tuple[DepType, List[str]]]):
"""
......@@ -383,8 +378,7 @@ class Component (object, metaclass=ABCMeta):
self.trans_dependencies[transition_name].append(self.st_dependencies[name])
for switch_name in switches:
self.place_dependencies[switch_name].append(self.st_dependencies[name])
elif type == DepType.USE:
places = []
transitions = []
......@@ -433,7 +427,6 @@ class Component (object, metaclass=ABCMeta):
for group_name in groups:
self.group_dependencies[group_name].append(self.st_dependencies[name])
def set_initial_place(self, name : str):
"""
This method allows to set the (unique) initial place of the component, if not already done
......@@ -447,7 +440,6 @@ class Component (object, metaclass=ABCMeta):
if self.initial_place is not None:
raise Exception("Trying to set place %s as intial place of component %s while %s is already the intial place." % (name, self.get_name(), self.initial_place))
self.initial_place = name
def get_places(self):
"""
......@@ -511,6 +503,7 @@ class Component (object, metaclass=ABCMeta):
else:
print(message)
# TODO: Rewrite
def is_connected(self, name : str):
"""
This method is used to know if a given dependency is connected or not
......@@ -550,6 +543,7 @@ class Component (object, metaclass=ABCMeta):
return check
# TODO: Rewrite
def check_connections(self):
"""
This method check connections once the component has been
......@@ -760,7 +754,6 @@ class Component (object, metaclass=ABCMeta):
return did_something
def _start_transition(self) -> bool:
"""
This method starts the transitions which are ready to run:
......
......@@ -152,4 +152,3 @@ class Dependency (object):
if c.is_locked():
return True
return False
class GanttChart:
def __init__(self, transitions=(), default_show_behaviors=True):
self._transitions = dict()
self._default_show_behaviors = default_show_behaviors
self.add_transitions(transitions)
@staticmethod
def from_json(filename):
gc = GanttChart()
with open(filename) as f:
import json
readable_dict = json.load(f)
for component, bt in readable_dict.items():
for behavior, transitions in bt.items():
gc.add_transitions([[component, behavior, t["name"], t["start"], t["end"]] for t in transitions])
return gc
def add_transition(self, component, behavior, transition_name, start_time, end_time):
if component not in self._transitions:
self._transitions[component] = dict()
if behavior not in self._transitions[component]:
self._transitions[component][behavior] = []
self._transitions[component][behavior].append((start_time, end_time, transition_name))
def add_transitions(self, transitions):
for t in transitions:
self.add_transition(*t)
def _flat_transitions(self):
components_list = []
for component, bt in self._transitions.items():
behaviors_list = []
for behavior, transitions in bt.items():
behaviors_list.append([(st, et, tn, behavior) for (st, et, tn) in sorted(transitions)])
behaviors_list.sort(key=lambda x: x[0])
behaviors_list = _flatten(behaviors_list)
components_list.append([(st, et, tn, bv, component) for (st, et, tn, bv) in behaviors_list])
components_list.sort(key=lambda x: x[0])
components_list = _flatten(components_list)
return components_list
def get_plotly(self, title="", show_behaviors=None, override_time_format=None):
""" Timeformat: https://github.com/d3/d3-time-format/blob/master/README.md """
import plotly.figure_factory as ff
from random import randint
from datetime import datetime
if show_behaviors is None:
show_behaviors = self._default_show_behaviors
def _to_time(nb_seconds):
return datetime.fromtimestamp(nb_seconds)
colors = [
'#1f77b4', # muted blue
'#ff7f0e', # safety orange
'#2ca02c', # cooked asparagus green
'#d62728', # brick red
'#9467bd', # muted purple
'#8c564b', # chestnut brown
'#e377c2', # raspberry yogurt pink
'#7f7f7f', # middle gray
'#bcbd22', # curry yellow-green
'#17becf' # blue-teal
]
while len(self._transitions) > len(colors):
colors.append(f'rgb({randint(0,256)}, {randint(0,256)}, {randint(0,256)})')
# for component_name in self._transitions:
# colors[component_name] = f'rgb({randint(0,256)}, {randint(0,256)}, {randint(0,256)})'
def _name_for_trans(t):
if show_behaviors:
return "%s.%s.%s" % (t[4], t[3], t[2])
else:
return "%s.%s" % (t[4], t[2])
df = [dict(Task=_name_for_trans(t), Start=_to_time(t[0]), Finish=_to_time(t[1]), Component=t[4]) for t in self._flat_transitions()]
fig = ff.create_gantt(df,
title=title,
index_col='Component',
group_tasks=True,
colors=colors,
showgrid_x=True,
showgrid_y=True)
max_time = max([_to_time(t[1]) for t in self._flat_transitions()])
if override_time_format:
fig.layout.xaxis.tickformat = override_time_format
elif max_time < datetime.fromtimestamp(60):
fig.layout.xaxis.tickformat = '%S.%L'
elif max_time < datetime.fromtimestamp(60*60):
fig.layout.xaxis.tickformat = '%M:%S'
elif max_time < datetime.fromtimestamp(24*60*60):
fig.layout.xaxis.tickformat = '%H:%M:%S'
elif max_time < datetime.fromtimestamp(365*24*60*60):
fig.layout.xaxis.tickformat = '%j %H:%M:%S'
else:
raise Exception("Error: maximum time is higher than 365 days, please provide a custom time format!")
return fig
def show_plotly(self, title="", show_behaviors=None, override_time_format=None):
""" Timeformat: https://github.com/d3/d3-time-format/blob/master/README.md """
fig = self.get_plotly(title, show_behaviors=show_behaviors, override_time_format=override_time_format)
fig.show()
def export_plotly(self, filename, title="", show_behaviors=None, override_time_format=None):
"""
Supported file formats: PNG, JPEG, WebP, SVG, PDF
Timeformat: https://github.com/d3/d3-time-format/blob/master/README.md
"""
fig = self.get_plotly(title, show_behaviors=show_behaviors, override_time_format=override_time_format)
fig.write_image(filename)
def export_json(self, filename):
readable_dict = dict()
for component, bt in self._transitions.items():
readable_dict[component] = dict()
for behavior, transitions in bt.items():
readable_dict[component][behavior] = [{"name": tn, "start": st, "end": et} for (st, et, tn) in sorted(transitions)]
with open(filename, "w") as f:
import json
json.dump(readable_dict, f, indent='\t')
def import_json(self, filename):
with open(filename) as f:
import json
self._transitions = json.load(f)
def _flatten(list_of_lists):
from itertools import chain
return chain(*list_of_lists)
def _unit_tests():
print("Running unit tests")
gc = GanttChart()
gc.add_transition("client", "deploy", "download", 0, 5)
gc.add_transition("client", "deploy", "mount", 0, 0.5)
gc.add_transition("client", "deploy", "prepare", 0.5, 3)
gc.add_transition("client", "deploy", "configure", 4, 5)
gc.add_transition("client", "deploy", "run", 6, 8)
gc.add_transition("server", "deploy", "download", 0, 5)
gc.add_transition("server", "deploy", "allocate", 0, 4)
gc.add_transition("server", "deploy", "run", 4, 5)
gc.add_transition("server", "deploy", "check1", 5, 6)
gc.add_transition("server", "deploy", "check2", 5, 5.5)
gc.add_transition("server", "deploy", "ctest", 7, 21)
gc.show_plotly(show_behaviors=False)
if __name__ == '__main__':
_unit_tests()
import csv
from collections import OrderedDict
from concerto.gnuplot.gnuplot_gantt import gnuplot_file_from_list
class GanttChart():
class GanttRecord:
START = 0
STOP = 1
CHANGE_BEHAVIOR = 2
......@@ -11,23 +9,24 @@ class GanttChart():
def __init__(self):
self.log = []
def start_transition(self,component,behavior,transition,time):
self.log.append((self.START,component,behavior,transition,time))
def start_transition(self, component, behavior, transition, time):
self.log.append((self.START, component, behavior, transition, time))
def stop_transition(self,component,behavior,transition,time):
self.log.append((self.STOP,component,behavior,transition,time))
def stop_transition(self, component, behavior, transition, time):
self.log.append((self.STOP, component, behavior, transition, time))
def push_b(self,component,behavior,time):
def push_b(self, component, behavior, time):
self.log.append((self.CHANGE_BEHAVIOR, component, behavior, None, time))
def dump(self, file_name):
import csv
file = open(file_name, "w")
fieldnames = ['action', 'component', 'behavior', 'transition', 'time']
writer = csv.DictWriter(file, fieldnames=fieldnames)
writer.writeheader()
for (action,component,behavior,transition,time) in self.log:
for (action, component, behavior, transition, time) in self.log:
row_dict = {
'action': action,
'component': component,
......@@ -50,26 +49,29 @@ class GanttChart():
return
temp_dict = {}
(_,_,_,_,min_time) = self.log[0]
for (action,component,behavior,transition,time) in self.log:
(_, _, _, _, min_time) = self.log[0]
for (action, component, behavior, transition, time) in self.log:
if action is self.START:
if (component,behavior,transition) in temp_dict:
if (component, behavior, transition) in temp_dict:
raise Exception("GanttChart: error during export to gnuplot: action already started")
start_time = time-min_time
temp_dict[(component,behavior,transition)] = start_time
if start_time > max_time: max_time = start_time
temp_dict[(component, behavior, transition)] = start_time
if start_time > max_time:
max_time = start_time
elif action is self.STOP:
if (component,behavior,transition) not in temp_dict:
if (component, behavior, transition) not in temp_dict:
raise Exception("GanttChart: error during export to gnuplot: action finished before being started")
start_time = temp_dict[(component,behavior,transition)]
start_time = temp_dict[(component, behavior, transition)]
end_time = time-min_time
transitions.append((component, behavior, start_time, end_time, transition))
if end_time > max_time: max_time = end_time
del temp_dict[(component,behavior,transition)]
if end_time > max_time:
max_time = end_time
del temp_dict[(component, behavior, transition)]
else:
start_time = time-min_time
push_bs.append((component, start_time, behavior))
if start_time > max_time: max_time = start_time
if start_time > max_time:
max_time = start_time
next_cb_dict = {}
new_push_bs = []
......@@ -79,13 +81,13 @@ class GanttChart():
else:
end_time = max_time
if behavior is not None:
new_push_bs.insert(0,(component, start_time, end_time, behavior))
new_push_bs.insert(0, (component, start_time, end_time, behavior))
next_cb_dict[component] = start_time
transitions.sort()
new_push_bs.sort()
return (transitions, new_push_bs)
return transitions, new_push_bs
def get_formatted(self):
(transitions, push_bs) = self._get_ordered_tuples()
......@@ -128,6 +130,11 @@ class GanttChart():
from json import dump
with open(file_name, "w") as f:
dump(self.get_formatted(), f, indent='\t')
def get_gantt_chart(self):
from concerto.exporters.gantt_chart import GanttChart as ExpGanttChart
egc = ExpGanttChart([(c, b, tn, st, et) for (c, b, st, et, tn) in self._get_ordered_tuples()[0]])
return egc
@staticmethod
def export_gnuplot_from_tuples(transitions, push_bs, file_name, title):
......@@ -141,28 +148,27 @@ class GanttChart():
for (component, behavior, start_time, end_time, transition) in transitions:
if component in push_bs_dict:
cn = component.replace("_","\\\\\\_")
cn = component.replace("_", "\\\\\\_")
for (s, e, b) in push_bs_dict[component]:
tuple_list.append((cn,s,e))
tuple_list.append((cn, s, e))
del push_bs_dict[component]
name = "%s.%s.%s"%(component,behavior,transition)
name = name.replace("_","\\\\\\_")
tuple_list.append((name,start_time,end_time))
name = "%s.%s.%s" % (component, behavior, transition)
name = name.replace("_", "\\\\\\_")
tuple_list.append((name, start_time, end_time))
for c in push_bs_dict:
cn = c.replace("_","\\\\\\_")
cn = c.replace("_", "\\\\\\_")
for (s, e, b) in push_bs_dict[c]:
tuple_list.append((cn,s,e))
tuple_list.append((cn, s, e))
gnuplot_file_from_list(tuple_list, file_name, title)
def export_gnuplot(self, file_name, title=''):
(transitions, push_bs) = self._get_ordered_tuples()
self.export_gnuplot_from_tuples(transitions, push_bs, file_name, title)
@staticmethod
def formatted_to_ordered_tuples(formatted, min_time=0., max_time=float('inf'), components_to_remove=[]):
def formatted_to_ordered_tuples(formatted, min_time=0., max_time=float('inf'), components_to_remove=()):
transitions = []
push_bs = []
for c_block in formatted["transitions"]:
......@@ -174,7 +180,7 @@ class GanttChart():
start = t_block["start"]
end = t_block["end"]
if start >= min_time and end <= max_time and component not in components_to_remove:
transitions.append((component,behavior,start-min_time,end-min_time,transition))
transitions.append((component, behavior, start-min_time, end-min_time, transition))
for c_block in formatted["behaviors"]:
component = c_block["component"]
for b_block in c_block["behaviors"]:
......@@ -182,18 +188,17 @@ class GanttChart():
start = b_block["start"]
end = b_block["end"]
if start >= min_time and end <= max_time and component not in components_to_remove:
push_bs.append((component,start-min_time,end-min_time,behavior))
push_bs.append((component, start-min_time, end-min_time, behavior))
transitions.sort()
push_bs.sort()
return (transitions, push_bs)
return transitions, push_bs
@staticmethod
def json_to_gnuplot(json_file_name, gnuplot_file_name, title='', min_time=0., max_time=float('inf'), components_to_remove=[]):
def json_to_gnuplot(json_file_name, gnuplot_file_name, title='', min_time=0., max_time=float('inf'), components_to_remove=()):
from json import load
with open(json_file_name) as f:
formatted = load(f)
transitions, push_bs = GanttChart.formatted_to_ordered_tuples(formatted, min_time=min_time, max_time=max_time, components_to_remove=components_to_remove)
GanttChart.export_gnuplot_from_tuples(transitions, push_bs, gnuplot_file_name, title)
transitions, push_bs = GanttRecord.formatted_to_ordered_tuples(formatted, min_time=min_time, max_time=max_time, components_to_remove=components_to_remove)
GanttRecord.export_gnuplot_from_tuples(transitions, push_bs, gnuplot_file_name, title)
This diff is collapsed.
......@@ -7,7 +7,7 @@
"""
import threading
from concerto.gantt_chart import GanttChart
class Transition (object):
"""This Transition class is used to create a transition.
......
......@@ -6,7 +6,7 @@ from concerto.all import *
from concerto.utility import Printer
from concerto.components.data_provider import DataProvider
from concerto.components.jinja2 import Jinja2, Jinja2Static