Commit e46d22b3 authored by Augustin Degomme's avatar Augustin Degomme

continuing the flood of your mailboxes

- a few warnings removed for windows again
- node selection while zooming now almost keeps the zoom (I don't get why the min changes a little bit, and don't know if it's possible to fix this)

new features:
-multithreaded Paje parser : this parser uses 3 threads :
   - the parsing thread to read the file and produce lines and tokens (lexical analysis), and aggregate them in blocks of 10000 lines 
   - the builder thread which handles these blocks of lines and calls store_event of the ParserEventPaje and transforms the tokens into the appropriate types, checks the correctness of the line (syntaxic analysis) but doesn't perform the calls to trace and the structural verifications 
  - the trace building thread which performs semantic analysis (if types, containers, exist) and performs the calls to the trace (adding events, states to the trace).

- file mapping : just for multithread version, the file is mapped into memory by chunks (100MB for the moment), which is faster and allows to handle larger files without using too much memory. The limit of 1Gb of the other version is removed, ViTE can now handle much larger files.


These features need more testing and feedback and can be activated at compile time by setting the flag MT_PARSING. 

note: the parser still uses a tokens number limit, this will be merged with the new version soon.


issues:
- gracefully stop parsing when cancel button is hit is not handled yet and causes segfaults
parent ead333e7
......@@ -114,6 +114,12 @@ SET(VITE_HDRS
parser/ParserEventPaje.hpp
parser/ParserPaje.hpp
parser/ParserVite.hpp
#Multithread Paje
parser/mt_ParserEventPaje.hpp
parser/mt_ParserPaje.hpp
parser/mt_PajeFileManager.hpp
parser/BuilderThread.hpp
trace/TraceBuilderThread.hpp
# Statistics headers
statistics/Stats_window.hpp
statistics/Statistic.hpp
......@@ -192,6 +198,12 @@ SET(VITE_SRCS
parser/ParserEventPaje.cpp
parser/ParserPaje.cpp
parser/ParserVite.cpp
#Multithread Paje Parser
parser/mt_ParserEventPaje.cpp
parser/mt_ParserPaje.cpp
parser/mt_PajeFileManager.cpp
parser/BuilderThread.cpp
trace/TraceBuilderThread.cpp
# Statistics code files
statistics/Statistic.cpp
statistics/Stats_window.cpp
......
......@@ -54,7 +54,9 @@
#include "common/Message.hpp"
#include "common/Errors.hpp"
/* -- */
#ifdef WIN32
#define sprintf sprintf_s
#endif
using namespace std;
......
......@@ -51,6 +51,7 @@
class stringstream;
class ostream;
class Interface;
#include <sstream>
class Message: public std::stringstream {
private:
......
......@@ -49,6 +49,10 @@
/* -- */
#include "common.hpp"
#include "Tools.hpp"
#ifdef WIN32
#define sscanf sscanf_s
#endif
/* -- */
using namespace std;
......
......@@ -878,7 +878,10 @@ void Core::launch_action(int state, void* arg) {
}
break;
}
case _STATE_ZOOM_IN_AN_INTERVAL:{
_render_opengl->apply_zoom_on_interval(((Element_pos*)arg)[0], ((Element_pos*)arg)[1]);
}
default:/* like _STATE_UNKNOWN */
display_help();
warning(string("Cannot determine the arguments past. Please check the correct syntax."));
......
......@@ -215,6 +215,8 @@ public:
static const int _DRAW_OPENGL_PARTIAL = 24;
static const int _STATE_CLEAN_RENDER_AREA = 25;
static const int _STATE_ZOOM_IN_AN_INTERVAL=26;
/*!
* \brief Launch an action according to the argument state value.
* \param state An integer corresponding to a kind of action which must be executed.
......
......@@ -260,11 +260,14 @@ void Stats_window::set_selected_nodes(string kind_of_state){
_nodes_displayed->clear();
//can we load that from _nodes_original instead ?
set_displayed_container_names();
Element_pos zoom[2]={Info::Render::_x_min_visible, Info::Render::_x_max_visible};
Info::Render::_x_min_visible = 0.0;
Info::Render::_x_max_visible = 0.0;
_trace->set_view_root_containers(_displayed_containers);
//_trace->set_selected_container(_trace->get_root_containers());
_console->get_console()->launch_action(Core:: _STATE_CLEAN_RENDER_AREA);
_console->get_console()->draw_trace(_console->get_filename(),Core::_DRAW_OPENGL);
_console->get_console()->launch_action(Core:: _STATE_ZOOM_IN_AN_INTERVAL, &zoom);
_console->get_console()->launch_action(Core:: _STATE_RENDER_UPDATE);
......@@ -325,15 +328,19 @@ void Node_select::on_load_button_clicked(){
_trace->set_selected_container(&_displayed_containers);*/
//use _view_root containers and _view_children in order not to modify the actual structure and limit modifications to do in drawtrace
_displayed_containers.clear();
build_displayed_nodes(_displayed_containers);
_displayed_containers.clear();
build_displayed_nodes(_displayed_containers);
_trace->set_view_root_containers(_displayed_containers);
//_trace->set_selected_container(_trace->get_root_containers());
Element_pos it[2]={Info::Render::_x_min_visible, Info::Render::_x_max_visible};
Info::Render::_x_min_visible = 0.0;
Info::Render::_x_max_visible = 0.0;
_console->get_console()->launch_action(Core:: _STATE_CLEAN_RENDER_AREA);
_console->get_console()->draw_trace(_console->get_filename(),Core::_DRAW_OPENGL);
_console->get_console()->launch_action(Core:: _STATE_RENDER_UPDATE);
_console->get_console()->launch_action(Core:: _STATE_ZOOM_IN_AN_INTERVAL, &it);
_console->get_console()->launch_action(Core:: _STATE_RENDER_UPDATE);
}
void Node_select::build_displayed_nodes( std::list<Container *>& displayed_containers){
......
/*
** This file is part of the ViTE project.
**
** This software is governed by the CeCILL-A license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-A license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
**
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
**
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
**
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-A license and that you accept its terms.
**
**
** ViTE developers are (for version 0.* to 1.0):
**
** - COULOMB Kevin
** - FAVERGE Mathieu
** - JAZEIX Johnny
** - LAGRASSE Olivier
** - MARCOUEILLE Jule
** - NOISETTE Pascal
** - REDONDY Arthur
** - VUCHENER Clément
**
*/
#include <fstream>
#include <list>
#include <map>
#include <queue>
/* -- */
#include "trace/values/Values.hpp"
#include "trace/EntityTypes.hpp"
#include "trace/Entitys.hpp"
#include "trace/Trace.hpp"
/* -- */
#include "parser/Parser.hpp"
#include "parser/mt_PajeFileManager.hpp"
#include "parser/ParserDefinitionPaje.hpp"
#include "parser/mt_ParserEventPaje.hpp"
/* -- */
#include "parser/BuilderThread.hpp"
#include "trace/TraceBuilderThread.hpp"
BuilderThread::BuilderThread(mt_ParserEventPaje *p, Trace * trace,QWaitCondition* cond,QWaitCondition* trace_cond, QSemaphore *sem1, QSemaphore *sem2){
_parser = p;
_cond=cond;
_trace_cond=trace_cond;
_freeSlots=sem1;
_linesProduced=sem2;
_trace=trace;
_is_finished=false;
}
void BuilderThread::run(int n, PajeLine* line) {
_freeSlots->acquire(); //do not produce too fast (5 blocks max at a time)
int n_without_errors=0;
Trace_builder_struct* tb_structs= new Trace_builder_struct[n];
int j;
for(j=0; j<n; j++){
if( _parser->store_event(&line[j],*_trace, &tb_structs[n_without_errors])==0)n_without_errors++;
free(line[j]._tokens); // release tokens allocated in the PajeFileManager
}
emit(build_trace(n_without_errors, tb_structs));
free(line);
_linesProduced->release();
}
bool BuilderThread::is_finished(){return _is_finished;}
void BuilderThread::finish_build(){
//quit();
QMutex mut;
//finish the TraceBuilderThread before closing this one
emit build_finished();
mut.lock();
_trace_cond->wait(&mut);
mut.unlock();
_is_finished=true;
_cond->wakeAll();
}
/*
** This file is part of the ViTE project.
**
** This software is governed by the CeCILL-A license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-A license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
**
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
**
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
**
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-A license and that you accept its terms.
**
**
** ViTE developers are (for version 0.* to 1.0):
**
** - COULOMB Kevin
** - FAVERGE Mathieu
** - JAZEIX Johnny
** - LAGRASSE Olivier
** - MARCOUEILLE Jule
** - NOISETTE Pascal
** - REDONDY Arthur
** - VUCHENER Clément
**
*/
#ifndef BUILDING_THREAD_HPP
#define BUILDING_THREAD_HPP
#include <QThread>
#include <QMutex>
#include <QWaitCondition>
#include <QSemaphore>
#include <queue>
#include <parser/mt_PajeFileManager.hpp>
class Trace;
class Parser;
class mt_ParserEventPaje;
struct Trace_builder_struct;
/*!
* \class ParsingThread
* \brief Thread to parse asynchronously a trace with any parser available
*/
#include <QtCore>
class BuilderThread : public QObject {
Q_OBJECT
private:
mt_ParserEventPaje *_parser;
std::queue<PajeLine*>* _q;
QWaitCondition* _cond;
QWaitCondition* _trace_cond;
//QMutex* _mutex;
QSemaphore *_freeSlots;
QSemaphore *_linesProduced;
Trace * _trace;
bool _is_finished;
public:
/*!
* \fn ParsingThread(Parser *p, Trace *t)
* \param p the parser used to parse the file.
* \param t the trace where we store data.
*/
//BuilderThread(ParserEventPaje *p, Trace * trace,std::queue<PajeLine*>* q ,QWaitCondition *cond, QMutex *mutex);
BuilderThread(mt_ParserEventPaje *p, Trace * trace,QWaitCondition *cond,QWaitCondition* trace_cond, QSemaphore *freeBytes, QSemaphore* linesProduced);
bool is_finished();
public slots:
void finish_build();
/*!
* \fn run()
* \brief run the thread.
*/
void run(int n, PajeLine* line);
signals:
void build_finished();
void build_trace(int, Trace_builder_struct*);
};
#endif // PARSING_THREAD_HPP
......@@ -48,12 +48,15 @@
#ifndef PARSER_HPP
#define PARSER_HPP
class Trace;
#include <QObject>
/*!
* \class Parser Parser.hpp "../parser/src/Parser.hpp"
* \brief Contains the definition of the parser interface.
*/
class Parser{
class Parser: public QObject{
Q_OBJECT
protected:
bool _is_finished;
bool _is_canceled;
......
......@@ -57,6 +57,11 @@
/* -- */
using namespace std;
#ifdef WIN32
#define sscanf sscanf_s
#define sprintf sprintf_s
#endif
#define _OUT_A_DEFINITION 0
#define _IN_A_DEFINITION 1
......@@ -315,7 +320,11 @@ void ParserDefinitionPaje::add_field_to_definition(const PajeLine_t *line){
if ( _FNameStr2Id.find(fieldname) == _FNameStr2Id.end() ) {
int size = strlen(fieldname)+1;
char *newfn = new char[size];
#ifndef WIN32
strcpy(newfn, fieldname);
#else
strcpy_s(newfn,size, fieldname);
#endif
_FieldNames[_nbFieldNames]._name = newfn;
_FieldNames[_nbFieldNames]._id = _nbFieldNames;
......
......@@ -61,6 +61,9 @@
#include "parser/ParserEventPaje.hpp"
/* -- */
using namespace std;
#ifdef WIN32
#define sscanf sscanf_s
#endif
ParserEventPaje::ParserEventPaje(ParserDefinitionPaje *defs) {
_Definitions = defs;
......
......@@ -56,6 +56,9 @@
/* -- */
#include "parser/Parser.hpp"
#include "parser/ParserPaje.hpp"
#ifdef MT_PARSING
#include "parser/mt_ParserPaje.hpp"
#endif
#include "parser/ParserVite.hpp"
#ifdef WITH_OTF
#include <otf.h>
......@@ -81,8 +84,12 @@ bool ParserFactory::create(Parser **parser,
ext = filename.substr(pos);
if(ext == ".trace") {
*parser = new ParserPaje(filename);
}
#ifdef MT_PARSING
*parser = new mt_ParserPaje(filename);
#else
*parser = new ParserPaje(filename);
#endif
}
else if(ext == ".ept") {
*parser = new ParserVite(filename);
}
......
......@@ -115,7 +115,8 @@ void ParserPaje::parse(Trace &trace,
_file->get_line(&line);
#endif
}
catch(char *str){
catch(char *){
Error::set(Error::_EXPECT_ID_DEF, 0, Error::_ERROR);
continue;
}
......
......@@ -65,7 +65,7 @@ void ParsingThread::run() {
try {
_parser->parse(*_trace);
}
catch (const std::string &error) {
catch (const std::string &) {
_parser->finish();
}
......
This diff is collapsed.
/*
** This file is part of the ViTE project.
**
** This software is governed by the CeCILL-A license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-A license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
**
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
**
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
**
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-A license and that you accept its terms.
**
**
** ViTE developers are (for version 0.* to 1.0):
**
** - COULOMB Kevin
** - FAVERGE Mathieu
** - JAZEIX Johnny
** - LAGRASSE Olivier
** - MARCOUEILLE Jule
** - NOISETTE Pascal
** - REDONDY Arthur
** - VUCHENER Clément
**
*/
/**
* \file mt_PajeFileManager.hpp
* Visual Trace Explorer
*
* Release Date: January, 2nd 2010
* ViTE is a software to vivisualize execution trace provided by
* a group of student from ENSEIRB and INRIA Bordeaux - Sud-Ouest
*
* @version 1.1
* @author Kevin Coulomb
* @author Mathieu Faverge
* @author Johnny Jazeix
* @author Olivier Lagrasse
* @author Jule Marcoueille
* @author Pascal Noisette
* @author Arthur Redondy
* @author Clément Vuchener
* @date 2010-01-02
*
**/
#ifndef FILE_HPP
#define FILE_HPP
#include <string.h>
#ifdef WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#endif
#include <stdio.h>
#include <fcntl.h>
#define _PAJE_BUFSIZE 256
#define _PAJE_NBMAXTKS 16
#include <cstring>
#include <string>
#include <fstream>
#include <iostream>
#include <QMetaType>
using namespace std;
/**
* \struct PajeLine
*
* Brief structure to store informations read
* on each line
*
* \param _id Line number
* \param _nbtks Number of tokens found on the line
* \param _tokens Pointers on the found tokens
*
*/
typedef struct PajeLine {
int _id;
int _nbtks;
char **_tokens;
} PajeLine_t;
//Q_DECLARE_METATYPE(PajeLine);
/**
* \class mt_PajeFileManager
*
* File manager to read files using Paje syntax.
* Each line is read one after one and stored in
* the PajeLine structure associated to the class.
*
* \sa Parser
* \sa ParserPaje
* \sa ParserVite
*/
class mt_PajeFileManager : public std::ifstream {
private:
std::string _filename;
unsigned long long _filesize;
int _total_filesize;
int _nbchunks;
int _current_chunk;
int _mapped_size;
bool _eof;
bool _eoc;
unsigned int _lineid;
int _nbtks;
char** _tokens;
char* _line;
char* _old_line;
#ifdef WIN32
HANDLE _fd;
HANDLE _fm;
unsigned long long _pageSize;
#else
FILE* _fd;
#endif
unsigned long long _offset;
unsigned long long _global_offset;
mt_PajeFileManager(const mt_PajeFileManager &);
public:
/*!
* \brief Constructor for the file
*/
mt_PajeFileManager();
/*!
* \brief Constructor for the file
* \param filename : a filename
*/
mt_PajeFileManager(const char * filename);
/*!
* \brief Destructor
* Destroy the file
*/
~mt_PajeFileManager();
/*!
* \fn open(const char * filename, ios_base::openmode mode)
* \brief Open the file
* \param filename the file name
* \param mode the opening mode
*/
void open (const char * filename);
/*!
* \fn close()
* \brief Close the file if already opened
*/
void close();
/*!
* \fn eof()
* \brief check the end of file
*/
bool eof();
/*!
* \fn eoc()
* \brief check the end of the opened chunk of file
*/
bool eoc();
/*!
* \fn close_old_chunk()
* \brief release the previous chunk of file
*/
void close_old_chunk();
/*!
* \fn get_filesize() const;
* \return The length of the file opened
*/
int get_filesize() const;
/*!
* \fn get_size_loaded();
* \return The size already loaded
*/
int get_size_loaded();
/*!
* \fn get_percent_loaded()
* \return The percent of the file loaded (between 0 and 1).
*/
float get_percent_loaded();
/*!
* \fn get_line(PajeLine *lineptr)
* \brief Get the next line.
* \param lineptr the line filled in
* \return the next line
*/
int get_line(PajeLine *lineptr);
/*!
* \fn print_line()
* \brief Print the current line for debug
*/
void print_line();
};
#endif // FILE_HPP
/*
** This file is part of the ViTE project.
**
** This software is governed by the CeCILL-A license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-A license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
**
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
**
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
**
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-A license and that you accept its terms.
**
**
** ViTE developers are (for version 0.* to 1.0):
**
** - COULOMB Kevin
** - FAVERGE Mathieu
** - JAZEIX Johnny
** - LAGRASSE Olivier
** - MARCOUEILLE Jule
** - NOISETTE Pascal
** - REDONDY Arthur
** - VUCHENER Clément
**
*/
#include <string>
#include <map>
#include <set>
#include <queue>
#include <list>
/* -- */
#include "common/Errors.hpp"
/* -- */
#include "trace/values/Values.hpp"
#include "trace/EntityValue.hpp"
#include "trace/EntityTypes.hpp"
#include "trace/Entitys.hpp"
#include "trace/Trace.hpp"
/* -- */
#include <fstream>
#include "parser/mt_PajeFileManager.hpp" // temporary
#include "parser/PajeDefinition.hpp"
#include "parser/ParserDefinitionPaje.hpp"
#include "parser/mt_ParserEventPaje.hpp"
#include "trace/TraceBuilderThread.hpp"
/* -- */
#include <QMetaType>
Q_DECLARE_METATYPE(Name);
Q_DECLARE_METATYPE(Date);
Q_DECLARE_METATYPE(String);
using namespace std;
#ifdef WIN32 // remove a warning and use a safe function
#define sscanf sscanf_s
#endif
mt_ParserEventPaje::mt_ParserEventPaje(ParserDefinitionPaje *defs) {
_Definitions = defs;
}
mt_ParserEventPaje::~mt_ParserEventPaje(){
_containers.clear();
}