Commit bc373d0b authored by Mathieu Faverge's avatar Mathieu Faverge

Merge branch 'otf2' into 'master'

Otf2

This branch adds a basic support for the OTF2 trace format. The main interesting events are supported (function entry/exit, MPI P2P communication, metrics) and I was able to open medium-sized traces (up to 500MB) on my laptop (for larger traces, you'll need lots of RAM, or a few optimizations in the code).

The code is stable enough and ViTE is usable on most OTF2 traces, so I think it's time to merge it into the master branch.

See merge request !8
parents f5eb21fa e0a4a90a
......@@ -130,6 +130,8 @@ cmake_dependent_option(VITE_DBG_MEMORY_TRACE
### Trace format options
option(VITE_ENABLE_OTF
"Enable the support of OTF file format." OFF)
option(VITE_ENABLE_OTF2
"Enable the support of OTF2 file format." OFF)
option(VITE_ENABLE_TAU
"Enable the support of TAU file format." OFF)
......@@ -190,6 +192,10 @@ if(VITE_ENABLE_OTF)
find_package(OTF)
endif(VITE_ENABLE_OTF)
if(VITE_ENABLE_OTF2)
find_package(OTF2)
endif(VITE_ENABLE_OTF2)
if(VITE_ENABLE_TAU)
find_package(TAU)
endif(VITE_ENABLE_TAU)
......
#
# Find the OTF2 libraries and include dir
#
# OTF2_INCLUDE_DIR - Directories to include to use OTF2
# OTF2_LIBRARY - Files to link against to use OTF2
# OTF2_FOUND - When false, don't try to use OTF2
#
# OTF2_DIR can be used to make it simpler to find the various include
# directories and compiled libraries when OTF2 was not installed in the
# usual/well-known directories (e.g. because you made an in tree-source
# compilation or because you installed it in an "unusual" directory).
# Just set OTF2_DIR it to your specific installation directory
#
FIND_LIBRARY(OTF2_LIBRARY otf2
PATHS
/usr/lib
/usr/local/lib
${OTF2_DIR}/lib
)
IF(OTF2_LIBRARY)
GET_FILENAME_COMPONENT(OTF2_GUESSED_INCLUDE_DIR_tmp "${OTF2_LIBRARY}" PATH)
STRING(REGEX REPLACE "lib$" "include" OTF2_GUESSED_INCLUDE_DIR "${OTF2_GUESSED_INCLUDE_DIR_tmp}")
ENDIF(OTF2_LIBRARY)
FIND_PATH( OTF2_INCLUDE_DIR otf2/otf2.h
PATHS
${OTF2_GUESSED_INCLUDE_DIR}
${OTF2_DIR}/include
/usr/include
/usr/local/include
${CMAKE_SOURCE_DIR}/externals/otf2/include/
)
IF( OTF2_INCLUDE_DIR )
IF( OTF2_LIBRARY )
SET( OTF2_FOUND "YES" )
MARK_AS_ADVANCED( OTF2_DIR )
MARK_AS_ADVANCED( OTF2_INCLUDE_DIR )
MARK_AS_ADVANCED( OTF2_LIBRARY )
ENDIF( OTF2_LIBRARY )
ENDIF( OTF2_INCLUDE_DIR )
IF( NOT OTF2_FOUND )
MESSAGE("OTF2 installation was not found. Please provide OTF2_DIR:")
MESSAGE(" - through the GUI when working with ccmake, ")
MESSAGE(" - as a command line argument when working with cmake e.g. ")
MESSAGE(" cmake .. -DOTF2_DIR:PATH=/usr/local/otf ")
MESSAGE("Note: the following message is triggered by cmake on the first ")
MESSAGE(" undefined necessary PATH variable (e.g. OTF2_INCLUDE_DIR).")
MESSAGE(" Providing OTF2_DIR (as above described) is probably the")
MESSAGE(" simplest solution unless you have a really customized/odd")
MESSAGE(" OTF2 installation...")
SET(OTF2_DIR "" CACHE PATH "Root of OTF2 install tree." )
ENDIF( NOT OTF2_FOUND )
......@@ -296,6 +296,30 @@ IF(VITE_ENABLE_OTF)
ENDIF(VITE_ENABLE_OTF)
#############################################
# OTF2 Parser
#############################################
IF(VITE_ENABLE_OTF2)
ADD_DEFINITIONS(-DWITH_OTF2)
SET(VITE_HDRS
${VITE_HDRS}
parser/OTF2Parser/ParserDefinitionOTF2.hpp
parser/OTF2Parser/ParserEventOTF2.hpp
parser/OTF2Parser/ParserOTF2.hpp
)
SET(VITE_SRCS
${VITE_SRCS}
parser/OTF2Parser/ParserDefinitionOTF2.cpp
parser/OTF2Parser/ParserEventOTF2.cpp
parser/OTF2Parser/ParserOTF2.cpp
)
INCLUDE_DIRECTORIES(${OTF2_INCLUDE_DIR})
ENDIF(VITE_ENABLE_OTF2)
#############################################
# TAU Parser
#############################################
......@@ -411,6 +435,10 @@ IF( VITE_ENABLE_OTF )
LINK_DIRECTORIES( ${OTF_LIBRARY_DIR} )
ENDIF( VITE_ENABLE_OTF )
IF(VITE_ENABLE_OTF2)
LINK_DIRECTORIES(${OTF2_LIBRARY_DIR} )
ENDIF(VITE_ENABLE_OTF2)
#resource
IF(WIN32)
set(VITE_RES
......@@ -476,6 +504,18 @@ IF(VITE_ENABLE_OTF)
ENDIF(WIN32)
ENDIF(VITE_ENABLE_OTF)
IF(VITE_ENABLE_OTF2)
TARGET_LINK_LIBRARIES(vite
${OTF2_LIBRARY}
)
# if OTF2 is compiled with zlib support we need to add it
IF(WIN32)
TARGET_LINK_LIBRARIES(vite
${ZLIB_LIBRARY}
)
ENDIF(WIN32)
ENDIF(VITE_ENABLE_OTF2)
IF(VITE_ENABLE_TAU)
TARGET_LINK_LIBRARIES(vite
${TAU_LIBRARY}
......
This diff is collapsed.
/*
** This file is part of the ViTE project.
**
** This software is governed by the CeCILL-A license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-A license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
**
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
**
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
**
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-A license and that you accept its terms.
**
**
** ViTE developers are (for version 0.* to 1.0):
**
** - COULOMB Kevin
** - FAVERGE Mathieu
** - JAZEIX Johnny
** - LAGRASSE Olivier
** - MARCOUEILLE Jule
** - NOISETTE Pascal
** - REDONDY Arthur
** - VUCHENER Clément
**
*/
/**
* @file ParserDefinitionOTF2.hpp
*
* @author François Trahay
* @author Lagrasse Olivier
* @author Johnny Jazeix
* @author Mathieu Faverge
*
*/
#ifndef PARSERDEFINITIONOTF2_HPP
#define PARSERDEFINITIONOTF2_HPP
#include <otf2/otf2.h>
class Trace;
/*!
* \def NB_COLORS
* The number of default colors.
* They are created in the constructor, deleted in destructor.
*/
#define NB_COLORS 10
struct OTF2_Location;
struct OTF2_LocationGroup;
/*!
* \struct OTF2_SystemTreeNode
* \brief Contains the definition of a machine (equivalent in Paje : Container)
*/
struct OTF2_SystemTreeNode {
/*! \brief Id of the node */
OTF2_SystemTreeNodeRef _node_id;
/*! \brief String that identifies the node */
std::string _id_string;
/*! \brief Name of the node */
uint32_t _name_id;
/*! \brief id of the parent node */
OTF2_SystemTreeNodeRef _parent;
/*! \brief child system tree nodes */
std::map<OTF2_SystemTreeNodeRef, OTF2_SystemTreeNode* > _child_nodes;
/*! \brief child location groups */
std::map<OTF2_LocationGroupRef, OTF2_LocationGroup* > _location_group;
};
/*!
* \struct OTF2_LocationGroup
* \brief Contains the definition of a processGroup (equivalent in Paje : Container)
*/
struct OTF2_LocationGroup {
/*! \brief Id of the LocationGroup */
OTF2_LocationGroupRef _group_id;
/*! \brief String that identifies the group */
std::string _id_string;
/*! \brief Name of the LocationGroup */
uint32_t _name_id;
/*! \brief Child locations */
std::map<OTF2_LocationRef, OTF2_Location* > _location;
/*! \brief Id of the parent node */
OTF2_SystemTreeNodeRef _node_id;
};
/*!
* \struct OTF2_Location
* \brief Contains the definition of a process (equivalent in Paje : Container)
*/
struct OTF2_Location {
/*! \brief Id of the Location */
OTF2_LocationRef _location_id;
/*! \brief String that identifies the location */
std::string _id_string;
/*! \brief Name of the Location */
uint32_t _name_id;
/*! \brief Id of the parent group */
OTF2_LocationGroupRef _group_id;
/*! \brief Number of events in this Location */
uint64_t _number_of_events;
/*! \brief Container corresponding to this Location */
Container* container;
};
/*!
* \struct OTF2_Function
* \brief Contains the definition of a function (equivalent in Paje : State)
*/
struct OTF2_Function {
/*! \brief Name of the state */
OTF2_StringRef _name_id;
/*! \brief Alternative name of the region (e.g. mangled name */
OTF2_StringRef _canonicalName;
/*! \brief A more detailed description of this region */
OTF2_StringRef _region_description;
/*! \brief Region role. */
OTF2_RegionRole _regionRole;
/*! \brief Paradigm. */
OTF2_Paradigm _paradigm;
/*! \brief Region flags. */
OTF2_RegionFlag _regionFlags;
/*! \brief The source file where this region was declared */
OTF2_StringRef _sourceFile;
/*! \brieg Starting line number of this region in the source file. */
uint32_t _begin_line_number;
/*! \brieg Ending line number of this region in the source file. */
uint32_t _end_line_number;
};
struct OTF2_MetricMember {
OTF2_MetricMemberRef _id;
OTF2_StringRef _name;
OTF2_StringRef _description;
OTF2_MetricType _metricType;
OTF2_MetricMode _metricMode;
OTF2_Type _valueType;
OTF2_Base _base;
int64_t _exponent;
OTF2_StringRef _unit;
};
struct OTF2_MetricClass {
OTF2_MetricRef _id;
std::vector<OTF2_MetricMemberRef> _metricMembers;
OTF2_MetricOccurrence _metricOccurrence;
OTF2_RecorderKind _recorderKind;
};
/*!
*
* \class ParserDefinitionOTF2
* \brief Parse the definitions of the trace and store them.
*
*/
class ParserDefinitionOTF2 {
private:
/*!
* Reader for the file
*/
OTF2_GlobalDefReader* _global_def_reader;
OTF2_GlobalDefReaderCallbacks* _global_def_callbacks;
/*!
* Maps in order to easily retrieve the events.
*/
static std::map<OTF2_SystemTreeNodeRef, OTF2_SystemTreeNode* > _system_tree_node;
static std::map<OTF2_LocationGroupRef, OTF2_LocationGroup* > _location_group;
static std::map<OTF2_LocationRef, OTF2_Location* > _location;
static std::map<OTF2_MetricMemberRef, OTF2_MetricMember > _metric_member;
static std::map<OTF2_MetricRef, OTF2_MetricClass > _metric_class;
static std::map<uint32_t, OTF2_Function > _functions;
static std::map<uint32_t, const char *> _strings;
static uint64_t _ticks_per_second;
static double _first_timestamp;
/*!
* Table containing default colors.
* We take the state id modulo the max number to get the color.
*/
static std::vector<Color *> _default_colors;
static OTF2_CallbackCode handler_DefTimerResolution(void *, uint64_t, uint64_t, uint64_t);
static OTF2_CallbackCode handler_DefString(void *, OTF2_StringRef, const char *);
// System tree callback
static OTF2_CallbackCode handler_DefSystemTreeNode(void */*userData */,
OTF2_SystemTreeNodeRef tree_node_id,
OTF2_StringRef name_id,
OTF2_StringRef class_id,
OTF2_SystemTreeNodeRef parent_node_id);
static OTF2_CallbackCode handler_DefLocationGroup(void */*userdata*/,
OTF2_LocationGroupRef location_group_identifier,
OTF2_StringRef name,
OTF2_LocationGroupType type,
OTF2_SystemTreeNodeRef system_tree_parent);
static OTF2_CallbackCode handler_DefLocation(void *userData,
OTF2_LocationRef locationIdentifier,
OTF2_StringRef name_id,
OTF2_LocationType location_type,
uint64_t numberOfEvents,
OTF2_LocationGroupRef locationGroup);
//Region callback
static OTF2_CallbackCode handler_DefState(void *, OTF2_RegionRef, OTF2_StringRef, OTF2_StringRef, OTF2_StringRef description, OTF2_RegionRole, OTF2_Paradigm, OTF2_RegionFlag, OTF2_StringRef, uint32_t, uint32_t);
static OTF2_CallbackCode handler_DefGroup(void *userData,
OTF2_GroupRef group_id,
OTF2_StringRef name_id,
OTF2_GroupType type,
OTF2_Paradigm paradigm,
OTF2_GroupFlag flags,
uint32_t numberOfMembers,
const uint64_t *members);
static OTF2_CallbackCode handler_DefMetricMember( void* userData,
OTF2_MetricMemberRef self,
OTF2_StringRef name,
OTF2_StringRef description,
OTF2_MetricType metricType,
OTF2_MetricMode metricMode,
OTF2_Type valueType,
OTF2_Base base,
int64_t exponent,
OTF2_StringRef unit );
static OTF2_CallbackCode handler_DefMetricClass( void* userData,
OTF2_MetricRef self,
uint8_t numberOfMetrics,
const OTF2_MetricMemberRef* metricMembers,
OTF2_MetricOccurrence metricOccurrence,
OTF2_RecorderKind recorderKind );
static OTF2_CallbackCode handler_DefMetricInstance( void* userData,
OTF2_MetricRef self,
OTF2_MetricRef metricClass,
OTF2_LocationRef recorder,
OTF2_MetricScope metricScope,
uint64_t scope );
static OTF2_CallbackCode handler_DefMetricClassRecorder( void* userData,
OTF2_MetricRef metric,
OTF2_LocationRef recorder );
ParserDefinitionOTF2(const ParserDefinitionOTF2&);
public:
/*!
* \fn ParserDefinitionOTF2(OTF2_Reader *reader)
* \brief constructor
*/
ParserDefinitionOTF2(OTF2_Reader *reader);
/*!
* \fn ~ParserDefinitionOTF2()
* \brief destructor
*/
~ParserDefinitionOTF2();
/*!
* \fn set_handlers(Trace *t)
* \brief Create and set the handlers for the definition parsing.
* \param t The trace we want to store in.
*/
void set_handlers(Trace *t);
/*!
* \fn read_definitions(OTF2_Reader *reader)
* \brief Begin the reading of the definitions
* \param reader The main otf2 file we want to read in.
*/
void read_definitions(OTF2_Reader *reader);
/*!
* \fn create_container_types(Trace *t)
* \brief Create all the container types needed for the trace
* It is run at the end of the definitions parsing.
* \param t The trace where we store data
*/
void create_container_types(Trace *t);
void create_metric_member(Trace *t, OTF2_MetricMember m);
void create_metric_class(Trace *t, OTF2_MetricClass m);
/*!
* \fn initialize_types(Trace *t)
* \brief Create all the types needed for the trace
* It is run at the end of the definitions parsing.
* \param t The trace where we store data
*/
void initialize_types(Trace *t);
void create_location(Trace *t, OTF2_Location *l);
void create_location_group(Trace *t, OTF2_LocationGroup *lg);
void create_system_tree_node(Trace *t, OTF2_SystemTreeNode *node);
static void generate_string_id(OTF2_Location* l);
static void generate_string_id(OTF2_LocationGroup* lg);
static void generate_string_id(OTF2_SystemTreeNode* n);
static std::string get_string_id(OTF2_Location* l);
static std::string get_string_id(OTF2_LocationGroup* lg);
static std::string get_string_id(OTF2_SystemTreeNode* n);
/*!
* \fn print_definitions()
* \brief Print all the definitions stored. Useful for debug
*/
void print_definitions();
static OTF2_MetricClass get_metric_class(const OTF2_MetricRef id);
static OTF2_MetricMember get_metric_member(const OTF2_MetricClass metric_class, int index);
/*!
* \fn get_system_tree_node_by_id(const uint32_t id)
* \brief Accessor for the system_tree_node map
* \param id the id we want the corresponding system tree node
* \return The system tree node associated to id
*/
static OTF2_SystemTreeNode* get_system_tree_node_by_id(const OTF2_SystemTreeNodeRef id);
/*!
* \fn get_location_group_by_id(const uint32_t id)
* \brief Accessor for the location_group map
* \param id the id we want the corresponding location group
* \return The location group associated to id
*/
static OTF2_LocationGroup *get_location_group_by_id(const OTF2_LocationGroupRef id);
/*!
* \fn get_location_by_id(const uint32_t id)
* \brief Accessor for the location map
* \param id the id we want the corresponding location
* \return The location associated to id
*/
static OTF2_Location *get_location_by_id(const OTF2_LocationRef id);
/*!
* \fn get_function_by_id(const uint32_t id)
* \brief Accessor for the function map
* \param id the id we want the corresponding function
* \return The OTF2_Function associated to id
*/
static OTF2_Function get_function_by_id(const uint32_t id);
/*!
* \fn get_ticks_per_second()
* \brief Accessor for the tick_per_second (equivalent to a time unit)
* \return The number of ticks per second
*/
static uint64_t get_ticks_per_second();
static double get_timestamp(OTF2_TimeStamp ts);
static Color *get_color(uint32_t func_id);
static const char *get_string_by_id(uint32_t id);
};
#endif // PARSERDEFINITIONOTF2_HPP
This diff is collapsed.
This diff is collapsed.
/*
** This file is part of the ViTE project.
**
** This software is governed by the CeCILL-A license under French law
** and abiding by the rules of distribution of free software. You can
** use, modify and/or redistribute the software under the terms of the
** CeCILL-A license as circulated by CEA, CNRS and INRIA at the following
** URL: "http://www.cecill.info".
**
** As a counterpart to the access to the source code and rights to copy,
** modify and redistribute granted by the license, users are provided
** only with a limited warranty and the software's author, the holder of
** the economic rights, and the successive licensors have only limited
** liability.
**
** In this respect, the user's attention is drawn to the risks associated
** with loading, using, modifying and/or developing or reproducing the
** software by the user in light of its specific status of free software,
** that may mean that it is complicated to manipulate, and that also
** therefore means that it is reserved for developers and experienced
** professionals having in-depth computer knowledge. Users are therefore
** encouraged to load and test the software's suitability as regards
** their requirements in conditions enabling the security of their
** systems and/or data to be ensured and, more generally, to use and
** operate it in the same conditions as regards security.
**
** The fact that you are presently reading this means that you have had
** knowledge of the CeCILL-A license and that you accept its terms.
**
**
** ViTE developers are (for version 0.* to 1.0):
**
** - COULOMB Kevin
** - FAVERGE Mathieu
** - JAZEIX Johnny
** - LAGRASSE Olivier
** - MARCOUEILLE Jule
** - NOISETTE Pascal
** - REDONDY Arthur
** - VUCHENER Clément
**
*/
/**
* @file ParserOTF2.cpp
*
* @author Lagrasse Olivier
* @author Johnny Jazeix
* @author Mathieu Faverge
*
*/
#include <iostream>
#include <fstream>
#include <string>
#include <map>
#include <list>
/* -- */
#include <otf2/otf2.h>
/* -- */
#include "trace/values/Values.hpp"
#include "trace/EntityTypes.hpp"
#include "trace/Entitys.hpp"
#include "trace/Trace.hpp"
/* -- */
#include "parser/Parser.hpp"
#include "parser/OTF2Parser/ParserOTF2.hpp"
#include "parser/OTF2Parser/ParserDefinitionOTF2.hpp"
#include "parser/OTF2Parser/ParserEventOTF2.hpp"
/* -- */
using namespace std;
#define VITE_OTF_2_MAXFILES_OPEN 100
#define VITE_ERR_OTF_2_OPENREADER "Failed to create the OTF2 Reader\n"
ParserOTF2::ParserOTF2() {}
ParserOTF2::ParserOTF2(const string &filename) : Parser(filename) {}
ParserOTF2::~ParserOTF2() {}
void ParserOTF2::parse(Trace &trace,
bool finish_trace_after_parse) {
ParserDefinitionOTF2 *parserdefinition;
ParserEventOTF2 *parserevent;
OTF2_Reader *reader;
string filename = get_next_file_to_parse();