Commit 41f95f22 authored by Serrière Guillaume's avatar Serrière Guillaume

Add automatique test for the kappa coefficient.

Fix Kappa coefficient.
Signed-off-by: default avatarSerrière Guillaume <guillaume.serriere@inria.fr>
parent 30ad4f1c
......@@ -49,8 +49,8 @@ VRPN_ExternalServerPort = 53883
Kernel_PluginsPatternLinux = ${Path_Lib}/libopenvibe-plugins-*.so
Kernel_PluginsPatternWindows = ${Path_Bin}/openvibe-plugins-*.dll
Kernel_Plugins = ${Kernel_PluginsPattern${OperatingSystem}}
Kernel_MainLogLevel = Information
Kernel_ConsoleLogLevel = Trace
Kernel_MainLogLevel = Debug
Kernel_ConsoleLogLevel = Information
Kernel_ConsoleLogWithHexa = False
Kernel_ConsoleLogTimeInSecond = True
Kernel_ConsoleLogTimePrecision = 3
......
......@@ -20,6 +20,13 @@ INCLUDE("FindOpenViBEModuleSystem")
INCLUDE("FindOpenViBEModuleXML")
INCLUDE("FindThirdPartyGTK")
# ---------------------------------
# Test applications
# ---------------------------------
IF(OV_COMPILE_TESTS)
ADD_SUBDIRECTORY(test)
ENDIF(OV_COMPILE_TESTS)
# ---------------------------------
# Target macros
# Defines target operating system
......
......@@ -174,6 +174,8 @@ boolean CAlgorithmConfusionMatrix::process(void)
}
if(l_bFound)
{
this->getLogManager() << LogLevel_Trace << "Result received : " << l_ui64StimulationFromClassifierIdentifier << ". Corresponding target : "
<< l_ui64StimulationTargeted << ".\n";
// now we found the target, let's update the confusion matrix
// we need to update the whole line vector for the targeted class
uint32 l_ui32OldAttemptCount = m_mapClassificationAttemptCountPerClass[l_ui64StimulationTargeted];
......
......@@ -35,7 +35,7 @@ boolean CBoxAlgorithmKappaCoefficient::initialize(void)
ip_bPercentages = false;
TParameterHandler<boolean> ip_bSums(m_pConfusionMatrixAlgorithm->getInputParameter(OVP_Algorithm_ConfusionMatrixAlgorithm_InputParameterId_Sums));
ip_bSums = false;
ip_bSums = true;
m_ui32AmountClass=getBoxAlgorithmContext()->getStaticBoxContext()->getSettingCount() - c_ui32ClassLabelOffset;
vector < uint64 > l_vClassCodes;
......@@ -99,6 +99,8 @@ boolean CBoxAlgorithmKappaCoefficient::initialize(void)
boolean CBoxAlgorithmKappaCoefficient::uninitialize(void)
{
//Log for the automatic test
this->getLogManager() << LogLevel_Trace << "[Test] Final value of Kappa " << m_f64KappaCoefficient << "\n";
m_pConfusionMatrixAlgorithm->uninitialize();
this->getAlgorithmManager().releaseAlgorithm(*m_pConfusionMatrixAlgorithm);
......@@ -174,11 +176,7 @@ boolean CBoxAlgorithmKappaCoefficient::process(void)
//The confusion matrix has changed so we need to update the kappa coefficient
float64* l_pConfusionMatrix = op_pConfusionMatrix->getBuffer();
//First we need the amount of sample that have been classified
uint32 l_ui32Total = 0;
for(size_t j =0; j < m_ui32AmountClass * m_ui32AmountClass ; ++j)
{
l_ui32Total += static_cast<uint32>(l_pConfusionMatrix[j]);
}
uint32 l_ui32Total = static_cast<uint32>(l_pConfusionMatrix[(m_ui32AmountClass+1) * (m_ui32AmountClass + 1) -1]);
//Now we gonna compute the two sum we need to compute the kappa coefficient
//It's more easy to use a double loop
......@@ -187,20 +185,19 @@ boolean CBoxAlgorithmKappaCoefficient::process(void)
for(size_t j =0; j < m_ui32AmountClass ; ++j)
{
l_f64ObservedAccurancy += l_pConfusionMatrix[j*m_ui32AmountClass + j];
//We need to take the column sum in account
l_f64ObservedAccurancy += l_pConfusionMatrix[j*(m_ui32AmountClass+1) + j];
for(size_t k =0; k < m_ui32AmountClass ; ++k)
{
l_f64ExpectedAccurancy += (l_pConfusionMatrix[m_ui32AmountClass* j + k] * l_pConfusionMatrix[m_ui32AmountClass * k + j]);
}
l_f64ExpectedAccurancy += (l_pConfusionMatrix[(m_ui32AmountClass+1)* j + m_ui32AmountClass] *
l_pConfusionMatrix[(m_ui32AmountClass+1) * m_ui32AmountClass + j]);
}
l_f64ObservedAccurancy /= l_ui32Total;
l_f64ExpectedAccurancy /= (l_ui32Total * l_ui32Total);
float64 l_f64KappaCoefficient = (l_f64ObservedAccurancy - l_f64ExpectedAccurancy)/(1 - l_f64ExpectedAccurancy);
m_f64KappaCoefficient = (l_f64ObservedAccurancy - l_f64ExpectedAccurancy)/(1 - l_f64ExpectedAccurancy);
updateKappaValue(l_f64KappaCoefficient);
m_oOutputMatrixEncoder.getInputMatrix()->getBuffer()[0] = l_f64KappaCoefficient;
updateKappaValue();
m_oOutputMatrixEncoder.getInputMatrix()->getBuffer()[0] = m_f64KappaCoefficient;
m_oOutputMatrixEncoder.encodeBuffer();
l_rDynamicBoxContext.markOutputAsReadyToSend(0, l_rDynamicBoxContext.getInputChunkStartTime(1, i), l_rDynamicBoxContext.getInputChunkEndTime(0, i));
}
......@@ -219,11 +216,11 @@ boolean CBoxAlgorithmKappaCoefficient::process(void)
return true;
}
void CBoxAlgorithmKappaCoefficient::updateKappaValue(float64 f64KappaValue)
void CBoxAlgorithmKappaCoefficient::updateKappaValue()
{
std::stringstream l_sStream;
l_sStream << std::fixed;
l_sStream << std::setprecision(2);
l_sStream << f64KappaValue;
l_sStream << m_f64KappaCoefficient;
gtk_label_set(GTK_LABEL(m_pKappaLabel), l_sStream.str().c_str());
}
......@@ -39,7 +39,7 @@ namespace OpenViBEPlugins
_IsDerivedFromClass_Final_(OpenViBEToolkit::TBoxAlgorithm < OpenViBE::Plugins::IBoxAlgorithm >, OVP_ClassId_BoxAlgorithm_KappaCoefficient)
protected:
void updateKappaValue(OpenViBE::float64 f64KappaValue);
void updateKappaValue();
OpenViBEToolkit::TStimulationDecoder < CBoxAlgorithmKappaCoefficient > m_oTargetStimulationDecoder;
OpenViBEToolkit::TStimulationDecoder < CBoxAlgorithmKappaCoefficient > m_oClassifierStimulationDecoder;
......@@ -52,6 +52,7 @@ namespace OpenViBEPlugins
OpenViBE::uint32 m_ui32AmountClass;
OpenViBE::uint64 m_ui64CurrentProcessingTimeLimit;
OpenViBE::float64 m_f64KappaCoefficient;
::GtkWidget* m_pKappaLabel;
};
......
PROJECT(test_evaluation)
IF(WIN32)
ADD_DEFINITIONS(-DTARGET_OS_Windows)
ENDIF(WIN32)
IF(UNIX)
ADD_DEFINITIONS(-DTARGET_OS_Linux)
ENDIF(UNIX)
ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE)
ADD_DEFINITIONS(-DTARGET_ARCHITECTURE_i386)
INCLUDE_DIRECTORIES(../src)
ADD_EXECUTABLE(${PROJECT_NAME} test_kappa.cpp)
#INCLUDE("FindOpenViBE")
# Unfortunately we need to install the tests as any application to find .dll/.so files
# on both Windows and Linux.
OV_INSTALL_LAUNCH_SCRIPT(${PROJECT_NAME})
INSTALL(TARGETS ${PROJECT_NAME}
RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR})
\ No newline at end of file
#blabla
# @FIXME there is a problem of using the global log, this will cause interference if any tests are run in parallel
IF(WIN32)
SET(EXT cmd)
SET(OS_FLAGS "--no-pause")
SET(OV_LOGFILE "$ENV{APPDATA}/openvibe/log/openvibe-designer.log")
ELSE(WIN32)
SET(EXT sh)
SET(OS_FLAGS "")
SET(OV_LOGFILE "$ENV{HOME}/.config/openvibe/log/openvibe-designer.log")
ENDIF(WIN32)
SET(TEST_SCENARIOS LDA-OneVsOne-HT-Test LDA-OneVsOne-PKPD-Test LDA-OneVsOne-Voting-Test LDA-OneVsAll-Test sLDA-OneVsOne-HT-Test sLDA-OneVsOne-PKPD-Test sLDA-OneVsOne-Voting-Test sLDA-OneVsAll-Test SVM-Native-Test SVM-OneVsOne-Voting-Test SVM-OneVsOne-HT-Test SVM-OneVsOne-PKPD-Test SVM-OneVsAll-Test MLP-Native-Test MLP-OneVsOne-Voting-Test MLP-OneVsOne-HT-Test MLP-OneVsOne-PKPD-Test MLP-OneVsAll-Test)
ADD_TEST(clean_Evaluation-kappa rm -f ${OV_LOGFILE})
ADD_TEST(run_Evaluation-kappa "$ENV{OV_BINARY_PATH}/openvibe-designer.${EXT}" ${OS_FLAGS} "--invisible" "--no-session-management" "--config" "$ENV{OV_BINARY_PATH}/share/openvibe/kernel/openvibe_test.conf" "--play-fast" Test-kappa.xml)
ADD_TEST(compare_Evaluation_${TEST_NAME} "$ENV{OV_BINARY_PATH}/test_evaluation.${EXT}" "${OS_FLAGS}" "${OV_LOGFILE}")
# It would be better to clean last, but we can't do this as it will delete the
# output we wish to include, and we can't prevent clean from running if a prev. test fails
# We need the clean to be sure that the comparator stage is not getting data from a previous run.
SET_TESTS_PROPERTIES(run_${TEST_NAME} PROPERTIES DEPENDS clean_${TEST_NAME})
SET_TESTS_PROPERTIES(run_${TEST_NAME} PROPERTIES ATTACHED_FILES_ON_FAIL ${OV_LOGFILE})
SET_TESTS_PROPERTIES(compare_${TEST_NAME} PROPERTIES DEPENDS run_${TEST_NAME}})
SET_TESTS_PROPERTIES(compare_${TEST_NAME} PROPERTIES ATTACHED_FILES_ON_FAIL ${OV_LOGFILE})
This diff is collapsed.
Time (s);Identifier;Duration
2.7050781250e-01;33026;0.0000000000e+00
3.3984375000e-01;33024;0.0000000000e+00
5.4199218750e-01;33026;0.0000000000e+00
6.7968750000e-01;33024;0.0000000000e+00
7.0996093750e-01;33025;0.0000000000e+00
8.1250000000e-01;33026;0.0000000000e+00
1.0195312500e+00;33024;0.0000000000e+00
1.0839843750e+00;33026;0.0000000000e+00
1.3544921875e+00;33026;0.0000000000e+00
1.3593750000e+00;33024;0.0000000000e+00
1.4199218750e+00;33025;0.0000000000e+00
1.6259765625e+00;33026;0.0000000000e+00
1.6992187500e+00;33024;0.0000000000e+00
1.8964843750e+00;33026;0.0000000000e+00
2.0390625000e+00;33024;0.0000000000e+00
2.1298828125e+00;33025;0.0000000000e+00
2.1679687500e+00;33026;0.0000000000e+00
2.3798828125e+00;33024;0.0000000000e+00
2.4384765625e+00;33026;0.0000000000e+00
2.7099609375e+00;33026;0.0000000000e+00
2.7197265625e+00;33024;0.0000000000e+00
2.8398437500e+00;33025;0.0000000000e+00
2.9804687500e+00;33026;0.0000000000e+00
3.0595703125e+00;33024;0.0000000000e+00
3.2519531250e+00;33026;0.0000000000e+00
3.3994140625e+00;33024;0.0000000000e+00
3.5224609375e+00;33026;0.0000000000e+00
3.5498046875e+00;33025;0.0000000000e+00
3.7392578125e+00;33024;0.0000000000e+00
3.7939453125e+00;33026;0.0000000000e+00
4.0644531250e+00;33026;0.0000000000e+00
4.0791015625e+00;33024;0.0000000000e+00
4.2597656250e+00;33025;0.0000000000e+00
4.3359375000e+00;33026;0.0000000000e+00
4.4199218750e+00;33024;0.0000000000e+00
4.6064453125e+00;33026;0.0000000000e+00
4.7597656250e+00;33024;0.0000000000e+00
4.8779296875e+00;33026;0.0000000000e+00
4.9697265625e+00;33025;0.0000000000e+00
5.0996093750e+00;33024;0.0000000000e+00
5.1484375000e+00;33026;0.0000000000e+00
5.4199218750e+00;33026;0.0000000000e+00
5.4394531250e+00;33024;0.0000000000e+00
5.6796875000e+00;33025;0.0000000000e+00
5.6904296875e+00;33026;0.0000000000e+00
5.7792968750e+00;33024;0.0000000000e+00
5.9619140625e+00;33026;0.0000000000e+00
6.1191406250e+00;33024;0.0000000000e+00
6.2324218750e+00;33026;0.0000000000e+00
6.3896484375e+00;33025;0.0000000000e+00
6.4599609375e+00;33024;0.0000000000e+00
6.5039062500e+00;33026;0.0000000000e+00
6.7744140625e+00;33026;0.0000000000e+00
6.7998046875e+00;33024;0.0000000000e+00
7.0458984375e+00;33026;0.0000000000e+00
7.0996093750e+00;33025;0.0000000000e+00
7.1396484375e+00;33024;0.0000000000e+00
7.3164062500e+00;33026;0.0000000000e+00
7.4794921875e+00;33024;0.0000000000e+00
7.5878906250e+00;33026;0.0000000000e+00
7.8095703125e+00;33025;0.0000000000e+00
7.8193359375e+00;33024;0.0000000000e+00
7.8583984375e+00;33026;0.0000000000e+00
8.1298828125e+00;33026;0.0000000000e+00
8.1591796875e+00;33024;0.0000000000e+00
8.4003906250e+00;33026;0.0000000000e+00
8.4990234375e+00;33024;0.0000000000e+00
8.5195312500e+00;33025;0.0000000000e+00
8.6718750000e+00;33026;0.0000000000e+00
8.8398437500e+00;33024;0.0000000000e+00
8.9423828125e+00;33026;0.0000000000e+00
9.1796875000e+00;33024;0.0000000000e+00
9.2138671875e+00;33026;0.0000000000e+00
9.2294921875e+00;33025;0.0000000000e+00
9.4843750000e+00;33026;0.0000000000e+00
9.5195312500e+00;33024;0.0000000000e+00
9.7558593750e+00;33026;0.0000000000e+00
9.8593750000e+00;33024;0.0000000000e+00
9.9394531250e+00;33025;0.0000000000e+00
1.0026367188e+01;33026;0.0000000000e+00
Time (s);Identifier;Duration
2.7150781250e-01;33026;0.0000000000e+00
3.4084375000e-01;33025;0.0000000000e+00
5.4299218750e-01;33026;0.0000000000e+00
6.8068750000e-01;33025;0.0000000000e+00
7.1096093750e-01;33025;0.0000000000e+00
8.1350000000e-01;33026;0.0000000000e+00
1.0295312500e+00;33024;0.0000000000e+00
1.0939843750e+00;33026;0.0000000000e+00
1.3569921875e+00;33026;0.0000000000e+00
1.3693750000e+00;33026;0.0000000000e+00
1.4299218750e+00;33025;0.0000000000e+00
1.6359765625e+00;33026;0.0000000000e+00
1.7092187500e+00;33024;0.0000000000e+00
1.9064843750e+00;33026;0.0000000000e+00
2.0490625000e+00;33024;0.0000000000e+00
2.1398828125e+00;33024;0.0000000000e+00
2.1779687500e+00;33026;0.0000000000e+00
2.3898828125e+00;33024;0.0000000000e+00
2.4484765625e+00;33026;0.0000000000e+00
2.7159609375e+00;33026;0.0000000000e+00
2.7297265625e+00;33024;0.0000000000e+00
2.8498437500e+00;33026;0.0000000000e+00
2.9904687500e+00;33026;0.0000000000e+00
3.0695703125e+00;33024;0.0000000000e+00
3.2619531250e+00;33024;0.0000000000e+00
3.4094140625e+00;33024;0.0000000000e+00
3.5324609375e+00;33024;0.0000000000e+00
3.5598046875e+00;33025;0.0000000000e+00
3.7492578125e+00;33024;0.0000000000e+00
3.8039453125e+00;33025;0.0000000000e+00
4.0744531250e+00;33026;0.0000000000e+00
4.0891015625e+00;33024;0.0000000000e+00
4.2697656250e+00;33025;0.0000000000e+00
4.3459375000e+00;33026;0.0000000000e+00
4.4299218750e+00;33024;0.0000000000e+00
4.6164453125e+00;33026;0.0000000000e+00
4.7697656250e+00;33024;0.0000000000e+00
4.8879296875e+00;33026;0.0000000000e+00
4.9797265625e+00;33025;0.0000000000e+00
5.1096093750e+00;33024;0.0000000000e+00
5.1584375000e+00;33026;0.0000000000e+00
5.4299218750e+00;33026;0.0000000000e+00
5.4494531250e+00;33024;0.0000000000e+00
5.6896875000e+00;33025;0.0000000000e+00
5.7004296875e+00;33026;0.0000000000e+00
5.7892968750e+00;33024;0.0000000000e+00
5.9719140625e+00;33026;0.0000000000e+00
6.1291406250e+00;33024;0.0000000000e+00
6.2424218750e+00;33026;0.0000000000e+00
6.3996484375e+00;33025;0.0000000000e+00
6.4699609375e+00;33024;0.0000000000e+00
6.5139062500e+00;33026;0.0000000000e+00
6.7844140625e+00;33026;0.0000000000e+00
6.8098046875e+00;33024;0.0000000000e+00
7.0558984375e+00;33026;0.0000000000e+00
7.1096093750e+00;33025;0.0000000000e+00
7.1496484375e+00;33024;0.0000000000e+00
7.3264062500e+00;33026;0.0000000000e+00
7.4894921875e+00;33024;0.0000000000e+00
7.5978906250e+00;33026;0.0000000000e+00
7.8155703125e+00;33025;0.0000000000e+00
7.8293359375e+00;33024;0.0000000000e+00
7.8683984375e+00;33026;0.0000000000e+00
8.1398828125e+00;33026;0.0000000000e+00
8.1691796875e+00;33024;0.0000000000e+00
8.4103906250e+00;33026;0.0000000000e+00
8.5090234375e+00;33024;0.0000000000e+00
8.5295312500e+00;33025;0.0000000000e+00
8.6818750000e+00;33026;0.0000000000e+00
8.8498437500e+00;33024;0.0000000000e+00
8.9523828125e+00;33026;0.0000000000e+00
9.1896875000e+00;33024;0.0000000000e+00
9.2238671875e+00;33026;0.0000000000e+00
9.2394921875e+00;33025;0.0000000000e+00
9.4943750000e+00;33026;0.0000000000e+00
9.5295312500e+00;33024;0.0000000000e+00
9.7658593750e+00;33026;0.0000000000e+00
9.8693750000e+00;33024;0.0000000000e+00
9.9494531250e+00;33025;0.0000000000e+00
1.0126367188e+01;33026;0.0000000000e+00
#include <fstream>
#include <sstream>
#include <string>
#include <iostream>
#include <cstring>
#include <cstdlib>
using namespace std;
int main (int argc, char** argv)
{
if(argc!=2)
{
cout << "Usage: test_evaluation <filename>\n";
return 3;
}
ifstream file(argv[1], ios::in);
if(file.good() && !file.bad() && file.is_open()) // ...
{
string line;
while(getline(file, line))
{
size_t pos;
if(line.find("Final value of Kappa") != string::npos)
{
cout << "Found kappa line " << line << endl;
pos = line.rfind(" ");
string cutline = line.substr(pos);
stringstream l_sKappa(cutline);
double l_fKappaCoefficient;
l_sKappa >> l_fKappaCoefficient;
if(l_fKappaCoefficient != 0.840677)
{
cout << "Wrong Kappa coefficient. Found " << l_fKappaCoefficient << " instead of 0.840677" << endl;
return 1;
}
else{
cout << "Test ok" << endl;
return 0;
}
}
}
}
cout << "Error: Problem opening [" << argv[1] << "]\n";
return 2;
}
......@@ -65,6 +65,7 @@ SUBDIRS("${CTEST_SOURCE_DIRECTORY}/plugins/processing/stream-codecs/test")
SUBDIRS("${CTEST_SOURCE_DIRECTORY}/plugins/processing/python/test")
SUBDIRS("${CTEST_SOURCE_DIRECTORY}/plugins/processing/streaming/test")
SUBDIRS("${CTEST_SOURCE_DIRECTORY}/plugins/processing/matlab/test")
SUBDIRS("${CTEST_SOURCE_DIRECTORY}/plugins/processing/evaluation/test")
# SUBDIRS("${CTEST_SOURCE_DIRECTORY}/applications/external-stimulation-connection-example/test")
SUBDIRS("${CTEST_SOURCE_DIRECTORY}/applications/platform/acquisition-server/test")
SUBDIRS("${CTEST_SOURCE_DIRECTORY}/applications/platform/designer/test")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment