CMakeLists has been updated. Added some files which were missing in previous...

CMakeLists has been updated. Added some files which were missing in previous commits. Checked workload validity. Lots of TODO have been done: updating the export.hpp documentation, better displays...

CMakeLists.txt

@@ -19,16 +19,16 @@ set(CMAKE_BUILD_TYPE Debug)
 #    message(STATUS "The compiler ${CMAKE_C_COMPILER} has no C11 nor C99 support. Please update your C compiler.")
 #endif()
 
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -std=c++14")
 
-add_executable(batsim batsim.c job.c utils.c export.c machines.c)
-add_executable(batexec batexec.c job.c utils.c export.c)
+add_executable(batsim batsim.cpp export.cpp ipp.cpp job_submitter.cpp jobs.cpp jobs_execution.cpp machines.cpp network.cpp profiles.cpp workload.cpp)
+#add_executable(batexec batexec.c job.c utils.c export.c)
 
 ### Add definitions for compile
-target_link_libraries(batsim simgrid jansson m)
-target_link_libraries(batexec simgrid jansson m)
+target_link_libraries(batsim simgrid boost_system boost_filesystem)
+#target_link_libraries(batexec simgrid jansson m)
 
 ## Add intall target
 INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/batsim
-    ${CMAKE_CURRENT_BINARY_DIR}/batexec
+    #${CMAKE_CURRENT_BINARY_DIR}/batexec
     DESTINATION bin)

batsim.cpp

@@ -15,9 +15,12 @@
 #include "machines.hpp"
 #include "network.hpp"
 #include "profiles.hpp"
+#include "workload.hpp"
 
 using namespace std;
 
+XBT_LOG_NEW_DEFAULT_CATEGORY(batsim, "batsim");
+
 /**
  * @brief The main function arguments (a.k.a. program arguments)
  */
@@ -123,17 +126,23 @@ int main(int argc, char * argv[])
     MSG_init(&argc, argv);
 
     BatsimContext context;
-    context.jobs.load_from_json(mainArgs.workloadFilename);
-    context.profiles.load_from_json(mainArgs.workloadFilename);
+
+    load_json_workload(&context, mainArgs.workloadFilename);
     context.jobs.setProfiles(&context.profiles);
     context.tracer.setFilename(mainArgs.exportPrefix + "_schedule.trace");
-    // TODO: check jobs & profile validity
     //context.jobs.displayDebug();
 
+    XBT_INFO("Checking whether SMPI is used or not...");
     bool smpi_used = context.jobs.containsSMPIJob();
     if (!smpi_used)
+    {
+        XBT_INFO("SMPI will NOT be used.");
         MSG_config("host/model", "ptask_L07");
+    }
+    else
+        XBT_INFO("SMPI will be used.");
 
+    XBT_INFO("Creating the machines...");
     MSG_create_environment(mainArgs.platformFilename.c_str());
 
     xbt_dynar_t hosts = MSG_hosts_as_dynar();
@@ -142,20 +151,24 @@ int main(int argc, char * argv[])
     const Machine * masterMachine = context.machines.masterMachine();
     context.machines.setTracer(&context.tracer);
     context.tracer.initialize(&context, MSG_get_clock());
-    //context.machines.displayDebug();
+    XBT_INFO("Machines created successfully. There are %d computing machines.", context.machines.machines().size());
 
     // Socket
     context.socket.create_socket(mainArgs.socketFilename);
     context.socket.accept_pending_connection();
 
     // Main processes running
+    XBT_INFO("Creating jobs_submitter process...");
     JobSubmitterProcessArguments * submitterArgs = new JobSubmitterProcessArguments;
     submitterArgs->context = &context;
     MSG_process_create("jobs_submitter", job_submitter_process, (void*)submitterArgs, masterMachine->host);
+    XBT_INFO("The jobs_submitter process has been created.");
 
+    XBT_INFO("Creating the uds_server process...");
     ServerProcessArguments * serverArgs = new ServerProcessArguments;
     serverArgs->context = &context;
     MSG_process_create("server", uds_server_process, (void*)serverArgs, masterMachine->host);
+    XBT_INFO("The uds_server process has been created.");
 
     msg_error_t res = MSG_main();
 

export.cpp

@@ -296,6 +296,7 @@ void PajeTracer::finalize(BatsimContext * context, double time)
 
 void PajeTracer::addJobLaunching(int jobID, const std::vector<int> & usedMachineIDs, double time)
 {
+    (void) jobID;
     xbt_assert(state == INITIALIZED, "Bad addJobLaunching call: the PajeTracer object is not initialized or had been finalized");
 
     const int bufSize = 64;
@@ -507,7 +508,7 @@ void PajeTracer::hsvToRgb(double h, double s, double v, double & r, double & g,
 
 void exportJobsToCSV(const char *filename)
 {
-
+    (void) filename;
 }
 /* TODO
 void exportJobsToCSV(const char *filename)

export.hpp

@@ -84,7 +84,7 @@ public:
     /**
      * @brief Initializes a PajeTracer.
      * @details This function must be called once before adding job launchings, runnings or endings.
-     * TODO UPDATE
+     * @param context The Batsim context
      * @param machines The machines
      */
     void initialize(BatsimContext * context, double time);
@@ -92,8 +92,7 @@ public:
     /**
      * @brief Finalizes a PajeTracer.
      * @details This function must be called before the PajeTracer's object destruction.
-     * @param machines The machines
-     * TODO UPDATE
+     * @param context The Batsim context
      * @param time The simulation time at which the finalization is done
      */
     void finalize(BatsimContext * context, double time);
@@ -101,8 +100,8 @@ public:
     /**
      * @brief Adds a job launch in the file trace.
      * @details Please note that this method can only be called when the PajeTracer object has been initialized and had not been finalized yet.
-     * @param job The job
-     * TODO UPDATE
+     * @param jobID The job unique number
+     * @param usedMachineIDs The machines which compute the job
      * @param time The simulation time at which the addition is done
      */
     void addJobLaunching(int jobID, const std::vector<int> & usedMachineIDs, double time);
@@ -114,8 +113,7 @@ public:
     /**
      * @brief Adds a job run in the file trace.
      * @details Please note that this method can only be called when the PajeTracer object has been initialized and had not been finalized yet.
-     * @param job The job
-     * TODO UPDATE
+     * @param jobID The job unique number
      * @param time The simulation time at which the addition is done
      */
     void addJobRunning(int jobID, const std::vector<int> & usedMachineIDs, double time);
@@ -123,18 +121,19 @@ public:
     /**
      * @brief Adds a job end in the file trace.
      * @details Please note that this method can only be called when the PajeTracer object has been initialized and had not been finalized yet.
-     * @param job The job
-     * TODO UPDATE
-     * @param time The simulation time at which the addition is done
+     * @param jobID The job unique number
+     * @param usedMachineIDs The machines which compute the job
+     * @param time The simulation time at which the kill is done
      */
     void addJobEnding(int jobID, const std::vector<int> & usedMachineIDs, double time);
 
     /**
      * @brief Adds a job kill in the file trace.
      * @details Please note that this method can only be called when the PajeTracer object has been initialized and had not been finalized yet.
-     * @param job The job that have been killed
-     * TODO UPDATE
+     * @param jobID The job unique number
+     * @param usedMachineIDs The machines which compute the job
      * @param time The simulation time at which the kill is done
+     * @param associateKillToMachines By default (false), one event is added in the killer container. If set to true, one event is added for every machine on which the kill occurs.
      */
     void addJobKill(int jobID, const std::vector<int> & usedMachineIDs, double time, bool associateKillToMachines = false);
 
@@ -185,7 +184,7 @@ private:
     const char * mstateWaiting = "w";
     const char * mstateLaunching = "l";
 
-    const char * varGlobalUtilization = "vgu";
+    //const char * varGlobalUtilization = "vgu";
 
     const char * root = "root";
     const char * scheduler = "sc";

job_submitter.cpp

+#include "job_submitter.hpp"
+
+#include <vector>
+#include <algorithm>
+
+#include "jobs.hpp"
+#include "ipp.hpp"
+#include "context.hpp"
+
+using namespace std;
+
+int job_submitter_process(int argc, char *argv[])
+{
+    (void) argc;
+    (void) argv;
+
+    JobSubmitterProcessArguments * args = (JobSubmitterProcessArguments *) MSG_process_get_data(MSG_process_self());
+    BatsimContext * context = args->context;
+
+    send_message("server", IPMessageType::SUBMITTER_HELLO);
+
+    double previousSubmissionDate = MSG_get_clock();
+
+    vector<const Job *> jobsVector;
+
+    const auto & jobs = context->jobs.jobs();
+    for (const auto & mit : jobs)
+    {
+        const Job * job = mit.second;
+        jobsVector.push_back(job);
+    }
+
+    sort(jobsVector.begin(), jobsVector.end(), job_comparator_subtime);
+
+    for (const Job * job : jobsVector)
+    {
+        if (job->submission_time > previousSubmissionDate)
+            MSG_process_sleep(job->submission_time - previousSubmissionDate);
+
+        JobSubmittedMessage * msg = new JobSubmittedMessage;
+        msg->job_id = job->id;
+
+        send_message("server", IPMessageType::JOB_SUBMITTED, (void*)msg);
+        previousSubmissionDate = MSG_get_clock();
+    }
+
+    send_message("server", IPMessageType::SUBMITTER_BYE);
+    delete args;
+    return 0;
+}

job_submitter.hpp

+#pragma once
+
+int job_submitter_process(int argc, char *argv[]);

jobs.cpp

@@ -41,23 +41,8 @@ void Jobs::setProfiles(Profiles *profiles)
     _profiles = profiles;
 }
 
-void Jobs::load_from_json(const std::string &filename)
+void Jobs::load_from_json(const Document &doc, const string &filename)
 {
-    // Let the file content be placed in a string
-    ifstream ifile(filename);
-    string content;
-
-    ifile.seekg(0, ios::end);
-    content.reserve(ifile.tellg());
-    ifile.seekg(0, ios::beg);
-
-    content.assign((std::istreambuf_iterator<char>(ifile)),
-                std::istreambuf_iterator<char>());
-
-    // JSON document creation
-    Document doc;
-    doc.Parse(content.c_str());
-
     xbt_assert(doc.IsObject());
     xbt_assert(doc.HasMember("jobs"), "Invalid JSON file '%s': the 'jobs' array is missing", filename.c_str());
     const Value & jobs = doc["jobs"];

jobs.hpp

@@ -7,6 +7,8 @@
 #include <map>
 #include <vector>
 
+#include <rapidjson/document.h>
+
 class Profiles;
 
 enum class JobState
@@ -42,7 +44,7 @@ public:
 
     void setProfiles(Profiles * profiles);
 
-    void load_from_json(const std::string & filename);
+    void load_from_json(const rapidjson::Document & doc, const std::string & filename);
 
     Job * operator[](int job_id);
     const Job * operator[](int job_id) const;

jobs_execution.cpp

@@ -157,7 +157,7 @@ int execute_profile(BatsimContext *context,
 
         for (int i = 0; i < data->repeat; i++)
         {
-            for (int j = 0; j < data->sequence.size(); j++)
+            for (unsigned int j = 0; j < data->sequence.size(); j++)
             {
                 if (execute_profile(context, data->sequence[j], allocation, remaining_time) == 0)
                     return 0;

network.cpp

@@ -48,6 +48,8 @@ UnixDomainSocket::~UnixDomainSocket()
 
 void UnixDomainSocket::create_socket(const string & filename)
 {
+    XBT_INFO("Creating UDS socket on '%s'", filename.c_str());
+
     _server_socket = socket(AF_UNIX, SOCK_STREAM, 0);
     xbt_assert(_server_socket != -1, "Impossible to create socket");
 

profiles.cpp

@@ -27,23 +27,8 @@ Profiles::~Profiles()
     }
 }
 
-void Profiles::load_from_json(const string & filename)
+void Profiles::load_from_json(const Document &doc, const string & filename)
 {
-    // Let the file content be placed in a string
-    ifstream ifile(filename);
-    string content;
-
-    ifile.seekg(0, ios::end);
-    content.reserve(ifile.tellg());
-    ifile.seekg(0, ios::beg);
-
-    content.assign((std::istreambuf_iterator<char>(ifile)),
-                   std::istreambuf_iterator<char>());
-
-    // JSON document creation
-    Document doc;
-    doc.Parse(content.c_str());
-
     xbt_assert(doc.IsObject());
     xbt_assert(doc.HasMember("profiles"), "Invalid JSON file '%s': the 'profiles' object is missing", filename.c_str());
     const Value & profiles = doc["profiles"];
@@ -194,6 +179,11 @@ bool Profiles::exists(const std::string &profile_name) const
     return mit != _profiles.end();
 }
 
+const std::map<string, Profile *> Profiles::profiles() const
+{
+    return _profiles;
+}
+
 
 MsgParallelProfileData::~MsgParallelProfileData()
 {

profiles.hpp

@@ -4,6 +4,8 @@
 #include <map>
 #include <vector>
 
+#include <rapidjson/document.h>
+
 enum class ProfileType
 {
     DELAY,
@@ -59,12 +61,13 @@ public:
     Profiles();
     ~Profiles();
 
-    void load_from_json(const std::string & filename);
+    void load_from_json(const rapidjson::Document & doc, const std::string & filename);
 
     Profile * operator[](const std::string & profile_name);
     const Profile * operator[](const std::string & profile_name) const;
     bool exists(const std::string & profile_name) const;
 
+    const std::map<std::string, Profile *> profiles() const;
 
 private:
     std::map<std::string, Profile*> _profiles;

workload.cpp

+#include "workload.hpp"
+
+#include <fstream>
+#include <streambuf>
+
+#include <rapidjson/document.h>
+
+#include "context.hpp"
+#include "jobs.hpp"
+#include "profiles.hpp"
+
+using namespace std;
+using namespace rapidjson;
+
+XBT_LOG_NEW_DEFAULT_CATEGORY(workload, "workload");
+
+void check_worload_validity(BatsimContext *context)
+{
+    // Let's check that every SEQUENCE-typed profile points to existing profiles
+    for (auto mit : context->profiles.profiles())
+    {
+        Profile * profile = mit.second;
+        if (profile->type == ProfileType::SEQUENCE)
+        {
+            SequenceProfileData * data = (SequenceProfileData *) profile->data;
+            for (const auto & prof : data->sequence)
+                xbt_assert(context->profiles.exists(prof), "Invalid composed profile '%s': the used profile '%s' does not exist", mit.first.c_str(), prof.c_str());
+        }
+    }
+
+    // TODO : check that there are no circular calls between composed profiles...
+    // TODO: compute the constraint of the profile number of resources, to check if it match the jobs that use it
+
+    // Let's check that the profile of each job exists
+    for (auto mit : context->jobs.jobs())
+    {
+        Job * job = mit.second;
+        xbt_assert(context->profiles.exists(job->profile), "Invalid job %d: the associated profile '%s' does not exist", job->id, job->profile.c_str());
+
+        Profile * profile = context->profiles[job->profile];
+        if (profile->type == ProfileType::MSG_PARALLEL)
+        {
+            MsgParallelProfileData * data = (MsgParallelProfileData *) profile->data;
+            xbt_assert(data->nb_res == job->required_nb_res, "Invalid job %d: the requested number of resources (%d) do NOT match"
+                       " the number of resources of the associated profile '%s' (%d)", job->id, job->required_nb_res, job->profile.c_str(), data->nb_res);
+        }
+        else if (profile->type == ProfileType::SEQUENCE)
+        {
+            // TODO: check if the number of resources matches a resource-constrained composed profile
+        }
+
+    }
+}
+
+void load_json_workload(BatsimContext *context, const std::string &filename)
+{
+    XBT_INFO("Loading JSON workload '%s'...", filename.c_str());
+    // Let the file content be placed in a string
+    ifstream ifile(filename);
+    xbt_assert(ifile.is_open(), "Cannot read file '%s'", filename.c_str());
+    string content;
+
+    ifile.seekg(0, ios::end);
+    content.reserve(ifile.tellg());
+    ifile.seekg(0, ios::beg);
+
+    content.assign((std::istreambuf_iterator<char>(ifile)),
+                std::istreambuf_iterator<char>());
+
+    // JSON document creation
+    Document doc;
+    doc.Parse(content.c_str());
+
+    xbt_assert(doc.IsObject());
+    context->jobs.load_from_json(doc, filename);
+    context->profiles.load_from_json(doc, filename);
+
+    XBT_INFO("JSON workload parsed sucessfully.");
+    XBT_INFO("Checking workload validity...");
+    check_worload_validity(context);
+    XBT_INFO("Workload seems to be valid.");
+}

workload.hpp

+#pragma once
+
+#include <string>
+
+struct BatsimContext;
+
+void check_worload_validity(BatsimContext * context);
+
+void load_json_workload(BatsimContext * context, const std::string & json_filename);