diff --git a/CMakeLists.txt b/CMakeLists.txt index ce928d1e63054d31244202fa13181ec43915609f..bb55e9bcd1ab7dddbaa3b28b43d89993a02364f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,6 @@ cmake_minimum_required(VERSION 3.0.2) project(pmtool) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}) add_definitions("--std=c++11") -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") include_directories("include") option(PROFILING "Include profiling information" OFF) @@ -15,6 +14,14 @@ if(PROFILING) SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg") endif() +option(DEBUG "Include debug information" ON) + +if(DEBUG) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") +else() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") +endif() + find_package(LibRec REQUIRED) include_directories(${LIBREC_INCLUDE_DIRS}) @@ -58,3 +65,5 @@ if(THREADS_FOUND) target_link_libraries(pmtool Threads::Threads) endif() +add_executable(instanceinfo instanceinfo.cpp) +target_link_libraries(instanceinfo core) diff --git a/FindCPLEX.cmake b/FindCPLEX.cmake index c5565878319520fab351ea9b9fe66ea16eb8c36b..56ead0f98ecaebbc294063b1e049364d1275acae 100644 --- a/FindCPLEX.cmake +++ b/FindCPLEX.cmake @@ -188,7 +188,7 @@ IF(CPLEX_FOUND) SET(CPLEX_INCLUDE_DIRS ${CPLEX_INCLUDE_DIR} ${CPLEX_CONCERT_INCLUDE_DIR}) SET(CPLEX_LIBRARIES ${CPLEX_CONCERT_LIBRARY} ${CPLEX_ILOCPLEX_LIBRARY} ${CPLEX_LIBRARY} ) IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") - SET(CPLEX_LIBRARIES "${CPLEX_LIBRARIES};m;pthread") + SET(CPLEX_LIBRARIES "${CPLEX_LIBRARIES};m;pthread;dl") ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") ENDIF(CPLEX_FOUND) diff --git a/FindLibRec.cmake b/FindLibRec.cmake index c648f80677cec74504f52cc27d240b0ed6d04ffb..f9df9e7c34bd6a4756b8cddac9da9adcaf664d96 100644 --- a/FindLibRec.cmake +++ b/FindLibRec.cmake @@ -1,18 +1,11 @@ # This module finds librec. # -# User can give LIBREC_ROOT_DIR as a hint stored in the cmake cache. +# User can give LIBREC_INSTALL_DIR as a hint stored in the cmake cache. # # It sets the following variables: # LIBREC_FOUND - Set to false, or undefined, if librec isn't found. # LIBREC_INCLUDE_DIRS - include directory # LIBREC_LIBRARIES - library files -# include(LibFindMacros) - -# Dependencies -# libfind_package(Magick++ Magick) - -# Use pkg-config to get hints about paths -# libfind_pkg_check_modules(librec_PKGCONF ImageMagick++) set(LIBREC_INSTALL_DIR "" CACHE PATH "Librec install directory") @@ -35,11 +28,10 @@ find_library(LIBREC_LIBRARY message(STATUS "librec library: ${LIBREC_LIBRARY}") - include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set LIBXML2_FOUND to TRUE +# handle the QUIETLY and REQUIRED arguments and set LIBREC_FOUND to TRUE # if all listed variables are TRUE -find_package_handle_standard_args(LIBREC DEFAULT_MSG +find_package_handle_standard_args(LibRec DEFAULT_MSG LIBREC_LIBRARY LIBREC_INCLUDE_DIR) mark_as_advanced(LIBREC_INCLUDE_DIR LIBREC_LIBRARY ) diff --git a/InputFormat.md b/InputFormat.md index 28cc04761d2e9dfce82aff02ded1db9f0b1018e2..e20eabbda4b44f71a7fe391850aecc5a2755dffa 100644 --- a/InputFormat.md +++ b/InputFormat.md @@ -1,69 +1,72 @@ # Input format specification `pmtool` accepts two different input formats for specifying instances: an in-house format, - and one based on `librec` which allows easy interaction with `StarPU`. Examples of each format - can be found in the `data/` directory: files with the `.pm` extension follow the in-house format, files with the `.rec` - extension follow the recfile format. +and one based on `librec` which allows easy interaction with `StarPU`. Examples of each format +can be found in the `data/` directory: files with the `.pm` extension follow the in-house format, files with the `.rec` +extension follow the recfile format. ## In-house format - In the default in-house format, all the information about an instance is specified in one file, - specified with several arrays. Each array starts with a `[`, ends with a `]`, and contains a comma-separated - list of values. +In the default in-house format, all the information about an instance +is specified in one file, specified with several arrays. Each array +starts with a `[`, ends with a `]`, and contains a comma-separated +list of values. - A file must contain, in this order, the following arrays: - + the number of ressource of each type. The length of this array specifies the number of resource - types, and its $`j`$-th coordinate indicates how many resources of type $`j`$ are available. - + one array per resource type, specifying the execution time of tasks on this resource type. - All these arrays should have the same length, equal to the number of task types. The $`i`$-th coordinate of the - $`j`$-th array specifies the execution time of task type $`i`$ on resource type $`j`$. - + one array specifies the type of each task. The length of this array is equal to the number $`n`$ of tasks in - the instance, and the $`k`$-th coordinate specifies the type of task $`k`$. Values in this array should be - between 0 and the length of the "execution time" arrays minus one. - + one array per task, specifying the precedence constraints. The $`k`$-th array indicates the indices (between $`1`$ and $`n`$) - of the predecessors of task $`k`$. - - Arrays do not have to be separated, however it is good practice in my opinion to keep one per line. - Task numbering can be arbitrary (and does not have to follow dependencies), `pmtool` automatically computes a - topological ordering of the graph anywhere it is needed, and thus checks that there is no cycle in the graph. +A file must contain, in this order, the following arrays: ++ the number of ressource of each type. The length of this array +specifies the number of resource types, and its $`j`$-th coordinate +indicates how many resources of type $`j`$ are available. ++ one array per resource type, specifying the execution time of tasks on this resource type. +All these arrays should have the same length, equal to the number of task types. The $`i`$-th coordinate of the +$`j`$-th array specifies the execution time of task type $`i`$ on resource type $`j`$. ++ one array specifies the type of each task. The length of this array is equal to the number $`n`$ of tasks in +the instance, and the $`k`$-th coordinate specifies the type of task $`k`$. Values in this array should be +between 0 and the length of the "execution time" arrays minus one. ++ one array per task, specifying the precedence constraints. The $`k`$-th array indicates the indices (between $`1`$ and $`n`$) +of the predecessors of task $`k`$. + +Arrays do not have to be separated, however it is good practice in my opinion to keep one per line. +Task numbering can be arbitrary (and does not have to follow dependencies), `pmtool` automatically computes a +topological ordering of the graph anywhere it is needed, and thus checks that there is no cycle in the graph. ## Recfile format - This format uses a more standard representation (the same that Debian uses in its package system, and that StarPU uses - as an output). It is more verbose, more extensible, and allows to differentiate between the graph and the - platform description. `pmtool` automatically uses this format if the file name ends with `.rec`. +This format uses a more standard representation (the same that Debian uses in its package system, and that StarPU uses +as an output). It is more verbose, more extensible, and allows to differentiate between the graph and the +platform description. `pmtool` automatically uses this format if the file name ends with `.rec`. - In the recfile format, information is specified as *records*, which contain one *key/value* pair per line. The precise - format is `Key: Value`, where the value may be anything (and may contain spaces). The key and value are separated by `: `. - Records are separated by double newlines. +In the recfile format, information is specified as *records*, which contain one *key/value* pair per line. The precise +format is `Key: Value`, where the value may be anything (and may contain spaces). The key and value are separated by `: `. +Records are separated by double newlines. ### The instance file - The format of the instance file is designed as to accept the `tasks.rec` files as provided by StarPU's export tools. - The instance file contains one record per task, with the following keys: - + `Name` and `Footprint` specify the task type. All tasks with identical names and footprints belong to the same task type. - + `JobId` is an integer identifier of the task - + `DependsOn` contains a space-separated list of the `JobId`s of the predecessor of the current task. - + `EstimatedTime` contains a space-separated list of the execution time of this task on each resource (if there are - several resources of the same type, the corresponding value is repeated as many times as needed). The number of values - in this field should be the same for all tasks. - - The fields `Name`, `Footprint` and `JobId` are compulsory. The field `EstimatedTime` is compulsory if no platform file - is provided. The field `DependsOn` is optional (defaults to no dependencies). Other optional fields exist: - - + `SubmitOrder` provides another, more robust identifier from StarPU, which allows to identify tasks from one StarPU run - to the next. It is used as an identifier of tasks in exported schedules if the `--subimt-order` option is specified - on the command line. It should be an integer. - + `Tag` is a custom information provided by the application programmer in StarPU, providing another way of identifying tasks - in a user-defined manner. It is appended to the task identifier (either job id or submit order) if the - `--use-tags` option is specified on the command line. - + `WorkerId`, `StartTime` and `EndTime` allow to specify a schedule, providing a convienient way to compare - the actual schedule from StarPU to schedules computed with `pmtool`'s algorithms. If these fields are specified for - all tasks, then a shared data is added to the instance with the key `rl`; the schedule can then be recovered - by adding the `rep` (reproduce) algorithm to the command line, with the option `key=rl`, in the following way: - ``` +The format of the instance file is designed as to accept the `tasks.rec` files as provided by StarPU's export tools. +The instance file contains one record per task, with the following keys: + ++ `Name` and `Footprint` specify the task type. All tasks with identical names and footprints belong to the same task type. ++ `JobId` is an integer identifier of the task ++ `DependsOn` contains a space-separated list of the `JobId`s of the predecessor of the current task. ++ `EstimatedTime` contains a space-separated list of the execution time of this task on each resource (if there are + several resources of the same type, the corresponding value is repeated as many times as needed). The number of values + in this field should be the same for all tasks. + +The fields `Name`, `Footprint` and `JobId` are compulsory. The field `EstimatedTime` is compulsory if no platform file +is provided. The field `DependsOn` is optional (defaults to no dependencies). Other optional fields exist: + ++ `SubmitOrder` provides another, more robust identifier from StarPU, which allows to identify tasks from one StarPU run +to the next. It is used as an identifier of tasks in exported schedules if the `--subimt-order` option is specified +on the command line. It should be an integer. ++ `Tag` is a custom information provided by the application programmer in StarPU, providing another way of identifying tasks +in a user-defined manner. It is appended to the task identifier (either job id or submit order) if the +`--use-tags` option is specified on the command line. ++ `WorkerId`, `StartTime` and `EndTime` allow to specify a schedule, providing a convienient way to compare +the actual schedule from StarPU to schedules computed with `pmtool`'s algorithms. If these fields are specified for +all tasks, then a shared data is added to the instance with the key `rl`; the schedule can then be recovered +by adding the `rep` (reproduce) algorithm to the command line, with the option `key=rl`, in the following way: +``` pmtool tasks.rec -a rep:key=rl ``` + `Handles`, `Modes` and `Sizes` allow to specify which data this task @@ -85,29 +88,31 @@ pmtool tasks.rec -a rep:key=rl wrote to this handle. Tasks which write to a handle may change its size. - Because of internal behavior of StarPU, this format allows *virtual* tasks to be added to the instance. A task is virtual - if it has no `Name` field. `JobId` is still compulsory for virtual tasks. Dependencies from real tasks go *through* virtual tasks - to make them depend on any real task that this virtual task depends on. +Because of internal behavior of StarPU, this format allows *virtual* tasks to be added to the instance. A task is virtual +if it has no `Name` field. `JobId` is still compulsory for virtual tasks. Dependencies from real tasks go *through* virtual tasks +to make them depend on any real task that this virtual task depends on. ### The platform file - To avoid specifying `EstimatedTime` for all tasks, it is possible to specify the platform file separately. In StarPU, - this file is provided by the `starpu_perfmodel_recdump` utility in the `tools/` directory. This file contains two parts: - first the number of resources of each type, then the timings of each type of task. +To avoid specifying `EstimatedTime` for all tasks, it is possible to specify the platform file separately. In StarPU, +this file is provided by the `starpu_perfmodel_recdump` utility in the `tools/` directory. This file contains two parts: +first the number of resources of each type, then the timings of each type of task. - The first part starts with ```%rec: worker_count``` on a separate line (this is a special feature of the recfile format to separate a file - into different databases). It then contains one record per resource type, with two fields: - + `Architecture` provides an identifier for this resource type - + `NbWorkers` should contain an integer specifying the number of resources of this type +The first part starts with ```%rec: worker_count``` on a separate line (this is a special feature of the recfile format to separate a file +into different databases). It then contains one record per resource type, with two fields: + ++ `Architecture` provides an identifier for this resource type ++ `NbWorkers` should contain an integer specifying the number of resources of this type - The second part starts with ```%rec: timing``` on a separate line, and contains one record for each task type/resource type combination. - Records contain the following fields: - + `Name` and `Footprint` represent the task type, similarly to the instance file. - + `Architecture` represent the resource type, as specified in the first part. - + `Mean` contain the execution time (which is computed as an average by StarPU). - + Files produced by StarPU also contain an `Stddev` field, which contains the standard deviation - of the measurements, but this field is ignored by `pmtool`. +The second part starts with ```%rec: timing``` on a separate line, and contains one record for each task type/resource type combination. +Records contain the following fields: + ++ `Name` and `Footprint` represent the task type, similarly to the instance file. ++ `Architecture` represent the resource type, as specified in the first part. ++ `Mean` contain the execution time (which is computed as an average by StarPU). ++ Files produced by StarPU also contain an `Stddev` field, which contains the standard deviation +of the measurements, but this field is ignored by `pmtool`. - \ No newline at end of file + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..01679d6a6fb05127a1a6155f5abd8f74615bfb8a --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + pmtool + Copyright (C) 2019 EYRAUD-DUBOIS Lionel + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + pmtool Copyright (C) 2019 EYRAUD-DUBOIS Lionel + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/ProgramOptions.cpp b/ProgramOptions.cpp index 86629fb06666848a3b51977c8cc94ffa72d99ac8..3b2e6353d8996b2e16d396095b061b03126056e6 100644 --- a/ProgramOptions.cpp +++ b/ProgramOptions.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include "algorithm.h" #include "listAlgorithm.h" @@ -20,6 +22,7 @@ #ifdef WITH_CPLEX +#include "CriticalPath.h" #include "AreaBound.h" #include "DepBound.h" #include "IterDepBound.h" @@ -39,6 +42,7 @@ static const int opt_bw = 15; static const int opt_thread = 16; static const int opt_tags = 17; static const int opt_submit = 18; +static const int opt_export_type = 19; static struct option long_options[] = { @@ -62,6 +66,7 @@ static struct option long_options[] = { {"threads", optional_argument, 0, opt_thread}, {"use-tags", no_argument, 0, opt_tags}, {"submit-order", no_argument, 0, opt_submit}, + {"export-type", no_argument, 0, opt_export_type}, {0, 0, 0, 0 } }; @@ -180,6 +185,9 @@ void ProgramOptions::parse(int argc, char** argv) { case opt_submit: useSubmitOrder = true; break; + case opt_export_type: + outputTypeInExport = true; + break; case '?': case 'h': usage(); @@ -248,6 +256,7 @@ void ProgramOptions::displayBoundList() { cerr << "Bounds available:" << endl; #ifdef WITH_CPLEX cerr << " area \t\t area bound, hybridized" << endl; + cerr << " cp \t\t critical path" << endl; cerr << " dep \t\t global area bound with dependencies, hybridized" << endl; cerr << " iterdep \t global area bound with dependencies, hybridized + iteratively adding local area bounds" << endl; cerr << " mixed \t compute time to end for all tasks, and area bounds for the beginning of the graph" << endl; @@ -273,6 +282,8 @@ Bound* createBound(const string& name, const AlgOptions& options) { bound = new HybridBound(new AreaStart(options), options); if(name == "interval") bound = new IntervalBound(options); + if(name == "cp") + bound = new CriticalPath(options); #endif if(bound == NULL){ cerr << "Unknown bound " << name <<". For a list of bounds, use --help" << endl; @@ -307,6 +318,10 @@ Algorithm* createAlg(const string& name, const AlgOptions& options) { alg = new OnlineECT(options); if(name == "erls") alg = new OnlineERLS(options); + if(name == "lg") + alg = new OnlineLG(options); + if(name == "mg") + alg = new OnlineMG(options); #ifdef WITH_CPLEX if(name == "lp") alg = new SchedLPIndep(options); diff --git a/README.md b/README.md index d7355faaa685c373dea0543e5c474f4862b59327..b7a2a58a7a0dc6baa28a08bc87fc5790b67f9eb6 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ compile with `make`. All bounds require CPLEX, which can be specified with the `-DCPLEX_ROOT_DIR=<...>` option to `cmake`. -Depends on `librec-dev`, and CPLEX depends on libgpg-error-dev and +Depends on `librec-dev` ([See here](https://www.gnu.org/software/recutils/)), and CPLEX depends on libgpg-error-dev and libgcrypt-dev. ## Table of contents @@ -292,11 +292,17 @@ resources. All of these only work with two types of resources. dual approximation, dynamic-programming based algorithm. Based on the simpler algorithm from [Scheduling Independent Tasks on Multi-cores with GPU Accelerators](https://hal.inria.fr/hal-00921357), described in Section 5.2. Also described in [Scheduling Independent Tasks on Multi-cores with GPU Accelerators](https://hal.inria.fr/hal-01081625), - Section 5.2. + Section 5.2. Additional option: `disc` selects the discretization precision (default 3.0) + `dualhp` dual approximation, heteroprio-based greedy algorithm. Inspired from [Scheduling Data Flow Program in XKaapi: A New Affinity Based Algorithm for Heterogeneous Architectures](https://hal.inria.fr/hal-01081629v1), with only the second part of the schedule. + + `dp3demi` + dual approximation, dynamic programming based algorithm. Based + on APPROX-3/2 from + [Scheduling Independent Moldable Tasks on Multi-Cores with GPUs](https://hal.inria.fr/hal-01516752), + but restricted to the non moldable case. Should also appear as a + more generic (2q+1)/(2q) approximation in IJFCS. + `accel` Accel algorithm from [Scheduling Independent Tasks on Multi-cores with GPU Accelerators](https://hal.inria.fr/hal-01081625), Section 4. @@ -324,6 +330,9 @@ resources. All of these only work with two types of resources. Implements the CLB2C strategy (see [Considerations on distributed load balancing for fully heterogeneous machines: Two particular cases.](https://doi.org/10.1109/IPDPSW.2015.36)). Not exactly equivalent to this strategy, since `indep` performs list scheduling based only on the assignment to different types of resources. + + `minmin` + Implements the MinMin strategy (see [A Comparison of Eleven Static Heuristics for Mapping a Class of Independent Tasks onto Heterogeneous Distributed Computing Systems](https://doi.org/10.1006/jpdc.2000.1714).) + + `rank` (except for style `strict`) as for the previous algorithms + `dosort` (except for `strict`, default `yes`) @@ -437,7 +446,10 @@ Reminder: all bounds require to compile with CPLEX. Simple area bound, with one variable per type of tasks and type of resource. Very quick. -* `dep`: mixed area-dependency bound +* `cp` + Simple critical path bound. Very quick. + +* `dep`: mixed area-dependency bound. Always better than `max(area, cp)`, but slower Options: + `mode`: `normal` (default) or `concurrent` if concurrent, uses Cplex's concurrent solving mode @@ -512,6 +524,10 @@ often: * `submit-order` Use the `SubmitOrder` field in `instance.rec` input files instead of `JobId`. +* `export-type` + When exporting in `.rec` format with the `export=` option of algorithms, specify + all workers of this type instead of the particular worker. + * `--bw ` Specify the bandwidth used for communications. Unit is not specified, since it depends on the unit used to specify data sizes diff --git a/bounds/CMakeLists.txt b/bounds/CMakeLists.txt index 538a9aa67f4858bd514f0f2b27856e64ee1ae6f3..70e3ca619648f4b734ee712c4aa6e464d893122e 100644 --- a/bounds/CMakeLists.txt +++ b/bounds/CMakeLists.txt @@ -5,6 +5,7 @@ set(BOUNDS_SRC HybridBound.cpp IntervalBound.cpp IterDepBound.cpp + CriticalPath.cpp ) add_library(bounds ${BOUNDS_SRC}) diff --git a/bounds/CriticalPath.cpp b/bounds/CriticalPath.cpp new file mode 100644 index 0000000000000000000000000000000000000000..937c72e4fcc84909f8bae408e439d0e622801c74 --- /dev/null +++ b/bounds/CriticalPath.cpp @@ -0,0 +1,15 @@ +#include +#include "algorithm.h" +#include "CriticalPath.h" +#include "util.h" + +using namespace std; + +CriticalPath::CriticalPath(const AlgOptions & options) { + +} + +double CriticalPath::compute(Instance& ins) { + vector ranks = ins.computeMinRank(); + return getMax(ranks); +} diff --git a/core/algoptions.cpp b/core/algoptions.cpp index 86fe2c016c3b0c203a6918ce4b75bdecccb050b1..bc38d6eff2b73454917f0a7070a19332dc95cfd5 100644 --- a/core/algoptions.cpp +++ b/core/algoptions.cpp @@ -115,6 +115,21 @@ double AlgOptions::asDouble(const string& key, const double def) const { return (stod(opt->second)); } + +void AlgOptions::updateValue(double & v, const std::string &key) const { + if(isPresent(key)) + v = asDouble(key); +} +void AlgOptions::updateValue(int & v, const std::string &key) const { + if(isPresent(key)) + v = asInt(key); +} +void AlgOptions::updateValue(string & v, const std::string &key) const { + if(isPresent(key)) + v = asString(key); +} + + AlgOptions::AlgOptions(string toParse) { parse(toParse); } diff --git a/core/instance.cpp b/core/instance.cpp index 313f31ff5a8a48592b04aeefb3b90d7cd01ccfd1..abbadfcdbd28cf7b05ddabfaac6bfac848f9749c 100644 --- a/core/instance.cpp +++ b/core/instance.cpp @@ -332,9 +332,12 @@ vector Instance::computeRanks(vector wbar, int to, int from) { int i = topOrder[j]; // Compute max double m = -std::numeric_limits::infinity(); - for(int k = 0; k < (int) revDep[i].size(); k++) + if(revDep[i].empty()) + m = 0; + else + for(int k = 0; k < (int) revDep[i].size(); k++) if(rank[revDep[i][k]] > m) - m = rank[revDep[i][k]]; + m = rank[revDep[i][k]]; rank[i] = wbar[i] + m; if (topOrder[j] == from) diff --git a/core/schedAction.cpp b/core/schedAction.cpp index 60dfcaa5b8a85bc37467327852db8b5df677abc8..5c31a7ac34b0b3d4afdd8e171716bbba25c5a9dd 100644 --- a/core/schedAction.cpp +++ b/core/schedAction.cpp @@ -105,8 +105,8 @@ string ExportToString::getResult() { /* ExportAlloc: exports in .rec format */ -ExportAlloc::ExportAlloc(string filename, Instance* ins, bool submitOrder) - : output(filename), instance(ins), submitOrder(submitOrder) { +ExportAlloc::ExportAlloc(string filename, Instance* ins, bool submitOrder, bool outputType) + : output(filename), instance(ins), submitOrder(submitOrder), outputType(outputType) { } void ExportAlloc::onSchedule(int i, int w, double s, double f) { @@ -128,11 +128,16 @@ void ExportAlloc::onSchedule(int i, int w, double s, double f) { int type = instance->getType(w); - output << "Workers: "; - for(auto& i: instance->workerIDs[type]) - output << i << " "; - output << endl; + if (outputType && instance->workerIDs[type].size() > 1) { + output << "Workers:"; + for(auto& i: instance->workerIDs[type]) + output << " " << i; + output << endl; + } else { + output << "SpecificWorker: " << w << endl; + } + if(instance->workerNames.size() > 0) { output << "Architecture: " << instance->workerNames[type] << endl; } else { diff --git a/core/util.cpp b/core/util.cpp index b1de670fda8748fab42fe567a5dc2ae75dbc26d2..75ec7a8a728fbb8eb1fccc3320a7c46fc4404383 100644 --- a/core/util.cpp +++ b/core/util.cpp @@ -1,5 +1,5 @@ #include "util.h" -#include "cmath" +#include #include using namespace std; diff --git a/include/CriticalPath.h b/include/CriticalPath.h new file mode 100644 index 0000000000000000000000000000000000000000..9318e216e0189fe0d2b2f985385cc9ad92a5a813 --- /dev/null +++ b/include/CriticalPath.h @@ -0,0 +1,17 @@ +#ifndef CRITICALPATH_H +#define CRITICALPATH_H + +#include "algorithm.h" + +class CriticalPath : public Bound { + + + public: + CriticalPath(const AlgOptions & options); + + double compute(Instance& ins); + std::string name() { return "cp"; }; + +}; + +#endif diff --git a/include/Dmdas.h b/include/Dmdas.h index 7829967b6e9a6174d9c514ac53a5fe5bf7b33223..1aace8c78078f118847f9e0ff6054c2807f317c5 100644 --- a/include/Dmdas.h +++ b/include/Dmdas.h @@ -4,7 +4,6 @@ #include "instance.h" #include "algorithm.h" #include "algoptions.h" -#include "listAlgorithm.h" class Dmdas : public Algorithm { diff --git a/include/IndepDP2.h b/include/IndepDP2.h index dfa05f1c321037fb8ea3d7d2573c1bf6611fe914..96d026cf374f24f30b9402d8eb488124d81429f5 100644 --- a/include/IndepDP2.h +++ b/include/IndepDP2.h @@ -1,23 +1,23 @@ #ifndef INDEPDP2_H #define INDEPDP2_H -#include "IndepAllocator.h" +#include "IndepDualGeneric.h" #include "instance.h" #include -extern double lowerBoundTwoResource(Instance& ins, std::vector taskSet, - double CPUload = 0, double GPUload = 0); - -class IndepDP2 : public IndepAllocator { +class IndepDP2 : public IndepDualGeneric { protected: - double tryGuess(Instance &, std::vector taskSet, double maxGPUload, double maxlen, - IndepResult & result, bool getResult); - double epsilon = 0.01; + double tryGuess(Instance &, std::vector taskSet, std::vector& loads, + double maxlen, IndepResult & result, bool getResult); + double discretizationConstant = 3.0; +#ifdef WITH_CPLEX + bool solveWithCplex; + bool cplexUseDiscretizedValues; +#endif public: IndepDP2(const AlgOptions& opt); - IndepResult compute(Instance &, std::vector &taskSet, std::vector &loads); }; diff --git a/include/IndepDP3Demi.h b/include/IndepDP3Demi.h new file mode 100644 index 0000000000000000000000000000000000000000..263421070f8aa636468a871b84ab5d556f7eba32 --- /dev/null +++ b/include/IndepDP3Demi.h @@ -0,0 +1,27 @@ +#ifndef INDEPDP3DEMI_H +#define INDEPDP3DEMI_H + +#include "IndepDualGeneric.h" +#include "instance.h" +#include + +class IndepDP3Demi : public IndepDualGeneric { + + protected: + double tryGuess(Instance &, std::vector taskSet, std::vector& loads, + double target, IndepResult & result, bool getResult); + double discretizationConstant = 3.0; +#ifdef WITH_CPLEX + bool solveWithCplex; + bool cplexUseDiscretizedValues; +#endif + + public: + IndepDP3Demi(const AlgOptions& opt); + +}; + + + + +#endif diff --git a/include/IndepDualGeneric.h b/include/IndepDualGeneric.h new file mode 100644 index 0000000000000000000000000000000000000000..3e683bb5513f50c9ecc3c879d739aa5a2d10c7a1 --- /dev/null +++ b/include/IndepDualGeneric.h @@ -0,0 +1,27 @@ +#ifndef INDEPDUALGENERIC_H +#define INDEPDUALGENERIC_H + +#include "IndepAllocator.h" +#include "instance.h" +#include + +extern double lowerBoundTwoResource(Instance& ins, std::vector taskSet, + double CPUload = 0, double GPUload = 0); + +class IndepDualGeneric : public IndepAllocator { + + protected: + virtual double tryGuess(Instance &, std::vector taskSet, std::vector& loads, + double maxlen, IndepResult & result, bool getResult) = 0; + double epsilon = 0.01; + + public: + IndepDualGeneric(const AlgOptions& opt); + IndepResult compute(Instance &, std::vector &taskSet, std::vector &loads); + +}; + + + + +#endif diff --git a/include/IndepMinMin.h b/include/IndepMinMin.h new file mode 100644 index 0000000000000000000000000000000000000000..41febe4c84d42f137722bcf864b356864cedd59f --- /dev/null +++ b/include/IndepMinMin.h @@ -0,0 +1,30 @@ +// +// Created by eyraud on 25/03/19. +// Based on A Comparison of Eleven Static Heuristics for +// Mapping a Class of Independent Tasks onto +// Heterogeneous Distributed Computing Systems +// doi:10.1006/jpdc.2000.1714 + + +#ifndef PMTOOL_MINMIN_H +#define PMTOOL_MINMIN_H + + +#include "IndepAllocator.h" + +class IndepMinMin : public IndepAllocator { +private: + + Instance* ins; + std::vector workerIndices; + std::vector bestWorkers; /* length = nb worker types */ + + int getEarliestWorker(std::vector &loads, int type); + inline double endTime(std::vector &loads, int workerType, int task); +public: + IndepResult compute(Instance &ins, std::vector &taskSet, std::vector &loads) override; + IndepMinMin(const AlgOptions &options); +}; + + +#endif //PMTOOL_MINMIN_H diff --git a/include/OnlineLG.h b/include/OnlineLG.h new file mode 100644 index 0000000000000000000000000000000000000000..004df44815fc39bc1752eb0e497cc371d0878b4d --- /dev/null +++ b/include/OnlineLG.h @@ -0,0 +1,28 @@ +// +// Created by eyraud on 11/12/17. +// + +#ifndef PMTOOL_ONLINELG_H +#define PMTOOL_ONLINELG_H + +#include "algorithm.h" +#include "OnlineGeneric.h" + +/* Online implementation of the algorithm from + * "Scheduling Problems on Two Sets of Identical Machines" + + Only works with two types of ressources + + */ + +class OnlineLG : public OnlineGeneric { + +public: + explicit OnlineLG(const AlgOptions &options); + + double compute(Instance& ins, SchedAction* action) override; + int assignTask(int task, double now) override; +}; + + +#endif //PMTOOL_ONLINEZHANG_H diff --git a/include/OnlineMG.h b/include/OnlineMG.h new file mode 100644 index 0000000000000000000000000000000000000000..9c48b166eee41dad9b6b0daba660ad07292d935c --- /dev/null +++ b/include/OnlineMG.h @@ -0,0 +1,37 @@ +// +// Created by eyraud on 11/12/17. +// + +#ifndef PMTOOL_ONLINEMG_H +#define PMTOOL_ONLINEMG_H + +#include "algorithm.h" +#include "OnlineGeneric.h" + +/* Online implementation of the algorithm from + * "Scheduling Problems on Two Sets of Identical Machines" + + Only works with two types of ressources + + */ + +class OnlineMG : public OnlineGeneric { + + protected: + double maxRTime; + double sumRTime; + int largestGroup; + int smallestGroup; + int m1, m2; + double alpha = 1.00; + double gamma = 1.00; + +public: + explicit OnlineMG(const AlgOptions &options); + + double compute(Instance& ins, SchedAction* action) override; + int assignTask(int task, double now) override; +}; + + +#endif //PMTOOL_ONLINEMG_H diff --git a/include/OnlineZhang.h b/include/OnlineZhang.h index 41ab2680fc6d8ea2563caffcf24f23b2039a238a..6413c35d2675363a1c49e62a73af85a20a69bcd5 100644 --- a/include/OnlineZhang.h +++ b/include/OnlineZhang.h @@ -6,8 +6,7 @@ #define PMTOOL_ONLINEZHANG_H #include "algorithm.h" -#include "GreedyAlgorithm.h" -#include "GreedyPerType.h" +#include "OnlineGeneric.h" /* Online implementation of the algorithm from * "ONLINE SCHEDULING OF MIXED CPU-GPU JOBS" @@ -18,14 +17,14 @@ Does not change an allocation decision once taken, every choice is made on task push */ -class OnlineZhang : public GreedyPerType { +class OnlineZhang : public OnlineGeneric { // Options double lambda = 1.69; double beta = 0.80; double theta = 1.04; double phi = 0.64; - void updateValue(double&v, const std::string &key, const AlgOptions& opt); + //void updateValue(double&v, const std::string &key, const AlgOptions& opt); // Runtime values int largestGroup; @@ -39,11 +38,7 @@ public: explicit OnlineZhang(const AlgOptions &options); double compute(Instance& ins, SchedAction* action) override; - int chooseTask(int worker, double now) override; - void onTaskPush(int task, double now) override; - - - + int assignTask(int task, double now) override; }; diff --git a/include/ProgramOptions.h b/include/ProgramOptions.h index cf2395bbe069d99f9faca41779cb026fec2d8f85..c65bf793c1542d4ad0a0afa78a3ed264017ff813 100644 --- a/include/ProgramOptions.h +++ b/include/ProgramOptions.h @@ -43,8 +43,9 @@ class ProgramOptions { double platform_bw = 1; std::string outputBestFile; int nbThreads; - bool appendTags; - bool useSubmitOrder; + bool appendTags; + bool useSubmitOrder; + bool outputTypeInExport; std::set optionKeys; std::vector algs; diff --git a/include/TrueHeteroPrio.h b/include/TrueHeteroPrio.h index 0d5d210e6344e9dd01b5c014446aa5a43d178c88..38a3252a919ce1e3da067d7889efec69b96d21ea 100644 --- a/include/TrueHeteroPrio.h +++ b/include/TrueHeteroPrio.h @@ -7,14 +7,19 @@ #include "GreedyAlgorithm.h" +typedef enum {LATEST, AF, PRIO} StealType; class TrueHeteroPrio : public GreedyAlgorithm { private: - RankComputer ranker; std::set *taskSet; Instance* ins; + std::vector priorities; + StealType stealType = LATEST; + std::string rankString = "min"; + // bool stealIfNonEmpty = false; + bool isBetterSteal(int taskA, double endA, int taskB, double endB, int wType); -public: + public: double compute(Instance &ins, SchedAction *action) override; TrueHeteroPrio(const AlgOptions &options); diff --git a/include/algoptions.h b/include/algoptions.h index 341e43e97e11827f224b2755f20789b8f51ce0ed..ff5b13f98e6a44c344a49a53426608bd1674cf92 100644 --- a/include/algoptions.h +++ b/include/algoptions.h @@ -15,6 +15,11 @@ class AlgOptions : public std::map { std::string asString(const std::string &key, const std::string& def = "") const; int asInt(const std::string &key, const int def = 0) const ; double asDouble(const std::string &key, const double def = 0.0) const; + void updateValue(double & v, const std::string &key) const; + void updateValue(int & v, const std::string &key) const; + void updateValue(std::string & v, const std::string &key) const; + + std::string parseWithName(const std::string &line); std::vector multiParseWithName(const std::string & line, std::string & name); diff --git a/include/availSequence.h b/include/availSequence.h index 40bacecc15654a9096b3772536ae2222405eb47c..43d338e6acf8105499f32732c26d661faaa7fd70 100644 --- a/include/availSequence.h +++ b/include/availSequence.h @@ -1,7 +1,7 @@ #ifndef AVAILSEQUENCE_H #define AVAILSEQUENCE_H -#include +#include #include class Period { @@ -11,7 +11,7 @@ class Period { Period() {} Period(double s, double e) : start(s), end(e) {} }; -typedef std::list timeSeq; +typedef std::vector timeSeq; class AvailSequence { diff --git a/include/schedAction.h b/include/schedAction.h index ce5f04464a1e2b9576089046df9299dc8c2c61a6..263167793e5f4e9dd7c2dd7ed9b3252f2417553d 100644 --- a/include/schedAction.h +++ b/include/schedAction.h @@ -58,8 +58,9 @@ class ExportAlloc : public SchedAction { std::ofstream output; Instance* instance; bool submitOrder; + bool outputType; public: - ExportAlloc(std::string filename, Instance* ins, bool submitOrder = false); + ExportAlloc(std::string filename, Instance* ins, bool submitOrder = false, bool outputType = false); void onSchedule(int i, int w, double s, double f); ~ExportAlloc(); }; diff --git a/instanceinfo.cpp b/instanceinfo.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bb0bcb1236d8d630c1e5809e5cecfc7224b5240d --- /dev/null +++ b/instanceinfo.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include + +#include "instance.h" +#include "util.h" + +using namespace std; + + +static const int opt_no_convert = 10; +static const int opt_no_header = 12; + +static struct option long_options[] = { + {"no-convert", no_argument, 0, opt_no_convert}, + {"no-header", no_argument, 0, opt_no_header}, + {0, 0, 0, 0 } +}; + +static const char* optstring = ""; + +int main(int argc, char* argv[]) { + + + bool convertIndices = true; + bool printHeader = true; + vector inputFiles; + + int longindex = 0; + int opt = getopt_long(argc, argv, optstring, long_options, & longindex); + while(opt != -1) { + switch(opt) { + case opt_no_convert: + convertIndices = false; + break; + case opt_no_header: + printHeader = false; + break; + default: + break; + } + opt = getopt_long(argc, argv, optstring, long_options, & longindex); + } + for(int a = optind; a < argc; ++a) + inputFiles.push_back(string(argv[a])); + + if(printHeader) { + cout << "input nbTasks depth" << endl; + } + + for(string &file: inputFiles) { + Instance* instance = new Instance(file, convertIndices); + int nbTasks = instance->nbTasks; + int depth = getMax(instance->computeUnitRank()); + cout << file << " " << nbTasks << " " << depth << endl; + delete instance; + } + +} diff --git a/pmtool.cpp b/pmtool.cpp index 3894ebc88046115b512ceb32960c02bb89b0c6d2..2e47a0050bb3cae4d70e5ce2bae225bade84d50d 100644 --- a/pmtool.cpp +++ b/pmtool.cpp @@ -52,7 +52,7 @@ void computeAlg(ProgramOptions& progOpt, Instance* ins, ExportAlloc* exportAlloc = NULL; if(opts.isPresent("export")) { - exportAlloc = new ExportAlloc(opts.asString("export"), ins, progOpt.useSubmitOrder); + exportAlloc = new ExportAlloc(opts.asString("export"), ins, progOpt.useSubmitOrder, progOpt.outputTypeInExport); seq.add(exportAlloc); } @@ -66,8 +66,8 @@ void computeAlg(ProgramOptions& progOpt, Instance* ins, try { result->makespan = alg->compute(*ins, &seq); - auto time = chrono::duration_cast(chrono::steady_clock::now() - start); - result->milliseconds = time.count(); + auto time = chrono::duration_cast(chrono::steady_clock::now() - start); + result->milliseconds = time.count()/ 1000.0; //cout << input_file << " " << name << " " << result << " " << time.count(); @@ -83,9 +83,9 @@ void computeAlg(ProgramOptions& progOpt, Instance* ins, } catch (int e) { - auto time = chrono::duration_cast(chrono::steady_clock::now() - start); + auto time = chrono::duration_cast(chrono::steady_clock::now() - start); result->makespan = -1; - result->milliseconds = time.count(); + result->milliseconds = time.count() / 1000.0; } if(localExport) free(localExport); @@ -271,9 +271,9 @@ int main(int argc, char** argv) { if(progOpt.mergeTolerance > 0) { instance->autoMerge(progOpt.mergeTolerance); } - auto time = chrono::duration_cast(chrono::steady_clock::now() - start); + auto time = chrono::duration_cast(chrono::steady_clock::now() - start); if(progOpt.verbosity >= 4) - cerr << " done (" << time.count() << " ms)." << endl; + cerr << " done (" << time.count()/1000.0 << " ms)." << endl; instance->display(progOpt.verbosity); double bestBoundValue = 0; @@ -291,15 +291,15 @@ int main(int argc, char** argv) { bestBoundValue = result; bestBound = &(*it); } - auto time = chrono::duration_cast(chrono::steady_clock::now() - start); + auto time = chrono::duration_cast(chrono::steady_clock::now() - start); // auto duration = cout << input_file << " " << (instance->platformFile != "" ? instance->platformFile: "NA"); - cout << " True " << name << " " << result << " " << time.count(); + cout << " True " << name << " " << result << " " << time.count() / 1000.0; } catch (int e) { - auto time = chrono::duration_cast(chrono::steady_clock::now() - start); + auto time = chrono::duration_cast(chrono::steady_clock::now() - start); cout << input_file << " " << (instance->platformFile != "" ? instance->platformFile: "NA"); cout << " True " << name << " " << "NA" << " " << - time.count(); + time.count() / 1000.0; } if(progOpt.repartitionFile != "") { diff --git a/schedulers/CMakeLists.txt b/schedulers/CMakeLists.txt index 2f9b07a8e9d598abd27b1a3e6cf26e809ec73305..7495c69e154333bb1031b69b0a54e773357c3716 100644 --- a/schedulers/CMakeLists.txt +++ b/schedulers/CMakeLists.txt @@ -4,7 +4,9 @@ set(SCHED_SRC PreAllocatedGreedy.cpp HeteroPrio.cpp IndepBalanced.cpp + IndepDualGeneric.cpp ../include/IndepDualGeneric.h IndepDP2.cpp + IndepDP3Demi.cpp IndepImreh.cpp IndepAccel.cpp IndepBased.cpp @@ -15,7 +17,18 @@ set(SCHED_SRC availSequence.cpp GreedyAlgorithm.cpp HeftAlgorithm.cpp - OnlineZhang.cpp ../include/OnlineZhang.h GreedyPerType.cpp ../include/GreedyPerType.h TrueHeteroPrio.cpp ../include/TrueHeteroPrio.h OnlineGeneric.cpp ../include/OnlineGeneric.h OnlineQA.cpp ../include/OnlineQA.h OnlineECT.cpp ../include/OnlineECT.h OnlineERLS.cpp ../include/OnlineERLS.h IndepCLB2C.cpp ../include/IndepCLB2C.h) + OnlineZhang.cpp ../include/OnlineZhang.h + GreedyPerType.cpp ../include/GreedyPerType.h + TrueHeteroPrio.cpp ../include/TrueHeteroPrio.h + OnlineGeneric.cpp ../include/OnlineGeneric.h + OnlineQA.cpp ../include/OnlineQA.h + OnlineECT.cpp ../include/OnlineECT.h + OnlineERLS.cpp ../include/OnlineERLS.h + OnlineLG.cpp ../include/OnlineLG.h + OnlineMG.cpp ../include/OnlineMG.h + IndepCLB2C.cpp ../include/IndepCLB2C.h + IndepMinMin.cpp ../include/IndepMinMin.h + ) if(CPLEX_FOUND) set(SCHED_SRC ${SCHED_SRC} SchedLPIndep.cpp ../include/SchedLPIndep.h AreaRound.cpp ../include/AreaRound.h) diff --git a/schedulers/Dmdas.cpp b/schedulers/Dmdas.cpp index 1c15d0a95406c9e1f134572df3065c87b0cd603d..0f088105ab58c1079ba977889ec44067a539fb59 100644 --- a/schedulers/Dmdas.cpp +++ b/schedulers/Dmdas.cpp @@ -1,5 +1,4 @@ #include "Dmdas.h" -#include "listAlgorithm.h" #include "util.h" #include @@ -81,24 +80,32 @@ double Dmdas::compute(Instance &ins, SchedAction* action) { // For now keep the "old" version of dmdas where the assigned load also // counts lower priority tasks - auto getFinishedTime = [&] (int task, int w) { - if(ins.isValidWorker(w, task)) { + auto getFinishedTime = [&] (int task, int w, int wType) { + double execTime = ins.execType(wType, task); + if(ins.isValidValue(execTime)) { double s = max(endtimesWorkers[w], currentTime); return (s + assignedLoad[w] - + ins.execWorker(w, task)); + + execTime); } else return std::numeric_limits::infinity(); }; auto getBestWorker = [&] (int task) { - double bestSoFar = getFinishedTime(task, 0); - int bestWorker = 0; - for(int i = 1; i < nbWorkers; i++) { - double t = getFinishedTime(task, i); - if(t < bestSoFar) { - bestSoFar = t; bestWorker = i; + double bestSoFar = std::numeric_limits::infinity(); + int bestWorker = -1; + int i = 0; + for(int wType = 0; wType < ins.nbWorkerTypes; ++wType) { + for(int j = 0; j < ins.nbWorkers[wType]; ++j, ++i) { + double t = getFinishedTime(task, i, wType); + if(t < bestSoFar) { + bestSoFar = t; bestWorker = i; + } } } + if(bestWorker < 0) { + cerr << "Dmdas: no valid worker for task " << task << endl; + throw(1); + } return bestWorker; }; @@ -115,31 +122,34 @@ double Dmdas::compute(Instance &ins, SchedAction* action) { // int idle = wStartIndex[wTypeOrdering[0]]; + while(tasksToSchedule > 0) { while(!readyTasks->empty()) assignFront(); - for(int w = 0; w < nbWorkers; w++) { - if((endtimesWorkers[w] <= currentTime) && (!localQueues[w]->empty())) { - int task = localQueues[w]->front(); - int wType = ins.getType(w); - double finishTime = currentTime + ins.execType(wType, task); - if(verbosity >= 1) - cout << "dmdas: starting " << task << " of type " << ins.taskTypes[task] << " at " << currentTime << " to end at " << finishTime << " on worker " << w << " of type " << wType << endl; + int w = 0; + for(int wType = 0; wType < ins.nbWorkerTypes; ++wType) { + for(int j = 0; j < ins.nbWorkers[wType]; ++j, ++w) { + if((endtimesWorkers[w] <= currentTime) && (!localQueues[w]->empty())) { + int task = localQueues[w]->front(); + double finishTime = currentTime + ins.execType(wType, task); + if(verbosity >= 1) + cout << "dmdas: starting " << task << " of type " << ins.taskTypes[task] << " at " << currentTime << " to end at " << finishTime << " on worker " << w << " of type " << wType << endl; - tasksToSchedule --; - if(action != NULL) - action->onSchedule(task, w, currentTime, finishTime); - - localQueues[w]->eraseFront(); - assignedLoad[w] -= ins.execType(wType, task); - endtimesWorkers[w] = finishTime; - endtimesTasks[task] = finishTime; - currentRunningTask[w] = task; + tasksToSchedule --; + if(action != NULL) + action->onSchedule(task, w, currentTime, finishTime); + + localQueues[w]->eraseFront(); + assignedLoad[w] -= ins.execType(wType, task); + endtimesWorkers[w] = finishTime; + endtimesTasks[task] = finishTime; + currentRunningTask[w] = task; + } } } - + double nextTime = -1; for(int w = 0; w < nbWorkers; w++) { if((endtimesWorkers[w] > currentTime) && @@ -163,8 +173,7 @@ double Dmdas::compute(Instance &ins, SchedAction* action) { cout << "List: Finishing task " << i << " at time " << currentTime << endl; finishedTasks[i] = true; currentRunningTask[j] = -1; - for(int j = 0; j < (int) revDependencies[i].size(); j++) { - int k = revDependencies[i][j]; + for(int & k: revDependencies[i]) { nbDep[k]--; if(verbosity >= 7) cout << " Dependent task: " << k << " remaining dependencies: " << nbDep[k] << endl; diff --git a/schedulers/HeftAlgorithm.cpp b/schedulers/HeftAlgorithm.cpp index 81ea2bc91828787fb0df12794012a022c2a0217f..65f64858a312ef3d4aeefb7f1da989114d9cc533 100644 --- a/schedulers/HeftAlgorithm.cpp +++ b/schedulers/HeftAlgorithm.cpp @@ -47,22 +47,21 @@ double HeftAlgorithm::compute(Instance& ins, SchedAction *action) { if(verbosity >= 4) cout << "HEFT: considering task " << i << endl; - k = 0; double depTime = 0; - for(j = 0; j < (int) ins.dependencies[i].size(); j++) { - depTime = std::max(depTime, endtimesTasks[ins.dependencies[i][j]]); - } + for(int& d: ins.dependencies[i]) + depTime = std::max(depTime, endtimesTasks[d]); double bestTime = std::numeric_limits::infinity(); double bestStart = -1; int bestK = 0; int bestT = 0; + k = 0; timeSeq::iterator bestPeriod; // Should belond to workerAvailability[bestK] for(t = 0; t < ins.nbWorkerTypes; t++) { for(j = 0; j < ins.nbWorkers[t]; j++, k++) { double e = -1; double l = ins.execType(t, i); if(ins.isValidValue(l)) { timeSeq::iterator p = workerAvailability[k].getAvail(depTime, l, e); - if(e +l < bestTime){ + if(e + l < bestTime){ bestK = k; bestTime = e + l; bestStart = e; @@ -83,12 +82,11 @@ double HeftAlgorithm::compute(Instance& ins, SchedAction *action) { workerAvailability[bestK].insertBusy(bestPeriod, startTime, ins.execType(bestT, i)); endtimesTasks[i] = bestTime; - for(j = 0; j < (int) revDependencies[i].size(); j++) { - int k = revDependencies[i][j]; - nbDep[k]--; - if(nbDep[k] == 0){ - queue->insert(k); - if(action != NULL) action->onTaskPush(i); + for(int &d: revDependencies[i]) { + nbDep[d]--; + if(nbDep[d] == 0){ + queue->insert(d); + if(action != NULL) action->onTaskPush(d); } } } diff --git a/schedulers/IndepAccel.cpp b/schedulers/IndepAccel.cpp index 414cf3179ed5d109dc70dd1d9aa78370c9e7a59b..8fb1e6956423c98a7299655eefc3b1c4739b687e 100644 --- a/schedulers/IndepAccel.cpp +++ b/schedulers/IndepAccel.cpp @@ -30,6 +30,10 @@ bool IndepAccel::tryGuess(Instance& ins, std::vector taskSet, double target if(verbosity >= 6) cout << "IndepAccel, target = " << target << endl; + // Sets G* go on GPU, sets Cµ* go on CPU + // Sets "1" contain long tasks than may disturb the schedule, we + // should have at most m (or k) of them + // Sets "2" are the short ones vector setG1, setG2, setC1, setC2; for(auto t: taskSet) { if(ins.execType(IA_CPU, t) <= target) @@ -43,6 +47,9 @@ bool IndepAccel::tryGuess(Instance& ins, std::vector taskSet, double target (cpuTime(ins, b) - gpuTime(ins, b)) ); }; sort(setC2.begin(), setC2.end(), surfaceCmp); + + // Tasks with CPU time > target can not go on CPU, they go at the end of G1, + /// and are sorted by decreasing GPU time. Line 131 is where we forbid them to move sort(setG1.begin(), setG1.end(), [&] (int a, int b) { if(cpuTime(ins, a) > target) { @@ -60,6 +67,8 @@ bool IndepAccel::tryGuess(Instance& ins, std::vector taskSet, double target if(verbosity >= 7) cout << "IndepAccel: start. C1; " << setC1 << " C2: " << setC2 << " G1: " << setG1 << " G2: " << setG2 << endl; + // Tasks in G1 with GPU time small enough go to G2: they do not disturb the schedule + // We find them at the end of G1 because of how we sorted G1. if(!setG1.empty()) { auto it = setG1.end() - 1; while (gpuTime(ins, *it) < target / 2.0) { @@ -83,7 +92,9 @@ bool IndepAccel::tryGuess(Instance& ins, std::vector taskSet, double target int nbCPU = ins.nbWorkers[IA_CPU], nbGPU = ins.nbWorkers[IA_GPU]; double CPUload = 0, GPUload = 0; for(int i: setG1) - GPUload += gpuTime(ins, i); + GPUload += gpuTime(ins, i); + for(int i: setG2) + GPUload += gpuTime(ins, i); for(int i: setC2) CPUload += cpuTime(ins, i); @@ -97,20 +108,31 @@ bool IndepAccel::tryGuess(Instance& ins, std::vector taskSet, double target while( (iterations <= nbCPU) && ((GPUload > nbGPU * target + target/2.0) || (CPUload > nbCPU * target + target/2.0)) ) { - if(verbosity >= 7) - cout << "IndepAccel: iteration "<< iterations << "CPUload= " << CPUload << " C1; " << setC1 << " C2: " << setC2 << " G1: " << setG1 << " G2: " << setG2 << endl; + if(verbosity >= 7) { + cout << "IndepAccel: iteration "<< iterations << "CPUload= " << CPUload << " GPUload= " << GPUload << endl; + cout << " C1: " << setC1 << endl; + cout << " C2: " << setC2 << endl; + cout << " G1: " << setG1 << endl; + cout << " G2: " << setG2 << endl; + } + bool changed = false; while ((GPUload <= nbGPU * target) && (iterC2 != setC2.end())) { setG2.push_back(*iterC2); GPUload += gpuTime(ins, *iterC2); CPUload -= cpuTime(ins, *iterC2); - iterC2++; + iterC2++; + changed = true; } + // Here is where we forbid tasks which were moved at the end of G1 to be reassigned if( (((int) setC1.size()) < nbCPU) && (iterG1 != setG1.end()) && (cpuTime(ins, *iterG1) <= target) ) { setC1.push_back(*iterG1); GPUload -= gpuTime(ins, *iterG1); CPUload += cpuTime(ins, *iterG1); - iterG1++; + iterG1++; + changed = true; } + if(!changed) + break; iterations++; } @@ -172,6 +194,9 @@ IndepResult IndepAccel::compute(Instance& instance, vector &taskSet, vector double low = lowerBoundTwoResource(instance, taskSet); double up = std::numeric_limits::infinity(); + if(verbosity >= 6) + cerr << "IndepAccel: Lower bound is " << low << endl; + double target; while(abs(up - low) > epsilon*low) { if(up != std::numeric_limits::infinity()) diff --git a/schedulers/IndepBased.cpp b/schedulers/IndepBased.cpp index 516951dc78c192fb55ea26805f8a11256c5f22a1..85215efe335926b04affa32d56e966b342dfe624 100644 --- a/schedulers/IndepBased.cpp +++ b/schedulers/IndepBased.cpp @@ -2,11 +2,14 @@ #include "GreedyAlgorithm.h" #include "util.h" #include "IndepDP2.h" +#include "IndepDP3Demi.h" #include "IndepAccel.h" #include "IndepDualHP.h" #include "IndepImreh.h" #include "IndepBalanced.h" #include "IndepZhang.h" +#include +#include #include #include @@ -16,8 +19,6 @@ #ifdef WITH_CPLEX #include -#include - #endif using namespace std; @@ -28,6 +29,8 @@ IndepBased::IndepBased(const AlgOptions& options) : GreedyAlgorithm(options) { string indepName = options.asString("indep", "dualhp"); if(indepName == "dp2") indep = new IndepDP2(options); + if(indepName == "dp3demi") + indep = new IndepDP3Demi(options); if(indepName == "dualhp") indep = new IndepDualHP(options); if(indepName == "accel") { @@ -47,6 +50,8 @@ IndepBased::IndepBased(const AlgOptions& options) : GreedyAlgorithm(options) { indep = new IndepZhang(options); if(indepName == "clb2c") indep = new IndepCLB2C(options); + if(indepName == "minmin") + indep = new IndepMinMin(options); #ifdef WITH_CPLEX if(indepName == "round") indep = new AreaRound(options); diff --git a/schedulers/IndepDP2.cpp b/schedulers/IndepDP2.cpp index faf5b1acfa5971fee7f3a8acb4fa2c66d328eefa..ca992a75a778e23acb0d50cfdb9d13896822efa0 100644 --- a/schedulers/IndepDP2.cpp +++ b/schedulers/IndepDP2.cpp @@ -6,35 +6,107 @@ #include #include "util.h" +#ifdef WITH_CPLEX +#include +ILOSTLBEGIN +#endif + using namespace std; -IndepDP2::IndepDP2(const AlgOptions& opt): IndepAllocator(opt) { - verbosity = opt.asInt("verb_DP2", verbosity); +IndepDP2::IndepDP2(const AlgOptions& opt): IndepDualGeneric(opt) { + discretizationConstant = opt.asDouble("disc", discretizationConstant); +#ifdef WITH_CPLEX + solveWithCplex = (opt.asString("cplex", "false") == "true") || (opt.asString("cplex", "false") == "discrete"); + cplexUseDiscretizedValues = opt.asString("cplex", "false") == "discrete"; +#else + if(opt.isPresent("cplex")) + cerr << "Warning: DP2: pmtool compiled without cplex support, ignoring 'cplex' option." << endl; +#endif } -// Arbitrary convention: CPU times are index 0, GPU times are index 1. Just a naming thing. - -// returns minimum CPU load -double IndepDP2::tryGuess(Instance& instance, std::vector taskSet, double maxGPUload, double maxlen, - IndepResult &result, bool getResult) { +// returns minimum CPU load, taking into account existing loads +double IndepDP2::tryGuess(Instance& instance, std::vector taskSet, vector& loads, + double maxlen, IndepResult &result, bool getResult) { // CPUload(i, g) := smallest load on CPU from the first i tasks, // with at most g load on GPU // So need to discretize the GPU load ? Yes. Paper says with a ratio of lambda/3n // For all tasks in taskSet: // forall g, CPUload(i, g) = min CPUload(i-1, g-T^G_i) CPUload(i-1, g) + T^C_i + + double existingCPUload = loads[0]; + double existingGPUload = loads[1]; + double maxGPUload = maxlen * instance.nbWorkers[1] - existingGPUload; if(maxGPUload < 0) maxGPUload = 1; - double ratio = maxGPUload / (50.0 * taskSet.size()); + double ratio = maxlen / (discretizationConstant * taskSet.size()); vector discreteGPUtimings(instance.nbTaskTypes); for(int i = 0; i < instance.nbTaskTypes; i++) discreteGPUtimings[i] = ceil(instance.execTimes[1][i] / ratio); const int N = ceil(maxGPUload / ratio); - /* for(auto && t : taskSet) - N += discreteGPUtimings[instance.taskTypes[t]]; */ + +#ifdef WITH_CPLEX + if(solveWithCplex) { + IloEnv env; + IloModel model = IloModel(env); + + IloNumVarArray affect(env, taskSet.size(), 0.0, 1.0, ILOINT); // 0 means on CPU, 1 on GPU + IloExpr totalCPULoad(env); + IloExpr totalGPULoad(env); + for(int i = 0; i < taskSet.size(); ++i) { + int t = taskSet[i]; + int taskType = instance.taskTypes[t]; + const double exec0 = instance.execTimes[0][taskType]; + const double exec1 = instance.execTimes[1][taskType]; + totalCPULoad += (1-affect[i])*exec0; + totalGPULoad += affect[i]*(cplexUseDiscretizedValues ? discreteGPUtimings[taskType] : exec1); + if(exec0 > maxlen) + model.add(affect[i] == 1); + if(exec1 > maxlen) + model.add(affect[i] == 0); + } + + if(cplexUseDiscretizedValues) + model.add(totalGPULoad <= N); + else + model.add(totalGPULoad <= maxGPUload); + model.add(IloMinimize(env, totalCPULoad)); + + IloCplex modelCplex = IloCplex(model); + if(verbosity < 8) + modelCplex.setOut(env.getNullStream()); + modelCplex.setParam(IloCplex::Param::MIP::Tolerances::MIPGap, 0.0000001); + + + IloBool feasible = modelCplex.solve(); + if(! feasible) + return std::numeric_limits::infinity(); + + double value = modelCplex.getObjValue(); + + if(getResult) { + result[0].clear(); + result[1].clear(); + + IloNumArray affectValues(env); + modelCplex.getValues(affectValues, affect); + for(int i = 0; i < taskSet.size(); ++i) + if(affectValues[i] > 0.5) + result[1].push_back(i); + else + result[0].push_back(i); + + // cout << "CPUResult: " << result[0] << endl; + // cout << "GPUResult: " << result[1] << endl; + + } + return value + existingCPUload; + } +#endif + int length = getResult ? taskSet.size() + 1 : 1; double** CPUload = new double*[length]; CPUload[0] = new double[length * (N+1)]; @@ -49,28 +121,58 @@ double IndepDP2::tryGuess(Instance& instance, std::vector taskSet, double m for(int i = 0; i <= N; i++) CPUload[index][i] = 0; for(int t : taskSet) { - int taskType = instance.taskTypes[t]; + int taskType = instance.taskTypes[t]; int nextIndex = getResult ? index+1: index; - for(int l = N; l >= 0; l--) { - if(instance.execTimes[0][taskType] > maxlen && instance.execTimes[1][taskType] > maxlen) - return -1; // Problem is not feasible: task t cannot be placed on any resource - double newLoad = std::numeric_limits::infinity(); - if(instance.execTimes[0][taskType] <= maxlen) - newLoad = CPUload[index][l] + instance.execTimes[0][taskType]; - if((instance.execTimes[1][taskType] <= maxlen) && (discreteGPUtimings[taskType] <= l)) - newLoad = min(newLoad, CPUload[index][l - discreteGPUtimings[taskType]]); - CPUload[nextIndex][l] = newLoad; + + double exec0 = instance.execTimes[0][taskType]; + double exec1 = instance.execTimes[1][taskType]; + int discreteGPUtime = discreteGPUtimings[taskType]; + + // Possible optimization if needed: run this test for all + // taskTypes (which appear in the taskSet) instead of for all + // tasks + if(exec0 > maxlen && exec1 > maxlen){ + delete[] CPUload[0]; + delete[] CPUload; + return -1; // Problem is not feasible: task t cannot be placed on any resource } - if(getResult) index++; + if((exec0 <= maxlen) && (exec1 <= maxlen)) { + for(int l = N; l >= discreteGPUtime; --l) { + CPUload[nextIndex][l] = min(CPUload[index][l] + exec0, CPUload[index][l - discreteGPUtime]); + } + for(int l = discreteGPUtime - 1; l >= 0; --l) { + CPUload[nextIndex][l] = CPUload[index][l] + exec0; + } + } else if ((exec0 <= maxlen) && (exec1 > maxlen)) { + for(int l = N; l >= 0; --l) { + CPUload[nextIndex][l] = CPUload[index][l] + exec0; + } + } else /* ((exec0 > maxlen) && (exec1 <= maxlen)) */ { + for(int l = N; l >= discreteGPUtime; --l) { + CPUload[nextIndex][l] = CPUload[index][l - discreteGPUtime]; + } + for(int l = discreteGPUtime - 1; l >= 0; l--) { + CPUload[nextIndex][l] = std::numeric_limits::infinity(); + } + } + + + // for(int l = N; l >= 0; l--) { + // double newLoad = std::numeric_limits::infinity(); + // if( <= maxlen) + // newLoad = CPUload[index][l] + instance.execTimes[0][taskType]; + // if((instance.execTimes[1][taskType] <= maxlen) && (discreteGPUtimings[taskType] <= l)) + // newLoad = min(newLoad, CPUload[index][l - discreteGPUtimings[taskType]]); + // CPUload[nextIndex][l] = newLoad; + // } + index = nextIndex; } double value = CPUload[index][N]; - if(value == std::numeric_limits::infinity()) { - // Problem not feasible. - return -1; - } + int gLoad = N; - if(getResult) { + + if(getResult && value != std::numeric_limits::infinity()) { result[0].clear(); result[1].clear(); @@ -88,162 +190,5 @@ double IndepDP2::tryGuess(Instance& instance, std::vector taskSet, double m delete[] CPUload[0]; delete[] CPUload; - return value; -} - - -IndepResult IndepDP2::compute(Instance& instance, vector &taskSet, vector &loads) { - if(instance.nbWorkerTypes != 2) { - cerr << "IndepDP2: only implemented for instances with 2 worker types" << endl; - throw(1); - } - IndepResult result(2); // 2 because there are two resource types. - - int nbCPU = instance.nbWorkers[0]; - int nbGPU = instance.nbWorkers[1]; - - if(verbosity >= 4) { - cout << "IndepDP2: called with TS=" << taskSet << " and loads=" << loads << endl; - cout << " CPU times: "; - for(int i : taskSet) cout << instance.execType(0, i) << " "; - cout << endl; - cout << " GPU times: "; - for(int i : taskSet) cout << instance.execType(1, i) << " "; - cout << endl; - - } - - - if(taskSet.size() == 0) - return result; - - double minload = min(loads[0], loads[1]); - loads[0] -= minload; - loads[1] -= minload; - - - double low = lowerBoundTwoResource(instance, taskSet, loads[0], loads[1]); - double up = std::numeric_limits::infinity(); - - // TODO: optim for the case where GPUarea <= min execution time on CPU: result = all on GPU ! - - /* double firstguess = low; - if(area > low) { - low = area; - firstguess = 1.15*area; - } - - bool haveResult = false; - - double target = firstguess; - double r = tryGuess(instance, taskSet, target * nbGPU - loads[1], target, result, target == low); - if(verbosity >= 6) - cout << "IndepDP2: firstguess = "<< firstguess << ", result = " << r << " mkspan = " << (r+loads[0])/ nbCPU << endl; - - if((r != -1) && (r + loads[0])/nbCPU <= target*(1+ epsilon)) { - up = firstguess; - haveResult = (target == low); - } else { - low = firstguess; - } - */ - double target; - bool haveResult = false; - // Then, dichotomy, as usual - while(abs(up - low) > epsilon*low) { - if(up != std::numeric_limits::infinity()) - target = (up + low) / 2; - else - target = 1.15*low; - - double r = tryGuess(instance, taskSet, target * nbGPU - loads[1], target, result, abs(target - low) <= 3*epsilon*low); - if(verbosity >= 6) - cout << "IndepDP2: TARGET = "<< target << ", result = " << r << " mkspan= " << (r+loads[0])/ nbCPU << endl; - if((r != -1) && (r + loads[0]) /nbCPU <= target * (1+ epsilon)) { - up = target; - haveResult = (abs(target - low) <= 3*epsilon*low); - } - else { - low = target; - } - } - - if(! haveResult) { - double r = tryGuess(instance, taskSet, up * nbGPU - loads[1], up, result, true); - if(verbosity >= 6) - cout << "IndepDP2: TARGET = "<< up << ", result = " << r << " mkspan= " << (r+loads[0])/ nbCPU << endl; - } - - /* if(verbosity >= 4) - cout << "Result of IndepDP2: " << result << endl; */ - return result; - -} - -double lowerBoundTwoResource(Instance& instance, vector taskSet, - double CPUload, double GPUload) { - if(instance.nbWorkerTypes != 2) { - cerr << "lowerBoundTwoResources: only implemented for instances with 2 worker types" << endl; - throw(1); - } - - int nbCPU = instance.nbWorkers[0]; - int nbGPU = instance.nbWorkers[1]; - - double longest = 0; - - struct TypeData { - int nbTasks; - double CPUtime; - double GPUtime; - double ratio; - }; - vector taskTypes(instance.nbTaskTypes); - for(auto &&t: taskSet) { - taskTypes[instance.taskTypes[t]].nbTasks++; - } - for(int i = 0; i < instance.nbTaskTypes; i++) { - TypeData &d = taskTypes[i]; - d.CPUtime = instance.execTimes[0][i] / nbCPU; - d.GPUtime = instance.execTimes[1][i] / nbGPU; - d.ratio = instance.execTimes[0][i] / instance.execTimes[1][i]; - if(d.nbTasks > 0) - longest = max(longest, - min(instance.execTimes[0][i], instance.execTimes[1][i])); - } - sort(taskTypes.begin(), taskTypes.end(), [&] (TypeData a, TypeData b) { - return (a.ratio > b.ratio); - }); - - double CPUlen = CPUload / nbCPU, GPUlen = GPUload / nbGPU; - int g = 0; int c = instance.nbTaskTypes - 1; - while(g != c) { - TypeData & d = taskTypes[g]; TypeData &e = taskTypes[c]; - if(GPUlen + d.nbTasks * d.GPUtime <= CPUlen + e.nbTasks * e.CPUtime) { - GPUlen += d.nbTasks * d.GPUtime; - g++; - } else { - CPUlen += e.nbTasks * e.CPUtime; - c--; - } - } - TypeData &d = taskTypes[g]; - double remCPU = d.nbTasks * d.CPUtime; - double remGPU = d.nbTasks * d.GPUtime; - double area; - if(remCPU + CPUlen <= GPUlen) - area = GPUlen ; - else if(remGPU + GPUlen <= CPUlen) - area = CPUlen; - else - area = (CPUlen*remGPU +remGPU * remCPU + GPUlen * remCPU) / (remCPU + remGPU); - - // cout << "LB: " << area << " " << longest << " " << remCPU << " " << remGPU << " " << CPUlen << " " << GPUlen << " " << g << " " << c << endl; - /*for(auto &&t: taskTypes) { - cout << " " << t.nbTasks << " " << t.CPUtime << " " << t.GPUtime <<" " << t.ratio << endl; - } */ - - return max(area, longest); - - + return value + existingCPUload; } diff --git a/schedulers/IndepDP3Demi.cpp b/schedulers/IndepDP3Demi.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ad250755087294f049a3c82abd6841342c6b20fa --- /dev/null +++ b/schedulers/IndepDP3Demi.cpp @@ -0,0 +1,302 @@ +#include "IndepDP3Demi.h" +#include +#include +#include +#include +#include +#include "util.h" + +#ifdef WITH_CPLEX +#include +ILOSTLBEGIN +#endif + + +using namespace std; + + +IndepDP3Demi::IndepDP3Demi(const AlgOptions& opt): IndepDualGeneric(opt) { + discretizationConstant = opt.asDouble("disc", discretizationConstant); +#ifdef WITH_CPLEX + solveWithCplex = (opt.asString("cplex", "false") == "true") || (opt.asString("cplex", "false") == "discrete"); + cplexUseDiscretizedValues = opt.asString("cplex", "false") == "discrete"; +#else + if(opt.isPresent("cplex")) + cerr << "Warning: DP3demi: pmtool compiled without cplex support, ignoring 'cplex' option." << endl; +#endif + +} + + +// returns minimum CPU load, taking into account existing loads +double IndepDP3Demi::tryGuess(Instance& instance, std::vector taskSet, vector& loads, + double target, IndepResult &result, bool getResult) { + // CPUload(i, g) := smallest load on CPU from the first i tasks, + // with at most g load on GPU + // So need to discretize the GPU load ? Yes. Paper says with a ratio of lambda/3n + // For all tasks in taskSet: + // forall g, CPUload(i, g) = min CPUload(i-1, g-T^G_i) CPUload(i-1, g) + T^C_i + + int nbCPU = instance.nbWorkers[0]; + int nbGPU = instance.nbWorkers[1]; + + double existingCPUload = loads[0]; + double existingGPUload = loads[1]; + double maxGPUload = target * nbGPU - existingGPUload; + + if(maxGPUload < 0) maxGPUload = 1; + + double ratio = target / (discretizationConstant * taskSet.size()); + vector discreteGPUtimings(instance.nbTaskTypes); + vector isLongOnCPU(instance.nbTaskTypes); + vector isLongOnGPU(instance.nbTaskTypes); + for(int i = 0; i < instance.nbTaskTypes; i++) { + discreteGPUtimings[i] = ceil(instance.execTimes[1][i] / ratio); + isLongOnCPU[i] = instance.execTimes[0][i] > (target/2); + isLongOnGPU[i] = instance.execTimes[1][i] > (target/2); + } + const int N = ceil(maxGPUload / ratio); + +#ifdef WITH_CPLEX + if(solveWithCplex) { + IloEnv env; + IloModel model = IloModel(env); + + IloNumVarArray affect(env, taskSet.size(), 0.0, 1.0, ILOINT); // 0 means on CPU, 1 on GPU + IloExpr nbTasksWithLargeCPUTime(env); + IloExpr nbTasksWithLargeGPUTime(env); + IloExpr totalCPULoad(env); + IloExpr totalGPULoad(env); + for(int i = 0; i < taskSet.size(); ++i) { + int t = taskSet[i]; + int taskType = instance.taskTypes[t]; + const double exec0 = instance.execTimes[0][taskType]; + const double exec1 = instance.execTimes[1][taskType]; + if(isLongOnCPU[taskType]) + nbTasksWithLargeCPUTime += (1-affect[i]); + if(isLongOnGPU[taskType]) + nbTasksWithLargeGPUTime += affect[i]; + totalCPULoad += (1-affect[i])*exec0; + totalGPULoad += affect[i]*(cplexUseDiscretizedValues ? discreteGPUtimings[taskType] : exec1); + if(exec0 > target) + model.add(affect[i] == 1); + if(exec1 > target) + model.add(affect[i] == 0); + } + + model.add(nbTasksWithLargeCPUTime <= nbCPU); + model.add(nbTasksWithLargeGPUTime <= nbGPU); + if(cplexUseDiscretizedValues) + model.add(totalGPULoad <= N); + else + model.add(totalGPULoad <= maxGPUload); + model.add(IloMinimize(env, totalCPULoad)); + + IloCplex modelCplex = IloCplex(model); + if(verbosity < 8) + modelCplex.setOut(env.getNullStream()); + modelCplex.setParam(IloCplex::Param::MIP::Tolerances::MIPGap, 0.000001); + + IloBool feasible = modelCplex.solve(); + if(! feasible) + return std::numeric_limits::infinity(); + + double value = modelCplex.getObjValue(); + + if(getResult) { + result[0].clear(); + result[1].clear(); + + IloNumArray affectValues(env); + modelCplex.getValues(affectValues, affect); + for(int i = 0; i < taskSet.size(); ++i) + if(affectValues[i] > 0.5) + result[1].push_back(i); + else + result[0].push_back(i); + } + return value + existingCPUload; + } +#endif + + + + int nbJobsWithLargeCPUTime = 0; + int nbJobsWithLargeGPUTime = 0; + for(int & t: taskSet) { + int taskType = instance.taskTypes[t]; + if(instance.execTimes[0][taskType] > (target / 2)) + ++ nbJobsWithLargeCPUTime; + if(instance.execTimes[1][taskType] > (target / 2)) + ++ nbJobsWithLargeGPUTime; + } + + const int maxMu = min(nbCPU, nbJobsWithLargeCPUTime); + const int maxNu = min(nbGPU, nbJobsWithLargeGPUTime); + const int stateSpaceSize = (N+1) * (maxMu + 1) * (maxNu + 1); + int length = getResult ? taskSet.size() + 1 : 1; + double** CPUload = new double*[length]; + if(verbosity >= 7) { + cerr << "DP3demi: N=" << N << " maxMu= " << maxMu << " maxNu = " << maxNu << endl; + cerr << "DP3demi: allocating " << length << " space= " << stateSpaceSize << ", total= " << length * stateSpaceSize << endl; + } + CPUload[0] = new double[length * stateSpaceSize]; + for(int i = 1; i < length; i++) + CPUload[i] = CPUload[i-1] + stateSpaceSize; + +#define getTabValue(tab, l, m, k) (tab[l + (N+1)*(m) + (N+1)*(maxMu+1)*(k)]) + + + if(verbosity >= 7) + cout << "IndepDP3Demi: maxGLoad = " << maxGPUload << ", ratio = " << ratio << " N= " << N << " gR: " << getResult << " mL " << target << endl; + + int index = 0; + for(int i = 0; i < stateSpaceSize; ++i) + CPUload[index][i] = 0; + for(int t : taskSet) { + const int taskType = instance.taskTypes[t]; + const int nextIndex = getResult ? index+1: index; + + const double exec0 = instance.execTimes[0][taskType]; + const double exec1 = instance.execTimes[1][taskType]; + const int discreteGPUtime = discreteGPUtimings[taskType]; + + const int muOffset = isLongOnCPU[taskType]; + const int nuOffset = isLongOnGPU[taskType]; + + if(exec0 > target && exec1 > target) { + delete[] CPUload[0]; + delete[] CPUload; + return -1; // Problem is not feasible: task t cannot be placed on any resource + } + + if(verbosity >= 8) + cout << "Task " << t << " dGPUTime="<< discreteGPUtime << " exec0=" << exec0 << " muOffset="<= 7) + cerr << "DP3demi: final value is " << value << endl; + + int gLoad = N; + int mu = maxMu; + int nu = maxNu; + if(getResult && (value != std::numeric_limits::infinity())) { + + result[0].clear(); + result[1].clear(); + + for(; index > 0; index--) { + const int taskType = instance.taskTypes[taskSet[index-1]]; + const double exec0 = instance.execTimes[0][taskType]; + const double exec1 = instance.execTimes[1][taskType]; + const int discreteGPUtime = discreteGPUtimings[taskType]; + + const int muOffset = isLongOnCPU[taskType]; + const int nuOffset = isLongOnGPU[taskType]; + + if((mu >= muOffset) && + (abs(getTabValue(CPUload[index], gLoad, mu, nu) - (getTabValue(CPUload[index-1], gLoad, mu-muOffset, nu) + exec0)) <= 1e-5)) { + mu -= muOffset; + result[0].push_back(taskSet[index-1]); + } + else { + gLoad -= discreteGPUtimings[taskType]; + nu -= nuOffset; + result[1].push_back(taskSet[index-1]); + } + } + double actualLoad = 0; + for(int & t: result[0]) + actualLoad += instance.execTimes[0][instance.taskTypes[t]]; + + if(abs(actualLoad-value) > 0.0001) + cerr << "DP3Demi Warning: Difference between computed load and actual load: " << value << " " << actualLoad << endl; + + } + delete[] CPUload[0]; + delete[] CPUload; + + return value + existingCPUload; +} diff --git a/schedulers/IndepDualGeneric.cpp b/schedulers/IndepDualGeneric.cpp new file mode 100644 index 0000000000000000000000000000000000000000..211da7a96f3cdd49c6f63227c2d069e498dd8cbe --- /dev/null +++ b/schedulers/IndepDualGeneric.cpp @@ -0,0 +1,154 @@ +#include "IndepDualGeneric.h" +#include +#include +#include +#include +#include +#include "util.h" + +using namespace std; + + +IndepDualGeneric::IndepDualGeneric(const AlgOptions& opt): IndepAllocator(opt) { + verbosity = opt.asInt("verb_dual", verbosity); + epsilon = opt.asDouble("eps", epsilon); +} + + +// Arbitrary convention: CPU times are index 0, GPU times are index 1. Just a naming thing. + +IndepResult IndepDualGeneric::compute(Instance& instance, vector &taskSet, vector &loads) { + + // This class could be generalized if needed, but it would require + // another lower bound, so it has a cost. Do it if needed only + if(instance.nbWorkerTypes != 2) { + cerr << "IndepDualGeneric: only implemented for instances with 2 worker types" << endl; + throw(1); + } + IndepResult result(2); // 2 because there are two resource types. + + int nbCPU = instance.nbWorkers[0]; + int nbGPU = instance.nbWorkers[1]; + + if(verbosity >= 4) { + cout << "IndepDualGeneric: called with TS=" << taskSet << " and loads=" << loads << endl; + // cout << " CPU times: "; + // for(int i : taskSet) cout << instance.execType(0, i) << " "; + // cout << endl; + // cout << " GPU times: "; + // for(int i : taskSet) cout << instance.execType(1, i) << " "; + // cout << endl; + + } + + + if(taskSet.size() == 0) + return result; + + double minload = min(loads[0], loads[1]); + loads[0] -= minload; + loads[1] -= minload; + + + double low = lowerBoundTwoResource(instance, taskSet, loads[0], loads[1]); + double up = std::numeric_limits::infinity(); + + double target; + bool haveResult = false; + + while(abs(up - low) > epsilon*low) { + if(up != std::numeric_limits::infinity()) + target = (up + low) / 2; + else + target = 1.15*low; + + bool askForResult = abs(target - low) <= 3*epsilon*low; + double r = tryGuess(instance, taskSet, loads, target, result, askForResult); + if(verbosity >= 6) + cout << "IndepDualGeneric: TARGET = "<< target << ", result = " << r << " mkspan= " << r/ nbCPU << endl; + if((r != -1) && (r/nbCPU <= target * (1+ epsilon))) { + up = target; + haveResult = askForResult; + } + else { + low = target; + } + } + + if(! haveResult) { + double r = tryGuess(instance, taskSet, loads, up, result, true); + if(verbosity >= 6) + cout << "IndepDualGeneric: TARGET = "<< up << ", result = " << r << " mkspan= " << (r+loads[0])/ nbCPU << endl; + } + + return result; + +} + +double lowerBoundTwoResource(Instance& instance, vector taskSet, + double CPUload, double GPUload) { + if(instance.nbWorkerTypes != 2) { + cerr << "lowerBoundTwoResources: only implemented for instances with 2 worker types" << endl; + throw(1); + } + + int nbCPU = instance.nbWorkers[0]; + int nbGPU = instance.nbWorkers[1]; + + double longest = 0; + + struct TypeData { + int nbTasks; + double CPUtime; + double GPUtime; + double ratio; + }; + vector taskTypes(instance.nbTaskTypes); + for(auto &&t: taskSet) { + taskTypes[instance.taskTypes[t]].nbTasks++; + } + for(int i = 0; i < instance.nbTaskTypes; i++) { + TypeData &d = taskTypes[i]; + d.CPUtime = instance.execTimes[0][i] / nbCPU; + d.GPUtime = instance.execTimes[1][i] / nbGPU; + d.ratio = instance.execTimes[0][i] / instance.execTimes[1][i]; + if(d.nbTasks > 0) + longest = max(longest, + min(instance.execTimes[0][i], instance.execTimes[1][i])); + } + sort(taskTypes.begin(), taskTypes.end(), [&] (TypeData a, TypeData b) { + return (a.ratio > b.ratio); + }); + + double CPUlen = CPUload / nbCPU, GPUlen = GPUload / nbGPU; + int g = 0; int c = instance.nbTaskTypes - 1; + while(g != c) { + TypeData & d = taskTypes[g]; TypeData &e = taskTypes[c]; + if(GPUlen + d.nbTasks * d.GPUtime <= CPUlen + e.nbTasks * e.CPUtime) { + GPUlen += d.nbTasks * d.GPUtime; + g++; + } else { + CPUlen += e.nbTasks * e.CPUtime; + c--; + } + } + TypeData &d = taskTypes[g]; + double remCPU = d.nbTasks * d.CPUtime; + double remGPU = d.nbTasks * d.GPUtime; + double area; + if(remCPU + CPUlen <= GPUlen) + area = GPUlen ; + else if(remGPU + GPUlen <= CPUlen) + area = CPUlen; + else + area = (CPUlen*remGPU +remGPU * remCPU + GPUlen * remCPU) / (remCPU + remGPU); + + // cout << "LB: " << area << " " << longest << " " << remCPU << " " << remGPU << " " << CPUlen << " " << GPUlen << " " << g << " " << c << endl; + /*for(auto &&t: taskTypes) { + cout << " " << t.nbTasks << " " << t.CPUtime << " " << t.GPUtime <<" " << t.ratio << endl; + } */ + + return max(area, longest); + + +} diff --git a/schedulers/IndepMinMin.cpp b/schedulers/IndepMinMin.cpp new file mode 100644 index 0000000000000000000000000000000000000000..baf665af866e89517219a88e2ccb2356deee231c --- /dev/null +++ b/schedulers/IndepMinMin.cpp @@ -0,0 +1,87 @@ + +// Created by eyraud on 25/03/19 + +#include "IndepMinMin.h" +#include +#include +#include "util.h" + +using namespace std; + +IndepMinMin::IndepMinMin(const AlgOptions& options) : IndepAllocator(options) { + needsPreciseLoads = true; +} + +int IndepMinMin::getEarliestWorker(vector &loads, int type) { + int best = workerIndices[type]; + int to = workerIndices[type+1]; + for(int i = workerIndices[type]; i < to; ++i){ + if(loads[i] < loads[best]) + best = i; + } + return best; +} + +double IndepMinMin::endTime(vector &loads, int workerType, int task) { + return loads[bestWorkers[workerType]] + ins->execType(workerType, task); +} + +IndepResult IndepMinMin::compute(Instance & ins, vector &taskSet, vector &loads) { + + this->ins = & ins; + + workerIndices.resize(ins.nbWorkerTypes + 1); + bestWorkers.resize(ins.nbWorkerTypes); + workerIndices[0] = 0; + for(int i = 0; i < ins.nbWorkerTypes; ++i) { + workerIndices[i+1] = workerIndices[i] + ins.nbWorkers[i]; + bestWorkers[i] = getEarliestWorker(loads, i); + } + + IndepResult result(ins.nbWorkerTypes); + + set tasks(taskSet.begin(), taskSet.end()); + int nbTasks = taskSet.size(); + + while(nbTasks > 0) { + int bestWorkerType = -1; + int bestTask = -1; + for(int t: tasks) { + int bestWorkerTypeForThisTask = 0; + for(int j = 1; j < ins.nbWorkerTypes; ++j) { + if(endTime(loads, j, t) < endTime(loads, bestWorkerTypeForThisTask, t)) + bestWorkerTypeForThisTask = j; + } + if(bestTask < 0 || + endTime(loads, bestWorkerTypeForThisTask, t) < endTime(loads, bestWorkerType, bestTask)) { + bestTask = t; + bestWorkerType = bestWorkerTypeForThisTask; + } + } + + result[bestWorkerType].push_back(bestTask); + loads[bestWorkers[bestWorkerType]] += ins.execType(bestWorkerType, bestTask); + bestWorkers[bestWorkerType] = getEarliestWorker(loads, bestWorkerType); + + nbTasks -= 1; + tasks.erase(bestTask); + + } + + return(result); +} + + +// Possibilities +// As an IndepAllocator, close to CLB2C +// +// O(n^2m') m' = # worker types +// Until done: O(n) +// find best worker for each type of worker O(m) +// Find smallest ready task for each type of worker O(nm') =>O(m'(1+update)) +// Assign. +// Keep ordered sets of tasks, one for each worker type ? Updates seem costly, but worst case is still better +// Keep best worker for each type ? Avoids recomputation. + + +// What about MaxMin ? diff --git a/schedulers/OnlineERLS.cpp b/schedulers/OnlineERLS.cpp index dd1159b26a75400628dd1ed2d8e118206d2a9120..cf180ef0f4932f3707356e3870fbcef64a166c33 100644 --- a/schedulers/OnlineERLS.cpp +++ b/schedulers/OnlineERLS.cpp @@ -21,7 +21,7 @@ int OnlineERLS::assignTask(int task, double now) { } double OnlineERLS::compute(Instance &instance, SchedAction *action) { - largestGroup = instance.nbWorkers[0] > instance.nbWorkers[1] ? 0 : 1; + largestGroup = instance.nbWorkers[0] >= instance.nbWorkers[1] ? 0 : 1; threshold = sqrt(instance.nbWorkers[largestGroup] / instance.nbWorkers[1-largestGroup]); return OnlineGeneric::compute(instance, action); } diff --git a/schedulers/OnlineGeneric.cpp b/schedulers/OnlineGeneric.cpp index 2d29c5d0811c66b957fb165d497e1d3e2bb158f6..46e6f03e6f84bc338d949ebe5fc595819a2486d9 100644 --- a/schedulers/OnlineGeneric.cpp +++ b/schedulers/OnlineGeneric.cpp @@ -14,8 +14,11 @@ double OnlineGeneric::compute(Instance &instance, SchedAction *action) { instance.getRevDependencies(); readyTasks = priority_queue, greater>(); + finishTimes.clear(); finishTimes.resize(instance.totalWorkers, 0); + readyTimes.clear(); readyTimes.resize(instance.nbTasks, -1); + nbScheduledPredecessors.clear(); nbScheduledPredecessors.resize(instance.nbTasks, 0); for(int t = 0; t < instance.nbTasks; ++t) { @@ -33,7 +36,7 @@ double OnlineGeneric::compute(Instance &instance, SchedAction *action) { double finishTime = startTime + instance.execWorker(worker, next.second); if(verbosity >= 4) - cout << "Scheduling task " << next.second << " at time " << startTime << " on worker " << worker << " until time " << finishTime << endl; + cout << "OG @ " << next.first << " Scheduling task " << next.second << " at time " << startTime << " on worker " << worker << " until time " << finishTime << endl; if(action) action->onSchedule(next.second, worker, startTime, finishTime); finishTimes[worker] = finishTime; diff --git a/schedulers/OnlineLG.cpp b/schedulers/OnlineLG.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ad91021a1ae7e993bce86f7dcdfaa8a5dc48a6b5 --- /dev/null +++ b/schedulers/OnlineLG.cpp @@ -0,0 +1,28 @@ +// +// Created by eyraud on 11/12/17. +// + +#include +#include "OnlineLG.h" + +OnlineLG::OnlineLG(const AlgOptions& options) : OnlineGeneric(options) { + +} + + +double OnlineLG::compute(Instance &instance, SchedAction *action) { + if(instance.nbWorkerTypes != 2) { + std::cerr << "OnlineLG: only for two types of ressources, please." << std::endl; + throw(1); + } + return OnlineGeneric::compute(instance, action); +} + +int OnlineLG::assignTask(int task, double now) { + + double p1 = ins->execType(0, task); + double p2 = ins->execType(1, task); + int target = p1/ins->nbWorkers[0] <= p2/ins->nbWorkers[1] ? 0 : 1; + return getEarliestWorker(target); +} + diff --git a/schedulers/OnlineMG.cpp b/schedulers/OnlineMG.cpp new file mode 100644 index 0000000000000000000000000000000000000000..32308b37c68420ec0353049d9c3cdab9b0912ce2 --- /dev/null +++ b/schedulers/OnlineMG.cpp @@ -0,0 +1,53 @@ +// +// Created by eyraud on 11/12/17. +// + +#include +#include // for max +#include "OnlineMG.h" + +using namespace std; + +OnlineMG::OnlineMG(const AlgOptions& options) : OnlineGeneric(options) { + options.updateValue(alpha, "alpha"); + options.updateValue(gamma, "gamma"); +} + + +double OnlineMG::compute(Instance &instance, SchedAction *action) { + if(instance.nbWorkerTypes != 2) { + std::cerr << "OnlineMG: only for two types of ressources, please." << std::endl; + throw(1); + } + + largestGroup = instance.nbWorkers[0] >= instance.nbWorkers[1]? 0 : 1; + smallestGroup = 1 - largestGroup; + m1 = instance.nbWorkers[largestGroup]; + m2 = instance.nbWorkers[smallestGroup]; + + maxRTime = 0; + sumRTime = 0; + + return OnlineGeneric::compute(instance, action); +} + +int OnlineMG::assignTask(int task, double now) { + + double p1 = ins->execType(largestGroup, task); + double p2 = ins->execType(smallestGroup, task); + int target; + if(p2/m2 <= gamma * (p1/m1)) { + target = smallestGroup; + } else { + double newMax = max(maxRTime, p2); + double newSum = sumRTime + p2; + if(max(newMax, newSum / m2) <= alpha * p1) { + target = smallestGroup; + maxRTime = newMax; sumRTime = newSum; + } else { + target = largestGroup; + } + } + return getEarliestWorker(target); +} + diff --git a/schedulers/OnlineZhang.cpp b/schedulers/OnlineZhang.cpp index 0a87ad49f4c57ff6f497fb54273c835b270cb97f..68f9f428b8008a3bf8861734b86548093e89b6e7 100644 --- a/schedulers/OnlineZhang.cpp +++ b/schedulers/OnlineZhang.cpp @@ -5,18 +5,18 @@ #include #include "OnlineZhang.h" -void OnlineZhang::updateValue(double & v, const std::string &key, const AlgOptions& opt ) { - if(opt.isPresent(key)) - v = opt.asDouble(key); -} +// void OnlineZhang::updateValue(double & v, const std::string &key, const AlgOptions& opt ) { +// if(opt.isPresent(key)) +// v = opt.asDouble(key); +// } -OnlineZhang::OnlineZhang(const AlgOptions &options) : GreedyPerType(options) { - updateValue(lambda, "lambda", options); - updateValue(beta, "beta", options); - updateValue(theta, "theta", options); - updateValue(phi, "phi", options); +OnlineZhang::OnlineZhang(const AlgOptions &options) : OnlineGeneric(options) { + options.updateValue(lambda, "lambda"); + options.updateValue(beta, "beta"); + options.updateValue(theta, "theta"); + options.updateValue(phi, "phi"); } double OnlineZhang::compute(Instance &instance, SchedAction *action) { @@ -33,16 +33,17 @@ double OnlineZhang::compute(Instance &instance, SchedAction *action) { ins = &instance; avgLoad.resize(2, 0); - return GreedyPerType::compute(instance, action); + return OnlineGeneric::compute(instance, action); } -void OnlineZhang::onTaskPush(int task, double now) { +int OnlineZhang::assignTask(int task, double now) { double p1 = ins->execType(largestGroup, task); double p2 = ins->execType(smallestGroup, task); int target; + double minLoadG2 = finishTimes[getEarliestWorker(smallestGroup)]; // Rule 1 - if( p1 >= beta * (now + + avgLoad[smallestGroup]/m2 + p2) ) target = smallestGroup; + if( p1 >= beta * (minLoadG2 + p2) ) target = smallestGroup; // Rule 2 else if ( p1/m1 <= theta * (p2/m2) ) target = largestGroup; else if ( p1/m1 >= lambda * (p2/m2) ) target = smallestGroup; @@ -56,13 +57,6 @@ void OnlineZhang::onTaskPush(int task, double now) { } avgLoad[target] += ins->execType(target, task); - queues[target]->insert(task); - } - -int OnlineZhang::chooseTask(int worker, double now) { - int task = GreedyPerType::chooseTask(worker, now); - int type = ins->getType(worker); - avgLoad[type] -= ins->execType(type, task); - return task; + return getEarliestWorker(target); } diff --git a/schedulers/TrueHeteroPrio.cpp b/schedulers/TrueHeteroPrio.cpp index af2e406b64b3c54bb0b70e4327191eb27d151475..8a5102e39b7c99a1fd2b7f1919a866fc74e333e2 100644 --- a/schedulers/TrueHeteroPrio.cpp +++ b/schedulers/TrueHeteroPrio.cpp @@ -3,48 +3,92 @@ // #include +#include #include "TrueHeteroPrio.h" using namespace std; +bool TrueHeteroPrio::isBetterSteal(int taskA, double endA, int taskB, double endB, int wType) { + double scoreA, scoreB; + if(stealType == AF) { + scoreA = ins->execType(1-wType, taskA) / ins->execType(wType, taskA); + scoreB = ins->execType(1-wType, taskB) / ins->execType(wType, taskB); + if (scoreA == scoreB) { + scoreA = priorities[taskA]; + scoreB = priorities[taskB]; + } + } + else if (stealType == LATEST) { + scoreA = endA; + scoreB = endB; + } + else { /* Steal by priority */ + scoreA = priorities[taskA]; + scoreB = priorities[taskB]; + } + return scoreA > scoreB; +} + int TrueHeteroPrio::chooseTask(int worker, double now) { int wType = ins->getType(worker); - if(taskSet->empty()) { + + bool chosenFromQueue = false; + int resultTask = -1; + auto positionInQueue = taskSet->begin(); + if(! taskSet->empty()) { + if (wType == 0) { + positionInQueue = taskSet->end(); + --positionInQueue; + } + chosenFromQueue = true; + resultTask = *positionInQueue; + } + + bool unfavorable = false; + if (resultTask != -1) + unfavorable = ins->execType(wType, resultTask) > ins->execType(1-wType, resultTask); + + bool otherTypeHasAnIdleProc = false; + int chosenVictim = -1; + if(resultTask == -1 || unfavorable) { int bestTask = -1; - int chosenVictim = -1; + // Steal: copy/paste from GreedyPerType - for (int victim = 0; victim < ins->totalWorkers; victim++) { - int victimType = ins->getType(victim); - // ins->getType not efficient, but easier to read - if (victimType != wType && runningTasks[victim] != -1 - && ins->isValidType(wType, runningTasks[victim]) - && now + ins->execType(wType, runningTasks[victim]) < endTimesWorkers[victim]) { - int task = runningTasks[victim]; - double taskAF = ins->execType(victimType, task) / ins->execType(wType, task); - double bestAF = ins->execType(victimType, bestTask) / ins->execType(wType, bestTask); - if (bestTask == -1 || (taskAF > bestAF) ) { - bestTask = task; - chosenVictim = victim; - } - } - } - if (bestTask != -1) { - runningTasks[chosenVictim] = -1; - return bestTask; - } else { - return -1; - } - } else { - auto pos = taskSet->begin(); - if (wType == 0) { - pos = taskSet->end(); - --pos; - } - int task = *pos; - taskSet->erase(pos); - - return task; + int victim = wType == 1 ? 0 : ins->nbWorkers[0]; + for(int i = 0; i < ins->nbWorkers[1-wType]; ++i,++victim){ + int& task = runningTasks[victim]; + if(task < 0) + otherTypeHasAnIdleProc = true; + else if(ins->isValidType(wType, task) + && now + ins->execType(wType, task) < endTimesWorkers[victim]) { + if(bestTask == -1 || isBetterSteal(task, endTimesWorkers[victim], bestTask, endTimesWorkers[chosenVictim], wType)) { + bestTask = task; + chosenVictim = victim; + } + } + } + + if(bestTask != -1) { + resultTask = bestTask; + chosenFromQueue = false; + unfavorable = false; + } } + + + if(resultTask == -1) + return -1; + + if(otherTypeHasAnIdleProc && unfavorable) + // Stay idle if another processor could do better than me + return -1; + + if(chosenFromQueue) + taskSet->erase(positionInQueue); + else if(chosenVictim != -1) + runningTasks[chosenVictim] = -1; + + return resultTask; } void TrueHeteroPrio::onTaskPush(int task, double now) { @@ -59,15 +103,44 @@ double TrueHeteroPrio::compute(Instance &ins, SchedAction *action) { vector accelFactors(ins.nbTasks); for(int i = 0; i < ins.nbTasks; i++) accelFactors[i] = ins.execType(0, i) / ins.execType(1, i); - rankCompare compareByAccelFactor = rankCompare(accelFactors); - strictCompare strict(&compareByAccelFactor, false); + rankCompare compareByAccelFactor(accelFactors, true); + if(rankString == "heft") + priorities = ins.computeHEFTRank(); + else if(rankString == "min") + priorities = ins.computeMinRank(); + else if(rankString == "unit") + priorities = ins.computeUnitRank(); + else if(rankString == "none") { + priorities = vector(ins.nbTasks, 1); + } else { + cerr << "TrueHP: Unknown rank " << rankString << endl; + throw(1); + } + rankCompare compareByPriorities(priorities, true); + vector list; + list.push_back(&compareByAccelFactor); + list.push_back(&compareByPriorities); + lexCompare combined(list); + strictCompare strict(&combined, true); taskSet = new set(strict); this->ins = &ins; - + return GreedyAlgorithm::compute(ins, action); } -TrueHeteroPrio::TrueHeteroPrio(const AlgOptions &options) : GreedyAlgorithm(options), ranker(options) { - +TrueHeteroPrio::TrueHeteroPrio(const AlgOptions &options) : GreedyAlgorithm(options) { + string steal = options.asString("steal", "latest"); + if(steal == "latest") + stealType = LATEST; + else if(steal == "prio") + stealType = PRIO; + else if(steal == "af") + stealType = AF; + else { + cerr << "TrueHP: unknown steal type " << steal << ", defaulting to latest" << endl; + stealType = LATEST; + } + rankString = options.asString("rank", "min"); + } diff --git a/schedulers/availSequence.cpp b/schedulers/availSequence.cpp index d479ff30726babf574634cdaeeeb9bf62a3fb3d5..49bd3c3cfd2f5284f82bab7c532a68a8a63ad382 100644 --- a/schedulers/availSequence.cpp +++ b/schedulers/availSequence.cpp @@ -78,12 +78,12 @@ void AvailSequence::insertBusy(timeSeq::iterator i, double start, double len) { Period orig = *i; auto j = seq.erase(i); if(orig.start < start) { - Period before(orig.start, start); - seq.insert(j, before); + // Period before(orig.start, start); + seq.emplace(j, orig.start, start); } if(orig.end > start + len) { - Period after(start + len, orig.end); - seq.insert(j, after); + // Period after(start + len, orig.end); + seq.emplace(j, start+len, orig.end); } } diff --git a/tools/.gitignore b/tools/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..ba0430d26c996e7f078385407f959c96c271087c --- /dev/null +++ b/tools/.gitignore @@ -0,0 +1 @@ +__pycache__/ \ No newline at end of file diff --git a/tools/resultStorage.md b/tools/resultStorage.md new file mode 100644 index 0000000000000000000000000000000000000000..e6f96756584e0e181313e60177cacdbaace91e05 --- /dev/null +++ b/tools/resultStorage.md @@ -0,0 +1,123 @@ +# resultStorage: storing results from pmtool runs + +**Purpose:** Make it easier to save the results of running pmtool with + many scheduling algorithms on many instances. + +**Usage:** This is a Python class that should be used in a Python + script. + +The main component is the `MakeStorage` function, which returns a +`Storage` class. `MakeStorage` has 4 parameters: + ++ `instanceParameters` is a list of strings, which indicate the names + of the parameters which describe the instances. The `Instance` class + of the resulting `Storage` is a named tuple whose field names are + those provided in `instanceParameters` in addition to the `platform` + field which describes the platform. ++ `getInstanceFile` is a function which, given an `Instance` object, + shoud return the name of the `.rec` file which contains the instance ++ `getPlatformFile` is a function which, given a platform name as a + string, should return the name of the corresponding platform file. ++ `commandArgs` is a list of strings indicating how to invoke + `pmtool`. The first string should be a path to the `pmtool` + executable, and the next ones should be default arguments for all + invocations. + +Once we get a `Storage` class, we can create an instance by providing +it with a filename which indicates where to store the results. + + +The code uses several concepts, all described by named tuples: + ++ An `Instance` specifies a pmtool problem instance, with an instance + and a platform file ++ An `Algorithm` specifies a pmtool algorithm, with three fields: + `alg` is the name of the algorithm, `isBound` is a boolean + indicating whether this is a bound or an algorithm, `params` + contains the param string of the algorithm, with the + `key1=value1:key2=value2` syntax of pmtool. ++ A `Question` is the concatenation of an `Instance` and an + `Algorithm` ++ A `Record` is a `Question` with an answer, i.e. with the following + additional fields: `mkspan` and `time` provide the result from + pmtool, `date` indicates the date at which the computation was + done. + +The code provides helper functions to generate `Algorithm`s and +`Instance`s: + ++ `makeAlgorithm(name, isBound=False, **args)` combines all named + args as parameters for the algorithm (the values should be strings) ++ `makeAlgorithms(name, isBound=False, **args)` in which the values of + named args should be iterables containing string; this function + returns a list of `Algorithm` objects, representing all possible + combinations of parameters, just like the `key1=v1,v2:key2=v3,v4` + syntax from pmtool. Examples: + +```python +makeAlgorithms("indep", indep=["dualhp", "balest"], rank=["heft", "min"]) +makeAlgorithms("heft", rank=["heft", "min", "none", "unit"]) +``` + ++ `Storage.makeInstances` works the same as `makeAlgorithms`. ++ `Storage.allCombinations(instances, algs)` generates all possible +`Question`s from the given lists of instances and algorithms. + + +The most useful functions of the `Storage` class are: + ++ `updateFile(questions)` updates the results file by running all + questions which have no valid answer. An answer is valid if there is + at least one, and the most recent answer is at least as recent as + the corresponding instance file. TODO: There is still no test for + the platform file. `updateFile` takes an optional `pred` argument + which can change this definition of "valid answer". ++ `cleanupFile()` keeps only the latest answer for each question in + the file. In addition, it accepts an optional `toBeRemoved` + predicate which given a record specifies if it should be removed or + not. ++ `displayFileContents` provides a human (and R) readable output of + the contents of the storage. It accepts a `file` optional argument + to specify the output file (default is standard output), and a + `pred` argument which given a record specifies if it should be + displayed. + + +## Usage example + +The `cholesky` archive from the +[Graph Market](http://starpu.gforge.inria.fr/market/) contains a set +of `.rec` files from StarPU, all corresponding to a cholesky execution +for different number of tiles and for different platforms. The file +for size `N` is stored at `NxN/tasks.rec` file. The code to create the +corresponding storage is: + +```python +import resultStorage +Ns = [10, 20, 30, 40, 50, 60, 70, 90, 100, 110] +platforms = ["attila", "idgraf", "sirocco", "mirage"] +def getInstanceFile(i): + return "./" + i.N + "x" + i.N + "/tasks.rec" +def getPlatformFile(p): + return "./"+p+".platform" +DagStore = resultStorage.MakeStorage(["N"], getInstanceFile, getPlatformFile, ["/path/to/pmtool", "-t", "0.1"]) +store = DagStore("./cholesky.raw") +``` + +Then, the following code can be used to populate the store with all +results: + +```python +instances = store.makeInstances("cholesky", N = map(str, Ns), platform = platforms) + +algorithms = resultStorage.makeAlgorithms("heft", ("rank", ["heft", "min", "none", "unit", "area"])) +algorithms += resultStorage.makeAlgorithms("dmdas", ("rank", ["heft", "min", "none", "unit", "area"])) +algorithms += resultStorage.makeAlgorithms("hetprio", ("rank",["heft", "min", "none", "unit", "area"])) +algorithms += resultStorage.makeAlgorithms("hetprio", ("rank",["heft", "min", "none", "unit", "area"]), ("ssbaf", ["yes"]), ("stealidle", ["yes"])) +algorithms.append(resultStorage.Algorithm(alg = "ect", isBound=False, params="")) +algorithms.append(resultStorage.Algorithm(alg = "area", isBound=True, params="hybrid=no")) + +if __name__ == "__main__": + store.updateFile(store.allCombinations(instances, algorithms)) + store.displayFileContents(file="./cholesky.dat") +``` diff --git a/tools/resultStorage.py b/tools/resultStorage.py new file mode 100644 index 0000000000000000000000000000000000000000..e070bb548206929d5f5760cd8f030b1b366c8c68 --- /dev/null +++ b/tools/resultStorage.py @@ -0,0 +1,264 @@ +import csv +import sys +from collections import namedtuple, OrderedDict +import subprocess +import shutil +from datetime import datetime +import os +import pickle + + + +dateformat = "%Y-%m-%dT%H:%M:%S" + +algorithmFields = ["isBound", "alg", "params"] +resultFields = ["mkspan", "time", "date"] +Algorithm = namedtuple("Algorithm", algorithmFields) +Result = namedtuple("Result", resultFields) + +def _makeAlgorithm(row): + return Algorithm(**{k:row[k] for k in algorithmFields}) + +def _makeResult(row): + return Result(**{k:row[k] for k in resultFields}) + +def makeAlgorithm(name, isBound=False, **args): + params = OrderedDict(**{k:v for (k, v) in sorted(args.items(), key=lambda c: c[0]) }) + return Algorithm(alg = name, isBound = isBound, params = ":".join('='.join(t) for t in p.items())) + +def makeAlgorithms(name, isBound=False, **args): + params = [{}] + for (k,v) in sorted(args.items(), key=lambda c: c[0]): + params = [OrderedDict(d, **{k:x}) for x in v for d in params] + + return [ Algorithm(alg = name, isBound = isBound, params = ":".join('='.join(t) for t in p.items())) for p in params ] + +def MakeStorage(instanceParameters, getInstanceFile, getPlatformFile, commandArgs): + class Storage: + instanceFields = ["platform"] + instanceParameters + questionFields = instanceFields + algorithmFields + recordFields = instanceFields + algorithmFields + resultFields + + Instance = namedtuple("Instance", instanceFields) + Question = namedtuple("Question", questionFields) + Record = namedtuple("Record", recordFields) + + @classmethod + def _makeInstance(cls, row): + return cls.Instance(**{k:row[k] for k in cls.instanceFields}) + @classmethod + def _makeQuestion(cls, row): + return cls.Question(**{k:row[k] for k in cls.questionFields}) + @classmethod + def _makeRecord(cls, row): + return cls.Record(**{k:row[k] for k in cls.recordFields}) + + @classmethod + def makeInstances(cls, **kwargs): + result = [{}] + for k,v in kwargs.items(): + result = [dict(d, **{k:x}) for x in v for d in result] + + return list(map(cls._makeInstance, result)) + + @classmethod + def allCombinations(cls, instances, algorithms): + return (cls.Question(**i._asdict(), **a._asdict()) for i in instances for a in algorithms) + + + + @classmethod + def convertRow(cls, row): + row["date"] = datetime.strptime(row["date"], dateformat) + row["isBound"] = True if row["isBound"] == "True" else False + + @classmethod + def parseRow(cls, row): + cls.convertRow(row) + return (cls._makeQuestion(row), _makeResult(row)) + + @classmethod + def parseRowToRecord(cls, row): + cls.convertRow(row) + return cls._makeRecord(row) + + + def __init__(self, filename): + self.filename = filename + + @classmethod + def getFilename(cls, i): + return getInstanceFile(i) + + def readFile(self): + allData = {} + try: + with open(self.filename, 'r') as fd: + for row in csv.DictReader(fd, fieldnames = self.recordFields): + (question, result) = self.parseRow(row) + if question in allData: + allData[question].append(result) + else: + allData[question] = [result] + except OSError: + pass + return allData + + def readRecords(self): + with open(self.filename, 'r') as fd: + for row in csv.DictReader(fd, fieldnames = self.recordFields): + record = self.parseRowToRecord(row) + yield record + + def _writeRecord(self, csvWriter, record): + d = record._asdict() + d["date"] = d["date"].strftime(dateformat) + csvWriter.writerow(d) + + def writeRecords(self, results, overwrite=False): + with open(self.filename, 'w' if overwrite else 'a') as fd: + w = csv.DictWriter(fd, fieldnames = self.recordFields) + for r in results: + self._writeRecord(w, r) + + @classmethod + def hasUpToDateAnswer(cls, q, data): + if q in data: + lastAnswer = max(data[q], key=lambda r:r.date) + src = cls.getFilename(q) + try: + srcTime = datetime.fromtimestamp(os.path.getmtime(src)) + except OSError as e: + print("Instance source file %s is not readable: %s\n" % (src, e), file=sys.stderr) + exit + return lastAnswer.date >= srcTime + else: + return False + + @classmethod + def hasValidAnswer(cls, q, data): + return q in data + + def updateFile(self, questions, pred = lambda q, data: not Storage.hasValidAnswer(q, data)): + data = self.readFile() + toRun = [ q for q in questions if pred(q, data)] + results = self.runMany(toRun) + + # Keep only the last answer for each question + def cleanupFile(self, toBeRemoved = lambda x: False): + data = self.readFile() + toWrite = [self.Record(*q, *max(data[q], key=lambda r:r.date)) for q in data ] + self.writeRecords((r for r in toWrite if not toBeRemoved(r)), overwrite=True) + + def runMany(self, questions): + separated = [ (self._makeInstance(q._asdict()), _makeAlgorithm(q._asdict())) for q in questions ] + + ## Was a good idea, but pmtool has too many memory problems to open so many instances at once. + ## Get list of instances + # allInstances = set(i for (i,a) in separated) + ## Group instances which require the same algorithms + # instancesPerAlgList = {} + # for i in allInstances: + # algs = tuple(a for (j, a) in separated if i == j) + # if algs in instancesPerAlgList: + # instancesPerAlgList[algs].append(i) + # else: + # instancesPerAlgList[algs] = [i] + + # results = results + sum((self.runOne(instances, algs) for (algs, instances) in instancesPerAlgList.items()), []) + + groupedByInstances = {} + for (i, a) in separated: + if i in groupedByInstances: + groupedByInstances[i].append(a) + else: + groupedByInstances[i] = [a] + + nbInstances = len(groupedByInstances) + results = sum( (self.runOne(i, algs, progress = (1+index)/nbInstances) + for (index, (i, algs)) in enumerate(groupedByInstances.items())), []) + + return results + + + def runOne(self, instance, algs, progress=None): + with open(self.filename, 'a') as output: + + writer = csv.DictWriter(output, fieldnames = self.recordFields) + + filename = self.getFilename(instance) + args = [] + args.extend(commandArgs) + args.extend(["--output-raw", "--no-header"]) + args.extend(['-p', getPlatformFile(instance.platform)]) + args.append(filename) + args.extend(sum([[("-b" if a.isBound else "-a"), + a.alg+(":"+a.params if a.params else "")] for a in algs], [])) + print("Running: " + (" [%.1f%%] " % (progress*100) if progress else "") + " ".join(args)) + with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) as child: + reader = csv.reader(child.stdout, delimiter=' ') + result = [] + for row in reader: + algNameAndParams=row[3].split(":", maxsplit=1) + assert row[0] == filename + rec = self.Record(*list(instance), True if row[2] == 'True' else False, algNameAndParams[0], + algNameAndParams[1] if len(algNameAndParams) >= 2 else "", + row[4], row[5], datetime.now().strftime(dateformat)) + writer.writerow(rec._asdict()) + output.flush() + result.append(rec) + returncode = child.wait() + if returncode != 0: + print("Wrong return code %d for command:\n %s\n" % (returncode, " ".join(args)), file=sys.stderr) + print(child.stderr, file=sys.stderr) + raise Exception + return result + + + def printRecords(self, records, file=""): + if file: + fd = open(file, "w") + else: + fd = sys.stdout + recordsWithParamDicts = [ r._replace(params=OrderedDict(eq.split('=') for eq in r.params.split(":")) if r.params else {}) for r in records ] + allParams = frozenset.union(*(frozenset(r.params.keys()) for r in recordsWithParamDicts)) + print(" ".join(instanceParameters), + "platform isBound algorithm", + " ".join(allParams), "makespan time date", file=fd) + for r in recordsWithParamDicts: + print(" ".join(getattr(r, p) if getattr(r, p) else "NA" for p in instanceParameters), + "%s" % (r.platform), + "%s %s" % (r.isBound, r.alg), + " ".join(r.params[k] if k in r.params else "NA" for k in allParams), + "%s %s %s" % (r.mkspan, r.time, r.date.strftime(dateformat)), file = fd) + + def displayFileContents(self, pred = lambda x: True, **kwargs): + data = self.readRecords() + self.printRecords(filter(pred, data), **kwargs) + + def getRecordsFromDatFile(self, file, questions): + knownFields = [("m", "m"), ("k", "k"), ("isBound", "isBound"), + ("algorithm", "alg")] + [(p, p) for p in instanceParameters] + resultFields = [ ("makespan", "mkspan"), ("time", "time"), ("date", "date") ] + newQuestions = [(q._replace(params=dict(eq.split('=') for eq in q.params.split(":")) if q.params else {}), q) for q in questions] + result = [] + with open(file, "r") as fd: + reader = csv.DictReader(fd, delimiter=' ') + for row in reader: + self.convertRow(row) + known = {p:row[p] for (p,z) in knownFields + resultFields} + for (p, z) in knownFields + resultFields: + del row[p] + row = {k:v for (k, v) in row.items() if v != "NA"} + for (q, origQ) in newQuestions: + if all(getattr(q, z) == known[p] for (p, z) in knownFields) and row == q.params: + result.append(self.Record(**origQ._asdict(), mkspan=known["makespan"], time=known["time"], + date = known["date"])) + break + return result + + return Storage + + + +