Commit b74e0173 authored by Mathieu Giraud's avatar Mathieu Giraud

Merge branch 'feature-a/3393-binreadstorage-use-bins-when-necessary' into 'dev'

feature-a/3393-binreadstorage-use-bins-when-necessary into dev

Closes #3393 and #3395

See merge request !267
parents d1d409aa 773f29b3
Pipeline #36122 passed with stage
in 5 seconds
......@@ -17,6 +17,11 @@ BinReadStorage::BinReadStorage()
nb_inserted(0), nb_stored(0), smallest_bin_not_empty(~0),label(),inited(false) {}
void BinReadStorage::init(size_t nb_bins, size_t max_score, const VirtualReadScore *vrs, bool no_list) {
this->max_bins = nb_bins;
__init(0, max_score, vrs, no_list);
}
void BinReadStorage::__init(size_t nb_bins, size_t max_score, const VirtualReadScore *vrs, bool no_list) {
this->all_read_lengths = 0;
if (inited)
return;
......@@ -37,7 +42,30 @@ void BinReadStorage::init(size_t nb_bins, size_t max_score, const VirtualReadSco
inited=true;
}
void BinReadStorage::reallocate(){
list<Sequence> tmpBin;
if (bins)
tmpBin = bins[0];
free_objects();
all_read_lengths = 0;
smallest_bin_not_empty = ~0;
total_nb_scores = 0;
nb_stored = 0;
inited = false;
__init(max_bins, max_score, scorer, tmpBin.size() == 0);
for(auto s : tmpBin){
this->add(s);
}
nb_inserted -= nb_stored;
}
BinReadStorage::~BinReadStorage() {
free_objects();
}
void BinReadStorage::free_objects() {
if (bins)
delete [] bins;
if (score_bins) {
......@@ -62,6 +90,9 @@ void BinReadStorage::addScore(size_t bin, float score) {
}
void BinReadStorage::add(Sequence &s) {
if(nb_stored == getMaxNbReadsStored() && nb_inserted == nb_stored){
reallocate();
}
float score = scorer->getScore(s);
size_t bin = scoreToBin(score);
addScore(bin, score);
......
......@@ -68,6 +68,7 @@ class VirtualReadStorage {
*/
class BinReadStorage: public VirtualReadStorage {
private:
size_t max_bins;
size_t nb_bins;
list<Sequence> *bins;
double *score_bins;
......@@ -87,6 +88,7 @@ public:
* nb_bins are created and the maximal score for the reads that will be added is assumed
* to be max_score. If higher score are met, they are put in the nb_bins+1 bin.
* The class doesn't destruct the VirtualReadScore. It is the responsability of the caller.
* May not build as many as nb_bins to save memory.
* @pre all scores must be >= 0
* @param no_list: don't create a list (useful for storing only stats,
* false by default: lists are created). If the option is set to true, the
......@@ -94,7 +96,24 @@ public:
*/
void init(size_t nb_bins, size_t max_score, const VirtualReadScore *vrs, bool no_list = false);
private:
/**
* Private function used by init() that actually does the job. It actually creates the number of bins
* given in parameter.
*/
void __init(size_t nb_bins, size_t max_score, const VirtualReadScore *vrs, bool no_list = false);
void free_objects();
public:
/**
* If the bin is full we need, the storage needs to be reallocated to have more memory.
*/
void reallocate();
~BinReadStorage();
void add(Sequence &s);
......
......@@ -266,12 +266,12 @@ void WindowsStorage::clearSequences(){
seqs_by_window.clear();
}
json WindowsStorage::sortedWindowsToJson(map <junction, json> json_data_segment, int max_json_output) {
json WindowsStorage::sortedWindowsToJson(map <junction, json> json_data_segment, int max_json_output, bool delete_all) {
json windowsArray;
int top = 1;
for (list<pair <junction, size_t> >::const_iterator it = sort_all_windows.begin();
it != sort_all_windows.end(); ++it)
it != sort_all_windows.end(); )
{
json windowsList;
......@@ -295,6 +295,13 @@ json WindowsStorage::sortedWindowsToJson(map <junction, json> json_data_segment,
windowsArray.push_back(windowsList);
if (delete_all) {
germline_by_window.erase(it->first);
status_by_window.erase(it->first);
it = sort_all_windows.erase(it);
} else {
it++;
}
if (top == max_json_output + 1)
break ;
}
......
......@@ -218,8 +218,13 @@ class WindowsStorage {
* @max_json_output: maximal number of windows to output (<= 0: no limit)
*/
ostream &printSortedWindows(ostream &os);
json sortedWindowsToJson(map<junction, json> json_data_segment, int max_json_output);
/**
* @param delete_all: Delete the objects while they are inserted into the JSON. This prevents the memory
* from continously increasing (see #2120, #3387)
* @return a JSON object containing all the information
*/
json sortedWindowsToJson(map<junction, json> json_data_segment, int max_json_output, bool delete_all=false);
/**
* Clear the seqs_by_window map.
......
......@@ -9,13 +9,13 @@ void testBinReadStorage() {
TAP_TEST_EQUAL(reads.scoreToBin(0), 0, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(2), 0, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(10), 2, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(11), 3, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(5), 1, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(10), 0, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(11), 0, TEST_BRS_SCORE_TO_BIN, "");
TAP_TEST_EQUAL(reads.scoreToBin(5), 0, TEST_BRS_SCORE_TO_BIN, "");
Sequence seq1 = {"label", "l", "GAGAG", "", 0};
reads.add(seq1);
TAP_TEST_EQUAL(reads.smallest_bin_not_empty, 1, TEST_BRS_SBNE, "");
TAP_TEST_EQUAL(reads.smallest_bin_not_empty, 0, TEST_BRS_SBNE, "");
TAP_TEST_EQUAL(reads.getNbInserted(), 1, TEST_BRS_GET_NB_INSERTED, "");
TAP_TEST_EQUAL(reads.getNbStored(), 1, TEST_BRS_GET_NB_STORED, "");
TAP_TEST(reads.getScoreBySeq(seq1) == 5
......@@ -27,9 +27,9 @@ void testBinReadStorage() {
TAP_TEST_EQUAL(reads.smallest_bin_not_empty, 0, TEST_BRS_SBNE, "");
TAP_TEST_EQUAL(reads.getNbInserted(), 2, TEST_BRS_GET_NB_INSERTED, "");
TAP_TEST_EQUAL(reads.getNbStored(), 2, TEST_BRS_GET_NB_STORED, "");
TAP_TEST(reads.getScoreBySeq(seq2) == 2
&& reads.getScoreByScore(2.) == 2
&& reads.getScore(0) == 2, TEST_BRS_GET_SCORE, "");
TAP_TEST_EQUAL(reads.getScoreBySeq(seq2), 7, TEST_BRS_GET_NB_STORED, "");
TAP_TEST_EQUAL(reads.getScoreByScore(2.), 7, TEST_BRS_GET_NB_STORED, "");
TAP_TEST_EQUAL(reads.getScore(0), 7, TEST_BRS_GET_NB_STORED, "");
TAP_TEST_EQUAL(reads.getScore(), 7, TEST_BRS_GET_SCORE, "");
TAP_TEST_EQUAL(reads.getAverageScore(), 3.5, TEST_BRS_GET_AVG_SCORE, "");
......@@ -39,9 +39,9 @@ void testBinReadStorage() {
TAP_TEST_EQUAL(reads.smallest_bin_not_empty, 0, TEST_BRS_SBNE, "");
TAP_TEST_EQUAL(reads.getNbInserted(), 3, TEST_BRS_GET_NB_INSERTED, "");
TAP_TEST_EQUAL(reads.getNbStored(), 3, TEST_BRS_GET_NB_STORED, "");
TAP_TEST(reads.getScoreBySeq(seq3) == 10
&& reads.getScoreByScore(10.) == 10
&& reads.getScore(2) == 10, TEST_BRS_GET_SCORE, "");
TAP_TEST_EQUAL(reads.getScoreBySeq(seq3), 17, TEST_BRS_GET_NB_STORED, "");
TAP_TEST_EQUAL(reads.getScoreByScore(10.), 17, TEST_BRS_GET_NB_STORED, "");
TAP_TEST_EQUAL(reads.getScore(2), 17, TEST_BRS_GET_SCORE, "");
Sequence seq4 = {"label4", "l4", "AGAGACAGTA", "", 0};
reads.add(seq4);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment