kmerstore.h 15.7 KB
Newer Older
Mikaël Salson's avatar
Mikaël Salson committed
1
2
3
4
5
6
7
8
#ifndef KMERSTORE_H
#define KMERSTORE_H

#include <map>
#include <string>
#include <list>
#include <stdexcept>
#include <stdint.h>
9
#include <math.h>
Mikaël Salson's avatar
Mikaël Salson committed
10
11
12
13
14
#include "fasta.h"
#include "tools.h"

using namespace std;

15
16
17
typedef
enum { KMER_INDEX, AC_AUTOMATON } IndexTypes;

Mikaël Salson's avatar
Mikaël Salson committed
18
19
20
21
22
class Kmer {
public:
  unsigned int count;

  Kmer();
23
24
25
26

  /**
   * This constructor is used via a IKmerStore<Kmer> index (hence the argument list)       
   */
27
  Kmer(const string &label, int strand=1, size_t length=0);
28

Mikaël Salson's avatar
Mikaël Salson committed
29
  Kmer &operator+=(const Kmer &);
30

31
32
33
34
35
  /**
   * @return the length of the kmer (only there for compatibility with KmerAffect)
   */
  size_t getLength() const;

36
37
38
39
  /**
   * When indexing revcomp, should the value be the same or not?
   * In other words, does the information stored by the class is strand-dependent?
   */
Mikaël Salson's avatar
Mikaël Salson committed
40
  static bool hasRevcompSymetry();
41
42
43
44

  /**
   * @return true if the element is the same as when initialised with default constructor.
   */
45
46
47
48
49
50
51
  bool isNull() const;

  /**
   * @return true iff the kmer is unknown (which doesn't make sense here but
   * it only there for a reason of compatibility with KmerAffect)
   */
  bool isUnknown() const;
Mikaël Salson's avatar
Mikaël Salson committed
52
53
} ;
ostream &operator<<(ostream &os, const Kmer &kmer);
54
55
56
57
58
59
60
bool operator==(const Kmer &k1, const Kmer &k2);
bool operator<(const Kmer &k1, const Kmer &k2);
bool operator>(const Kmer &k1, const Kmer &k2);
bool operator<=(const Kmer &k1, const Kmer &k2);
bool operator>=(const Kmer &k1, const Kmer &k2);
bool operator!=(const Kmer &k1, const Kmer &k2);

Mikaël Salson's avatar
Mikaël Salson committed
61
62
63
64
65
66
67
68
69
70
71


/* K-mer indexing */

template <class T> class IKmerStore
{
protected:
  bool revcomp_indexed;
  int k; // weight of the seed
  int s; // span of the seed (s >= k)
  string seed ;
72
  size_t nb_kmers_inserted;
73
  size_t max_size_indexing;
74
  bool finished_building;
Mikaël Salson's avatar
Mikaël Salson committed
75
76
77
78
79

public:

  virtual ~IKmerStore();

80
81
  static int last_id;
  int id; // id of this index
82
  int refs; // number of germlines using this index
83

84
  list< pair <T, Fasta> > labels;
85

86
87
  IKmerStore();

Mikaël Salson's avatar
Mikaël Salson committed
88
89
90
  /**
   * @param input: A single FASTA file
   * @param label: label that must be associated to the given files
91
   * @param seed: the seed to use for indexing. By default it will be the seed of the index
92
   * @post All the sequences in the FASTA files have been indexed, and the label is stored in the list of labels
Mikaël Salson's avatar
Mikaël Salson committed
93
   */
94
  void insert(Fasta& input, const string& label="", int keep_only = 0, string seed = "");
Mikaël Salson's avatar
Mikaël Salson committed
95
96
97
98

  /**
   * @param input: A list of FASTA files
   * @param label: label that must be associated to the given files
99
   * @param seed: the seed to use for indexing. By default it will be the seed of the index
100
   * @post All the sequences in the FASTA files have been indexed, and the label is stored in the list of labels
Mikaël Salson's avatar
Mikaël Salson committed
101
   */
102
  void insert(list<Fasta>& input, const string& label="", int keep_only = 0, string seed = "");
Mikaël Salson's avatar
Mikaël Salson committed
103
104
105
106
  
  /**
   * @param input: A sequence to be cut in k-mers
   * @param label: label that must be associated to the given files
107
108
109
110
   * @param keep_only: if > 0 will keep at most the last keep_only nucleotides
   *                   of the sequence. if < 0 will keep at most the first
   *                   keep_only nucleotides of the sequence. if == 0,
   *                   will keep all the sequence.
111
   * @param seed: the seed to use for indexing. By default it will be the seed of the index. Also note that when the index ! hasDifferentKmerTypes(), the seed of the index will always be used (and therefore the parameter will be ignored)
Mikaël Salson's avatar
Mikaël Salson committed
112
113
   * @post All the k-mers in the sequence have been indexed.
   */
114
115
116
117
118
119
120
121
122
123
  virtual void insert(const seqtype &sequence,
                      const string &label,
                      bool ignore_extended_nucleotides=true, int keep_only = 0,
                      string seed="");

  /**
   * Perform extra steps to finish the building of the index. After using
   * that function, we are not supposed to insert any other sequence.
   * No query function should be called before calling that function.
   */
124
  virtual void finish_building();
Mikaël Salson's avatar
Mikaël Salson committed
125
126
127
128
129

  /**
   * @param word: a k-mer
   * @return the value for the kmer as stored in the structure (don't do revcomp)
   */ 
130
  virtual T& get(seqtype &word) = 0;
Mikaël Salson's avatar
Mikaël Salson committed
131

132
  /**
133
134
135
136
   * @return the percentage of kmers that are set in the index.
   *         When ! hasDifferentKmerTypes(), the index load is always the same
   *         (apart for the unknown kmer).
   *         When kmer.isUnknown(), we return the load of all th other kmers.
137
   */
138
  virtual float getIndexLoad(T kmer) const;
139

140
141
142
143
144
145
  /**
   * @return the given integer (size of a read),
   *         but limit this size to max_size_indexing if it was defined
   */
  int atMostMaxSizeIndexing(int n) const;

146
147
148
  /**
   * @return probability that the number of kmers is 'at_least' or more in a sequence of length 'length'
   */
149
  double getProbabilityAtLeastOrAbove(T kmer, int at_least, int length) const;
150

Mikaël Salson's avatar
Mikaël Salson committed
151
152
153
154
155
156
157
158
159
160
  /**
   * @return the value of k
   */
  int getK() const;

  /**
   * @return the value of s
   */
  int getS() const;

161
162
163
164
165
  /**
   * @return the seed used
   */
  string getSeed() const;

166
167
168
169
  /**
   * @param kmer: a kmer
   * @return one label associated with the kmer
   */
170
  Fasta getLabel(T kmer) const;
171

172
173
174
175
176
  /**
   * @return whether the index differentiate kmer types
   */
  virtual bool hasDifferentKmerTypes() const;

Mikaël Salson's avatar
Mikaël Salson committed
177
178
179
180
  /**
   * @param seq: a sequence
   * @param no_revcomp: force not to revcomp the sequence, even if
   *                    the index was built with revcomp.
181
182
   * @return a vector of length seq.length() - d, where d is the span
   *         of the seed (minus one) or is 0 when it doesn't apply.
183
   * containing for each position the corresponding value in the index.
Mikaël Salson's avatar
Mikaël Salson committed
184
   */
185
  virtual vector<T> getResults(const seqtype &seq, bool no_revcomp=false, string seed="") = 0;
Mikaël Salson's avatar
Mikaël Salson committed
186
187
188
189
190
191
192
193
194
195
196
197

  /**
   * @return true iff the revcomp is indexed
   */
  bool isRevcomp() const;

  /**
   * @param string: a k-mer
   * @return the value in the index associated to the given k-mer.
   *         isRevcomp() ==> the revcomp of the k-mer may be considered
   * @pre word.length() == this->k
   */
198
  virtual T& operator[](seqtype& word) = 0;
Mikaël Salson's avatar
Mikaël Salson committed
199
200
};

201
202
203
template<class T>
IKmerStore<T>::IKmerStore() {
  id = ++last_id;
Mikaël Salson's avatar
Mikaël Salson committed
204
  refs = 0;
205
  finished_building = false;
206
207
}

208
template<class T> int IKmerStore<T>::last_id = 0;
Mikaël Salson's avatar
Mikaël Salson committed
209
210
211
212
213

template <class T> 
class MapKmerStore : public IKmerStore<T>
{
public:	
214
  map<seqtype, T> store;
Mikaël Salson's avatar
Mikaël Salson committed
215
216
217
218
219
220
221

  /**
   * @param bool: if true, will also index revcomp
   * @param seed: the seed
   */
  MapKmerStore(string seed, bool=false);
  MapKmerStore(int k, bool=false);
222

223
  vector<T> getResults(const seqtype &seq, bool no_revcomp=false, string seed="");
224
225
  T& get(seqtype &word);
  T& operator[](seqtype & word);
226
227
228

 private:
  void init();
Mikaël Salson's avatar
Mikaël Salson committed
229
230
231
232
233
234
235
};

template <class T> 
class ArrayKmerStore : public IKmerStore<T>
{
  T* store;
	
236
  int index(const seqtype& word) const;
237
238
239
240
241
242
243
244

  /**
   * Allocates memory for store
   * @throws bad_alloc: when there is not enough memory or when the
   *                    the total number of kmers cannot be stored in an 
   *                    unsigned int
   */
  void init();
Mikaël Salson's avatar
Mikaël Salson committed
245
246
247
248
249
250
251
252
public:
  /**
   * @param bool: if true, will also index revcomp
   * @param seed: the seed
   */
  ArrayKmerStore(string seed, bool=false);
  ArrayKmerStore(int k, bool=false);
  ~ArrayKmerStore();
253

254
  vector<T> getResults(const seqtype &seq, bool no_revcomp=false, const string seed="");
255
256
  T& get(seqtype &word);
  T& operator[](seqtype & word);
257
  T& operator[](int word);
Mikaël Salson's avatar
Mikaël Salson committed
258
259
260
261
262
263
264
265
266
267
};


// IKmerStore

template<class T>
IKmerStore<T>::~IKmerStore(){}

template<class T> 
void IKmerStore<T>::insert(list<Fasta>& input,
268
                           const string &label,
269
270
                           int keep_only,
                           string seed){
Mikaël Salson's avatar
Mikaël Salson committed
271
  for(list<Fasta>::iterator it = input.begin() ; it != input.end() ; it++){
272
    insert(*it, label, keep_only, seed);
Mikaël Salson's avatar
Mikaël Salson committed
273
274
275
276
277
  }
}

template<class T> 
void IKmerStore<T>::insert(Fasta& input,
278
                           const string &label,
279
280
                           int keep_only,
                           string seed){
Mikaël Salson's avatar
Mikaël Salson committed
281
  for (int r = 0; r < input.size(); r++) {
282
    insert(input.sequence(r), label, true, keep_only, seed);
Mikaël Salson's avatar
Mikaël Salson committed
283
  }
284

285
  labels.push_back(make_pair(T(label, 1, seed.size()), input)) ;
286
287

  if (revcomp_indexed  && ! T::hasRevcompSymetry()) {
288
    labels.push_back(make_pair(T(label, -1, seed.size()), input)) ;
289
  }
Mikaël Salson's avatar
Mikaël Salson committed
290
291
292
}

template<class T> 
293
void IKmerStore<T>::insert(const seqtype &sequence,
294
                           const string &label,
295
                           bool ignore_extended_nucleotides,
296
297
                           int keep_only,
                           string seed){
298
299
300
301
302
303
304
  size_t start_indexing = 0;
  size_t end_indexing = sequence.length();
  if (keep_only > 0 && sequence.length() > (size_t)keep_only) {
    start_indexing = sequence.length() - keep_only;
  } else if (keep_only < 0 && sequence.length() > (size_t) -keep_only) {
    end_indexing = -keep_only;
  }
305

306
  if (seed.empty() || ! this->hasDifferentKmerTypes())
307
308
309
    seed = this->seed;

  size_t seed_span = seed.length();
310
311
312
313
314
  size_t size_indexing = end_indexing - start_indexing;
  if (size_indexing > max_size_indexing) {
    max_size_indexing = size_indexing;
  }

315
316
  for(size_t i = start_indexing ; i + seed_span < end_indexing + 1 ; i++) {
    seqtype substr = sequence.substr(i, seed_span);
317
    seqtype kmer = spaced(substr, seed);
318

319
    if (ignore_extended_nucleotides && has_extended_nucleotides(kmer))
320
321
      continue;

Mikaël Salson's avatar
Mikaël Salson committed
322
323
    int strand = 1;
    if (revcomp_indexed && T::hasRevcompSymetry()) {
324
      seqtype rc_kmer = revcomp(kmer);
Mikaël Salson's avatar
Mikaël Salson committed
325
326
327
328
329
      if (rc_kmer.compare(kmer) < 0) {
        strand = -1;
        kmer = rc_kmer;
      }
    }
330
331
332
333
    T &this_kmer = this->get(kmer);
    if (this_kmer.isNull()) {
      nb_kmers_inserted++;
    }
334
    this_kmer += T(label, strand, seed.size());
Mikaël Salson's avatar
Mikaël Salson committed
335
    if (revcomp_indexed && ! T::hasRevcompSymetry()) {
336
      seqtype rc_kmer = spaced(revcomp(substr), seed);
337
338
339
      T &this_rc_kmer = this->get(rc_kmer);
      if (this_rc_kmer.isNull())
        nb_kmers_inserted++;
340
      this_rc_kmer += T(label, -1, seed.size());
Mikaël Salson's avatar
Mikaël Salson committed
341
342
343
    }
  }
}
344

345
template<class T>
346
347
348
void IKmerStore<T>::finish_building() {
  finished_building = true;
}
349

350
351
template<class T>
float IKmerStore<T>::getIndexLoad(const T kmer) const {
352
  float index_load = nb_kmers_inserted / pow(4.0, k);
353
  return (kmer.isUnknown()) ? 1 - index_load : index_load;
354
}
Mikaël Salson's avatar
Mikaël Salson committed
355

356
357
358
359
360
361
362
template<class T>
int IKmerStore<T>::atMostMaxSizeIndexing(int n) const {
  if (!max_size_indexing || n < max_size_indexing)
    return n ;

  return max_size_indexing ;
}
363
364

template<class T>
365
double IKmerStore<T>::getProbabilityAtLeastOrAbove(const T kmer, int at_least, int length) const {
366
367
368

  // n: number of kmers in the sequence
  int n = length - getS() + 1;
369
  float index_load = getIndexLoad(kmer) ;
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384

  double proba = 0;
  double probability_having_system = pow(index_load, at_least);
  double probability_not_having_system = pow(1 - index_load, n - at_least);
  for (int i=at_least; i<=n; i++) {
    proba += nChoosek(n, i) * probability_having_system * probability_not_having_system;
    probability_having_system *= index_load;
    probability_not_having_system /= (1 - index_load);
  }

  return proba;
}



Mikaël Salson's avatar
Mikaël Salson committed
385
386
387
388
389
390
391
392
393
394
template<class T>
int IKmerStore<T>::getK() const {
  return k;
}

template<class T>
int IKmerStore<T>::getS() const {
  return s;
}

395
396
397
398
399
template<class T>
string IKmerStore<T>::getSeed() const {
  return seed;
}

400
template<class T>
401
402
Fasta IKmerStore<T>::getLabel(T kmer) const {
  for (typename list< pair<T, Fasta> >::const_iterator it = labels.begin(); it != labels.end(); ++it)
403
404
    if (it->first == kmer)
      return it->second ;
405
406
407
408
409
410
  // Nothing interesting found
  // Try by ignoring length if the index is not able to deal with different lengths
  if (! hasDifferentKmerTypes() && kmer.getLength() != (unsigned char)~0) {
    kmer.setLength(~0);
    return getLabel(kmer);
  }
411
  return FASTA_AMBIGUOUS ;
412
}
413

414
415
416
417
418
template<class T>
bool IKmerStore<T>::hasDifferentKmerTypes() const {
  return false;
}

419
// .getResults()
Mikaël Salson's avatar
Mikaël Salson committed
420
template<class T>
421
vector<T> MapKmerStore<T>::getResults(const seqtype &seq, bool no_revcomp, string seed) {
422

Mikaël Salson's avatar
Mikaël Salson committed
423
  string local_seed = seed;
424
  if (! seed.size())
Mikaël Salson's avatar
Mikaël Salson committed
425
426
    local_seed = IKmerStore<T>::seed;
  int s = local_seed.size();
427

Mikaël Salson's avatar
Mikaël Salson committed
428
429
430
431
432
  if ((int)seq.length() < s - 1) {
    return vector<T>(0);
  }
  vector<T> result(seq.length() - s + 1);
  for (size_t i=0; i + s < seq.length() + 1; i++) {
Mikaël Salson's avatar
Mikaël Salson committed
433
    seqtype kmer = spaced(seq.substr(i, s), local_seed);
434
    //    seqtype kmer = seq.substr(i, s);
Mikaël Salson's avatar
Mikaël Salson committed
435
    // cout << kmer << endl << kmer0 << endl << endl ;
436
    if (IKmerStore<T>::revcomp_indexed && no_revcomp) {
Mikaël Salson's avatar
Mikaël Salson committed
437
438
439
440
441
442
443
444
      result[i] = get(kmer);
    } else {
      result[i] = (*this)[kmer];
    }
  }
  return result;
}

445
template<class T>
446
vector<T> ArrayKmerStore<T>::getResults(const seqtype &seq, bool no_revcomp, string seed) {
Mikaël Salson's avatar
Mikaël Salson committed
447
448

  string local_seed = seed;
449
  if (! seed.size())
Mikaël Salson's avatar
Mikaël Salson committed
450
451
    local_seed = IKmerStore<T>::seed;
  int s = local_seed.size();
452
453
454
455
456
457
458
459
460
461
462
463

  int N = (int)seq.length();

  if (N < s - 1) {
    return vector<T>(0);
  }
  vector<T> result(N - s + 1);

  /* Read once the sequence, convert it to int* */
  int* intseq = new int[N];
  for (int i=0; i<N; i++)
    {
464
      intseq[i] = nuc_to_int(seq[i]);
465
466
467
468
    }

  /* Compute results */
  for (size_t i=0; (int) i+s < N+1; i++) {
Mikaël Salson's avatar
Mikaël Salson committed
469
    int kmer = spaced_int(intseq + i, local_seed);
470
471
472
473
    if (IKmerStore<T>::revcomp_indexed && no_revcomp) {
      result[i] = store[kmer]; // getfromint(kmer); // store[kmer];
      // cout << i << "/" << N << "  " << kmer << result[i] << endl ;
    } else {
474
      result[i] = (*this)[kmer]; // Deals with revcomp
475
476
477
478
479
480
481
    }
  }

  delete[] intseq ;
  return result;
}

Mikaël Salson's avatar
Mikaël Salson committed
482
483
484
485
486
487
488
489
490

template<class T>
bool IKmerStore<T>::isRevcomp() const {
  return revcomp_indexed;
}

// MapKmerStore

template <class T>
491
MapKmerStore<T>::MapKmerStore(string seed, bool revcomp):IKmerStore<T>(){
Mikaël Salson's avatar
Mikaël Salson committed
492
493
494
495
496
  this->seed = seed;   
  int k = seed_weight(seed);
  this->k = k;
  this->s = seed.size();
  this->revcomp_indexed = revcomp;
497
  init();
Mikaël Salson's avatar
Mikaël Salson committed
498
499
500
}

template <class T>
501
MapKmerStore<T>::MapKmerStore(int k, bool revcomp):IKmerStore<T>(){
Mikaël Salson's avatar
Mikaël Salson committed
502
503
504
505
  this->seed = seed_contiguous(k);
  this->k = k;
  this->s = k;
  this->revcomp_indexed = revcomp;
506
507
508
509
510
511
  init();
}

template <class T>
void MapKmerStore<T>::init() {
  this->nb_kmers_inserted = 0;
512
  this->max_size_indexing = 0;
Mikaël Salson's avatar
Mikaël Salson committed
513
514
515
}

template <class T> 
516
T& MapKmerStore<T>::get(seqtype& word){
Mikaël Salson's avatar
Mikaël Salson committed
517
518
519
520
  return store[word];
}

template <class T> 
521
T& MapKmerStore<T>::operator[](seqtype& word){
Mikaël Salson's avatar
Mikaël Salson committed
522
  if (this->isRevcomp() && T::hasRevcompSymetry()) {
523
    seqtype rc_kmer = revcomp(word);
Mikaël Salson's avatar
Mikaël Salson committed
524
525
526
527
528
529
530
531
532
    if (rc_kmer.compare(word) < 0)
      word = rc_kmer;
  }
  return store[word];
}

// ArrayKmerStore

template <class T> 
533
ArrayKmerStore<T>::ArrayKmerStore(int k, bool revcomp):IKmerStore<T>() {
Mikaël Salson's avatar
Mikaël Salson committed
534
535
536
537
  this->seed = seed_contiguous(k);
  this->k = k;
  this->s = k;
  this->revcomp_indexed = revcomp;
538
  init();
Mikaël Salson's avatar
Mikaël Salson committed
539
540
541
542
}


template <class T> 
543
ArrayKmerStore<T>::ArrayKmerStore(string seed, bool revcomp):IKmerStore<T>(){
Mikaël Salson's avatar
Mikaël Salson committed
544
545
546
547
548
  this->seed = seed; 
  int k = seed_weight(seed);
  this->k = k;
  this->s = seed.size();
  this->revcomp_indexed = revcomp;
549
550
551
552
553
  init();
}

template <class T>
void ArrayKmerStore<T>::init() {
554
  this->nb_kmers_inserted = 0;
555
  this->max_size_indexing = 0;
556
  if ((size_t)(this->k << 1) >= sizeof(int) * 8)
Mikaël Salson's avatar
Mikaël Salson committed
557
    throw std::bad_alloc();
558
  store = new T[(unsigned int)1 << (this->k << 1)];
Mikaël Salson's avatar
Mikaël Salson committed
559
560
561
562
563
564
565
566
567
568
569
570
571
572
  if (! store)
    throw std::bad_alloc();
}

template <class T> 
ArrayKmerStore<T>::~ArrayKmerStore(){
  delete [] store;
}

/**	Returns index of word in an array of size 4^k
	Considering B(A) = 0, B(C) = 1, B(T) = 2, B(G) = 3
	index_word = \sum_{i=0}^{k-1}B(word[i])*4^(k-i-1)
**/
template <class T> 
573
int ArrayKmerStore<T>::index(const seqtype& word) const{
Mikaël Salson's avatar
Mikaël Salson committed
574
  return dna_to_int(word, this->k);
Mikaël Salson's avatar
Mikaël Salson committed
575
576
577
}

template <class T> 
578
T& ArrayKmerStore<T>::get(seqtype& word){
Mikaël Salson's avatar
Mikaël Salson committed
579
580
581
582
  return store[index(word)];
}

template <class T> 
583
T& ArrayKmerStore<T>::operator[](seqtype& word){
584
585
586
587
588
  return (*this)[index(word)];
}

template <class T> 
T& ArrayKmerStore<T>::operator[](int word){
Mikaël Salson's avatar
Mikaël Salson committed
589
  if (this->isRevcomp() && T::hasRevcompSymetry()) {
590
591
    int rc_kmer = revcomp_int(word, IKmerStore<T>::k);
    if (rc_kmer < word)
Mikaël Salson's avatar
Mikaël Salson committed
592
593
      word = rc_kmer;
  }
594
  return store[word];
Mikaël Salson's avatar
Mikaël Salson committed
595
596
597
}

#endif