Allow more than 2^32 sequences to be clustered

src/CMakeLists.txt

@@ -68,8 +68,8 @@ add_dependencies(mmseqs-framework generated)
 #    endif ()
 #endif ()
 
-append_target_property(mmseqs-framework COMPILE_FLAGS ${MMSEQS_CXX_FLAGS} -pedantic -Wall -Wextra -Wdisabled-optimization)
-append_target_property(mmseqs-framework LINK_FLAGS ${MMSEQS_CXX_FLAGS} -pedantic -Wall -Wextra -Wdisabled-optimization)
+append_target_property(mmseqs-framework COMPILE_FLAGS ${MMSEQS_CXX_FLAGS} -pedantic -Wall -Wshorten-64-to-32 -Wextra -Wdisabled-optimization)
+append_target_property(mmseqs-framework LINK_FLAGS ${MMSEQS_CXX_FLAGS} -pedantic -Wall -Wshorten-64-to-32  -Wextra -Wdisabled-optimization)
 if (NOT EMSCRIPTEN)
     append_target_property(mmseqs-framework COMPILE_FLAGS -fno-exceptions)
     append_target_property(mmseqs-framework LINK_FLAGS -fno-exceptions)

src/alignment/Alignment.cpp

@@ -55,13 +55,13 @@ Alignment::Alignment(const std::string &querySeqDB, const std::string &targetSeq
         }
     }
 
-    uint16_t extended = DBReader<unsigned int>::getExtendedDbtype(FileUtil::parseDbType(prefDB.c_str()));
+    uint16_t extended = DBReader<KeyType>::getExtendedDbtype(FileUtil::parseDbType(prefDB.c_str()));
     bool touch = (par.preloadMode != Parameters::PRELOAD_MODE_MMAP);
     tDbrIdx = new IndexReader(targetSeqDB, par.threads,
                               extended & Parameters::DBTYPE_EXTENDED_INDEX_NEED_SRC ? IndexReader::SRC_SEQUENCES : IndexReader::SEQUENCES,
                               (touch) ? (IndexReader::PRELOAD_INDEX | IndexReader::PRELOAD_DATA) : 0);
     tdbr = tDbrIdx->sequenceReader;
-    targetSeqType = tdbr->getDbtype();
+    targetSeqType = tdbr->getDbtype();  
     sameQTDB = (targetSeqDB.compare(querySeqDB) == 0);
     if (sameQTDB == true) {
         qDbrIdx = tDbrIdx;
@@ -136,8 +136,8 @@ Alignment::Alignment(const std::string &querySeqDB, const std::string &targetSeq
     Debug(Debug::INFO) << "Query database size: "  << qdbr->getSize() << " type: " << Parameters::getDbTypeName(querySeqType) << "\n";
     Debug(Debug::INFO) << "Target database size: " << tdbr->getSize() << " type: " << Parameters::getDbTypeName(targetSeqType) << "\n";
 
-    prefdbr = new DBReader<unsigned int>(prefDB.c_str(), prefDBIndex.c_str(), threads, DBReader<unsigned int>::USE_DATA|DBReader<unsigned int>::USE_INDEX);
-    prefdbr->open(DBReader<unsigned int>::LINEAR_ACCCESS);
+    prefdbr = new DBReader<KeyType>(prefDB.c_str(), prefDBIndex.c_str(), threads, DBReader<KeyType>::USE_DATA | DBReader<KeyType>::USE_INDEX);
+    prefdbr->open(DBReader<KeyType>::LINEAR_ACCCESS);
     reversePrefilterResult = Parameters::isEqualDbtype(prefdbr->getDbtype(), Parameters::DBTYPE_PREFILTER_REV_RES);
 
     correlationScoreWeight = par.correlationScoreWeight;
@@ -250,7 +250,7 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
     if (alignmentOutputMode == Parameters::ALIGNMENT_OUTPUT_CLUSTER) {
         dbtype = Parameters::DBTYPE_CLUSTER_RES;
     }
-    dbtype = DBReader<unsigned int>::setExtendedDbtype(dbtype, DBReader<unsigned int>::getExtendedDbtype(prefdbr->getDbtype()));
+    dbtype = DBReader<KeyType>::setExtendedDbtype(dbtype, DBReader<KeyType>::getExtendedDbtype(prefdbr->getDbtype()));
     DBWriter dbw(outDB.c_str(), outDBIndex.c_str(), threads, compressed, dbtype);
     dbw.open();
 
@@ -316,11 +316,11 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
                 // get the prefiltering list
                 char *data, *origData;
                 data = origData = prefdbr->getData(id, thread_idx);
-                unsigned int queryDbKey = prefdbr->getDbKey(id);
+                KeyType queryDbKey = prefdbr->getDbKey(id);
                 size_t origQueryLen = 0;
                 // only load query data if data != \0
                 if (*data != '\0') {
-                    size_t qId = qdbr->getId(queryDbKey);
+                    KeyType qId = qdbr->getId(queryDbKey);
                     char *querySeqData = qdbr->getData(qId, thread_idx);
                     if (querySeqData == NULL) {
                         Debug(Debug::ERROR) << "Query sequence " << queryDbKey
@@ -345,7 +345,7 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
                 unsigned int rejected = 0;
                 while (*data != '\0' && passedNum < maxAccept && rejected < maxReject) {
                     Util::parseKey(data, buffer);
-                    const unsigned int dbKey = (unsigned int) strtoul(buffer, NULL, 10);
+                    const KeyType dbKey = (KeyType) strtoul(buffer, NULL, 10);
                     size_t elements = Util::getWordsOfLine(data, words, 10);
 
                     short diagonal = 0;
@@ -358,7 +358,7 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
                     }
                     data = Util::skipLine(data);
 
-                    size_t dbId = tdbr->getId(dbKey);
+                    KeyType dbId = tdbr->getId(dbKey);
                     char *dbSeqData = tdbr->getData(dbId, thread_idx);
                     if (dbSeqData == NULL) {
                         Debug(Debug::ERROR) << "Sequence " << dbKey << " is required in the prefiltering, but is not contained in the target sequence database!\nPlease check your database.\n";
@@ -409,7 +409,7 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
                     realigner->initQuery(&qSeq);
                     int realignAccepted = 0;
                     for (size_t result = 0; result < swResults.size() && realignAccepted < realignMaxSeqs; result++) {
-                        size_t dbId = tdbr->getId(swResults[result].dbKey);
+                        KeyType dbId = tdbr->getId(swResults[result].dbKey);
                         char *dbSeqData = tdbr->getData(dbId, thread_idx);
                         if (dbSeqData == NULL) {
                             Debug(Debug::ERROR) << "Sequence " << swResults[result].dbKey <<" is required in the prefiltering, but is not contained in the target sequence database!\nPlease check your database.\n";
@@ -443,8 +443,8 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
 
                 if (lcaAlign == true && swRealignResults.size() > 0) {
                     Matcher::result_t& topHit = swRealignResults[0];
-                    const unsigned int topHitKey = topHit.dbKey;
-                    size_t dbId = tdbr->getId(topHitKey);
+                    const KeyType topHitKey = topHit.dbKey;
+                    KeyType dbId = tdbr->getId(topHitKey);
                     char *qSeqData = tdbr->getData(dbId, thread_idx);
                     if (qSeqData == NULL) {
                         Debug(Debug::ERROR) << "Sequence " << topHitKey << " is required in the prefiltering, but is not contained in the target sequence database!\nPlease check your database.\n";
@@ -460,7 +460,7 @@ void Alignment::run(const std::string &outDB, const std::string &outDBIndex, con
                     unsigned int rejected = 0;
                     while (*data != '\0' && rejected < maxReject) {
                         Util::parseKey(data, buffer);
-                        const unsigned int dbKey = (unsigned int) strtoul(buffer, NULL, 10);
+                        const KeyType dbKey = (KeyType) strtoul(buffer, NULL, 10);
 //                        size_t elements = Util::getWordsOfLine(data, words, 10);
 //                        short diagonal = 0;
 //                        bool isReverse = false;
@@ -566,7 +566,7 @@ bool Alignment::checkCriteria(Matcher::result_t &res, bool isIdentity, double ev
     }
 }
 
-void Alignment::computeAlternativeAlignment(unsigned int queryDbKey, Sequence &dbSeq, std::vector<Matcher::result_t> &swResults,
+void Alignment::computeAlternativeAlignment(KeyType queryDbKey, Sequence &dbSeq, std::vector<Matcher::result_t> &swResults,
                                             Matcher &matcher, float covThr, float evalThr, int swMode, int thread_idx) {
     const unsigned char xIndex = m->aa2num[static_cast<int>('X')];
     const size_t firstItResSize = swResults.size();
@@ -575,7 +575,7 @@ void Alignment::computeAlternativeAlignment(unsigned int queryDbKey, Sequence &d
         if (isIdentity == true) {
             continue;
         }
-        size_t dbId = tdbr->getId(swResults[i].dbKey);
+        KeyType dbId = tdbr->getId(swResults[i].dbKey);
         char *dbSeqData = tdbr->getData(dbId, thread_idx);
         if (dbSeqData == NULL) {
             Debug(Debug::ERROR) << "Sequence " << swResults[i].dbKey << " is required in the prefiltering, but is not contained in the target sequence database!\nPlease check your database.\n";

src/alignment/Alignment.h

@@ -110,19 +110,19 @@ class Alignment {
     // needed for realignment
     BaseMatrix *realign_m;
 
-    DBReader<unsigned int> *qdbr;
+    DBReader<KeyType> *qdbr;
     IndexReader * qDbrIdx;
 
-    DBReader<unsigned int> *tdbr;
+    DBReader<KeyType> *tdbr;
     IndexReader * tDbrIdx;
 
-    DBReader<unsigned int> *prefdbr;
+    DBReader<KeyType> *prefdbr;
 
     bool reversePrefilterResult;
 
     static size_t estimateHDDMemoryConsumption(int dbSize, int maxSeqs);
 
-    void computeAlternativeAlignment(unsigned int queryDbKey, Sequence &dbSeq,
+    void computeAlternativeAlignment(KeyType queryDbKey, Sequence &dbSeq,
                                      std::vector<Matcher::result_t> &vector, Matcher &matcher,
                                      float covThr, float evalThr, int swMode, int thread_idx);
 };

src/alignment/CompressedA3M.cpp

@@ -36,8 +36,8 @@ void readU32(const char **ptr, uint32_t &result) {
 }
 
 std::string CompressedA3M::extractA3M(const char *data, size_t data_size,
-                                      DBReader<unsigned int>& sequenceReader,
-                                      DBReader<unsigned int>& headerReader, int thread_idx) {
+                                      DBReader<KeyType>& sequenceReader,
+                                      DBReader<KeyType>& headerReader, int thread_idx) {
     std::ostringstream output;
 
     //read stuff till compressed part
@@ -146,8 +146,8 @@ std::string CompressedA3M::extractA3M(const char *data, size_t data_size,
     return output.str();
 }
 
-void CompressedA3M::extractMatcherResults(unsigned int &key, std::vector<Matcher::result_t> &results,
-        const char *data, size_t dataSize, DBReader<unsigned int> &sequenceReader, bool skipFirst) {
+void CompressedA3M::extractMatcherResults(KeyType &key, std::vector<Matcher::result_t> &results,
+                                          const char *data, size_t dataSize, DBReader<KeyType> &sequenceReader, bool skipFirst) {
     //read stuff till compressed part
     char lastChar = '\0';
     size_t index = 0;

src/alignment/CompressedA3M.h

@@ -11,11 +11,11 @@ class CompressedA3M {
     static void hitToBuffer(unsigned int targetId, const Matcher::result_t& hit, std::string& buffer);
 
     static std::string extractA3M(const char *data, size_t data_size,
-                                  DBReader<unsigned int>& sequenceReader,
-                                  DBReader<unsigned int>& headerReader, int thread_idx);
+                                  DBReader<KeyType>& sequenceReader,
+                                  DBReader<KeyType>& headerReader, int thread_idx);
 
-    static void extractMatcherResults(unsigned int &key, std::vector<Matcher::result_t> &results,
-                                      const char *data, size_t dataSize, DBReader<unsigned int>& sequenceReader, bool skipFirst);
+    static void extractMatcherResults(KeyType &key, std::vector<Matcher::result_t> &results,
+                                      const char *data, size_t dataSize, DBReader<KeyType>& sequenceReader, bool skipFirst);
 };
 
 #endif

src/alignment/Fwbw.cpp

@@ -1071,12 +1071,12 @@ int fwbw(int argc, const char **argv, const Command &command) {
     //Prepare the parameters & DB
     Parameters &par = Parameters::getInstance();
     par.parseParameters(argc, argv, command, true, 0, MMseqsParameter::COMMAND_ALIGN);
-    DBReader<unsigned int> qdbr(par.db1.c_str(), par.db1Index.c_str(), par.threads, DBReader<unsigned int>::USE_DATA | DBReader<unsigned int>::USE_INDEX);
-    qdbr.open(DBReader<unsigned int>::NOSORT);
-    DBReader<unsigned int> tdbr(par.db2.c_str(), par.db2Index.c_str(), par.threads, DBReader<unsigned int>::USE_DATA | DBReader<unsigned int>::USE_INDEX);
-    tdbr.open(DBReader<unsigned int>::NOSORT);
-    DBReader<unsigned int> alnRes (par.db3.c_str(), par.db3Index.c_str(), par.threads, DBReader<unsigned int>::USE_DATA | DBReader<unsigned int>::USE_INDEX);
-    alnRes.open(DBReader<unsigned int>::LINEAR_ACCCESS);
+    DBReader<KeyType> qdbr(par.db1.c_str(), par.db1Index.c_str(), par.threads, DBReader<KeyType>::USE_DATA | DBReader<KeyType>::USE_INDEX);
+    qdbr.open(DBReader<KeyType>::NOSORT);
+    DBReader<KeyType> tdbr(par.db2.c_str(), par.db2Index.c_str(), par.threads, DBReader<KeyType>::USE_DATA | DBReader<KeyType>::USE_INDEX);
+    tdbr.open(DBReader<KeyType>::NOSORT);
+    DBReader<KeyType> alnRes (par.db3.c_str(), par.db3Index.c_str(), par.threads, DBReader<KeyType>::USE_DATA | DBReader<KeyType>::USE_INDEX);
+    alnRes.open(DBReader<KeyType>::LINEAR_ACCCESS);
 
     DBWriter fwbwAlnWriter(par.db4.c_str(), par.db4Index.c_str(), par.threads, par.compressed, Parameters::DBTYPE_ALIGNMENT_RES);
     fwbwAlnWriter.open();
@@ -1116,8 +1116,8 @@ int fwbw(int argc, const char **argv, const Command &command) {
 #pragma omp for schedule(dynamic,1)
             for (size_t id = start; id < (start + bucketSize); id++) {
                 progress.updateProgress();
-                unsigned int key = alnRes.getDbKey(id);
-                const size_t queryId = qdbr.getId(key);
+                KeyType key = alnRes.getDbKey(id);
+                const KeyType queryId = qdbr.getId(key);
                 char *alnData = alnRes.getData(id, thread_idx);
                 localFwbwResults.clear();
 
@@ -1130,8 +1130,8 @@ int fwbw(int argc, const char **argv, const Command &command) {
 
                 while (*alnData != '\0'){
                     Util::parseKey(alnData, entrybuffer);
-                    unsigned int targetKey = (unsigned int) strtoul(entrybuffer, NULL, 10);
-                    const size_t targetId = tdbr.getId(targetKey);
+                    KeyType targetKey = (KeyType) strtoul(entrybuffer, NULL, 10);
+                    const KeyType targetId = tdbr.getId(targetKey);
                     const char* targetSeq = tdbr.getData(targetId, thread_idx);
                     size_t targetLen = tdbr.getSeqLen(targetId);
 

src/alignment/Matcher.cpp

@@ -213,7 +213,7 @@ Matcher::result_t Matcher::parseAlignmentRecord(const char *data, bool readCompr
     strncpy(key, data, keySize);
     key[keySize] = '\0';
 
-    unsigned int targetId = Util::fast_atoi<unsigned int>(key);
+    KeyType targetId = Util::fast_atoi<KeyType>(key);
     int score = Util::fast_atoi<int>(entry[1]);
     double seqId;
     fast_float::from_chars(entry[2], entry[3] - 1, seqId);
@@ -279,7 +279,10 @@ Matcher::result_t Matcher::parseAlignmentRecord(const char *data, bool readCompr
 
 size_t Matcher::resultToBuffer(char * buff1, const result_t &result, bool addBacktrace, bool compress, bool addOrfPosition) {
     char * basePos = buff1;
-    char * tmpBuff = Itoa::u32toa_sse2((uint32_t) result.dbKey, buff1);
+    constexpr bool keyIsU32 = std::is_same<KeyType, unsigned int>::value;
+    char * tmpBuff = keyIsU32
+              ? Itoa::u32toa_sse2(static_cast<std::uint32_t>(result.dbKey), buff1)
+              : Itoa::u64toa_sse2(static_cast<std::uint64_t>(result.dbKey), buff1);
     *(tmpBuff-1) = '\t';
     tmpBuff = Itoa::i32toa_sse2(result.score, tmpBuff);
     *(tmpBuff-1) = '\t';

src/alignment/Matcher.h

@@ -30,7 +30,7 @@ class Matcher{
     const static int ALN_RES_WITH_ORF_AND_BT_COL_CNT = 15;
 
     struct result_t {
-        unsigned int dbKey;
+        KeyType dbKey;
         int score;
         float qcov;
         float dbcov;
@@ -48,7 +48,7 @@ class Matcher{
         int dbOrfStartPos;
         int dbOrfEndPos;
         std::string backtrace;
-        result_t(unsigned int dbkey,int score,
+        result_t(KeyType dbkey,int score,
                  float qcov, float dbcov,
                  float seqId, double eval,
                  unsigned int alnLength,
@@ -70,7 +70,7 @@ class Matcher{
                                           dbOrfStartPos(dbOrfStartPos), dbOrfEndPos(dbOrfEndPos),
                                           backtrace(backtrace) {};
 
-        result_t(unsigned int dbkey,int score,
+        result_t(KeyType dbkey,int score,
                  float qcov, float dbcov,
                  float seqId, double eval,
                  unsigned int alnLength,

src/alignment/rescorediagonal.cpp

@@ -44,13 +44,13 @@ float parsePrecisionLib(const std::string &scoreFile, double targetSeqid, double
 
 int doRescorediagonal(Parameters &par,
                       DBWriter &resultWriter,
-                      DBReader<unsigned int> &resultReader,
+                      DBReader<KeyType> &resultReader,
               const size_t dbFrom, const size_t dbSize) {
 
 
     IndexReader * qDbrIdx = NULL;
-    DBReader<unsigned int> * qdbr = NULL;
-    DBReader<unsigned int> * tdbr = NULL;
+    DBReader<KeyType> * qdbr = NULL;
+    DBReader<KeyType> * tdbr = NULL;
     bool touch = (par.preloadMode != Parameters::PRELOAD_MODE_MMAP);
     IndexReader * tDbrIdx = new IndexReader(par.db2, par.threads, IndexReader::SEQUENCES,   (touch) ? (IndexReader::PRELOAD_INDEX | IndexReader::PRELOAD_DATA) : 0 );
     int querySeqType = 0;
@@ -147,7 +147,7 @@ int doRescorediagonal(Parameters &par,
                 progress.updateProgress();
 
                 char *data = resultReader.getData(id, thread_idx);
-                size_t queryKey = resultReader.getDbKey(id);
+                KeyType queryKey = resultReader.getDbKey(id);
 
                 char *querySeq = NULL;
                 std::string queryToWrap; // needed only for wrapped end-start scoring
@@ -201,7 +201,7 @@ int doRescorediagonal(Parameters &par,
                         }
                     }
 
-                    unsigned int targetId = tdbr->getId(results[entryIdx].seqId);
+                    KeyType targetId = tdbr->getId(results[entryIdx].seqId);
                     const bool isIdentity = (queryId == targetId && (par.includeIdentity || sameQTDB)) ? true : false;
                     char *targetSeq = tdbr->getData(targetId, thread_idx);
                     int dbLen = static_cast<int>(tdbr->getSeqLen(targetId));
@@ -391,8 +391,8 @@ int rescorediagonal(int argc, const char **argv, const Command &command) {
     Parameters &par = Parameters::getInstance();
     par.parseParameters(argc, argv, command, true, 0, 0);
 
-    DBReader<unsigned int> resultReader(par.db3.c_str(), par.db3Index.c_str(), par.threads, DBReader<unsigned int>::USE_INDEX|DBReader<unsigned int>::USE_DATA);
-    resultReader.open(DBReader<unsigned int>::LINEAR_ACCCESS);
+    DBReader<KeyType> resultReader(par.db3.c_str(), par.db3Index.c_str(), par.threads, DBReader<KeyType>::USE_INDEX | DBReader<KeyType>::USE_DATA);
+    resultReader.open(DBReader<KeyType>::LINEAR_ACCCESS);
     int dbtype = resultReader.getDbtype(); // this is DBTYPE_PREFILTER_RES || DBTYPE_PREFILTER_REV_RES
     if(par.rescoreMode == Parameters::RESCORE_MODE_ALIGNMENT ||
        par.rescoreMode == Parameters::RESCORE_MODE_END_TO_END_ALIGNMENT ||