pass a char buffer to simplecpp instead of a stream

democlient/democlient.cpp

@@ -66,8 +66,8 @@ class CppcheckExecutor : public ErrorLogger {
         , cppcheck(settings, supprs, *this, false, nullptr)
     {}
 
-    void run(const char code[]) {
-        cppcheck.check(FileWithDetails("test.cpp", Standards::Language::CPP, 0), code);
+    void run(const char* code) {
+        cppcheck.checkBuffer(FileWithDetails("test.cpp", Standards::Language::CPP, 0), reinterpret_cast<const uint8_t*>(code), strlen(code));
     }
 
     void reportOut(const std::string & /*outmsg*/, Color /*c*/) override {}

gui/mainwindow.cpp

@@ -729,8 +729,11 @@ void MainWindow::analyzeCode(const QString& code, const QString& filename)
     checkLockDownUI();
     clearResults();
     mUI->mResults->checkingStarted(1);
-    // TODO: apply enforcedLanguage?
-    cppcheck.check(FileWithDetails(filename.toStdString(), Path::identify(filename.toStdString(), false), 0), code.toStdString());
+    {
+        const std::string code_s = code.toStdString();
+        // TODO: apply enforcedLanguage?
+        cppcheck.checkBuffer(FileWithDetails(filename.toStdString(), Path::identify(filename.toStdString(), false), 0), reinterpret_cast<const std::uint8_t*>(code_s.data()), code_s.size());
+    }
     analysisDone();
 
     // Expand results

lib/cppcheck.cpp

@@ -799,10 +799,9 @@ unsigned int CppCheck::check(const FileWithDetails &file)
     return returnValue;
 }
 
-unsigned int CppCheck::check(const FileWithDetails &file, const std::string &content)
+unsigned int CppCheck::checkBuffer(const FileWithDetails &file, const uint8_t* data, std::size_t size)
 {
-    std::istringstream iss(content);
-    return checkFile(file, "", 0, &iss);
+    return checkBuffer(file, "", 0, data, size);
 }
 
 unsigned int CppCheck::check(const FileSettings &fs)
@@ -851,14 +850,6 @@ unsigned int CppCheck::check(const FileSettings &fs)
     return returnValue;
 }
 
-static simplecpp::TokenList createTokenList(const std::string& filename, std::vector<std::string>& files, simplecpp::OutputList* outputList, std::istream* fileStream)
-{
-    if (fileStream)
-        return {*fileStream, files, filename, outputList};
-
-    return {filename, files, outputList};
-}
-
 std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const simplecpp::TokenList& tokens, const std::string& filePath) const
 {
     std::ostringstream toolinfo;
@@ -880,7 +871,23 @@ std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const simp
     return preprocessor.calculateHash(tokens, toolinfo.str());
 }
 
-unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string &cfgname, int fileIndex, std::istream* fileStream)
+unsigned int CppCheck::checkBuffer(const FileWithDetails &file, const std::string &cfgname, int fileIndex, const uint8_t* data, std::size_t size)
+{
+    const auto f = [&file, data, size](std::vector<std::string>& files, simplecpp::OutputList* outputList) {
+        return simplecpp::TokenList{data, size, files, file.spath(), outputList};
+    };
+    return checkInternal(file, cfgname, fileIndex, f);
+}
+
+unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string &cfgname, int fileIndex)
+{
+    const auto f = [&file](std::vector<std::string>& files, simplecpp::OutputList* outputList) {
+        return simplecpp::TokenList{file.spath(), files, outputList};
+    };
+    return checkInternal(file, cfgname, fileIndex, f);
+}
+
+unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::string &cfgname, int fileIndex, const CreateTokenListFn& createTokenList)
 {
     // TODO: move to constructor when CppCheck no longer owns the settings
     if (mSettings.checks.isEnabled(Checks::unusedFunction) && !mUnusedFunctionsCheck)
@@ -931,24 +938,13 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
                 std::size_t hash = 0;
                 // markup files are special and do not adhere to the enforced language
                 TokenList tokenlist{mSettings, Standards::Language::C};
-                if (fileStream) {
-                    std::vector<std::string> files;
-                    simplecpp::TokenList tokens(*fileStream, files, file.spath());
-                    if (analyzerInformation) {
-                        const Preprocessor preprocessor(mSettings, mErrorLogger, Standards::Language::C);
-                        hash = calculateHash(preprocessor, tokens);
-                    }
-                    tokenlist.createTokens(std::move(tokens));
-                }
-                else {
-                    std::vector<std::string> files;
-                    simplecpp::TokenList tokens(file.spath(), files);
-                    if (analyzerInformation) {
-                        const Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
-                        hash = calculateHash(preprocessor, tokens);
-                    }
-                    tokenlist.createTokens(std::move(tokens));
+                std::vector<std::string> files;
+                simplecpp::TokenList tokens = createTokenList(files, nullptr);
+                if (analyzerInformation) {
+                    const Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
+                    hash = calculateHash(preprocessor, tokens);
                 }
+                tokenlist.createTokens(std::move(tokens));
                 // this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
                 Tokenizer tokenizer(std::move(tokenlist), mErrorLogger);
                 mUnusedFunctionsCheck->parseTokens(tokenizer, mSettings);
@@ -967,7 +963,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
 
         simplecpp::OutputList outputList;
         std::vector<std::string> files;
-        simplecpp::TokenList tokens1 = createTokenList(file.spath(), files, &outputList, fileStream);
+        simplecpp::TokenList tokens1 = createTokenList(files, &outputList);
 
         // If there is a syntax error, report it and stop
         const auto output_it = std::find_if(outputList.cbegin(), outputList.cend(), [](const simplecpp::Output &output){
@@ -1075,8 +1071,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
                     code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
             }
             TokenList tokenlist(mSettings, file.lang());
-            std::istringstream istr2(code);
-            tokenlist.createTokens(istr2); // TODO: check result?
+            tokenlist.createTokensFromBuffer(code.data(), code.size()); // TODO: check result?
             executeRules("define", tokenlist);
         }
 #endif

lib/cppcheck.h

@@ -45,7 +45,10 @@ class Settings;
 struct Suppressions;
 class Preprocessor;
 
-namespace simplecpp { class TokenList; }
+namespace simplecpp {
+    class TokenList;
+    struct Output;
+}
 
 /// @addtogroup Core
 /// @{
@@ -100,12 +103,13 @@ class CPPCHECKLIB CppCheck {
      * the disk but the content is given in @p content. In errors the @p path
      * is used as a filename.
      * @param file The file to check.
-     * @param content File content as a string.
+     * @param data File content as a buffer.
+     * @param size Size of buffer.
      * @return amount of errors found or 0 if none were found.
      * @note You must set settings before calling this function (by calling
      *  settings()).
      */
-    unsigned int check(const FileWithDetails &file, const std::string &content);
+    unsigned int checkBuffer(const FileWithDetails &file, const uint8_t* data, std::size_t size);
 
     /**
      * @brief Returns current version number as a string.
@@ -174,14 +178,35 @@ class CPPCHECKLIB CppCheck {
      */
     std::size_t calculateHash(const Preprocessor &preprocessor, const simplecpp::TokenList &tokens, const std::string& filePath = {}) const;
 
+    /**
+     * @brief Check a file
+     * @param file the file
+     * @param cfgname  cfg name
+     * @return number of errors found
+     */
+    unsigned int checkFile(const FileWithDetails& file, const std::string &cfgname, int fileIndex);
+
+    /**
+     * @brief Check a file using buffer
+     * @param file the file
+     * @param cfgname  cfg name
+     * @param data the data to be read
+     * @param size the size of the data to be read
+     * @return number of errors found
+     */
+    unsigned int checkBuffer(const FileWithDetails& file, const std::string &cfgname, int fileIndex, const uint8_t* data, std::size_t size);
+
+    // TODO: should use simplecpp::OutputList
+    using CreateTokenListFn = std::function<simplecpp::TokenList (std::vector<std::string>&, std::list<simplecpp::Output>*)>;
+
     /**
      * @brief Check a file using stream
      * @param file the file
      * @param cfgname  cfg name
-     * @param fileStream stream the file content can be read from
+     * @param createTokenList a function to create the simplecpp::TokenList with
      * @return number of errors found
      */
-    unsigned int checkFile(const FileWithDetails& file, const std::string &cfgname, int fileIndex, std::istream* fileStream = nullptr);
+    unsigned int checkInternal(const FileWithDetails& file, const std::string &cfgname, int fileIndex, const CreateTokenListFn& createTokenList);
 
     /**
      * @brief Check normal tokens

lib/importproject.cpp

@@ -533,8 +533,7 @@ namespace {
             // TODO: improve evaluation
             const Settings s;
             TokenList tokenlist(s, Standards::Language::C);
-            std::istringstream istr(c);
-            tokenlist.createTokens(istr); // TODO: check result
+            tokenlist.createTokensFromBuffer(c.data(), c.size()); // TODO: check result
             // TODO: put in a helper
             // generate links
             {

lib/library.cpp

@@ -35,7 +35,6 @@
 #include <iostream>
 #include <list>
 #include <memory>
-#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <unordered_set>
@@ -178,8 +177,8 @@ static std::vector<std::string> getnames(const char *names)
 
 static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
 {
-    std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr); // TODO: check result?
+    const std::string str(valid + ',');
+    tokenList.createTokensFromBuffer(str.data(), str.size()); // TODO: check result?
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));

lib/programmemory.cpp

@@ -1822,8 +1822,7 @@ static std::shared_ptr<Token> createTokenFromExpression(const std::string& retur
     std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings, cpp ? Standards::Language::CPP : Standards::Language::C);
     {
         const std::string code = "return " + returnValue + ";";
-        std::istringstream istr(code);
-        if (!tokenList->createTokens(istr))
+        if (!tokenList->createTokensFromBuffer(code.data(), code.size()))
             return nullptr;
     }
 

lib/symboldatabase.cpp

@@ -7756,8 +7756,8 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 if (!typestr.empty()) {
                     ValueType valuetype;
                     TokenList tokenList(mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
-                    std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr); // TODO: check result?
+                    const std::string str(typestr+";");
+                    tokenList.createTokensFromBuffer(str.data(), str.size()); // TODO: check result?
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings)) {
                         valuetype.originalTypeName = typestr;
@@ -7846,8 +7846,8 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     continue;
                 }
                 TokenList tokenList(mSettings, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
-                std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr)) {
+                const std::string str(typestr+";");
+                if (tokenList.createTokensFromBuffer(str.data(), str.size())) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();

lib/tokenlist.cpp

@@ -321,17 +321,17 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 
 //---------------------------------------------------------------------------
 
-bool TokenList::createTokens(std::istream &code)
+bool TokenList::createTokensFromBuffer(const uint8_t* data, size_t size)
 {
-    return createTokensInternal(code, mFiles.empty() ? "" : *mFiles.cbegin());
+    return createTokensFromBufferInternal(data, size, mFiles.empty() ? "" : *mFiles.cbegin());
 }
 
 //---------------------------------------------------------------------------
 
-bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
+bool TokenList::createTokensFromBufferInternal(const uint8_t* data, size_t size, const std::string& file0)
 {
     simplecpp::OutputList outputList;
-    simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
+    simplecpp::TokenList tokens(data, size, mFiles, file0, &outputList);
 
     createTokens(std::move(tokens));
 

lib/tokenlist.h

@@ -98,9 +98,16 @@ class CPPCHECKLIB TokenList {
      * - multiline strings are not handled.
      * - UTF in the code are not handled.
      * - comments are not handled.
-     * @param code input stream for code
      */
-    bool createTokens(std::istream &code);
+    bool createTokensFromBuffer(const uint8_t* data, size_t size);
+    bool createTokensFromBuffer(const char* data, size_t size) {
+        return createTokensFromBuffer(reinterpret_cast<const uint8_t*>(data), size);
+    }
+    template<size_t size>
+    // cppcheck-suppress unusedFunction - used in tests only
+    bool createTokensFromString(const char (&data)[size]) {
+        return createTokensFromBuffer(reinterpret_cast<const uint8_t*>(data), size-1);
+    }
 
     void createTokens(simplecpp::TokenList&& tokenList);
 
@@ -208,7 +215,7 @@ class CPPCHECKLIB TokenList {
     }
 
 private:
-    bool createTokensInternal(std::istream &code, const std::string& file0);
+    bool createTokensFromBufferInternal(const uint8_t* data, std::size_t size, const std::string& file0);
 
     /** Token list */
     std::shared_ptr<TokensFrontBack> mTokensFrontBack;

lib/valueflow.cpp

@@ -124,7 +124,6 @@
 #include <memory>
 #include <numeric>
 #include <set>
-#include <sstream>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -1997,8 +1996,7 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
 static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp, const Settings& settings)
 {
     TokenList tokenList(settings, cpp ? Standards::Language::CPP : Standards::Language::C);
-    std::istringstream istr(y);
-    tokenList.createTokens(istr); // TODO: check result?
+    tokenList.createTokensFromBuffer(y.data(), y.size()); // TODO: check result?
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
 static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp, const Settings& settings)
@@ -7117,8 +7115,8 @@ static bool getMinMaxValues(const std::string& typestr,
                             MathLib::bigint& maxvalue)
 {
     TokenList typeTokens(settings, cpp ? Standards::Language::CPP : Standards::Language::C);
-    std::istringstream istr(typestr + ";");
-    if (!typeTokens.createTokens(istr))
+    const std::string str(typestr + ";");
+    if (!typeTokens.createTokensFromBuffer(str.data(), str.size()))
         return false;
     typeTokens.simplifyPlatformTypes();
     typeTokens.simplifyStdType();

oss-fuzz/main.cpp

@@ -60,11 +60,11 @@ static const Settings s_settings = create_settings();
 static DummyErrorLogger s_errorLogger;
 static const FileWithDetails s_file("test.cpp", Standards::Language::CPP, 0);
 
-static void doCheck(const std::string& code)
+static void doCheck(const uint8_t *data, size_t dataSize)
 {
     Suppressions supprs;
     CppCheck cppcheck(s_settings, supprs, s_errorLogger, false, nullptr);
-    cppcheck.check(s_file, code);
+    cppcheck.checkBuffer(s_file, data, dataSize);
 }
 
 #ifndef NO_FUZZ
@@ -74,7 +74,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t dataSize)
 {
     if (dataSize < 10000) {
         const std::string code = generateCode2(data, dataSize);
-        doCheck(code);
+        doCheck(reinterpret_cast<const unsigned char*>(code.data()), code.size());
     }
     return 0;
 }
@@ -98,7 +98,7 @@ int main(int argc, char * argv[])
 
     const std::string code = oss.str();
     for (int i = 0; i < cnt; ++i)
-        doCheck(code);
+        doCheck(reinterpret_cast<const unsigned char*>(code.data()), code.size());
 
     return EXIT_SUCCESS;
 }