optimized handling of some preprocessor directives

.github/workflows/CI-unixish.yml

@@ -100,9 +100,10 @@ jobs:
       if: matrix.os == 'ubuntu-24.04'
       run: |
           make clean
-          make -j$(nproc)
+          make -j$(nproc) CXXOPTS="-O1"
           valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner
-          valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./simplecpp simplecpp.cpp -e
+          # TODO: run Python tests with valgrind
+          VALGRIND_TOOL=memcheck ./selfcheck.sh
 
     - name: Run with libstdc++ debug mode
       if: matrix.os == 'ubuntu-24.04' && matrix.compiler == 'g++'
@@ -146,10 +147,13 @@ jobs:
         tar xvf 1.5.1.tar.gz
         make clean
         make -j$(nproc) CXXOPTS="-O2 -g3"
-        valgrind --tool=callgrind ./simplecpp -e simplecpp-1.5.1/simplecpp.cpp 2>callgrind.log || (cat callgrind.log && false)
+        VALGRIND_TOOL=callgrind SIMPLECPP_PATH=simplecpp-1.5.1 ./selfcheck.sh >callgrind.log || (cat callgrind.log && false)
         cat callgrind.log
-        callgrind_annotate --auto=no > callgrind.annotated.log
-        head -50 callgrind.annotated.log
+        for f in callgrind.out.*;
+        do
+          callgrind_annotate --auto=no $f > $f.annotated.log
+          head -50 $f.annotated.log
+        done
 
     - uses: actions/upload-artifact@v4
       if: matrix.os == 'ubuntu-24.04'

selfcheck.sh

@@ -1,7 +1,28 @@
 #!/bin/bash
 
-output=$(./simplecpp simplecpp.cpp -e -f 2>&1)
+if [ -z "$SIMPLECPP_PATH" ]; then
+  SIMPLECPP_PATH=.
+fi
+
+if [ -n "$VALGRIND_TOOL" ]; then
+  if [ "$VALGRIND_TOOL" = "memcheck" ]; then
+    VALGRIND_OPTS="--error-limit=yes --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42"
+  elif [ "$VALGRIND_TOOL" = "callgrind" ]; then
+    VALGRIND_OPTS="--tool=callgrind"
+  else
+    echo "unsupported valgrind tool '$VALGRIND_TOOL'"
+    exit 1
+  fi
+  VALGRIND_CMD="valgrind --tool=$VALGRIND_TOOL --log-fd=9 $VALGRIND_OPTS"
+  VALGRIND_REDIRECT="valgrind_$VALGRIND_TOOL.log"
+else
+  VALGRIND_CMD=
+  VALGRIND_REDIRECT="/dev/null"
+fi
+
+output=$($VALGRIND_CMD ./simplecpp "$SIMPLECPP_PATH/simplecpp.cpp" -e -f 2>&1 9> "$VALGRIND_REDIRECT")
 ec=$?
+cat "$VALGRIND_REDIRECT"
 errors=$(echo "$output" | grep -v 'Header not found: <')
 if [ $ec -ne 0 ]; then
   # only fail if we got errors which do not refer to missing system includes
@@ -104,8 +125,9 @@ else
 fi
 
 # run with -std=gnuc++* so __has_include(...) is available
-./simplecpp simplecpp.cpp -e -f -std=gnu++11 $defs $inc
+$VALGRIND_CMD ./simplecpp "$SIMPLECPP_PATH/simplecpp.cpp" -e -f -std=gnu++11 $defs $inc 9> "$VALGRIND_REDIRECT"
 ec=$?
+cat "$VALGRIND_REDIRECT"
 if [ $ec -ne 0 ]; then
   exit $ec
 fi

simplecpp.cpp

@@ -694,33 +694,55 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
 
             if (oldLastToken != cback()) {
                 oldLastToken = cback();
-                if (!isLastLinePreprocessor())
+                const Token * const llTok = isLastLinePreprocessor();
+                if (!llTok)
                     continue;
-                const std::string lastline(lastLine());
-                if (lastline == "# file %str%") {
-                    const Token *strtok = cback();
-                    while (strtok->comment)
-                        strtok = strtok->previous;
-                    loc.push(location);
-                    location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U));
-                    location.line = 1U;
-                } else if (lastline == "# line %num%") {
-                    const Token *numtok = cback();
-                    while (numtok->comment)
-                        numtok = numtok->previous;
-                    lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), &location);
-                } else if (lastline == "# %num% %str%" || lastline == "# line %num% %str%") {
-                    const Token *strtok = cback();
-                    while (strtok->comment)
-                        strtok = strtok->previous;
-                    const Token *numtok = strtok->previous;
-                    while (numtok->comment)
-                        numtok = numtok->previous;
-                    lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")),
-                                  std::atol(numtok->str().c_str()), &location);
+                const Token * const llNextToken = llTok->next;
+                if (!llTok->next)
+                    continue;
+                if (llNextToken->next) {
+                    // #file "file.c"
+                    if (llNextToken->str() == "file" &&
+                        llNextToken->next->str()[0] == '\"')
+                    {
+                        const Token *strtok = cback();
+                        while (strtok->comment)
+                            strtok = strtok->previous;
+                        loc.push(location);
+                        location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U));
+                        location.line = 1U;
+                    }
+                    // #3 "file.c"
+                    // #line 3 "file.c"
+                    else if ((llNextToken->number &&
+                              llNextToken->next->str()[0] == '\"') ||
+                             (llNextToken->str() == "line" &&
+                              llNextToken->next->number &&
+                              llNextToken->next->next &&
+                              llNextToken->next->next->str()[0] == '\"'))
+                    {
+                        const Token *strtok = cback();
+                        while (strtok->comment)
+                            strtok = strtok->previous;
+                        const Token *numtok = strtok->previous;
+                        while (numtok->comment)
+                            numtok = numtok->previous;
+                        lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")),
+                                      std::atol(numtok->str().c_str()), &location);
+                    }
+                    // #line 3
+                    else if (llNextToken->str() == "line" &&
+                             llNextToken->next->number)
+                    {
+                        const Token *numtok = cback();
+                        while (numtok->comment)
+                            numtok = numtok->previous;
+                        lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), &location);
+                    }
                 }
                 // #endfile
-                else if (lastline == "# endfile" && !loc.empty()) {
+                else if (llNextToken->str() == "endfile" && !loc.empty())
+                {
                     location = loc.top();
                     loc.pop();
                 }
@@ -737,8 +759,8 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
         TokenString currentToken;
 
         if (cback() && cback()->location.line == location.line && cback()->previous && cback()->previous->op == '#') {
-            const Token* const llTok = lastLineTok();
-            if (llTok && llTok->op == '#' && llTok->next && (llTok->next->str() == "error" || llTok->next->str() == "warning")) {
+            const Token* const ppTok = cback()->previous;
+            if (ppTok->next && (ppTok->next->str() == "error" || ppTok->next->str() == "warning")) {
                 char prev = ' ';
                 while (stream.good() && (prev == '\\' || (ch != '\r' && ch != '\n'))) {
                     currentToken += ch;
@@ -1415,34 +1437,6 @@ std::string simplecpp::TokenList::readUntil(Stream &stream, const Location &loca
     return ret;
 }
 
-std::string simplecpp::TokenList::lastLine(int maxsize) const
-{
-    std::string ret;
-    int count = 0;
-    for (const Token *tok = cback(); ; tok = tok->previous) {
-        if (!sameline(tok, cback())) {
-            break;
-        }
-        if (tok->comment)
-            continue;
-        if (++count > maxsize)
-            return "";
-        if (!ret.empty())
-            ret += ' ';
-        // add tokens in reverse for performance reasons
-        if (tok->str()[0] == '\"')
-            ret += "%rts%"; // %str%
-        else if (tok->number)
-            ret += "%mun%"; // %num%
-        else {
-            ret += tok->str();
-            std::reverse(ret.end() - tok->str().length(), ret.end());
-        }
-    }
-    std::reverse(ret.begin(), ret.end());
-    return ret;
-}
-
 const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const
 {
     const Token* prevTok = nullptr;
@@ -1459,10 +1453,12 @@ const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const
     return prevTok;
 }
 
-bool simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const
+const simplecpp::Token* simplecpp::TokenList::isLastLinePreprocessor(int maxsize) const
 {
     const Token * const prevTok = lastLineTok(maxsize);
-    return prevTok && prevTok->op == '#';
+    if (prevTok && prevTok->op == '#')
+        return prevTok;
+    return nullptr;
 }
 
 unsigned int simplecpp::TokenList::fileIndex(const std::string &filename)

simplecpp.h

@@ -361,9 +361,8 @@ namespace simplecpp {
         std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList);
         void lineDirective(unsigned int fileIndex, unsigned int line, Location *location);
 
-        std::string lastLine(int maxsize=1000) const;
         const Token* lastLineTok(int maxsize=1000) const;
-        bool isLastLinePreprocessor(int maxsize=1000) const;
+        const Token* isLastLinePreprocessor(int maxsize=1000) const;
 
         unsigned int fileIndex(const std::string &filename);