work done: refactor cpp parser for project

RedPandaIDE/parser/cppparser.cpp

@@ -55,6 +55,8 @@ CppParser::CppParser(QObject *parent) : QObject(parent),
     mCppTypeKeywords = CppTypeKeywords;
     mEnabled = true;
 
+    internalClear();
+
     //mNamespaces;
     //mBlockBeginSkips;
     //mBlockEndSkips;
@@ -812,8 +814,10 @@ void CppParser::parseFile(const QString &fileName, bool inProject, bool onlyIfNo
         if (onlyIfNotParsed && mPreprocessor.scannedFiles().contains(fName))
             return;
 
-        QSet<QString> files = calculateFilesToBeReparsed(fileName);
+        QSet<QString> filesToReparsed = calculateFilesToBeReparsed(fileName);
-        internalInvalidateFiles(files);
+        internalInvalidateFiles(filesToReparsed);
+
+        QStringList files = sortFilesByIncludeRelations(filesToReparsed);
 
         if (inProject)
             mProjectFiles.insert(fileName);
@@ -825,9 +829,7 @@ void CppParser::parseFile(const QString &fileName, bool inProject, bool onlyIfNo
         mFilesToScanCount = files.count();
         mFilesScannedCount = 0;
 
-        // parse header files in the first parse
         foreach (const QString& file,files) {
-            if (isHFile(file)) {
             mFilesScannedCount++;
             emit onProgress(file,mFilesToScanCount,mFilesScannedCount);
             if (!mPreprocessor.scannedFiles().contains(file)) {
@@ -835,17 +837,6 @@ void CppParser::parseFile(const QString &fileName, bool inProject, bool onlyIfNo
             }
         }
     }
-        //we only parse CFile in the second parse
-        foreach (const QString& file,files) {
-            if (!isHFile(file)) {
-                mFilesScannedCount++;
-                emit onProgress(file,mFilesToScanCount,mFilesScannedCount);
-                if (!mPreprocessor.scannedFiles().contains(file)) {
-                    internalParse(file);
-                }
-            }
-        }
-    }
 }
 
 void CppParser::parseFileList(bool updateView)
@@ -873,26 +864,16 @@ void CppParser::parseFileList(bool updateView)
         // Support stopping of parsing when files closes unexpectedly
         mFilesScannedCount = 0;
         mFilesToScanCount = mFilesToScan.count();
+
+        QStringList files = sortFilesByIncludeRelations(mFilesToScan);
         // parse header files in the first parse
-        foreach (const QString& file, mFilesToScan) {
+        foreach (const QString& file, files) {
-            if (isHFile(file)) {
             mFilesScannedCount++;
             emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
             if (!mPreprocessor.scannedFiles().contains(file)) {
                 internalParse(file);
             }
         }
-        }
-        //we only parse CFile in the second parse
-        foreach (const QString& file,mFilesToScan) {
-            if (isCFile(file)) {
-                mFilesScannedCount++;
-                emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
-                if (!mPreprocessor.scannedFiles().contains(file)) {
-                    internalParse(file);
-                }
-            }
-        }
         mFilesToScan.clear();
     }
 }
@@ -1210,15 +1191,10 @@ PStatement CppParser::addStatement(const PStatement& parent,
             if (oldStatement  && !oldStatement->hasDefinition) {
                 oldStatement->hasDefinition = true;
                 if (oldStatement->fileName!=fileName) {
-                    PFileIncludes fileIncludes1=mPreprocessor.includesList().value(fileName);
+                    PFileIncludes fileIncludes=mPreprocessor.includesList().value(fileName);
-                    if (fileIncludes1) {
+                    if (fileIncludes) {
-                        fileIncludes1->statements.insert(oldStatement->fullName,
+                        fileIncludes->statements.insert(oldStatement->fullName,
                                                          oldStatement);
-                        fileIncludes1->dependingFiles.insert(oldStatement->fileName);
-                        PFileIncludes fileIncludes2=mPreprocessor.includesList().value(oldStatement->fileName);
-                        if (fileIncludes2) {
-                            fileIncludes2->dependedFiles.insert(fileName);
-                        }
                     }
                 }
                 oldStatement->definitionLine = line;
@@ -1275,7 +1251,6 @@ PStatement CppParser::addStatement(const PStatement& parent,
         PFileIncludes fileIncludes = mPreprocessor.includesList().value(fileName);
         if (fileIncludes) {
             fileIncludes->statements.insert(result->fullName,result);
-            fileIncludes->declaredStatements.insert(result->fullName,result);
         }
     }
     return result;
@@ -1385,7 +1360,6 @@ void CppParser::removeScopeLevel(int line)
             mStatementList.deleteStatement(currentScope);
         } else {
             fileIncludes->statements.insert(currentScope->fullName,currentScope);
-            fileIncludes->declaredStatements.insert(currentScope->fullName,currentScope);
         }
     }
     mCurrentScope.pop_back();
@@ -1453,6 +1427,55 @@ void CppParser::internalClear()
     mInlineNamespaceEndSkips.clear();
 }
 
+QStringList CppParser::sortFilesByIncludeRelations(const QSet<QString> &files)
+{
+    QStringList result;
+
+    //rebuild file include relations
+    foreach(const QString& file, files) {
+        //already removed in interalInvalidateFiles
+        //mPreprocessor.removeScannedFile(file);
+        QStringList buffer;
+        if (mOnGetFileStream) {
+            mOnGetFileStream(file,buffer);
+        }
+        mPreprocessor.setScanOptions(mParseGlobalHeaders, mParseLocalHeaders);
+        mPreprocessor.preprocess(file);
+        mPreprocessor.clearTempResults();
+    }
+
+    QSet<QString> fileSet=files;
+    while (!fileSet.isEmpty()) {
+        bool found=false;
+        foreach (const QString& file,fileSet) {
+            PFileIncludes fileIncludes = mPreprocessor.includesList().value(file);
+            bool hasInclude=false;
+            foreach(const QString& inc,fileIncludes->includeFiles.keys()) {
+                if (fileSet.contains(inc)) {
+                    hasInclude=true;
+                    break;
+                }
+            }
+            if (!hasInclude) {
+                result.push_front(file);
+                fileSet.remove(file);
+                found=true;
+                break;
+            }
+        }
+        if (!found) {
+            foreach (const QString& file,fileSet) {
+                result.push_front(file);
+                fileSet.remove(file);
+            }
+        }
+    }
+    foreach(const QString& file, files) {
+        mPreprocessor.removeScannedFile(file);
+    }
+    return result;
+}
+
 bool CppParser::checkForCatchBlock()
 {
 //    return  mIndex < mTokenizer.tokenCount() &&
@@ -2581,7 +2604,7 @@ void CppParser::handlePreprocessor()
                 emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
             }
         }
-    } else if (text.startsWith("define") || text.startsWith("define")) {
+    } else if (text.startsWith("define")) {
 
       // format: #define A B, remove define keyword
       QString s = text.mid(QString("define").length());
@@ -2592,6 +2615,7 @@ void CppParser::handlePreprocessor()
       QString name,args,value;
       mPreprocessor.getDefineParts(s,name,args,value);
 
+      qDebug()<<"add define "<<name<<mCurrentFile<<mTokenizer[mIndex]->line<<mIndex;
       addStatement(
         nullptr, // defines don't belong to any scope
         mCurrentFile,
@@ -3271,13 +3295,13 @@ void CppParser::internalParse(const QString &fileName)
         mPreprocessor.preprocess(fileName, buffer);
 
         QStringList preprocessResult = mPreprocessor.result();
-        //reduce memory usage
-        mPreprocessor.clearTempResults();
 #ifdef QT_DEBUG
-//        stringsToFile(mPreprocessor.result(),"r:\\preprocess.txt");
+//        stringsToFile(mPreprocessor.result(),QString("r:\\preprocess-%1.txt").arg(extractFileName(fileName)));
 //        mPreprocessor.dumpDefinesTo("r:\\defines.txt");
 //        mPreprocessor.dumpIncludesListTo("r:\\includes.txt");
 #endif
+        //reduce memory usage
+        mPreprocessor.clearTempResults();
 
         // Tokenize the preprocessed buffer file
         mTokenizer.tokenize(preprocessResult);
@@ -3287,7 +3311,6 @@ void CppParser::internalParse(const QString &fileName)
             return;
 
         // Process the token list
-        internalClear();
         while(true) {
             if (!handleStatement())
                 break;
@@ -3299,21 +3322,12 @@ void CppParser::internalParse(const QString &fileName)
 //
 //      mStatementList.dumpAll("r:\\all-stats.txt");
 #endif
-        //reduce memory usage
-        mTokenizer.clear();
     }
 }
 
 void CppParser::inheritClassStatement(const PStatement& derived, bool isStruct,
                                       const PStatement& base, StatementClassScope access)
 {
-    PFileIncludes fileIncludes1=mPreprocessor.includesList().value(derived->fileName);
-    PFileIncludes fileIncludes2=mPreprocessor.includesList().value(base->fileName);
-    if (fileIncludes1 && fileIncludes2) {
-        //derived class depeneds on base class
-        fileIncludes1->dependingFiles.insert(base->fileName);
-        fileIncludes2->dependedFiles.insert(derived->fileName);
-    }
     //differentiate class and struct
     if (access == StatementClassScope::scsNone) {
         if (isStruct)
@@ -4271,8 +4285,6 @@ void CppParser::internalInvalidateFile(const QString &fileName)
             mNamespaces.remove(key);
         }
     }
-    // delete it from scannedfiles
-    mPreprocessor.scannedFiles().remove(fileName);
 
     // remove its include files list
     PFileIncludes p = findFileIncludes(fileName, true);
@@ -4281,25 +4293,12 @@ void CppParser::internalInvalidateFile(const QString &fileName)
         //p->includeFiles.clear();
         //p->usings.clear();
         for (PStatement& statement:p->statements) {
-            if ((statement->kind == StatementKind::skFunction
-                 || statement->kind == StatementKind::skConstructor
-                 || statement->kind == StatementKind::skDestructor
-                 || statement->kind == StatementKind::skVariable)
-                    && (fileName != statement->fileName)) {
-                statement->hasDefinition = false;
-            }
-        }
-
-        for (PStatement& statement:p->declaredStatements) {
             mStatementList.deleteStatement(statement);
         }
-
+        p->statements.clear();
-        //p->declaredStatements.clear();
-        //p->statements.clear();
-        //p->scopes.clear();
-        //p->dependedFiles.clear();
-        //p->dependingFiles.clear();
     }
+    // delete it from scannedfiles
+    mPreprocessor.removeScannedFile(fileName);
 }
 
 void CppParser::internalInvalidateFiles(const QSet<QString> &files)
@@ -4312,22 +4311,14 @@ QSet<QString> CppParser::calculateFilesToBeReparsed(const QString &fileName)
 {
     if (fileName.isEmpty())
         return QSet<QString>();
-    QQueue<QString> queue;
+    QSet<QString> result;
-    QSet<QString> processed;
+    result.insert(fileName);
-    queue.enqueue(fileName);
+    foreach (const QString& file,mPreprocessor.includesList().keys()) {
-    while (!queue.isEmpty()) {
+        PFileIncludes fileIncludes = mPreprocessor.includesList()[file];
-        QString name = queue.dequeue();
+        if (fileIncludes->includeFiles.contains(fileName))
-        processed.insert(name);
+            result.insert(file);
-        PFileIncludes p=mPreprocessor.includesList().value(name);
-        if (!p)
-          continue;
-        foreach (const QString& s,p->dependedFiles) {
-            if (!processed.contains(s)) {
-                queue.enqueue(s);
     }
-        }
+    return result;
-    }
-    return processed;
 }
 
 int CppParser::calcKeyLenForStruct(const QString &word)
@@ -4738,6 +4729,7 @@ int CppParser::parserId() const
 void CppParser::setOnGetFileStream(const GetFileStreamCallBack &newOnGetFileStream)
 {
     mOnGetFileStream = newOnGetFileStream;
+    mPreprocessor.setOnGetFileStream(newOnGetFileStream);
 }
 
 const QSet<QString> &CppParser::filesToScan() const

RedPandaIDE/parser/cppparser.h

@@ -29,7 +29,6 @@ class CppParser : public QObject
 {
     Q_OBJECT
 
-    using GetFileStreamCallBack = std::function<bool (const QString&, QStringList&)>;
 public:
     explicit CppParser(QObject *parent = nullptr);
     ~CppParser();
@@ -190,6 +189,8 @@ private:
 
     void internalClear();
 
+    QStringList sortFilesByIncludeRelations(const QSet<QString> &files);
+
     bool checkForCatchBlock();
     bool checkForEnum();
     bool checkForForBlock();

RedPandaIDE/parser/cpppreprocessor.cpp

@@ -249,28 +249,12 @@ void CppPreprocessor::dumpIncludesListTo(const QString &fileName) const
         #else
                          <<endl;
         #endif
-            foreach (const QString& s,fileIncludes->dependingFiles) {
-                stream<<"\t^^"+s
-        #if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
-                         <<Qt::endl;
-        #else
-                         <<endl;
-        #endif
-            }
             stream<<"\t**depended by:**"
         #if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
                          <<Qt::endl;
         #else
                          <<endl;
         #endif
-            foreach (const QString& s,fileIncludes->dependedFiles) {
-                stream<<"\t&&"+s
-        #if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
-                         <<Qt::endl;
-        #else
-                         <<endl;
-        #endif
-            }
             stream<<"\t**using:**"
         #if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
                          <<Qt::endl;
@@ -490,7 +474,11 @@ void CppPreprocessor::handleInclude(const QString &line, bool fromNext)
         return;
 
     PFileIncludes oldCurrentIncludes = mCurrentIncludes;
-    openInclude(fileName);
+    QStringList buffer;
+    if (mOnGetFileStream) {
+        mOnGetFileStream(fileName,buffer);
+    }
+    openInclude(fileName, buffer);
     //oldCurrentIncludes->includeFiles.insert(fileName,true);
 }
 
@@ -745,7 +733,6 @@ void CppPreprocessor::openInclude(const QString &fileName, QStringList bufferedT
         //mCurrentIncludes->includeFiles;
         //mCurrentIncludes->usings;
         //mCurrentIncludes->statements;
-        //mCurrentIncludes->declaredStatements;
         //mCurrentIncludes->scopes;
         //mCurrentIncludes->dependedFiles;
         //mCurrentIncludes->dependingFiles;
@@ -755,7 +742,7 @@ void CppPreprocessor::openInclude(const QString &fileName, QStringList bufferedT
     parsedFile->fileIncludes = mCurrentIncludes;
 
     // Don't parse stuff we have already parsed
-    if ((!bufferedText.isEmpty()) || !mScannedFiles.contains(fileName)) {
+    if (!mScannedFiles.contains(fileName)) {
         // Parse ONCE
         //if not Assigned(Stream) then
         mScannedFiles.insert(fileName);
@@ -1842,6 +1829,11 @@ int CppPreprocessor::evaluateExpression(QString line)
     return result;
 }
 
+void CppPreprocessor::setOnGetFileStream(const GetFileStreamCallBack &newOnGetFileStream)
+{
+    mOnGetFileStream = newOnGetFileStream;
+}
+
 const QList<QString> &CppPreprocessor::projectIncludePathList() const
 {
     return mProjectIncludePathList;

RedPandaIDE/parser/cpppreprocessor.h

@@ -75,6 +75,7 @@ public:
     void addProjectIncludePath(const QString& fileName);
     void clearIncludePaths();
     void clearProjectIncludePaths();
+    void removeScannedFile(const QString& filename);
 
     QStringList result() const;
 
@@ -91,6 +92,8 @@ public:
     const QList<QString> &includePathList() const;
 
     const QList<QString> &projectIncludePathList() const;
+    void setOnGetFileStream(const GetFileStreamCallBack &newOnGetFileStream);
+
 private:
     void preprocessBuffer();
     void skipToEndOfPreprocessor();
@@ -221,6 +224,8 @@ private:
 
     bool mParseSystem;
     bool mParseLocal;
+
+    GetFileStreamCallBack mOnGetFileStream;
 };
 
 using PCppPreprocessor = std::shared_ptr<CppPreprocessor>;

RedPandaIDE/parser/parserutils.cpp

@@ -666,3 +666,15 @@ bool isCppControlKeyword(const QString &word)
 {
     return CppControlKeyWords.contains(word);
 }
+
+static int counter=0;
+Statement::Statement()
+{
+    counter++;
+}
+
+Statement::~Statement()
+{
+    counter--;
+    qDebug()<<"statement deleted:"<<counter<<fullName<<kind<<extractFileName(fileName)<<line;
+}

RedPandaIDE/parser/parserutils.h

@@ -22,6 +22,8 @@
 #include <QVector>
 #include <memory>
 
+using GetFileStreamCallBack = std::function<bool (const QString&, QStringList&)>;
+
 enum class ParserLanguage {
     C,
     CPlusPlus
@@ -111,15 +113,6 @@ enum class MemberOperatorType {
   otOther
 };
 
-struct RemovedStatement{
-  QString type; // type "int"
-  QString command; // identifier/name of statement "foo"
-  int definitionLine; // definition
-  QString definitionFileName; // definition
-  QString fullName; // fullname(including class and namespace)
-  QString noNameArgs; // Args without name
-};
-
 enum class EvalStatementKind {
     Namespace,
     Type,
@@ -128,8 +121,6 @@ enum class EvalStatementKind {
     Function
 };
 
-using PRemovedStatement = std::shared_ptr<RemovedStatement>;
-
 struct StatementMatchPosition{
     int start;
     int end;
@@ -143,6 +134,8 @@ using StatementList = QList<PStatement>;
 using PStatementList = std::shared_ptr<StatementList>;
 using StatementMap = QMultiMap<QString, PStatement>;
 struct Statement {
+    Statement();
+    ~Statement();
     std::weak_ptr<Statement> parentScope; // parent class/struct/namespace scope, don't use auto pointer to prevent circular reference
     QString type; // type "int"
     QString command; // identifier/name of statement "foo"
@@ -234,8 +227,6 @@ struct FileIncludes {
     StatementMap statements; // but we don't save temporary statements (full name as key)
     StatementMap declaredStatements; // statements declared in this file (full name as key)
     CppScopes scopes; // int is start line of the statement scope
-    QSet<QString> dependingFiles; // The files I depeneds on
-    QSet<QString> dependedFiles; // the files depends on me
 };
 using PFileIncludes = std::shared_ptr<FileIncludes>;