rabix
/
RedPanda-CPP
mirror of https://github.com/Reduced-panda/RedPanda-CPP
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

work done: refactor cpp parser for project

This commit is contained in:
Roy Qu 2022-10-22 19:33:20 +08:00
parent dbf34548d8
commit d1d68758aa
6 changed files with 119 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
RedPandaIDE/parser/cppparser.cpp
View File

@ -55,6 +55,8 @@ CppParser::CppParser(QObject *parent) : QObject(parent),
mCppTypeKeywords = CppTypeKeywords;
mEnabled = true;
internalClear();
//mNamespaces;
//mBlockBeginSkips;
//mBlockEndSkips;
@ -812,8 +814,10 @@ void CppParser::parseFile(const QString &fileName, bool inProject, bool onlyIfNo
if (onlyIfNotParsed && mPreprocessor.scannedFiles().contains(fName))
return;
QSet<QString> files = calculateFilesToBeReparsed(fileName);
internalInvalidateFiles(files);
QSet<QString> filesToReparsed = calculateFilesToBeReparsed(fileName);
internalInvalidateFiles(filesToReparsed);
QStringList files = sortFilesByIncludeRelations(filesToReparsed);
if (inProject)
mProjectFiles.insert(fileName);
@ -825,24 +829,11 @@ void CppParser::parseFile(const QString &fileName, bool inProject, bool onlyIfNo
mFilesToScanCount = files.count();
mFilesScannedCount = 0;
// parse header files in the first parse
foreach (const QString& file,files) {
if (isHFile(file)) {
mFilesScannedCount++;
emit onProgress(file,mFilesToScanCount,mFilesScannedCount);
if (!mPreprocessor.scannedFiles().contains(file)) {
internalParse(file);
}
}
}
//we only parse CFile in the second parse
foreach (const QString& file,files) {
if (!isHFile(file)) {
mFilesScannedCount++;
emit onProgress(file,mFilesToScanCount,mFilesScannedCount);
if (!mPreprocessor.scannedFiles().contains(file)) {
internalParse(file);
}
mFilesScannedCount++;
emit onProgress(file,mFilesToScanCount,mFilesScannedCount);
if (!mPreprocessor.scannedFiles().contains(file)) {
internalParse(file);
}
}
}
@ -873,24 +864,14 @@ void CppParser::parseFileList(bool updateView)
// Support stopping of parsing when files closes unexpectedly
mFilesScannedCount = 0;
mFilesToScanCount = mFilesToScan.count();
QStringList files = sortFilesByIncludeRelations(mFilesToScan);
// parse header files in the first parse
foreach (const QString& file, mFilesToScan) {
if (isHFile(file)) {
mFilesScannedCount++;
emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
if (!mPreprocessor.scannedFiles().contains(file)) {
internalParse(file);
}
}
}
//we only parse CFile in the second parse
foreach (const QString& file,mFilesToScan) {
if (isCFile(file)) {
mFilesScannedCount++;
emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
if (!mPreprocessor.scannedFiles().contains(file)) {
internalParse(file);
}
foreach (const QString& file, files) {
mFilesScannedCount++;
emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
if (!mPreprocessor.scannedFiles().contains(file)) {
internalParse(file);
}
}
mFilesToScan.clear();
@ -1210,15 +1191,10 @@ PStatement CppParser::addStatement(const PStatement& parent,
if (oldStatement && !oldStatement->hasDefinition) {
oldStatement->hasDefinition = true;
if (oldStatement->fileName!=fileName) {
PFileIncludes fileIncludes1=mPreprocessor.includesList().value(fileName);
if (fileIncludes1) {
fileIncludes1->statements.insert(oldStatement->fullName,
PFileIncludes fileIncludes=mPreprocessor.includesList().value(fileName);
if (fileIncludes) {
fileIncludes->statements.insert(oldStatement->fullName,
oldStatement);
fileIncludes1->dependingFiles.insert(oldStatement->fileName);
PFileIncludes fileIncludes2=mPreprocessor.includesList().value(oldStatement->fileName);
if (fileIncludes2) {
fileIncludes2->dependedFiles.insert(fileName);
}
}
}
oldStatement->definitionLine = line;
@ -1275,7 +1251,6 @@ PStatement CppParser::addStatement(const PStatement& parent,
PFileIncludes fileIncludes = mPreprocessor.includesList().value(fileName);
if (fileIncludes) {
fileIncludes->statements.insert(result->fullName,result);
fileIncludes->declaredStatements.insert(result->fullName,result);
}
}
return result;
@ -1385,7 +1360,6 @@ void CppParser::removeScopeLevel(int line)
mStatementList.deleteStatement(currentScope);
} else {
fileIncludes->statements.insert(currentScope->fullName,currentScope);
fileIncludes->declaredStatements.insert(currentScope->fullName,currentScope);
}
}
mCurrentScope.pop_back();
@ -1453,6 +1427,55 @@ void CppParser::internalClear()
mInlineNamespaceEndSkips.clear();
}
QStringList CppParser::sortFilesByIncludeRelations(const QSet<QString> &files)
{
QStringList result;
//rebuild file include relations
foreach(const QString& file, files) {
//already removed in interalInvalidateFiles
//mPreprocessor.removeScannedFile(file);
QStringList buffer;
if (mOnGetFileStream) {
mOnGetFileStream(file,buffer);
}
mPreprocessor.setScanOptions(mParseGlobalHeaders, mParseLocalHeaders);
mPreprocessor.preprocess(file);
mPreprocessor.clearTempResults();
}
QSet<QString> fileSet=files;
while (!fileSet.isEmpty()) {
bool found=false;
foreach (const QString& file,fileSet) {
PFileIncludes fileIncludes = mPreprocessor.includesList().value(file);
bool hasInclude=false;
foreach(const QString& inc,fileIncludes->includeFiles.keys()) {
if (fileSet.contains(inc)) {
hasInclude=true;
break;
}
}
if (!hasInclude) {
result.push_front(file);
fileSet.remove(file);
found=true;
break;
}
}
if (!found) {
foreach (const QString& file,fileSet) {
result.push_front(file);
fileSet.remove(file);
}
}
}
foreach(const QString& file, files) {
mPreprocessor.removeScannedFile(file);
}
return result;
}
bool CppParser::checkForCatchBlock()
{
// return mIndex < mTokenizer.tokenCount() &&
@ -2581,7 +2604,7 @@ void CppParser::handlePreprocessor()
emit onProgress(mCurrentFile,mFilesToScanCount,mFilesScannedCount);
}
}
} else if (text.startsWith("define") || text.startsWith("define")) {
} else if (text.startsWith("define")) {
// format: #define A B, remove define keyword
QString s = text.mid(QString("define").length());
@ -2592,6 +2615,7 @@ void CppParser::handlePreprocessor()
QString name,args,value;
mPreprocessor.getDefineParts(s,name,args,value);
qDebug()<<"add define "<<name<<mCurrentFile<<mTokenizer[mIndex]->line<<mIndex;
addStatement(
nullptr, // defines don't belong to any scope
mCurrentFile,
@ -3271,13 +3295,13 @@ void CppParser::internalParse(const QString &fileName)
mPreprocessor.preprocess(fileName, buffer);
QStringList preprocessResult = mPreprocessor.result();
//reduce memory usage
mPreprocessor.clearTempResults();
#ifdef QT_DEBUG
// stringsToFile(mPreprocessor.result(),"r:\\preprocess.txt");
// stringsToFile(mPreprocessor.result(),QString("r:\\preprocess-%1.txt").arg(extractFileName(fileName)));
// mPreprocessor.dumpDefinesTo("r:\\defines.txt");
// mPreprocessor.dumpIncludesListTo("r:\\includes.txt");
#endif
//reduce memory usage
mPreprocessor.clearTempResults();
// Tokenize the preprocessed buffer file
mTokenizer.tokenize(preprocessResult);
@ -3287,7 +3311,6 @@ void CppParser::internalParse(const QString &fileName)
return;
// Process the token list
internalClear();
while(true) {
if (!handleStatement())
break;
@ -3299,21 +3322,12 @@ void CppParser::internalParse(const QString &fileName)
//
// mStatementList.dumpAll("r:\\all-stats.txt");
#endif
//reduce memory usage
mTokenizer.clear();
}
}
void CppParser::inheritClassStatement(const PStatement& derived, bool isStruct,
const PStatement& base, StatementClassScope access)
{
PFileIncludes fileIncludes1=mPreprocessor.includesList().value(derived->fileName);
PFileIncludes fileIncludes2=mPreprocessor.includesList().value(base->fileName);
if (fileIncludes1 && fileIncludes2) {
//derived class depeneds on base class
fileIncludes1->dependingFiles.insert(base->fileName);
fileIncludes2->dependedFiles.insert(derived->fileName);
}
//differentiate class and struct
if (access == StatementClassScope::scsNone) {
if (isStruct)
@ -4271,8 +4285,6 @@ void CppParser::internalInvalidateFile(const QString &fileName)
mNamespaces.remove(key);
}
}
// delete it from scannedfiles
mPreprocessor.scannedFiles().remove(fileName);
// remove its include files list
PFileIncludes p = findFileIncludes(fileName, true);
@ -4281,25 +4293,12 @@ void CppParser::internalInvalidateFile(const QString &fileName)
//p->includeFiles.clear();
//p->usings.clear();
for (PStatement& statement:p->statements) {
if ((statement->kind == StatementKind::skFunction
|| statement->kind == StatementKind::skConstructor
|| statement->kind == StatementKind::skDestructor
|| statement->kind == StatementKind::skVariable)
&& (fileName != statement->fileName)) {
statement->hasDefinition = false;
}
}
for (PStatement& statement:p->declaredStatements) {
mStatementList.deleteStatement(statement);
}
//p->declaredStatements.clear();
//p->statements.clear();
//p->scopes.clear();
//p->dependedFiles.clear();
//p->dependingFiles.clear();
p->statements.clear();
}
// delete it from scannedfiles
mPreprocessor.removeScannedFile(fileName);
}
void CppParser::internalInvalidateFiles(const QSet<QString> &files)
@ -4312,22 +4311,14 @@ QSet<QString> CppParser::calculateFilesToBeReparsed(const QString &fileName)
{
if (fileName.isEmpty())
return QSet<QString>();
QQueue<QString> queue;
QSet<QString> processed;
queue.enqueue(fileName);
while (!queue.isEmpty()) {
QString name = queue.dequeue();
processed.insert(name);
PFileIncludes p=mPreprocessor.includesList().value(name);
if (!p)
continue;
foreach (const QString& s,p->dependedFiles) {
if (!processed.contains(s)) {
queue.enqueue(s);
}
}
QSet<QString> result;
result.insert(fileName);
foreach (const QString& file,mPreprocessor.includesList().keys()) {
PFileIncludes fileIncludes = mPreprocessor.includesList()[file];
if (fileIncludes->includeFiles.contains(fileName))
result.insert(file);
}
return processed;
return result;
}
int CppParser::calcKeyLenForStruct(const QString &word)
@ -4738,6 +4729,7 @@ int CppParser::parserId() const
void CppParser::setOnGetFileStream(const GetFileStreamCallBack &newOnGetFileStream)
{
mOnGetFileStream = newOnGetFileStream;
mPreprocessor.setOnGetFileStream(newOnGetFileStream);
}
const QSet<QString> &CppParser::filesToScan() const

3
RedPandaIDE/parser/cppparser.h
View File

@ -29,7 +29,6 @@ class CppParser : public QObject
{
Q_OBJECT
using GetFileStreamCallBack = std::function<bool (const QString&, QStringList&)>;
public:
explicit CppParser(QObject *parent = nullptr);
~CppParser();
@ -190,6 +189,8 @@ private:
void internalClear();
QStringList sortFilesByIncludeRelations(const QSet<QString> &files);
bool checkForCatchBlock();
bool checkForEnum();
bool checkForForBlock();

30
RedPandaIDE/parser/cpppreprocessor.cpp
View File

@ -249,28 +249,12 @@ void CppPreprocessor::dumpIncludesListTo(const QString &fileName) const
#else
<<endl;
#endif
foreach (const QString& s,fileIncludes->dependingFiles) {
stream<<"\t^^"+s
#if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
<<Qt::endl;
#else
<<endl;
#endif
}
stream<<"\t**depended by:**"
#if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
<<Qt::endl;
#else
<<endl;
#endif
foreach (const QString& s,fileIncludes->dependedFiles) {
stream<<"\t&&"+s
#if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
<<Qt::endl;
#else
<<endl;
#endif
}
stream<<"\t**using:**"
#if QT_VERSION >= QT_VERSION_CHECK(5,15,0)
<<Qt::endl;
@ -490,7 +474,11 @@ void CppPreprocessor::handleInclude(const QString &line, bool fromNext)
return;
PFileIncludes oldCurrentIncludes = mCurrentIncludes;
openInclude(fileName);
QStringList buffer;
if (mOnGetFileStream) {
mOnGetFileStream(fileName,buffer);
}
openInclude(fileName, buffer);
//oldCurrentIncludes->includeFiles.insert(fileName,true);
}
@ -745,7 +733,6 @@ void CppPreprocessor::openInclude(const QString &fileName, QStringList bufferedT
//mCurrentIncludes->includeFiles;
//mCurrentIncludes->usings;
//mCurrentIncludes->statements;
//mCurrentIncludes->declaredStatements;
//mCurrentIncludes->scopes;
//mCurrentIncludes->dependedFiles;
//mCurrentIncludes->dependingFiles;
@ -755,7 +742,7 @@ void CppPreprocessor::openInclude(const QString &fileName, QStringList bufferedT
parsedFile->fileIncludes = mCurrentIncludes;
// Don't parse stuff we have already parsed
if ((!bufferedText.isEmpty()) || !mScannedFiles.contains(fileName)) {
if (!mScannedFiles.contains(fileName)) {
// Parse ONCE
//if not Assigned(Stream) then
mScannedFiles.insert(fileName);
@ -1842,6 +1829,11 @@ int CppPreprocessor::evaluateExpression(QString line)
return result;
}
void CppPreprocessor::setOnGetFileStream(const GetFileStreamCallBack &newOnGetFileStream)
{
mOnGetFileStream = newOnGetFileStream;
}
const QList<QString> &CppPreprocessor::projectIncludePathList() const
{
return mProjectIncludePathList;

5
RedPandaIDE/parser/cpppreprocessor.h
View File

@ -75,6 +75,7 @@ public:
void addProjectIncludePath(const QString& fileName);
void clearIncludePaths();
void clearProjectIncludePaths();
void removeScannedFile(const QString& filename);
QStringList result() const;
@ -91,6 +92,8 @@ public:
const QList<QString> &includePathList() const;
const QList<QString> &projectIncludePathList() const;
void setOnGetFileStream(const GetFileStreamCallBack &newOnGetFileStream);
private:
void preprocessBuffer();
void skipToEndOfPreprocessor();
@ -221,6 +224,8 @@ private:
bool mParseSystem;
bool mParseLocal;
GetFileStreamCallBack mOnGetFileStream;
};
using PCppPreprocessor = std::shared_ptr<CppPreprocessor>;

12
RedPandaIDE/parser/parserutils.cpp
View File

@ -666,3 +666,15 @@ bool isCppControlKeyword(const QString &word)
{
return CppControlKeyWords.contains(word);
}
static int counter=0;
Statement::Statement()
{
counter++;
}
Statement::~Statement()
{
counter--;
qDebug()<<"statement deleted:"<<counter<<fullName<<kind<<extractFileName(fileName)<<line;
}

17
RedPandaIDE/parser/parserutils.h
View File

@ -22,6 +22,8 @@
#include <QVector>
#include <memory>
using GetFileStreamCallBack = std::function<bool (const QString&, QStringList&)>;
enum class ParserLanguage {
C,
CPlusPlus
@ -111,15 +113,6 @@ enum class MemberOperatorType {
otOther
};
struct RemovedStatement{
QString type; // type "int"
QString command; // identifier/name of statement "foo"
int definitionLine; // definition
QString definitionFileName; // definition
QString fullName; // fullname(including class and namespace)
QString noNameArgs; // Args without name
};
enum class EvalStatementKind {
Namespace,
Type,
@ -128,8 +121,6 @@ enum class EvalStatementKind {
Function
};
using PRemovedStatement = std::shared_ptr<RemovedStatement>;
struct StatementMatchPosition{
int start;
int end;
@ -143,6 +134,8 @@ using StatementList = QList<PStatement>;
using PStatementList = std::shared_ptr<StatementList>;
using StatementMap = QMultiMap<QString, PStatement>;
struct Statement {
Statement();
~Statement();
std::weak_ptr<Statement> parentScope; // parent class/struct/namespace scope, don't use auto pointer to prevent circular reference
QString type; // type "int"
QString command; // identifier/name of statement "foo"
@ -234,8 +227,6 @@ struct FileIncludes {
StatementMap statements; // but we don't save temporary statements (full name as key)
StatementMap declaredStatements; // statements declared in this file (full name as key)
CppScopes scopes; // int is start line of the statement scope
QSet<QString> dependingFiles; // The files I depeneds on
QSet<QString> dependedFiles; // the files depends on me
};
using PFileIncludes = std::shared_ptr<FileIncludes>;