Skip to content

Commit 7761784

Browse files
firewavefirewave
authored
Tokenizer: get settings from TokenList (#7516)
1 parent 5962f16 commit 7761784

9 files changed

+32
-28
lines changed

‎lib/cppcheck.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -717,7 +717,7 @@ unsigned int CppCheck::checkClang(const FileWithDetails &file)
717717
try {
718718
TokenList tokenlist{mSettings, file.lang()};
719719
tokenlist.appendFileIfNew(file.spath());
720-
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
720+
Tokenizer tokenizer(std::move(tokenlist), mErrorLogger);
721721
std::istringstream ast(output2);
722722
clangimport::parseClangAstDump(tokenizer, ast);
723723
ValueFlow::setValues(tokenizer.list,
@@ -930,7 +930,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
930930
tokenlist.createTokens(std::move(tokens));
931931
}
932932
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
933-
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
933+
Tokenizer tokenizer(std::move(tokenlist), mErrorLogger);
934934
mUnusedFunctionsCheck->parseTokens(tokenizer, mSettings);
935935

936936
if (analyzerInformation) {
@@ -1141,7 +1141,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
11411141
});
11421142
hasValidConfig = true;
11431143

1144-
Tokenizer tokenizer(std::move(tokenlist), mSettings, mErrorLogger);
1144+
Tokenizer tokenizer(std::move(tokenlist), mErrorLogger);
11451145
try {
11461146
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
11471147
tokenizer.setTimerResults(&s_timerResults);

‎lib/tokenize.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,9 @@ static bool isClassStructUnionEnumStart(const Token * tok)
115115

116116
//---------------------------------------------------------------------------
117117

118-
Tokenizer::Tokenizer(TokenList tokenList, const Settings &settings, ErrorLogger &errorLogger) :
118+
Tokenizer::Tokenizer(TokenList tokenList, ErrorLogger &errorLogger) :
119119
list(std::move(tokenList)),
120-
mSettings(settings),
120+
mSettings(list.getSettings()),
121121
mErrorLogger(errorLogger),
122122
mTemplateSimplifier(new TemplateSimplifier(*this))
123123
{}
@@ -10875,7 +10875,7 @@ bool Tokenizer::isPacked(const Token * bodyStart) const
1087510875
void Tokenizer::getErrorMessages(ErrorLogger& errorLogger, const Settings& settings)
1087610876
{
1087710877
TokenList tokenlist{settings, Standards::Language::C};
10878-
Tokenizer tokenizer(std::move(tokenlist), settings, errorLogger);
10878+
Tokenizer tokenizer(std::move(tokenlist), errorLogger);
1087910879
tokenizer.invalidConstFunctionTypeError(nullptr);
1088010880
// checkLibraryNoReturn
1088110881
tokenizer.unhandled_macro_class_x_y(nullptr, "", "", "", "");

‎lib/tokenize.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class CPPCHECKLIB Tokenizer {
5454
friend class TestTokenizer;
5555

5656
public:
57-
Tokenizer(TokenList tokenList, const Settings & settings, ErrorLogger &errorLogger);
57+
Tokenizer(TokenList tokenList, ErrorLogger &errorLogger);
5858
~Tokenizer();
5959

6060
void setTimerResults(TimerResults *tr) {

‎lib/tokenlist.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,10 @@ class CPPCHECKLIB TokenList {
203203
*/
204204
static const Token * isFunctionHead(const Token *tok, const std::string &endsWith);
205205

206+
const Settings& getSettings() const {
207+
return mSettings;
208+
}
209+
206210
private:
207211
bool createTokensInternal(std::istream &code, const std::string& file0);
208212

‎test/helpers.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,15 @@ namespace tinyxml2 {
4545
class SimpleTokenizer : public Tokenizer {
4646
public:
4747
explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true)
48-
: Tokenizer{TokenList{s_settings, cpp ? Standards::Language::CPP : Standards::Language::C}, s_settings, errorlogger}
48+
: Tokenizer{TokenList{s_settings, cpp ? Standards::Language::CPP : Standards::Language::C}, errorlogger}
4949
{}
5050

5151
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true)
52-
: Tokenizer{TokenList{settings, cpp ? Standards::Language::CPP : Standards::Language::C}, settings, errorlogger}
52+
: Tokenizer{TokenList{settings, cpp ? Standards::Language::CPP : Standards::Language::C}, errorlogger}
5353
{}
5454

5555
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename)
56-
: Tokenizer{TokenList{settings, Path::identify(filename, false)}, settings, errorlogger}
56+
: Tokenizer{TokenList{settings, Path::identify(filename, false)}, errorlogger}
5757
{
5858
list.appendFileIfNew(filename);
5959
}
@@ -234,14 +234,14 @@ class SimpleTokenizer2 : public Tokenizer {
234234
public:
235235
template<size_t size>
236236
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0)
237-
: Tokenizer{TokenList{settings, Path::identify(file0, false)}, settings, errorlogger}
237+
: Tokenizer{TokenList{settings, Path::identify(file0, false)}, errorlogger}
238238
{
239239
preprocess(code, mFiles, file0, *this, errorlogger);
240240
}
241241

242242
// TODO: get rid of this
243243
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0)
244-
: Tokenizer{TokenList{settings, Path::identify(file0, false)}, settings, errorlogger}
244+
: Tokenizer{TokenList{settings, Path::identify(file0, false)}, errorlogger}
245245
{
246246
preprocess(code, mFiles, file0, *this, errorlogger);
247247
}

‎test/testclangimport.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ class TestClangImport : public TestFixture {
141141
std::string parse(const char clang[]) {
142142
const Settings settings = settingsBuilder().clang().build();
143143
TokenList tokenlist{settings, Standards::Language::CPP};
144-
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
144+
Tokenizer tokenizer(std::move(tokenlist), *this);
145145
std::istringstream istr(clang);
146146
clangimport::parseClangAstDump(tokenizer, istr);
147147
if (!tokenizer.tokens()) {
@@ -1062,7 +1062,7 @@ class TestClangImport : public TestFixture {
10621062
#define GET_SYMBOL_DB(AST) \
10631063
const Settings settings = settingsBuilder().clang().platform(Platform::Type::Unix64).build(); \
10641064
TokenList tokenlist{settings, Standards::Language::CPP}; \
1065-
Tokenizer tokenizer(std::move(tokenlist), settings, *this); \
1065+
Tokenizer tokenizer(std::move(tokenlist), *this); \
10661066
{ \
10671067
std::istringstream istr(AST); \
10681068
clangimport::parseClangAstDump(tokenizer, istr); \

‎test/testsimplifytemplate.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5433,7 +5433,7 @@ class TestSimplifyTemplate : public TestFixture {
54335433
tokenlist.appendFileIfNew("test.cpp");
54345434
if (!tokenlist.createTokens(istr))
54355435
return false;
5436-
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
5436+
Tokenizer tokenizer(std::move(tokenlist), *this);
54375437
tokenizer.createLinks();
54385438
tokenizer.splitTemplateRightAngleBrackets(false);
54395439

@@ -5503,7 +5503,7 @@ class TestSimplifyTemplate : public TestFixture {
55035503
tokenlist.appendFileIfNew("test.cpp");
55045504
if (!tokenlist.createTokens(istr))
55055505
return false;
5506-
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
5506+
Tokenizer tokenizer(std::move(tokenlist), *this);
55075507
tokenizer.createLinks();
55085508
tokenizer.splitTemplateRightAngleBrackets(false);
55095509

@@ -5574,7 +5574,7 @@ class TestSimplifyTemplate : public TestFixture {
55745574
std::istringstream istr(code);
55755575
if (!TokenListHelper::createTokens(tokenlist, istr, "test.cpp"))
55765576
return false;
5577-
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
5577+
Tokenizer tokenizer(std::move(tokenlist), *this);
55785578
tokenizer.createLinks();
55795579
tokenizer.splitTemplateRightAngleBrackets(false);
55805580

@@ -5605,7 +5605,7 @@ class TestSimplifyTemplate : public TestFixture {
56055605
std::istringstream istr(code);
56065606
if (!TokenListHelper::createTokens(tokenlist, istr, "test.cpp"))
56075607
return false;
5608-
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
5608+
Tokenizer tokenizer(std::move(tokenlist), *this);
56095609
tokenizer.createLinks();
56105610
tokenizer.splitTemplateRightAngleBrackets(false);
56115611

‎test/testsimplifytypedef.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ class TestSimplifyTypedef : public TestFixture {
277277
std::istringstream istr(code);
278278
if (!tokenlist.createTokens(istr))
279279
return "";
280-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
280+
Tokenizer tokenizer(std::move(tokenlist), *this);
281281
tokenizer.createLinks();
282282
tokenizer.simplifyTypedef();
283283

@@ -312,7 +312,7 @@ class TestSimplifyTypedef : public TestFixture {
312312
std::istringstream istr(code);
313313
if (!TokenListHelper::createTokens(tokenlist, istr, "file.c"))
314314
return "";
315-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
315+
Tokenizer tokenizer(std::move(tokenlist), *this);
316316
tokenizer.createLinks();
317317
tokenizer.simplifyTypedef();
318318
try {
@@ -328,7 +328,7 @@ class TestSimplifyTypedef : public TestFixture {
328328
std::istringstream istr(code);
329329
if (!TokenListHelper::createTokens(tokenlist, istr, "file.c"))
330330
return {};
331-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
331+
Tokenizer tokenizer(std::move(tokenlist), *this);
332332
tokenizer.createLinks();
333333
tokenizer.simplifyTypedef();
334334
try {
@@ -4456,7 +4456,7 @@ class TestSimplifyTypedef : public TestFixture {
44564456
TokenList tokenlist{settings1, Standards::Language::C};
44574457
std::istringstream istr(code);
44584458
ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c"));
4459-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
4459+
Tokenizer tokenizer(std::move(tokenlist), *this);
44604460
tokenizer.createLinks();
44614461
tokenizer.simplifyTypedef();
44624462

@@ -4499,7 +4499,7 @@ class TestSimplifyTypedef : public TestFixture {
44994499
TokenList tokenlist{settings1, Standards::Language::C};
45004500
std::istringstream istr(code);
45014501
ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c"));
4502-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
4502+
Tokenizer tokenizer(std::move(tokenlist), *this);
45034503
tokenizer.createLinks();
45044504
tokenizer.simplifyTypedef();
45054505

@@ -4518,7 +4518,7 @@ class TestSimplifyTypedef : public TestFixture {
45184518
TokenList tokenlist{settings1, Standards::Language::C};
45194519
std::istringstream istr(code);
45204520
ASSERT(TokenListHelper::createTokens(tokenlist, istr, "file.c"));
4521-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
4521+
Tokenizer tokenizer(std::move(tokenlist), *this);
45224522
tokenizer.createLinks();
45234523
tokenizer.simplifyTypedef();
45244524

‎test/testtokenize.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -558,7 +558,7 @@ class TestTokenizer : public TestFixture {
558558
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
559559

560560
TokenList tokenlist{settings, Path::identify(filename, false)};
561-
Tokenizer tokenizer(std::move(tokenlist), settings, *this);
561+
Tokenizer tokenizer(std::move(tokenlist), *this);
562562
tokenizer.setDirectives(std::move(directives));
563563

564564
tokenizer.dump(ostr);
@@ -870,7 +870,7 @@ class TestTokenizer : public TestFixture {
870870
std::istringstream istr(code);
871871
tokenlist.appendFileIfNew("test.h");
872872
ASSERT(tokenlist.createTokens(istr));
873-
Tokenizer tokenizer(std::move(tokenlist), settings1, *this);
873+
Tokenizer tokenizer(std::move(tokenlist), *this);
874874
ASSERT_THROW_INTERNAL(tokenizer.simplifyTokens1(""), SYNTAX);
875875
}
876876
}
@@ -3689,7 +3689,7 @@ class TestTokenizer : public TestFixture {
36893689

36903690
void simplifyString() {
36913691
TokenList tokenlist{settings0, Standards::Language::CPP};
3692-
Tokenizer tokenizer(std::move(tokenlist), settings0, *this);
3692+
Tokenizer tokenizer(std::move(tokenlist), *this);
36933693
ASSERT_EQUALS("\"abc\"", tokenizer.simplifyString("\"abc\""));
36943694
ASSERT_EQUALS("\"\n\"", tokenizer.simplifyString("\"\\xa\""));
36953695
ASSERT_EQUALS("\"3\"", tokenizer.simplifyString("\"\\x33\""));
@@ -6143,7 +6143,7 @@ class TestTokenizer : public TestFixture {
61436143
if (!tokenlist.createTokens(istr))
61446144
return "ERROR";
61456145

6146-
Tokenizer tokenizer(std::move(tokenlist), settings0, *this);
6146+
Tokenizer tokenizer(std::move(tokenlist), *this);
61476147
tokenizer.combineStringAndCharLiterals();
61486148
tokenizer.combineOperators();
61496149
tokenizer.simplifySpaceshipOperator();

0 commit comments

Comments
 (0)