Skip to content

Commit 44d097d

Browse files
firewavefirewave
authored
testrunner: some Tokenizer related refactoring (danmar#7427)
1 parent c78d1f2 commit 44d097d

22 files changed

+188
-227
lines changed

‎Makefile‎

Lines changed: 50 additions & 50 deletions
Large diffs are not rendered by default.

‎test/helpers.cpp‎

Lines changed: 2 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -162,13 +162,11 @@ std::map<std::string, std::string> PreprocessorHelper::getcode(const Settings& s
162162
return cfgcode;
163163
}
164164

165-
void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger)
165+
void SimpleTokenizer2::preprocess(const char code[], std::vector<std::string> &files, const std::string& file0, Tokenizer& tokenizer, ErrorLogger& errorlogger)
166166
{
167-
// TODO: make sure the given Tokenizer has not been used yet
168-
169167
// TODO: get rid of stream
170168
std::istringstream istr(code);
171-
const simplecpp::TokenList tokens1(istr, files, files[0]);
169+
const simplecpp::TokenList tokens1(istr, files, file0);
172170

173171
Preprocessor preprocessor(tokenizer.getSettings(), errorlogger);
174172
simplecpp::TokenList tokens2 = preprocessor.preprocess(tokens1, "", files, true);
@@ -180,40 +178,6 @@ void PreprocessorHelper::preprocess(const char code[], std::vector<std::string>
180178
tokenizer.setDirectives(std::move(directives));
181179
}
182180

183-
// TODO: get rid of this
184-
void PreprocessorHelper::preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger, const simplecpp::DUI& dui)
185-
{
186-
// TODO: make sure the given Tokenizer has not been used yet
187-
188-
std::istringstream istr(code);
189-
const simplecpp::TokenList tokens1(istr, files, files[0]);
190-
191-
// Preprocess..
192-
simplecpp::TokenList tokens2(files);
193-
std::map<std::string, simplecpp::TokenList*> filedata;
194-
// TODO: provide and handle outputList
195-
simplecpp::preprocess(tokens2, tokens1, files, filedata, dui);
196-
197-
// Tokenizer..
198-
tokenizer.list.createTokens(std::move(tokens2));
199-
200-
const Preprocessor preprocessor(tokenizer.getSettings(), errorlogger);
201-
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
202-
tokenizer.setDirectives(std::move(directives));
203-
}
204-
205-
std::vector<RemarkComment> PreprocessorHelper::getRemarkComments(const char code[], ErrorLogger& errorLogger)
206-
{
207-
std::vector<std::string> files{"test.cpp"};
208-
std::istringstream istr(code);
209-
const simplecpp::TokenList tokens1(istr, files, files[0]);
210-
211-
const Settings settings;
212-
213-
const Preprocessor preprocessor(settings, errorLogger);
214-
return preprocessor.getRemarkComments(tokens1);
215-
}
216-
217181
bool LibraryHelper::loadxmldata(Library &lib, const char xmldata[], std::size_t len)
218182
{
219183
tinyxml2::XMLDocument doc;

‎test/helpers.h‎

Lines changed: 56 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
#define helpersH
2121

2222
#include "library.h"
23-
#include "preprocessor.h"
2423
#include "settings.h"
2524
#include "standards.h"
2625
#include "tokenize.h"
@@ -37,9 +36,6 @@
3736
class Token;
3837
class SuppressionList;
3938
class ErrorLogger;
40-
namespace simplecpp {
41-
struct DUI;
42-
}
4339
namespace tinyxml2 {
4440
class XMLDocument;
4541
}
@@ -69,39 +65,59 @@ class SimpleTokenizer : public Tokenizer {
6965
}
7066
*/
7167

72-
/**
73-
* Tokenize code
74-
* @param code The code
75-
* @param cpp Indicates if the code is C++
76-
* @param configuration E.g. "A" for code where "#ifdef A" is true
77-
* @return false if source code contains syntax errors
78-
*/
68+
template<size_t size>
69+
bool tokenize(const char (&code)[size],
70+
const std::string& filename,
71+
const std::string &configuration = "")
72+
{
73+
std::istringstream istr(code);
74+
return tokenize(istr, filename, configuration);
75+
}
76+
7977
template<size_t size>
8078
bool tokenize(const char (&code)[size],
8179
bool cpp = true,
8280
const std::string &configuration = "")
8381
{
8482
std::istringstream istr(code);
85-
if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
86-
return false;
83+
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
84+
}
8785

88-
return simplifyTokens1(configuration);
86+
bool tokenize(const std::string& code,
87+
const std::string& filename,
88+
const std::string &configuration = "")
89+
{
90+
std::istringstream istr(code);
91+
return tokenize(istr, filename, configuration);
8992
}
9093

91-
// TODO: get rid of this
9294
bool tokenize(const std::string& code,
9395
bool cpp = true,
9496
const std::string &configuration = "")
9597
{
9698
std::istringstream istr(code);
97-
if (!list.createTokens(istr, cpp ? "test.cpp" : "test.c"))
99+
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
100+
}
101+
102+
private:
103+
/**
104+
* Tokenize code
105+
* @param istr The code as stream
106+
* @param filename Indicates if the code is C++
107+
* @param configuration E.g. "A" for code where "#ifdef A" is true
108+
* @return false if source code contains syntax errors
109+
*/
110+
bool tokenize(std::istream& istr,
111+
const std::string& filename,
112+
const std::string &configuration = "")
113+
{
114+
if (!list.createTokens(istr, filename))
98115
return false;
99116

100117
return simplifyTokens1(configuration);
101118
}
102119

103-
private:
104-
// TODO. find a better solution
120+
// TODO: find a better solution
105121
static const Settings s_settings;
106122
};
107123

@@ -171,12 +187,6 @@ class PreprocessorHelper
171187
static std::string getcode(const Settings& settings, ErrorLogger& errorlogger, const std::string &filedata, const std::string &cfg, const std::string &filename, SuppressionList *inlineSuppression = nullptr);
172188
static std::map<std::string, std::string> getcode(const Settings& settings, ErrorLogger& errorlogger, const char code[], const std::string &filename = "file.c", SuppressionList *inlineSuppression = nullptr);
173189

174-
static void preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger);
175-
static void preprocess(const char code[], std::vector<std::string> &files, Tokenizer& tokenizer, ErrorLogger& errorlogger, const simplecpp::DUI& dui);
176-
177-
/** get remark comments */
178-
static std::vector<RemarkComment> getRemarkComments(const char code[], ErrorLogger& errorLogger);
179-
180190
private:
181191
static std::map<std::string, std::string> getcode(const Settings& settings, ErrorLogger& errorlogger, const char code[], std::set<std::string> cfgs, const std::string &filename = "file.c", SuppressionList *inlineSuppression = nullptr);
182192
};
@@ -246,4 +256,26 @@ struct LibraryHelper
246256
static Library::Error loadxmldoc(Library &lib, const tinyxml2::XMLDocument& doc);
247257
};
248258

259+
class SimpleTokenizer2 : public Tokenizer {
260+
public:
261+
template<size_t size>
262+
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char (&code)[size], const std::string& file0)
263+
: Tokenizer{settings, errorlogger}
264+
{
265+
preprocess(code, mFiles, file0, *this, errorlogger);
266+
}
267+
268+
// TODO: get rid of this
269+
SimpleTokenizer2(const Settings &settings, ErrorLogger &errorlogger, const char code[], const std::string& file0)
270+
: Tokenizer{settings, errorlogger}
271+
{
272+
preprocess(code, mFiles, file0, *this, errorlogger);
273+
}
274+
275+
private:
276+
static void preprocess(const char code[], std::vector<std::string> &files, const std::string& file0, Tokenizer& tokenizer, ErrorLogger& errorlogger);
277+
278+
std::vector<std::string> mFiles;
279+
};
280+
249281
#endif // helpersH

‎test/testbufferoverrun.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
#include "platform.h"
2626
#include "settings.h"
2727
#include "fixture.h"
28-
#include "tokenize.h"
2928

3029
#include <cstddef>
3130
#include <list>
@@ -76,9 +75,7 @@ class TestBufferOverrun : public TestFixture {
7675
{
7776
const Settings settings = settingsBuilder(settings0).severity(Severity::performance).certainty(Certainty::inconclusive).build();
7877

79-
std::vector<std::string> files(1, "test.cpp");
80-
Tokenizer tokenizer(settings, *this);
81-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
78+
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");
8279

8380
// Tokenizer..
8481
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

‎test/testclass.cpp‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,9 @@
2323
#include "fixture.h"
2424
#include "helpers.h"
2525
#include "settings.h"
26-
#include "tokenize.h"
27-
#include "tokenlist.h"
2826

2927
#include <cstddef>
3028
#include <list>
31-
#include <sstream>
3229
#include <string>
3330
#include <vector>
3431

@@ -8703,9 +8700,7 @@ class TestClass : public TestFixture {
87038700
void checkUselessOverride_(const char* file, int line, const char code[]) {
87048701
const Settings settings = settingsBuilder().severity(Severity::style).build();
87058702

8706-
std::vector<std::string> files(1, "test.cpp");
8707-
Tokenizer tokenizer(settings, *this);
8708-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
8703+
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");
87098704

87108705
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
87118706

@@ -9085,11 +9080,9 @@ class TestClass : public TestFixture {
90859080
// getFileInfo
90869081
std::list<Check::FileInfo*> fileInfo;
90879082
for (const std::string& c: code) {
9088-
Tokenizer tokenizer(settingsDefault, *this);
9089-
std::istringstream istr(c);
9083+
SimpleTokenizer tokenizer{settingsDefault, *this};
90909084
const std::string filename = std::to_string(fileInfo.size()) + ".cpp";
9091-
ASSERT(tokenizer.list.createTokens(istr, filename));
9092-
ASSERT(tokenizer.simplifyTokens1(""));
9085+
ASSERT(tokenizer.tokenize(c, filename));
90939086
fileInfo.push_back(check.getFileInfo(tokenizer, settingsDefault));
90949087
}
90959088

@@ -9151,9 +9144,7 @@ class TestClass : public TestFixture {
91519144
void checkReturnByReference_(const char* file, int line, const char (&code)[size]) {
91529145
const Settings settings = settingsBuilder().severity(Severity::performance).library("std.cfg").build();
91539146

9154-
std::vector<std::string> files(1, "test.cpp");
9155-
Tokenizer tokenizer(settings, *this);
9156-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
9147+
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");
91579148

91589149
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
91599150

‎test/testcondition.cpp‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
#include "helpers.h"
2323
#include "platform.h"
2424
#include "settings.h"
25-
#include "tokenize.h"
2625

2726
#include <cstddef>
2827
#include <limits>
@@ -139,9 +138,8 @@ class TestCondition : public TestFixture {
139138
#define check(...) check_(__FILE__, __LINE__, __VA_ARGS__)
140139
void check_(const char* file, int line, const char code[], const CheckOptions& options = make_default_obj()) {
141140
const Settings settings = settingsBuilder(options.s ? *options.s : settings0).certainty(Certainty::inconclusive, options.inconclusive).build();
142-
Tokenizer tokenizer(settings, *this);
143-
std::vector<std::string> files(1, options.cpp ? "test.cpp" : "test.c");
144-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
141+
142+
SimpleTokenizer2 tokenizer(settings, *this, code, options.cpp ? "test.cpp" : "test.c");
145143

146144
// Tokenizer..
147145
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
@@ -155,9 +153,7 @@ class TestCondition : public TestFixture {
155153
{
156154
const Settings settings = settingsBuilder(settings0).severity(Severity::performance).certainty(Certainty::inconclusive).build();
157155

158-
std::vector<std::string> files(1, "test.cpp");
159-
Tokenizer tokenizer(settings, *this);
160-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
156+
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");
161157

162158
// Tokenizer..
163159
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

‎test/testincompletestatement.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
#include "helpers.h"
2222
#include "settings.h"
2323
#include "fixture.h"
24-
#include "tokenize.h"
2524

2625
#include <string>
2726
#include <vector>
@@ -44,9 +43,7 @@ class TestIncompleteStatement : public TestFixture {
4443
void check_(const char* file, int line, const char code[], const CheckOptions& options = make_default_obj()) {
4544
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, options.inconclusive).build();
4645

47-
std::vector<std::string> files(1, options.cpp ? "test.cpp" : "test.c");
48-
Tokenizer tokenizer(settings1, *this);
49-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
46+
SimpleTokenizer2 tokenizer(settings1, *this, code, options.cpp ? "test.cpp" : "test.c");
5047

5148
// Tokenize..
5249
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

‎test/testleakautovar.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
#include "fixture.h"
2121
#include "helpers.h"
2222
#include "settings.h"
23-
#include "tokenize.h"
2423

2524
#include <cstddef>
2625
#include <string>
@@ -3195,9 +3194,7 @@ class TestLeakAutoVarRecursiveCountLimit : public TestFixture {
31953194

31963195
#define checkP(...) checkP_(__FILE__, __LINE__, __VA_ARGS__)
31973196
void checkP_(const char* file, int line, const char code[], bool cpp = false) {
3198-
std::vector<std::string> files(1, cpp?"test.cpp":"test.c");
3199-
Tokenizer tokenizer(settings, *this);
3200-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
3197+
SimpleTokenizer2 tokenizer(settings, *this, code, cpp?"test.cpp":"test.c");
32013198

32023199
// Tokenizer..
32033200
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

‎test/testnullpointer.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
#include "library.h"
2626
#include "settings.h"
2727
#include "token.h"
28-
#include "tokenize.h"
2928

3029
#include <cstddef>
3130
#include <list>
@@ -201,9 +200,7 @@ class TestNullPointer : public TestFixture {
201200
void checkP_(const char* file, int line, const char (&code)[size]) {
202201
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, false).build();
203202

204-
std::vector<std::string> files(1, "test.cpp");
205-
Tokenizer tokenizer(settings1, *this);
206-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
203+
SimpleTokenizer2 tokenizer(settings1, *this, code, "test.cpp");
207204

208205
// Tokenizer..
209206
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

‎test/testother.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
#include "platform.h"
2424
#include "settings.h"
2525
#include "standards.h"
26-
#include "tokenize.h"
2726

2827
#include <cstddef>
2928
#include <string>
@@ -354,9 +353,7 @@ class TestOther : public TestFixture {
354353
settings->standards.cpp = Standards::CPPLatest;
355354
settings->certainty.enable(Certainty::inconclusive);
356355

357-
std::vector<std::string> files(1, options.cpp ? "test.cpp" : "test.c");
358-
Tokenizer tokenizer(*settings, *this);
359-
PreprocessorHelper::preprocess(code, files, tokenizer, *this);
356+
SimpleTokenizer2 tokenizer(*settings, *this, code, options.cpp ? "test.cpp" : "test.c");
360357

361358
// Tokenizer..
362359
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);

0 commit comments

Comments
 (0)