Skip to content

improved simplecpp::TokenList constructors #599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
firewave merged 2 commits into from
Dec 16, 2025
Merged

improved simplecpp::TokenList constructors #599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
097f9c7
added `simplecpp::View` to be able to write portable `simplecpp::Toke…
firewave Dec 8, 2025
c482be2
make `simplecpp::TokenList` constructors which take an array always a…
firewave Dec 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 44 additions & 7 deletions simplecpp.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,46 @@ namespace simplecpp {
enum cppstd_t : std::int8_t { CPPUnknown=-1, CPP03, CPP11, CPP14, CPP17, CPP20, CPP23, CPP26 };

using TokenString = std::string;

#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
using View = std::string_view;
#else
struct View
{
// cppcheck-suppress noExplicitConstructor
View(const char* data)
: mData(data)
, mSize(strlen(data))
{}

// only provide when std::span is not available so using untyped initilization won't use View
#if !defined(__cpp_lib_span)
View(const char* data, std::size_t size)
: mData(data)
, mSize(size)
{}

// cppcheck-suppress noExplicitConstructor
View(const std::string& str)
: mData(str.data())
, mSize(str.size())
{}
#endif // !defined(__cpp_lib_span)

const char* data() const {
return mData;
}

std::size_t size() const {
return mSize;
}

private:
const char* mData;
std::size_t mSize;
};
#endif // defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)

class Macro;

/**
Expand Down Expand Up @@ -217,7 +257,6 @@ namespace simplecpp {
explicit TokenList(std::vector<std::string> &filenames);
/** generates a token list from the given std::istream parameter */
TokenList(std::istream &istr, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR
/** generates a token list from the given buffer */
template<size_t size>
TokenList(const char (&data)[size], std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
Expand All @@ -228,7 +267,7 @@ namespace simplecpp {
TokenList(const unsigned char (&data)[size], std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
: TokenList(data, size-1, filenames, filename, outputList, 0)
{}

#ifdef SIMPLECPP_TOKENLIST_ALLOW_PTR
/** generates a token list from the given buffer */
TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
: TokenList(data, size, filenames, filename, outputList, 0)
Expand All @@ -237,13 +276,11 @@ namespace simplecpp {
TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
: TokenList(reinterpret_cast<const unsigned char*>(data), size, filenames, filename, outputList, 0)
{}
#endif
#if defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
#endif // SIMPLECPP_TOKENLIST_ALLOW_PTR
/** generates a token list from the given buffer */
TokenList(std::string_view data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
TokenList(View data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
: TokenList(reinterpret_cast<const unsigned char*>(data.data()), data.size(), filenames, filename, outputList, 0)
{}
#endif
#ifdef __cpp_lib_span
/** generates a token list from the given buffer */
TokenList(std::span<const char> data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
Expand All @@ -254,7 +291,7 @@ namespace simplecpp {
TokenList(std::span<const unsigned char> data, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr)
: TokenList(data.data(), data.size(), filenames, filename, outputList, 0)
{}
#endif
#endif // __cpp_lib_span

/** generates a token list from the given filename parameter */
TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList = nullptr);
Expand Down
94 changes: 85 additions & 9 deletions test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3299,20 +3299,97 @@ static void preprocess_files()
}
}

static void safe_api()
static void tokenlist_api()
{
// this test is to make sure the safe APIs are compiling
#if defined(__cpp_lib_string_view) || defined(__cpp_lib_span)
std::vector<std::string> filenames;
# if defined(__cpp_lib_string_view)
# if !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
// sized array + size
{
char input[] = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList(input,sizeof(input),filenames,"");
}
{
const char input[] = "code";
simplecpp::TokenList(input,sizeof(input),filenames,"");
}
{
unsigned char input[] = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList(input,sizeof(input),filenames,"");
}
{
const unsigned char input[] = "code";
simplecpp::TokenList(input,sizeof(input),filenames,"");
}
#endif // !defined(__cpp_lib_string_view) && !defined(__cpp_lib_span)
// pointer via View
{
const char * const input = "code";
simplecpp::TokenList({input},filenames,"");
}
// sized array via View
{
char input[] = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList(simplecpp::View{input},filenames,"");
}
{
const char input[] = "code";
simplecpp::TokenList(simplecpp::View{input},filenames,"");
}
// sized array + size via View/std::span
{
char input[] = "code"; // NOLINT(misc-const-correctness)
Copy link
Collaborator Author

@firewave firewave Dec 10, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not actually reported but pre-emptively added in case it is detected in the future. I added a comment about it upstream: llvm/llvm-project#56758 (comment)

simplecpp::TokenList({input,sizeof(input)},filenames,"");
}
{
const char input[] = "code";
simplecpp::TokenList({input,sizeof(input)},filenames,"");
}
// sized array
{
char input[] = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList(input,filenames,"");
}
{
const char input[] = "code";
simplecpp::TokenList(input,filenames,"");
}
{
unsigned char input[] = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList(input,filenames,"");
}
{
const unsigned char input[] = "code";
simplecpp::TokenList(input,filenames,"");
}
// std::string via View/std::span (implicit)
{
std::string input = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList(input,filenames,"");
}
{
const std::string input = "code";
simplecpp::TokenList(input,filenames,"");
}
// std::string via View/std::span (explicit)
{
std::string input = "code"; // NOLINT(misc-const-correctness)
simplecpp::TokenList({input},filenames,"");
}
{
const std::string input = "code";
simplecpp::TokenList({input},filenames,"");
}

// this test is to make sure the safe APIs are compiling
#ifdef __cpp_lib_string_view
{
const char input[] = "code";
const std::string_view sv = input;
// std::string_view can be implicitly converted into a std::span
simplecpp::TokenList(sv,filenames,"");
}
# endif
# ifdef __cpp_lib_span
#endif // __cpp_lib_string_view
#ifdef __cpp_lib_span
{
char input[] = "code";
const std::span sp = input;
Expand All @@ -3333,8 +3410,7 @@ static void safe_api()
const std::span sp = input;
simplecpp::TokenList(sp,filenames,"");
}
# endif
#endif
#endif // __cpp_lib_span
}

static void isAbsolutePath() {
Expand Down Expand Up @@ -3660,7 +3736,7 @@ int main(int argc, char **argv)

TEST_CASE(preprocess_files);

TEST_CASE(safe_api);
TEST_CASE(tokenlist_api);

TEST_CASE(isAbsolutePath);

Expand Down