Initial string literal concat support (except interpolation not yet working)

Allows code like

    std::cout << "Hello " "world\n";

Note that in Cpp2 this is done in the grammar inside the language, whereas in Cpp2 it's done in the preprocessor outside the language.

Author: hsutter

source/lex.h

@@ -1659,8 +1659,8 @@ auto lex_line(
                 }
 
                 //G string-literal:
-                //G     encoding-prefix? '"' s-char-seq? '"'
-                //G     encoding-prefix? 'R"' d-char-seq? '(' s-char-seq? ')' d-char-seq? '"'
+                //G     string-literal? encoding-prefix? '"' s-char-seq? '"'
+                //G     string-literal? encoding-prefix? 'R"' d-char-seq? '(' s-char-seq? ')' d-char-seq? '"'
                 //G
                 //G s-char-seq:
                 //G     interpolation? s-char

source/parse.h

@@ -186,44 +186,70 @@ struct primary_expression_node
 
 
 struct literal_node {
-    token const* literal             = {};
-    token const* user_defined_suffix = {};
+    //  A literal is represented as a sequence of tokens:
+    //      - length 1: a literal (most common)
+    //      - length 2: a literal and a user-defined suffix
+    //      - length >= 2: a series of one or more of either of the above for string literals
+    std::vector <token const*> pieces = {};
 
     //  API
     //
     auto get_token() const
         -> token const*
     {
-        return literal;
+        assert(!pieces.empty());
+        return pieces.front();
     }
 
     auto to_string() const
         -> std::string
     {
-        assert (literal);
-        auto ret = literal->to_string();
-        if (user_defined_suffix) {
-            ret += user_defined_suffix->to_string();
+        assert(!pieces.empty());
+        auto ret = std::string{};
+
+        for (bool first = true; auto p : pieces)
+        {
+            assert(p);
+
+            //  Add a space to non-first pieces that start with " (i.e., not a UDL suffix)
+            if (
+                !std::exchange(first, false)
+                && p->as_string_view().starts_with("\"")
+                ) 
+            {
+                ret += " ";
+            }
+
+            ret += *p;
         }
+
         return ret;
     }
 
+    auto has_user_defined_suffix()
+        -> bool
+    {
+        return
+            std::ssize(pieces) > 1
+            && !pieces[1]->as_string_view().starts_with("\"")
+            ;
+    }
+
     //  Internals
     //
     auto position() const
         -> source_position
     {
-        assert (literal);
-        return literal->position();
+        assert(!pieces.empty());
+        return get_token()->position();
     }
 
     auto visit(auto& v, int depth) -> void
     {
         v.start(*this, depth);
-        assert (literal);
-        literal->visit(v, depth+1);
-        if (user_defined_suffix) {
-            user_defined_suffix->visit(v, depth+1);
+        for (auto p : pieces) {
+            assert(p);
+            p->visit(v, depth+1);
         }
         v.end(*this, depth);
     }
@@ -4617,18 +4643,7 @@ auto pretty_print_visualize(primary_expression_node const& n, int indent)
 auto pretty_print_visualize(literal_node const& n, int)
     -> std::string
 {
-    //  TODO: This is an initial visualizer implementation, and still
-    //        skips a few rarer things (such as raw string literals)
-
-    assert(n.literal);
-
-    auto ret = n.literal->to_string();
-
-    if (n.user_defined_suffix) {
-        ret += n.user_defined_suffix->as_string_view();
-    }
-
-    return ret;
+    return n.to_string();
 }
 
 
@@ -6908,12 +6923,26 @@ class parser
     {
         if (is_literal(curr().type())) {
             auto n = std::make_unique<literal_node>();
-            n->literal = &curr();
+            n->pieces.push_back( &curr() );
             next();
             if (curr().type() == lexeme::UserDefinedLiteralSuffix) {
-                n->user_defined_suffix = &curr();
+                n->pieces.push_back(&curr());
                 next();
             }
+
+            //  String literals can have multiple chunks, such as "xyzzy" "plugh"
+            //  (in Cpp2 these are merged in the preprocessor, in Cpp2 they're in the grammar)
+            if (n->pieces.front()->type() == lexeme::StringLiteral) {
+                while (curr().type() == lexeme::StringLiteral) {
+                    n->pieces.push_back(&curr());
+                    next();
+                    if (curr().type() == lexeme::UserDefinedLiteralSuffix) {
+                        n->pieces.push_back(&curr());
+                        next();
+                    }
+                }
+            }
+
             return n;
         }
         return {};

source/to_cpp1.h

@@ -1699,10 +1699,9 @@ class cppfront
             pos = n.position();
         }
 
-        assert(n.literal);
-        emit(*n.literal);
-        if (n.user_defined_suffix) {
-            emit(*n.user_defined_suffix);
+        for (auto p : n.pieces) {
+            assert(p);
+            emit(*p);
         }
     }
 
@@ -3458,7 +3457,7 @@ class cppfront
                 {
                     if (auto lit = i->expr_list->expressions.front().expr->get_literal();
                         lit
-                        && lit->literal->type() == lexeme::DecimalLiteral
+                        && lit->get_token()->type() == lexeme::DecimalLiteral
                         )
                     {
                         prefix.emplace_back( "CPP2_ASSERT_IN_BOUNDS_LITERAL(", i->op->position() );
@@ -3561,7 +3560,7 @@ class cppfront
                 && is_literal(t->type())
                 && t->type() != lexeme::StringLiteral
                 && t->type() != lexeme::FloatLiteral
-                && !std::get<primary_expression_node::literal>(p->expr)->user_defined_suffix
+                && !std::get<primary_expression_node::literal>(p->expr)->has_user_defined_suffix()
                 && std::ssize(n.ops) > 0
                 && *n.ops[0].op == "as"
                 )