Rollup merge of #98278 - nnethercote:some-token-stream-cleanups, r=petrochenkov

Some token stream cleanups

Best reviewed one commit at a time.

r? ```@petrochenkov```

Author: JohnTitor

compiler/rustc_ast/src/tokenstream.rs

@@ -25,7 +25,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use rustc_span::{Span, DUMMY_SP};
 use smallvec::{smallvec, SmallVec};
 
-use std::{fmt, iter, mem};
+use std::{fmt, iter};
 
 /// When the main Rust parser encounters a syntax-extension invocation, it
 /// parses the arguments to the invocation as a token tree. This is a very
@@ -399,45 +399,6 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream {
-        match streams.len() {
-            0 => TokenStream::default(),
-            1 => streams.pop().unwrap(),
-            _ => {
-                // We are going to extend the first stream in `streams` with
-                // the elements from the subsequent streams. This requires
-                // using `make_mut()` on the first stream, and in practice this
-                // doesn't cause cloning 99.9% of the time.
-                //
-                // One very common use case is when `streams` has two elements,
-                // where the first stream has any number of elements within
-                // (often 1, but sometimes many more) and the second stream has
-                // a single element within.
-
-                // Determine how much the first stream will be extended.
-                // Needed to avoid quadratic blow up from on-the-fly
-                // reallocations (#57735).
-                let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum();
-
-                // Get the first stream. If it's `None`, create an empty
-                // stream.
-                let mut iter = streams.drain(..);
-                let mut first_stream_lrc = iter.next().unwrap().0;
-
-                // Append the elements to the first stream, after reserving
-                // space for them.
-                let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc);
-                first_vec_mut.reserve(num_appends);
-                for stream in iter {
-                    first_vec_mut.extend(stream.0.iter().cloned());
-                }
-
-                // Create the final `TokenStream`.
-                TokenStream(first_stream_lrc)
-            }
-        }
-    }
-
     pub fn trees(&self) -> CursorRef<'_> {
         CursorRef::new(self)
     }
@@ -562,50 +523,65 @@ impl TokenStreamBuilder {
     }
 
     pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
-        let mut stream = stream.into();
-
-        // If `self` is not empty and the last tree within the last stream is a
-        // token tree marked with `Joint`...
-        if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut()
-            && let Some((TokenTree::Token(last_token), Spacing::Joint)) = last_stream_lrc.last()
-            // ...and `stream` is not empty and the first tree within it is
-            // a token tree...
-            && let TokenStream(ref mut stream_lrc) = stream
-            && let Some((TokenTree::Token(token), spacing)) = stream_lrc.first()
-            // ...and the two tokens can be glued together...
-            && let Some(glued_tok) = last_token.glue(&token)
-        {
-            // ...then do so, by overwriting the last token
-            // tree in `self` and removing the first token tree
-            // from `stream`. This requires using `make_mut()`
-            // on the last stream in `self` and on `stream`,
-            // and in practice this doesn't cause cloning 99.9%
-            // of the time.
-
-            // Overwrite the last token tree with the merged
-            // token.
-            let last_vec_mut = Lrc::make_mut(last_stream_lrc);
-            *last_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing);
-
-            // Remove the first token tree from `stream`. (This
-            // is almost always the only tree in `stream`.)
-            let stream_vec_mut = Lrc::make_mut(stream_lrc);
-            stream_vec_mut.remove(0);
-
-            // Don't push `stream` if it's empty -- that could
-            // block subsequent token gluing, by getting
-            // between two token trees that should be glued
-            // together.
-            if !stream.is_empty() {
-                self.0.push(stream);
-            }
-            return;
-        }
-        self.0.push(stream);
+        self.0.push(stream.into());
     }
 
     pub fn build(self) -> TokenStream {
-        TokenStream::from_streams(self.0)
+        let mut streams = self.0;
+        match streams.len() {
+            0 => TokenStream::default(),
+            1 => streams.pop().unwrap(),
+            _ => {
+                // We will extend the first stream in `streams` with the
+                // elements from the subsequent streams. This requires using
+                // `make_mut()` on the first stream, and in practice this
+                // doesn't cause cloning 99.9% of the time.
+                //
+                // One very common use case is when `streams` has two elements,
+                // where the first stream has any number of elements within
+                // (often 1, but sometimes many more) and the second stream has
+                // a single element within.
+
+                // Determine how much the first stream will be extended.
+                // Needed to avoid quadratic blow up from on-the-fly
+                // reallocations (#57735).
+                let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum();
+
+                // Get the first stream, which will become the result stream.
+                // If it's `None`, create an empty stream.
+                let mut iter = streams.drain(..);
+                let mut res_stream_lrc = iter.next().unwrap().0;
+
+                // Append the subsequent elements to the result stream, after
+                // reserving space for them.
+                let res_vec_mut = Lrc::make_mut(&mut res_stream_lrc);
+                res_vec_mut.reserve(num_appends);
+                for stream in iter {
+                    let stream_iter = stream.0.iter().cloned();
+
+                    // If (a) `res_mut_vec` is not empty and the last tree
+                    // within it is a token tree marked with `Joint`, and (b)
+                    // `stream` is not empty and the first tree within it is a
+                    // token tree, and (c) the two tokens can be glued
+                    // together...
+                    if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last()
+                        && let Some((TokenTree::Token(tok), spacing)) = stream.0.first()
+                        && let Some(glued_tok) = last_tok.glue(&tok)
+                    {
+                        // ...then overwrite the last token tree in
+                        // `res_vec_mut` with the glued token, and skip the
+                        // first token tree from `stream`.
+                        *res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing);
+                        res_vec_mut.extend(stream_iter.skip(1));
+                    } else {
+                        // Append all of `stream`.
+                        res_vec_mut.extend(stream_iter);
+                    }
+                }
+
+                TokenStream(res_stream_lrc)
+            }
+        }
     }
 }
 
@@ -679,20 +655,6 @@ impl Cursor {
         })
     }
 
-    pub fn index(&self) -> usize {
-        self.index
-    }
-
-    pub fn append(&mut self, new_stream: TokenStream) {
-        if new_stream.is_empty() {
-            return;
-        }
-        let index = self.index;
-        let stream = mem::take(&mut self.stream);
-        *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees();
-        self.index = index;
-    }
-
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
         self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
     }