Make TokenStream less recursive.

`TokenStream` is currently recursive in *two* ways:

- the `TokenTree` variant contains a `ThinTokenStream`, which can
  contain a `TokenStream`;

- the `TokenStream` variant contains a `Vec<TokenStream>`.

The latter is not necessary and causes significant complexity. This
commit replaces it with the simpler `Vec<(TokenTree, IsJoint)>`.

This reduces complexity significantly. In particular, `StreamCursor` is
eliminated, and `Cursor` becomes much simpler, consisting now of just a
`TokenStream` and an index.

The commit also removes the `Extend` impl for `TokenStream`, because it
is only used in tests. (The commit also removes those tests.)

Overall, the commit reduces the number of lines of code by almost 200.

src/libsyntax/parse/lexer/tokentrees.rs

@@ -1,7 +1,7 @@
 use print::pprust::token_to_string;
 use parse::lexer::StringReader;
 use parse::{token, PResult};
-use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree};
+use tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
 
 impl<'a> StringReader<'a> {
     // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
@@ -33,7 +33,7 @@ impl<'a> StringReader<'a> {
         }
     }
 
-    fn parse_token_tree(&mut self) -> PResult<'a, TokenStream> {
+    fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
         let sm = self.sess.source_map();
         match self.token {
             token::Eof => {
@@ -156,7 +156,7 @@ impl<'a> StringReader<'a> {
                 Ok(TokenTree::Delimited(
                     delim_span,
                     delim,
-                    tts.into(),
+                    tts.into()
                 ).into())
             },
             token::CloseDelim(_) => {
@@ -176,7 +176,7 @@ impl<'a> StringReader<'a> {
                 let raw = self.span_src_raw;
                 self.real_token();
                 let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
-                Ok(TokenStream::Tree(tt, if is_joint { Joint } else { NonJoint }))
+                Ok((tt, if is_joint { Joint } else { NonJoint }))
             }
         }
     }

src/libsyntax/parse/parser.rs

@@ -2914,7 +2914,7 @@ impl<'a> Parser<'a> {
                 TokenTree::Delimited(
                     frame.span,
                     frame.delim,
-                    frame.tree_cursor.original_stream().into(),
+                    frame.tree_cursor.stream.into(),
                 )
             },
             token::CloseDelim(_) | token::Eof => unreachable!(),