Language.Rust.Parser.Lexer

As much as possible, this follows Rust's choices for tokenization, including punting some things to the parser. For instance, the last two > in Vec<Option<i32>> are lexed as a single GreaterGreater token while the last two tokens of Vec<Option<Option<i32>>> are GreaterGreater and Greater.

Yet weirder (but very useful in parsing for dealing with conflicts and precedences of logical and, bitwise and, and unary reference), &&&x&&&y lexes into AmpersandAmpersand, Ampersand, IdentTok "x", AmpersandAmpersand, Ampersand, IdentTok "y". Although the parser sometimes needs to "break apart" tokens, it never has to think about putting them together. That means it can easily figure out that &&&x&&&y parses as &(&(&x)) && (&y) and not &(&(&x)) & (&(&y)) even if bitwise conjunctions bind more tightly that logical conjunctions.

This sort of amguity where one token need to be broken up by the parser occurs for

• && in patterns like &&mut x
• || in closures with no arguments like || x
• << in qualified type paths like FromIterator<<A as IntoIterator>::Item>
• >> in qualified paths like <Self as Foo<T>>::Bar
• >= in equality predicates like F<A>=i32
• >>= in equality predicates like F<G<A>>=i32

Lexing

lexToken :: P (Spanned Token)

Lexer for one Token. The only token this cannot produce is Interpolated.

lexNonSpace :: P (Spanned Token)

Lexer for one non-whitespace Token. The only tokens this cannot produce are Interpolated and Space (which includes comments that aren't doc comments).

lexTokens :: P (Spanned Token) -> P [Spanned Token]

Apply the given lexer repeatedly until (but not including) the Eof token. Meant for debugging purposes - in general this defeats the point of a threaded lexer.

lexShebangLine :: P (Maybe String)

Lex the first line, if it immediately starts with #! (but not #![ - that should be an inner attribute). If this fails to find a shebang line, it consumes no input.

Tokens

data Token

A general token (based on syntax::parse::token::Token).

Unlike its libsyntax counterpart, Token has folded in syntax::parse::token::BinOpToken and syntax::parse::token::BinOpEqToken as regular tokens.

Constructors

Equal

= token

Less

< token

Greater

> token

Ampersand

& token

Pipe

| token

Exclamation

! token

Tilde

~ token

Plus

+ token

Minus

- token

Star

* token

Slash

/ token

Percent

% token

Caret

^ token

GreaterEqual

>= token

GreaterGreaterEqual

>>= token

AmpersandAmpersand

&& token

PipePipe

|| token

LessLess

<< token

GreaterGreater

>> token

EqualEqual

== token

NotEqual

!= token

LessEqual

<= token

LessLessEqual

<<= token

MinusEqual

-= token

AmpersandEqual

&= token

PipeEqual

|= token

PlusEqual

+= token

StarEqual

*= token

SlashEqual

/= token

CaretEqual

^= token

PercentEqual

%= token

At

@ token

Dot

. token

DotDot

.. token

DotDotEqual

..= token

DotDotDot

... token

Comma

, token

Semicolon

; token

Colon

: token

ModSep

:: token

RArrow

-> token

LArrow

<- token

FatArrow

=> token

Pound

# token

Dollar

$ token

Question

? token

OpenDelim !Delim

One of (, [, {

CloseDelim !Delim

One of ), ], }

LiteralTok LitTok (Maybe Name)

a literal token with an optional suffix (something like i32)

IdentTok Ident

an arbitrary identifier (something like x or foo or and_then)

LifetimeTok Ident

a lifetime (something like 'a or 'static)

Space Space Name

whitespace

Doc String !AttrStyle !Bool

doc comment with its contents, whether it is outer/inner, and whether it is inline or not

Shebang

#! shebang token

Eof

end of file token

Interpolated (Nonterminal Span)

can be expanded into several tokens in macro-expansion

Instances

Eq Token

Methods

(==) :: Token -> Token -> Bool

(/=) :: Token -> Token -> Bool

Data Token

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Token -> c Token

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Token

toConstr :: Token -> Constr

dataTypeOf :: Token -> DataType

dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c Token)

dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)

gmapT :: (forall b. Data b => b -> b) -> Token -> Token

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r

gmapQ :: (forall d. Data d => d -> u) -> Token -> [u]

gmapQi :: Int -> (forall d. Data d => d -> u) -> Token -> u

gmapM :: Monad m => (forall d. Data d => d -> m d) -> Token -> m Token

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token

Ord Token

Methods

compare :: Token -> Token -> Ordering

(<) :: Token -> Token -> Bool

(<=) :: Token -> Token -> Bool

(>) :: Token -> Token -> Bool

(>=) :: Token -> Token -> Bool

max :: Token -> Token -> Token

min :: Token -> Token -> Token

Show Token

This instance is only for error messages and debugging purposes.

Methods

showsPrec :: Int -> Token -> ShowS

show :: Token -> String

showList :: [Token] -> ShowS

Generic Token

Associated Types

type Rep Token :: * -> *

Methods

from :: Token -> Rep Token x

to :: Rep Token x -> Token

NFData Token

Methods

rnf :: Token -> ()

Pretty Token

Methods

prettyUnresolved :: Token -> Doc b

type Rep Token

type Rep Token = D1 * (MetaData "Token" "Language.Rust.Syntax.Token" "language-rust-0.2.0.27-AdTGPQ0ClkM3k0DGaVHmew" False) ((:+:) * ((:+:) * ((:+:) * ((:+:) * ((:+:) * (C1 * (MetaCons "Equal" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "Less" PrefixI False) (U1 *)) (C1 * (MetaCons "Greater" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "Ampersand" PrefixI False) (U1 *)) (C1 * (MetaCons "Pipe" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "Exclamation" PrefixI False) (U1 *)) (C1 * (MetaCons "Tilde" PrefixI False) (U1 *))))) ((:+:) * ((:+:) * (C1 * (MetaCons "Plus" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "Minus" PrefixI False) (U1 *)) (C1 * (MetaCons "Star" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "Slash" PrefixI False) (U1 *)) (C1 * (MetaCons "Percent" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "Caret" PrefixI False) (U1 *)) (C1 * (MetaCons "GreaterEqual" PrefixI False) (U1 *)))))) ((:+:) * ((:+:) * ((:+:) * (C1 * (MetaCons "GreaterGreaterEqual" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "AmpersandAmpersand" PrefixI False) (U1 *)) (C1 * (MetaCons "PipePipe" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "LessLess" PrefixI False) (U1 *)) (C1 * (MetaCons "GreaterGreater" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "EqualEqual" PrefixI False) (U1 *)) (C1 * (MetaCons "NotEqual" PrefixI False) (U1 *))))) ((:+:) * ((:+:) * (C1 * (MetaCons "LessEqual" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "LessLessEqual" PrefixI False) (U1 *)) (C1 * (MetaCons "MinusEqual" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "AmpersandEqual" PrefixI False) (U1 *)) (C1 * (MetaCons "PipeEqual" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "PlusEqual" PrefixI False) (U1 *)) (C1 * (MetaCons "StarEqual" PrefixI False) (U1 *))))))) ((:+:) * ((:+:) * ((:+:) * ((:+:) * (C1 * (MetaCons "SlashEqual" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "CaretEqual" PrefixI False) (U1 *)) (C1 * (MetaCons "PercentEqual" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "At" PrefixI False) (U1 *)) (C1 * (MetaCons "Dot" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "DotDot" PrefixI False) (U1 *)) (C1 * (MetaCons "DotDotEqual" PrefixI False) (U1 *))))) ((:+:) * ((:+:) * (C1 * (MetaCons "DotDotDot" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "Comma" PrefixI False) (U1 *)) (C1 * (MetaCons "Semicolon" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "Colon" PrefixI False) (U1 *)) (C1 * (MetaCons "ModSep" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "RArrow" PrefixI False) (U1 *)) (C1 * (MetaCons "LArrow" PrefixI False) (U1 *)))))) ((:+:) * ((:+:) * ((:+:) * (C1 * (MetaCons "FatArrow" PrefixI False) (U1 *)) ((:+:) * (C1 * (MetaCons "Pound" PrefixI False) (U1 *)) (C1 * (MetaCons "Dollar" PrefixI False) (U1 *)))) ((:+:) * ((:+:) * (C1 * (MetaCons "Question" PrefixI False) (U1 *)) (C1 * (MetaCons "OpenDelim" PrefixI False) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness SourceStrict DecidedStrict) (Rec0 * Delim)))) ((:+:) * (C1 * (MetaCons "CloseDelim" PrefixI False) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness SourceStrict DecidedStrict) (Rec0 * Delim))) (C1 * (MetaCons "LiteralTok" PrefixI False) ((:*:) * (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * LitTok)) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * (Maybe Name)))))))) ((:+:) * ((:+:) * (C1 * (MetaCons "IdentTok" PrefixI False) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * Ident))) ((:+:) * (C1 * (MetaCons "LifetimeTok" PrefixI False) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * Ident))) (C1 * (MetaCons "Space" PrefixI False) ((:*:) * (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * Space)) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * Name)))))) ((:+:) * ((:+:) * (C1 * (MetaCons "Doc" PrefixI False) ((:*:) * (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * String)) ((:*:) * (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness SourceStrict DecidedStrict) (Rec0 * AttrStyle)) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness SourceStrict DecidedStrict) (Rec0 * Bool))))) (C1 * (MetaCons "Shebang" PrefixI False) (U1 *))) ((:+:) * (C1 * (MetaCons "Eof" PrefixI False) (U1 *)) (C1 * (MetaCons "Interpolated" PrefixI False) (S1 * (MetaSel (Nothing Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * (Nonterminal Span))))))))))

Error reporting

lexicalError :: P a

Signal a lexical error.