Language.Python.Common.Token

Lexical tokens for the Python lexer. Contains the superset of tokens from version 2 and version 3 of Python (they are mostly the same).

The tokens

data Token

Lexical tokens.

Constructors

IndentToken

Indentation: increase.

Fields

• token_span :: !SrcSpan

DedentToken

Indentation: decrease.

Fields

• token_span :: !SrcSpan

NewlineToken

Newline.

Fields

• token_span :: !SrcSpan

LineJoinToken

Line join (backslash at end of line).

Fields

• token_span :: !SrcSpan

CommentToken

Single line comment.

Fields

• token_span :: !SrcSpan
• token_literal :: !String

IdentifierToken

Identifier.

Fields

• token_span :: !SrcSpan
• token_literal :: !String

StringToken

Literal: string.

Fields

• token_span :: !SrcSpan
• token_literal :: !String

ByteStringToken

Literal: byte string.

Fields

• token_span :: !SrcSpan
• token_literal :: !String

UnicodeStringToken

Literal: unicode string, version 2 only.

Fields

• token_span :: !SrcSpan
• token_literal :: !String

IntegerToken

Literal: integer.

Fields

• token_span :: !SrcSpan
• token_literal :: !String
• token_integer :: !Integer

LongIntegerToken

Literal: long integer. Version 2 only.

Fields

• token_span :: !SrcSpan
• token_literal :: !String
• token_integer :: !Integer

FloatToken

Literal: floating point.

Fields

• token_span :: !SrcSpan
• token_literal :: !String
• token_double :: !Double

ImaginaryToken

Literal: imaginary number.

Fields

• token_span :: !SrcSpan
• token_literal :: !String
• token_double :: !Double

DefToken

Keyword: 'def'.

Fields

• token_span :: !SrcSpan

WhileToken

Keyword: 'while'.

Fields

• token_span :: !SrcSpan

IfToken

Keyword: 'if'.

Fields

• token_span :: !SrcSpan

TrueToken

Keyword: 'True'.

Fields

• token_span :: !SrcSpan

FalseToken

Keyword: 'False'.

Fields

• token_span :: !SrcSpan

ReturnToken

Keyword: 'Return'.

Fields

• token_span :: !SrcSpan

TryToken

Keyword: 'try'.

Fields

• token_span :: !SrcSpan

ExceptToken

Keyword: 'except'.

Fields

• token_span :: !SrcSpan

RaiseToken

Keyword: 'raise'.

Fields

• token_span :: !SrcSpan

InToken

Keyword: 'in'.

Fields

• token_span :: !SrcSpan

IsToken

Keyword: 'is'.

Fields

• token_span :: !SrcSpan

LambdaToken

Keyword: 'lambda'.

Fields

• token_span :: !SrcSpan

ClassToken

Keyword: 'class'.

Fields

• token_span :: !SrcSpan

FinallyToken

Keyword: 'finally'.

Fields

• token_span :: !SrcSpan

NoneToken

Keyword: 'None'.

Fields

• token_span :: !SrcSpan

ForToken

Keyword: 'for'.

Fields

• token_span :: !SrcSpan

FromToken

Keyword: 'from'.

Fields

• token_span :: !SrcSpan

GlobalToken

Keyword: 'global'.

Fields

• token_span :: !SrcSpan

WithToken

Keyword: 'with'.

Fields

• token_span :: !SrcSpan

AsToken

Keyword: 'as'.

Fields

• token_span :: !SrcSpan

ElifToken

Keyword: 'elif'.

Fields

• token_span :: !SrcSpan

YieldToken

Keyword: 'yield'.

Fields

• token_span :: !SrcSpan

AssertToken

Keyword: 'assert'.

Fields

• token_span :: !SrcSpan

ImportToken

Keyword: 'import'.

Fields

• token_span :: !SrcSpan

PassToken

Keyword: 'pass'.

Fields

• token_span :: !SrcSpan

BreakToken

Keyword: 'break'.

Fields

• token_span :: !SrcSpan

ContinueToken

Keyword: 'continue'.

Fields

• token_span :: !SrcSpan

DeleteToken

Keyword: 'del'.

Fields

• token_span :: !SrcSpan

ElseToken

Keyword: 'else'.

Fields

• token_span :: !SrcSpan

NotToken

Keyword: 'not'.

Fields

• token_span :: !SrcSpan

AndToken

Keyword: boolean conjunction 'and'.

Fields

• token_span :: !SrcSpan

OrToken

Keyword: boolean disjunction 'or'. Version 3.x only:

Fields

• token_span :: !SrcSpan

NonLocalToken

Keyword: 'nonlocal' (Python 3.x only) Version 2.x only:

Fields

• token_span :: !SrcSpan

PrintToken

Keyword: 'print'. (Python 2.x only)

Fields

• token_span :: !SrcSpan

ExecToken

Keyword: 'exec'. (Python 2.x only)

Fields

• token_span :: !SrcSpan

AtToken

Delimiter: at sign '@'.

Fields

• token_span :: !SrcSpan

LeftRoundBracketToken

Delimiter: left round bracket '('.

Fields

• token_span :: !SrcSpan

RightRoundBracketToken

Delimiter: right round bracket ')'.

Fields

• token_span :: !SrcSpan

LeftSquareBracketToken

Delimiter: left square bracket '['.

Fields

• token_span :: !SrcSpan

RightSquareBracketToken

Delimiter: right square bracket ']'.

Fields

• token_span :: !SrcSpan

LeftBraceToken

Delimiter: left curly bracket '{'.

Fields

• token_span :: !SrcSpan

RightBraceToken

Delimiter: right curly bracket '}'.

Fields

• token_span :: !SrcSpan

DotToken

Delimiter: dot (full stop) '.'.

Fields

• token_span :: !SrcSpan

CommaToken

Delimiter: comma ','.

Fields

• token_span :: !SrcSpan

SemiColonToken

Delimiter: semicolon ';'.

Fields

• token_span :: !SrcSpan

ColonToken

Delimiter: colon ':'.

Fields

• token_span :: !SrcSpan

EllipsisToken

Delimiter: ellipses (three dots) '...'.

Fields

• token_span :: !SrcSpan

RightArrowToken

Delimiter: right facing arrow '->'.

Fields

• token_span :: !SrcSpan

AssignToken

Delimiter: assignment '='.

Fields

• token_span :: !SrcSpan

PlusAssignToken

Delimiter: plus assignment '+='.

Fields

• token_span :: !SrcSpan

MinusAssignToken

Delimiter: minus assignment '-='.

Fields

• token_span :: !SrcSpan

MultAssignToken

Delimiter: multiply assignment '*='

Fields

• token_span :: !SrcSpan

DivAssignToken

Delimiter: divide assignment '/='.

Fields

• token_span :: !SrcSpan

ModAssignToken

Delimiter: modulus assignment '%='.

Fields

• token_span :: !SrcSpan

PowAssignToken

Delimiter: power assignment '**='.

Fields

• token_span :: !SrcSpan

BinAndAssignToken

Delimiter: binary-and assignment '&='.

Fields

• token_span :: !SrcSpan

BinOrAssignToken

Delimiter: binary-or assignment '|='.

Fields

• token_span :: !SrcSpan

BinXorAssignToken

Delimiter: binary-xor assignment '^='.

Fields

• token_span :: !SrcSpan

LeftShiftAssignToken

Delimiter: binary-left-shift assignment '<<='.

Fields

• token_span :: !SrcSpan

RightShiftAssignToken

Delimiter: binary-right-shift assignment '>>='.

Fields

• token_span :: !SrcSpan

FloorDivAssignToken

Delimiter: floor-divide assignment '//='.

Fields

• token_span :: !SrcSpan

BackQuoteToken

Delimiter: back quote character '`\'.

Fields

• token_span :: !SrcSpan

PlusToken

Operator: plus '+'.

Fields

• token_span :: !SrcSpan

MinusToken

Operator: minus: '-'.

Fields

• token_span :: !SrcSpan

MultToken

Operator: multiply '*'.

Fields

• token_span :: !SrcSpan

DivToken

Operator: divide '/'.

Fields

• token_span :: !SrcSpan

GreaterThanToken

Operator: greater-than '>'.

Fields

• token_span :: !SrcSpan

LessThanToken

Operator: less-than '<'.

Fields

• token_span :: !SrcSpan

EqualityToken

Operator: equals '=='.

Fields

• token_span :: !SrcSpan

GreaterThanEqualsToken

Operator: greater-than-or-equals '>='.

Fields

• token_span :: !SrcSpan

LessThanEqualsToken

Operator: less-than-or-equals '<='.

Fields

• token_span :: !SrcSpan

ExponentToken

Operator: exponential '**'.

Fields

• token_span :: !SrcSpan

BinaryOrToken

Operator: binary-or '|'.

Fields

• token_span :: !SrcSpan

XorToken

Operator: binary-xor '^'.

Fields

• token_span :: !SrcSpan

BinaryAndToken

Operator: binary-and '&'.

Fields

• token_span :: !SrcSpan

ShiftLeftToken

Operator: binary-shift-left '<<'.

Fields

• token_span :: !SrcSpan

ShiftRightToken

Operator: binary-shift-right '>>'.

Fields

• token_span :: !SrcSpan

ModuloToken

Operator: modulus '%'.

Fields

• token_span :: !SrcSpan

FloorDivToken

Operator: floor-divide '//'.

Fields

• token_span :: !SrcSpan

TildeToken

Operator: tilde '~'.

Fields

• token_span :: !SrcSpan

NotEqualsToken

Operator: not-equals '!='.

Fields

• token_span :: !SrcSpan

NotEqualsV2Token

Operator: not-equals '<>'. Version 2 only.

Fields

• token_span :: !SrcSpan

EOFToken

End of file

Fields

• token_span :: !SrcSpan

Instances

Eq Token

Methods

(==) :: Token -> Token -> Bool

(/=) :: Token -> Token -> Bool

Data Token

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Token -> c Token

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Token

toConstr :: Token -> Constr

dataTypeOf :: Token -> DataType

dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c Token)

dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)

gmapT :: (forall b. Data b => b -> b) -> Token -> Token

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r

gmapQ :: (forall d. Data d => d -> u) -> Token -> [u]

gmapQi :: Int -> (forall d. Data d => d -> u) -> Token -> u

gmapM :: Monad m => (forall d. Data d => d -> m d) -> Token -> m Token

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token

Ord Token

Methods

compare :: Token -> Token -> Ordering

(<) :: Token -> Token -> Bool

(<=) :: Token -> Token -> Bool

(>) :: Token -> Token -> Bool

(>=) :: Token -> Token -> Bool

max :: Token -> Token -> Token

min :: Token -> Token -> Token

Show Token

Methods

showsPrec :: Int -> Token -> ShowS

show :: Token -> String

showList :: [Token] -> ShowS

Span Token

Methods

getSpan :: Token -> SrcSpan

String conversion

debugTokenString :: Token -> String

Produce a string from a token containing detailed information. Mainly intended for debugging.

tokenString :: Token -> String

Produce a string from a token which is suitable for printing as Python concrete syntax. Invisible tokens yield an empty string.

Classification

hasLiteral :: Token -> Bool

Test if a token contains its literal source text.

data TokenClass

Classification of tokens

Constructors

Comment

Number

Identifier

Punctuation

Bracket

Layout

Keyword

String

Operator

Assignment

Instances

Eq TokenClass

Methods

(==) :: TokenClass -> TokenClass -> Bool

(/=) :: TokenClass -> TokenClass -> Bool

Ord TokenClass

Methods

compare :: TokenClass -> TokenClass -> Ordering

(<) :: TokenClass -> TokenClass -> Bool

(<=) :: TokenClass -> TokenClass -> Bool

(>) :: TokenClass -> TokenClass -> Bool

(>=) :: TokenClass -> TokenClass -> Bool

max :: TokenClass -> TokenClass -> TokenClass

min :: TokenClass -> TokenClass -> TokenClass

Show TokenClass

Methods

showsPrec :: Int -> TokenClass -> ShowS

show :: TokenClass -> String

showList :: [TokenClass] -> ShowS

classifyToken :: Token -> TokenClass