Language.Python.Common.Token

Lexical tokens for the Python lexer. Contains the superset of tokens from version 2 and version 3 of Python (they are mostly the same).

The tokens

data Token

Lexical tokens.

Constructors

IndentToken

Indentation: increase.

Fields

token_span :: !SrcSpan

DedentToken

Indentation: decrease.

Fields

token_span :: !SrcSpan

NewlineToken

Newline.

Fields

token_span :: !SrcSpan

LineJoinToken

Line join (backslash at end of line).

Fields

token_span :: !SrcSpan

CommentToken

Single line comment.

Fields

token_span :: !SrcSpan

token_literal :: !String

IdentifierToken

Identifier.

Fields

token_span :: !SrcSpan

token_literal :: !String

StringToken

Literal: string.

Fields

token_span :: !SrcSpan

token_literal :: !String

ByteStringToken

Literal: byte string.

Fields

token_span :: !SrcSpan

token_literal :: !String

UnicodeStringToken

Literal: unicode string, version 2 only.

Fields

token_span :: !SrcSpan

token_literal :: !String

IntegerToken

Literal: integer.

Fields

token_span :: !SrcSpan

token_literal :: !String

token_integer :: !Integer

LongIntegerToken

Literal: long integer. Version 2 only.

Fields

token_span :: !SrcSpan

token_literal :: !String

token_integer :: !Integer

FloatToken

Literal: floating point.

Fields

token_span :: !SrcSpan

token_literal :: !String

token_double :: !Double

ImaginaryToken

Literal: imaginary number.

Fields

token_span :: !SrcSpan

token_literal :: !String

token_double :: !Double

DefToken

Keyword: 'def'.

Fields

token_span :: !SrcSpan

WhileToken

Keyword: 'while'.

Fields

token_span :: !SrcSpan

IfToken

Keyword: 'if'.

Fields

token_span :: !SrcSpan

TrueToken

Keyword: 'True'.

Fields

token_span :: !SrcSpan

FalseToken

Keyword: 'False'.

Fields

token_span :: !SrcSpan

ReturnToken

Keyword: 'Return'.

Fields

token_span :: !SrcSpan

TryToken

Keyword: 'try'.

Fields

token_span :: !SrcSpan

ExceptToken

Keyword: 'except'.

Fields

token_span :: !SrcSpan

RaiseToken

Keyword: 'raise'.

Fields

token_span :: !SrcSpan

InToken

Keyword: 'in'.

Fields

token_span :: !SrcSpan

IsToken

Keyword: 'is'.

Fields

token_span :: !SrcSpan

LambdaToken

Keyword: 'lambda'.

Fields

token_span :: !SrcSpan

ClassToken

Keyword: 'class'.

Fields

token_span :: !SrcSpan

FinallyToken

Keyword: 'finally'.

Fields

token_span :: !SrcSpan

NoneToken

Keyword: 'None'.

Fields

token_span :: !SrcSpan

ForToken

Keyword: 'for'.

Fields

token_span :: !SrcSpan

FromToken

Keyword: 'from'.

Fields

token_span :: !SrcSpan

GlobalToken

Keyword: 'global'.

Fields

token_span :: !SrcSpan

WithToken

Keyword: 'with'.

Fields

token_span :: !SrcSpan

AsToken

Keyword: 'as'.

Fields

token_span :: !SrcSpan

ElifToken

Keyword: 'elif'.

Fields

token_span :: !SrcSpan

YieldToken

Keyword: 'yield'.

Fields

token_span :: !SrcSpan

AssertToken

Keyword: 'assert'.

Fields

token_span :: !SrcSpan

ImportToken

Keyword: 'import'.

Fields

token_span :: !SrcSpan

PassToken

Keyword: 'pass'.

Fields

token_span :: !SrcSpan

BreakToken

Keyword: 'break'.

Fields

token_span :: !SrcSpan

ContinueToken

Keyword: 'continue'.

Fields

token_span :: !SrcSpan

DeleteToken

Keyword: 'del'.

Fields

token_span :: !SrcSpan

ElseToken

Keyword: 'else'.

Fields

token_span :: !SrcSpan

NotToken

Keyword: 'not'.

Fields

token_span :: !SrcSpan

AndToken

Keyword: boolean conjunction 'and'.

Fields

token_span :: !SrcSpan

OrToken

Keyword: boolean disjunction 'or'. Version 3.x only:

Fields

token_span :: !SrcSpan

NonLocalToken

Keyword: 'nonlocal' (Python 3.x only) Version 2.x only:

Fields

token_span :: !SrcSpan

PrintToken

Keyword: 'print'. (Python 2.x only)

Fields

token_span :: !SrcSpan

ExecToken

Keyword: 'exec'. (Python 2.x only)

Fields

token_span :: !SrcSpan

AtToken

Delimiter: at sign '@'.

Fields

token_span :: !SrcSpan

LeftRoundBracketToken

Delimiter: left round bracket '('.

Fields

token_span :: !SrcSpan

RightRoundBracketToken

Delimiter: right round bracket ')'.

Fields

token_span :: !SrcSpan

LeftSquareBracketToken

Delimiter: left square bracket '['.

Fields

token_span :: !SrcSpan

RightSquareBracketToken

Delimiter: right square bracket ']'.

Fields

token_span :: !SrcSpan

LeftBraceToken

Delimiter: left curly bracket '{'.

Fields

token_span :: !SrcSpan

RightBraceToken

Delimiter: right curly bracket '}'.

Fields

token_span :: !SrcSpan

DotToken

Delimiter: dot (full stop) '.'.

Fields

token_span :: !SrcSpan

CommaToken

Delimiter: comma ','.

Fields

token_span :: !SrcSpan

SemiColonToken

Delimiter: semicolon ';'.

Fields

token_span :: !SrcSpan

ColonToken

Delimiter: colon ':'.

Fields

token_span :: !SrcSpan

EllipsisToken

Delimiter: ellipses (three dots) '...'.

Fields

token_span :: !SrcSpan

RightArrowToken

Delimiter: right facing arrow '->'.

Fields

token_span :: !SrcSpan

AssignToken

Delimiter: assignment '='.

Fields

token_span :: !SrcSpan

PlusAssignToken

Delimiter: plus assignment '+='.

Fields

token_span :: !SrcSpan

MinusAssignToken

Delimiter: minus assignment '-='.

Fields

token_span :: !SrcSpan

MultAssignToken

Delimiter: multiply assignment '*='

Fields

token_span :: !SrcSpan

DivAssignToken

Delimiter: divide assignment '/='.

Fields

token_span :: !SrcSpan

ModAssignToken

Delimiter: modulus assignment '%='.

Fields

token_span :: !SrcSpan

PowAssignToken

Delimiter: power assignment '**='.

Fields

token_span :: !SrcSpan

BinAndAssignToken

Delimiter: binary-and assignment '&='.

Fields

token_span :: !SrcSpan

BinOrAssignToken

Delimiter: binary-or assignment '|='.

Fields

token_span :: !SrcSpan

BinXorAssignToken

Delimiter: binary-xor assignment '^='.

Fields

token_span :: !SrcSpan

LeftShiftAssignToken

Delimiter: binary-left-shift assignment '<<='.

Fields

token_span :: !SrcSpan

RightShiftAssignToken

Delimiter: binary-right-shift assignment '>>='.

Fields

token_span :: !SrcSpan

FloorDivAssignToken

Delimiter: floor-divide assignment '//='.

Fields

token_span :: !SrcSpan

BackQuoteToken

Delimiter: back quote character '`\'.

Fields

token_span :: !SrcSpan

PlusToken

Operator: plus '+'.

Fields

token_span :: !SrcSpan

MinusToken

Operator: minus: '-'.

Fields

token_span :: !SrcSpan

MultToken

Operator: multiply '*'.

Fields

token_span :: !SrcSpan

DivToken

Operator: divide '/'.

Fields

token_span :: !SrcSpan

GreaterThanToken

Operator: greater-than '>'.

Fields

token_span :: !SrcSpan

LessThanToken

Operator: less-than '<'.

Fields

token_span :: !SrcSpan

EqualityToken

Operator: equals '=='.

Fields

token_span :: !SrcSpan

GreaterThanEqualsToken

Operator: greater-than-or-equals '>='.

Fields

token_span :: !SrcSpan

LessThanEqualsToken

Operator: less-than-or-equals '<='.

Fields

token_span :: !SrcSpan

ExponentToken

Operator: exponential '**'.

Fields

token_span :: !SrcSpan

BinaryOrToken

Operator: binary-or '|'.

Fields

token_span :: !SrcSpan

XorToken

Operator: binary-xor '^'.

Fields

token_span :: !SrcSpan

BinaryAndToken

Operator: binary-and '&'.

Fields

token_span :: !SrcSpan

ShiftLeftToken

Operator: binary-shift-left '<<'.

Fields

token_span :: !SrcSpan

ShiftRightToken

Operator: binary-shift-right '>>'.

Fields

token_span :: !SrcSpan

ModuloToken

Operator: modulus '%'.

Fields

token_span :: !SrcSpan

FloorDivToken

Operator: floor-divide '//'.

Fields

token_span :: !SrcSpan

TildeToken

Operator: tilde '~'.

Fields

token_span :: !SrcSpan

NotEqualsToken

Operator: not-equals '!='.

Fields

token_span :: !SrcSpan

NotEqualsV2Token

Operator: not-equals '<>'. Version 2 only.

Fields

token_span :: !SrcSpan

EOFToken

End of file

Fields

token_span :: !SrcSpan

Instances

Eq Token

Data Token

Ord Token

Show Token

Pretty Token

Span Token

Typeable * Token

String conversion

debugTokenString :: Token -> String

Produce a string from a token containing detailed information. Mainly intended for debugging.

tokenString :: Token -> String

Produce a string from a token which is suitable for printing as Python concrete syntax. Invisible tokens yield an empty string.

Classification

hasLiteral :: Token -> Bool

Test if a token contains its literal source text.

data TokenClass

Classification of tokens

Constructors

Comment

Number

Identifier

Punctuation

Bracket

Layout

Keyword

String

Operator

Assignment

Instances

Eq TokenClass

Ord TokenClass

Show TokenClass

classifyToken :: Token -> TokenClass