Language.Python.Internal.Lexer

Documentation

tokenizeWithTabs

Arguments

:: (AsLexicalError s Char, AsTabError s SrcInfo, AsIncorrectDedent s SrcInfo)

=> FilePath

File name

-> Text

Input to tokenize

-> Either s [PyToken SrcInfo]

Tokenize an input file, inserting indent/level/dedent tokens in appropriate positions according to the block structure.

Source Information

data SrcInfo

Constructors

SrcInfo

Fields

• _srcInfoName :: FilePath
• _srcInfoLineStart :: !Int
• _srcInfoLineEnd :: !Int
• _srcInfoColStart :: !Int
• _srcInfoColEnd :: !Int
• _srcInfoOffsetStart :: !Int
• _srcInfoOffsetEnd :: !Int

Instances

Eq SrcInfo

Defined in Language.Python.Internal.Lexer

Methods

(==) :: SrcInfo -> SrcInfo -> Bool

(/=) :: SrcInfo -> SrcInfo -> Bool

Show SrcInfo

Defined in Language.Python.Internal.Lexer

Methods

showsPrec :: Int -> SrcInfo -> ShowS

show :: SrcInfo -> String

showList :: [SrcInfo] -> ShowS

Semigroup SrcInfo

Defined in Language.Python.Internal.Lexer

Methods

(<>) :: SrcInfo -> SrcInfo -> SrcInfo

sconcat :: NonEmpty SrcInfo -> SrcInfo

stimes :: Integral b => b -> SrcInfo -> SrcInfo

initialSrcInfo :: FilePath -> SrcInfo

withSrcInfo :: MonadParsec e s m => m (SrcInfo -> a) -> m a

Errors

class AsLexicalError s t | s -> t where

Methods

_LexicalError :: Prism' s (NonEmpty SourcePos, Maybe (ErrorItem t), Set (ErrorItem t))

Instances

AsLexicalError (ParseError a) Char

Defined in Language.Python.Parse.Error

Methods

_LexicalError :: Prism' (ParseError a) (NonEmpty SourcePos, Maybe (ErrorItem Char), Set (ErrorItem Char))

unsafeFromLexicalError :: (HasCallStack, AsLexicalError s t) => ParseError t Void -> s

Convert a concrete ParseError to a value that has an instance of AsLexicalError

This function is partial, because our parser will never use FancyError

class AsTabError s a | s -> a where

Methods

_TabError :: Prism' s a

Instances

AsTabError (ParseError a) a

Defined in Language.Python.Parse.Error

Methods

_TabError :: Prism' (ParseError a) a

AsTabError (IndentationError a) a

Defined in Language.Python.Validate.Indentation.Error

Methods

_TabError :: Prism' (IndentationError a) a

AsTabError (ValidationError a) a

Defined in Language.Python.Validate.Error

Methods

_TabError :: Prism' (ValidationError a) a

class AsIncorrectDedent s a | s -> a where

Methods

_IncorrectDedent :: Prism' s a

Instances

AsIncorrectDedent (ParseError a) a

Defined in Language.Python.Parse.Error

Methods

_IncorrectDedent :: Prism' (ParseError a) a

fromTabError :: (AsTabError s a, AsIncorrectDedent s a) => TabError a -> s

Convert a concrete TabError to a value that has an instance of AsTabError

data TabError a

Constructors

TabError a

Tabs and spaces were used inconsistently

IncorrectDedent a

The dedent at the end of a block doesn't match and preceding indents

e.g.

def a():
    if b:
        pass
    else:
        pass
  pass

The final line will cause an IncorrectDedent error

Instances

Eq a => Eq (TabError a)

Defined in Language.Python.Internal.Lexer

Methods

(==) :: TabError a -> TabError a -> Bool

(/=) :: TabError a -> TabError a -> Bool

Show a => Show (TabError a)

Defined in Language.Python.Internal.Lexer

Methods

showsPrec :: Int -> TabError a -> ShowS

show :: TabError a -> String

showList :: [TabError a] -> ShowS

Miscellaneous

tokenize

Arguments

:: AsLexicalError e Char

=> FilePath

File name

-> Text

Input to tokenize

-> Either e [PyToken SrcInfo]

Convert some input to a sequence of tokens. Indent and dedent tokens are not added (see insertTabs)

insertTabs

Arguments

:: (Semigroup a, AsTabError s a, AsIncorrectDedent s a)

=> a

Initial source annotation

-> [PyToken a]

Token stream

-> Either s [PyToken a]

Insert indent and dedent tokens

https://docs.python.org/3.5/reference/lexical_analysis.html#indentation

Megaparsec re-exports

data ParseError t e

ParseError t e represents a parse error parametrized over the token type t and the custom data e.

Note that the stack of source positions contains current position as its head, and the rest of positions allows to track full sequence of include files with topmost source file at the end of the list.

Semigroup and Monoid instances of the data type allow to merge parse errors from different branches of parsing. When merging two ParseErrors, the longest match is preferred; if positions are the same, custom data sets and collections of message items are combined. Note that fancy errors take precedence over trivial errors in merging.

Since: megaparsec-6.0.0

Constructors

TrivialError (NonEmpty SourcePos) (Maybe (ErrorItem t)) (Set (ErrorItem t))

Trivial errors, generated by Megaparsec's machinery. The data constructor includes the stack of source positions, unexpected token (if any), and expected tokens.

FancyError (NonEmpty SourcePos) (Set (ErrorFancy e))

Fancy, custom errors.

Instances

(Eq t, Eq e) => Eq (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

(==) :: ParseError t e -> ParseError t e -> Bool

(/=) :: ParseError t e -> ParseError t e -> Bool

(Data t, Data e, Ord t, Ord e) => Data (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> ParseError t e -> c (ParseError t e)

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c (ParseError t e)

toConstr :: ParseError t e -> Constr

dataTypeOf :: ParseError t e -> DataType

dataCast1 :: Typeable t0 => (forall d. Data d => c (t0 d)) -> Maybe (c (ParseError t e))

dataCast2 :: Typeable t0 => (forall d e0. (Data d, Data e0) => c (t0 d e0)) -> Maybe (c (ParseError t e))

gmapT :: (forall b. Data b => b -> b) -> ParseError t e -> ParseError t e

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> ParseError t e -> r

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> ParseError t e -> r

gmapQ :: (forall d. Data d => d -> u) -> ParseError t e -> [u]

gmapQi :: Int -> (forall d. Data d => d -> u) -> ParseError t e -> u

gmapM :: Monad m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e)

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e)

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e)

(Read t, Read e, Ord t, Ord e) => Read (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

readsPrec :: Int -> ReadS (ParseError t e)

readList :: ReadS [ParseError t e]

readPrec :: ReadPrec (ParseError t e)

readListPrec :: ReadPrec [ParseError t e]

(Show t, Show e) => Show (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

showsPrec :: Int -> ParseError t e -> ShowS

show :: ParseError t e -> String

showList :: [ParseError t e] -> ShowS

Generic (ParseError t e)

Defined in Text.Megaparsec.Error

Associated Types

type Rep (ParseError t e) :: Type -> Type

Methods

from :: ParseError t e -> Rep (ParseError t e) x

to :: Rep (ParseError t e) x -> ParseError t e

(Ord t, Ord e) => Semigroup (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

(<>) :: ParseError t e -> ParseError t e -> ParseError t e

sconcat :: NonEmpty (ParseError t e) -> ParseError t e

stimes :: Integral b => b -> ParseError t e -> ParseError t e

(Ord t, Ord e) => Monoid (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

mempty :: ParseError t e

mappend :: ParseError t e -> ParseError t e -> ParseError t e

mconcat :: [ParseError t e] -> ParseError t e

(Show t, Ord t, ShowToken t, Typeable t, Show e, ShowErrorComponent e, Typeable e) => Exception (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

toException :: ParseError t e -> SomeException

fromException :: SomeException -> Maybe (ParseError t e)

displayException :: ParseError t e -> String

(NFData t, NFData e) => NFData (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

rnf :: ParseError t e -> ()

type Rep (ParseError t e)

Defined in Text.Megaparsec.Error

type Rep (ParseError t e) = D1 (MetaData "ParseError" "Text.Megaparsec.Error" "megaparsec-6.5.0-d919d8e9b2d559a41469a5f4cb583e9cc78df1a606e6731f314f6df926990c52" False) (C1 (MetaCons "TrivialError" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (NonEmpty SourcePos)) :*: (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Maybe (ErrorItem t))) :*: S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Set (ErrorItem t))))) :+: C1 (MetaCons "FancyError" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (NonEmpty SourcePos)) :*: S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Set (ErrorFancy e)))))