Language.Python.Internal.Lexer

Documentation

tokenizeWithTabs

Arguments

:: (AsLexicalError s Char, AsTabError s SrcInfo, AsIncorrectDedent s SrcInfo)

=> FilePath

File name

-> Text

Input to tokenize

-> Either s [PyToken SrcInfo]

Tokenize an input file, inserting indent/level/dedent tokens in appropriate positions according to the block structure.

Source Information

data SrcInfo

Constructors

SrcInfo

Fields

• _srcInfoName :: FilePath
• _srcInfoLineStart :: !Int
• _srcInfoLineEnd :: !Int
• _srcInfoColStart :: !Int
• _srcInfoColEnd :: !Int
• _srcInfoOffsetStart :: !Int
• _srcInfoOffsetEnd :: !Int

Instances

Eq SrcInfo

Defined in Language.Python.Internal.Lexer

Methods

(==) :: SrcInfo -> SrcInfo -> Bool

(/=) :: SrcInfo -> SrcInfo -> Bool

Show SrcInfo

Defined in Language.Python.Internal.Lexer

Methods

showsPrec :: Int -> SrcInfo -> ShowS

show :: SrcInfo -> String

showList :: [SrcInfo] -> ShowS

Semigroup SrcInfo

Defined in Language.Python.Internal.Lexer

Methods

(<>) :: SrcInfo -> SrcInfo -> SrcInfo

sconcat :: NonEmpty SrcInfo -> SrcInfo

stimes :: Integral b => b -> SrcInfo -> SrcInfo

initialSrcInfo :: FilePath -> SrcInfo

withSrcInfo :: MonadParsec e s m => m (SrcInfo -> a) -> m a

Errors

class AsLexicalError s t | s -> t where

Methods

_LexicalError :: Prism' s (NonEmpty SourcePos, Maybe (ErrorItem t), Set (ErrorItem t))

Instances

AsLexicalError (ParseError a) Char

Defined in Language.Python.Parse.Error

Methods

_LexicalError :: Prism' (ParseError a) (NonEmpty SourcePos, Maybe (ErrorItem Char), Set (ErrorItem Char))

unsafeFromLexicalError :: (HasCallStack, AsLexicalError s t) => ParseError t Void -> s

Convert a concrete ParseError to a value that has an instance of AsLexicalError

This function is partial, because our parser will never use FancyError

class AsTabError s a | s -> a where

Methods

_TabError :: Prism' s a

Instances

AsTabError (ParseError a) a

Defined in Language.Python.Parse.Error

Methods

_TabError :: Prism' (ParseError a) a

AsTabError (IndentationError a) a

Defined in Language.Python.Validate.Indentation.Error

Methods

_TabError :: Prism' (IndentationError a) a

AsTabError (ValidationError a) a

Defined in Language.Python.Validate.Error

Methods

_TabError :: Prism' (ValidationError a) a

class AsIncorrectDedent s a | s -> a where

Methods

_IncorrectDedent :: Prism' s a

Instances

AsIncorrectDedent (ParseError a) a

Defined in Language.Python.Parse.Error

Methods

_IncorrectDedent :: Prism' (ParseError a) a

fromTabError :: (AsTabError s a, AsIncorrectDedent s a) => TabError a -> s

Convert a concrete TabError to a value that has an instance of AsTabError

data TabError a

Constructors

TabError a

Tabs and spaces were used inconsistently

IncorrectDedent a

The dedent at the end of a block doesn't match and preceding indents

e.g.

def a():
    if b:
        pass
    else:
        pass
  pass

The final line will cause an IncorrectDedent error

Instances

Eq a => Eq (TabError a)

Defined in Language.Python.Internal.Lexer

Methods

(==) :: TabError a -> TabError a -> Bool

(/=) :: TabError a -> TabError a -> Bool

Show a => Show (TabError a)

Defined in Language.Python.Internal.Lexer

Methods

showsPrec :: Int -> TabError a -> ShowS

show :: TabError a -> String

showList :: [TabError a] -> ShowS

Miscellaneous

tokenize

Arguments

:: AsLexicalError e Char

=> FilePath

File name

-> Text

Input to tokenize

-> Either e [PyToken SrcInfo]

Convert some input to a sequence of tokens. Indent and dedent tokens are not added (see insertTabs)

insertTabs

Arguments

:: (Semigroup a, AsTabError s a, AsIncorrectDedent s a)

=> a

Initial source annotation

-> [PyToken a]

Token stream

-> Either s [PyToken a]

Insert indent and dedent tokens

https://docs.python.org/3.5/reference/lexical_analysis.html#indentation

Megaparsec re-exports

data ParseError t e

Constructors

TrivialError (NonEmpty SourcePos) (Maybe (ErrorItem t)) (Set (ErrorItem t))

FancyError (NonEmpty SourcePos) (Set (ErrorFancy e))

Instances

(Eq t, Eq e) => Eq (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

(==) :: ParseError t e -> ParseError t e -> Bool

(/=) :: ParseError t e -> ParseError t e -> Bool

(Data t, Data e, Ord t, Ord e) => Data (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> ParseError t e -> c (ParseError t e)

gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c (ParseError t e)

toConstr :: ParseError t e -> Constr

dataTypeOf :: ParseError t e -> DataType

dataCast1 :: Typeable t0 => (forall d. Data d => c (t0 d)) -> Maybe (c (ParseError t e))

dataCast2 :: Typeable t0 => (forall d e0. (Data d, Data e0) => c (t0 d e0)) -> Maybe (c (ParseError t e))

gmapT :: (forall b. Data b => b -> b) -> ParseError t e -> ParseError t e

gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> ParseError t e -> r

gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> ParseError t e -> r

gmapQ :: (forall d. Data d => d -> u) -> ParseError t e -> [u]

gmapQi :: Int -> (forall d. Data d => d -> u) -> ParseError t e -> u

gmapM :: Monad m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e)

gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e)

gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> ParseError t e -> m (ParseError t e)

(Read t, Read e, Ord t, Ord e) => Read (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

readsPrec :: Int -> ReadS (ParseError t e)

readList :: ReadS [ParseError t e]

readPrec :: ReadPrec (ParseError t e)

readListPrec :: ReadPrec [ParseError t e]

(Show t, Show e) => Show (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

showsPrec :: Int -> ParseError t e -> ShowS

show :: ParseError t e -> String

showList :: [ParseError t e] -> ShowS

Generic (ParseError t e)

Defined in Text.Megaparsec.Error

Associated Types

type Rep (ParseError t e) :: Type -> Type

Methods

from :: ParseError t e -> Rep (ParseError t e) x

to :: Rep (ParseError t e) x -> ParseError t e

(Ord t, Ord e) => Semigroup (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

(<>) :: ParseError t e -> ParseError t e -> ParseError t e

sconcat :: NonEmpty (ParseError t e) -> ParseError t e

stimes :: Integral b => b -> ParseError t e -> ParseError t e

(Ord t, Ord e) => Monoid (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

mempty :: ParseError t e

mappend :: ParseError t e -> ParseError t e -> ParseError t e

mconcat :: [ParseError t e] -> ParseError t e

(Show t, Ord t, ShowToken t, Typeable t, Show e, ShowErrorComponent e, Typeable e) => Exception (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

toException :: ParseError t e -> SomeException

fromException :: SomeException -> Maybe (ParseError t e)

displayException :: ParseError t e -> String

(NFData t, NFData e) => NFData (ParseError t e)

Defined in Text.Megaparsec.Error

Methods

rnf :: ParseError t e -> ()

type Rep (ParseError t e)

Defined in Text.Megaparsec.Error

type Rep (ParseError t e) = D1 (MetaData "ParseError" "Text.Megaparsec.Error" "megaparsec-6.5.0-7b380a41c7db9c6a4aad5bc0349f059f6770288935e8a88dbe061dc3475e1e3e" False) (C1 (MetaCons "TrivialError" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (NonEmpty SourcePos)) :*: (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Maybe (ErrorItem t))) :*: S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Set (ErrorItem t))))) :+: C1 (MetaCons "FancyError" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (NonEmpty SourcePos)) :*: S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Set (ErrorFancy e)))))