Yi.Lexer.Alex

Utilities to turn a lexer generated by Alex into a Scanner that can be used by Yi. Most lexers will use the types defined here. Some things are exported for use by lexers themselves through the use of YiLexerscommon.hsinc.

Documentation

utf8Encode :: Char -> [Word8]

Encode a Haskell String to a list of Word8 values, in UTF8 format.

type Byte = Word8

type IndexedStr = [(Point, Char)]

type AlexInput = (Char, [Byte], IndexedStr)

type Action hlState token = IndexedStr -> hlState -> (hlState, token)

data AlexState lexerState

Lexer state

Constructors

AlexState

Fields

• stLexer :: lexerState
• lookedOffset :: !Point
• stPosn :: !Posn

Instances

Show lexerState => Show (AlexState lexerState)

Methods

showsPrec :: Int -> AlexState lexerState -> ShowS

show :: AlexState lexerState -> String

showList :: [AlexState lexerState] -> ShowS

data Tok t

Constructors

Tok

Fields

• tokT :: t
• tokLen :: Size
• tokPosn :: Posn

Instances

Functor Tok

Methods

fmap :: (a -> b) -> Tok a -> Tok b

(<$) :: a -> Tok b -> Tok a

Eq (Tok a)

Methods

(==) :: Tok a -> Tok a -> Bool

(/=) :: Tok a -> Tok a -> Bool

Show t => Show (Tok t)

Methods

showsPrec :: Int -> Tok t -> ShowS

show :: Tok t -> String

showList :: [Tok t] -> ShowS

tokToSpan :: Tok t -> Span t

tokFromT :: t -> Tok t

tokBegin :: Tok t -> Point

tokEnd :: Tok t -> Point

data Posn

Constructors

Posn

Fields

• posnOfs :: !Point
• posnLine :: !Int
• posnCol :: !Int

Instances

Eq Posn

Methods

(==) :: Posn -> Posn -> Bool

(/=) :: Posn -> Posn -> Bool

Ord Posn

Methods

compare :: Posn -> Posn -> Ordering

(<) :: Posn -> Posn -> Bool

(<=) :: Posn -> Posn -> Bool

(>) :: Posn -> Posn -> Bool

(>=) :: Posn -> Posn -> Bool

max :: Posn -> Posn -> Posn

min :: Posn -> Posn -> Posn

Show Posn

Methods

showsPrec :: Int -> Posn -> ShowS

show :: Posn -> String

showList :: [Posn] -> ShowS

Ix Posn

Methods

range :: (Posn, Posn) -> [Posn]

index :: (Posn, Posn) -> Posn -> Int

unsafeIndex :: (Posn, Posn) -> Posn -> Int

inRange :: (Posn, Posn) -> Posn -> Bool

rangeSize :: (Posn, Posn) -> Int

unsafeRangeSize :: (Posn, Posn) -> Int

startPosn :: Posn

moveStr :: Posn -> IndexedStr -> Posn

moveCh :: Posn -> Char -> Posn

alexGetChar :: AlexInput -> Maybe (Char, AlexInput)

alexGetByte :: AlexInput -> Maybe (Word8, AlexInput)

alexCollectChar :: AlexInput -> [Char]

alexInputPrevChar :: AlexInput -> Char

actionConst :: token -> Action lexState token

Return a constant token

actionAndModify :: (lexState -> lexState) -> token -> Action lexState token

Return a constant token, and modify the lexer state

actionStringAndModify :: (s -> s) -> (String -> token) -> Action s token

Convert the parsed string into a token, and also modify the lexer state

actionStringConst :: (String -> token) -> Action lexState token

Convert the parsed string into a token

type ASI s = (AlexState s, AlexInput)

type TokenLexer l s t i = (l s, i) -> Maybe (t, (l s, i))

Function to (possibly) lex a single token and give us the remaining input.

type CharScanner = Scanner Point Char

Handy alias

data Lexer l s t i

Generalises lexers. This allows us to easily use lexers which don't want to be cornered into the types we have predefined here and use in common.hsinc.

Constructors

Lexer

Fields

• _step :: TokenLexer l s t i
• _starting :: s -> Point -> Posn -> l s
• _withChars :: Char -> [(Point, Char)] -> i
• _looked :: l s -> Point
• _statePosn :: l s -> Posn
• _lexEmpty :: t
• _startingState :: s

data StyleLexer l s t i

Just like Lexer but also knows how to turn its tokens into StyleNames.

Constructors

StyleLexer

Fields

• _tokenToStyle :: t -> StyleName
• _styleLexer :: Lexer l s (Tok t) i

type StyleLexerASI s t = StyleLexer AlexState s t AlexInput

StyleLexer over ASI.

commonLexer :: (ASI s -> Maybe (Tok t, ASI s)) -> s -> Lexer AlexState s (Tok t) AlexInput

Defines a Lexer for ASI. This exists to make using the new lexScanner easier if you're using ASI as all our lexers do today, 23-08-2014.

lexScanner :: Lexer l s t i -> CharScanner -> Scanner (l s) t

Combine a character scanner with a lexer to produce a token scanner. May be used together with mkHighlighter to produce a Highlighter, or with linearSyntaxMode to produce a Mode.

unfoldLexer :: ((state, input) -> Maybe (token, (state, input))) -> (state, input) -> [(state, token)]

unfold lexer into a function that returns a stream of (state, token)

Lenses

posnOfsA :: Lens' Posn Point

posnLineA :: Lens' Posn Int

posnColA :: Lens' Posn Int

tokTA :: forall t t. Lens (Tok t) (Tok t) t t

tokPosnA :: forall t. Lens' (Tok t) Posn

tokLenA :: forall t. Lens' (Tok t) Size

withChars :: forall l s t i. Lens' (Lexer l s t i) (Char -> [(Point, Char)] -> i)

step :: forall l s t i. Lens' (Lexer l s t i) (TokenLexer l s t i)

statePosn :: forall l s t i. Lens' (Lexer l s t i) (l s -> Posn)

startingState :: forall l s t i. Lens' (Lexer l s t i) s

starting :: forall l s t i. Lens' (Lexer l s t i) (s -> Point -> Posn -> l s)

looked :: forall l s t i. Lens' (Lexer l s t i) (l s -> Point)

lexEmpty :: forall l s t i. Lens' (Lexer l s t i) t

tokenToStyle :: forall l s t i. Lens' (StyleLexer l s t i) (t -> StyleName)

styleLexer :: forall l s t i l s i. Lens (StyleLexer l s t i) (StyleLexer l s t i) (Lexer l s (Tok t) i) (Lexer l s (Tok t) i)

(+~) :: SemiNum absolute relative => absolute -> relative -> absolute

(~-) :: SemiNum absolute relative => absolute -> absolute -> relative

newtype Size

Size of a buffer region

Constructors

Size

Fields

• fromSize :: Int

Instances

Enum Size

Methods

succ :: Size -> Size

pred :: Size -> Size

toEnum :: Int -> Size

fromEnum :: Size -> Int

enumFrom :: Size -> [Size]

enumFromThen :: Size -> Size -> [Size]

enumFromTo :: Size -> Size -> [Size]

enumFromThenTo :: Size -> Size -> Size -> [Size]

Eq Size

Methods

(==) :: Size -> Size -> Bool

(/=) :: Size -> Size -> Bool

Integral Size

Methods

quot :: Size -> Size -> Size

rem :: Size -> Size -> Size

div :: Size -> Size -> Size

mod :: Size -> Size -> Size

quotRem :: Size -> Size -> (Size, Size)

divMod :: Size -> Size -> (Size, Size)

toInteger :: Size -> Integer

Num Size

Methods

(+) :: Size -> Size -> Size

(-) :: Size -> Size -> Size

(*) :: Size -> Size -> Size

negate :: Size -> Size

abs :: Size -> Size

signum :: Size -> Size

fromInteger :: Integer -> Size

Ord Size

Methods

compare :: Size -> Size -> Ordering

(<) :: Size -> Size -> Bool

(<=) :: Size -> Size -> Bool

(>) :: Size -> Size -> Bool

(>=) :: Size -> Size -> Bool

max :: Size -> Size -> Size

min :: Size -> Size -> Size

Real Size

Methods

toRational :: Size -> Rational

Show Size

Methods

showsPrec :: Int -> Size -> ShowS

show :: Size -> String

showList :: [Size] -> ShowS

Binary Size

Methods

put :: Size -> Put

get :: Get Size

putList :: [Size] -> Put

SemiNum Point Size

Methods

(+~) :: Point -> Size -> Point

(-~) :: Point -> Size -> Point

(~-) :: Point -> Point -> Size

type Stroke = Span StyleName