Text.Tokenizer.Types

This module contains common types used by uniqueness checking and tokenizing algorithms

Documentation

newtype Alt a

Type synonym for list monad used as a collection of alternatives

Constructors

Alt [a]

Instances

Monad Alt

Defined in Text.Tokenizer.Types

Methods

(>>=) :: Alt a -> (a -> Alt b) -> Alt b

(>>) :: Alt a -> Alt b -> Alt b

return :: a -> Alt a

Functor Alt

Defined in Text.Tokenizer.Types

Methods

fmap :: (a -> b) -> Alt a -> Alt b

(<$) :: a -> Alt b -> Alt a

Applicative Alt

Defined in Text.Tokenizer.Types

Methods

pure :: a -> Alt a

(<*>) :: Alt (a -> b) -> Alt a -> Alt b

liftA2 :: (a -> b -> c) -> Alt a -> Alt b -> Alt c

(*>) :: Alt a -> Alt b -> Alt b

(<*) :: Alt a -> Alt b -> Alt a

Foldable Alt

Defined in Text.Tokenizer.Types

Methods

fold :: Monoid m => Alt m -> m

foldMap :: Monoid m => (a -> m) -> Alt a -> m

foldMap' :: Monoid m => (a -> m) -> Alt a -> m

foldr :: (a -> b -> b) -> b -> Alt a -> b

foldr' :: (a -> b -> b) -> b -> Alt a -> b

foldl :: (b -> a -> b) -> b -> Alt a -> b

foldl' :: (b -> a -> b) -> b -> Alt a -> b

foldr1 :: (a -> a -> a) -> Alt a -> a

foldl1 :: (a -> a -> a) -> Alt a -> a

toList :: Alt a -> [a]

null :: Alt a -> Bool

length :: Alt a -> Int

elem :: Eq a => a -> Alt a -> Bool

maximum :: Ord a => Alt a -> a

minimum :: Ord a => Alt a -> a

sum :: Num a => Alt a -> a

product :: Num a => Alt a -> a

Traversable Alt

Defined in Text.Tokenizer.Types

Methods

traverse :: Applicative f => (a -> f b) -> Alt a -> f (Alt b)

sequenceA :: Applicative f => Alt (f a) -> f (Alt a)

mapM :: Monad m => (a -> m b) -> Alt a -> m (Alt b)

sequence :: Monad m => Alt (m a) -> m (Alt a)

Alternative Alt

Defined in Text.Tokenizer.Types

Methods

empty :: Alt a

(<|>) :: Alt a -> Alt a -> Alt a

some :: Alt a -> Alt [a]

many :: Alt a -> Alt [a]

Eq a => Eq (Alt a)

Defined in Text.Tokenizer.Types

Methods

(==) :: Alt a -> Alt a -> Bool

(/=) :: Alt a -> Alt a -> Bool

Ord a => Ord (Alt a)

Defined in Text.Tokenizer.Types

Methods

compare :: Alt a -> Alt a -> Ordering

(<) :: Alt a -> Alt a -> Bool

(<=) :: Alt a -> Alt a -> Bool

(>) :: Alt a -> Alt a -> Bool

(>=) :: Alt a -> Alt a -> Bool

max :: Alt a -> Alt a -> Alt a

min :: Alt a -> Alt a -> Alt a

Show a => Show (Alt a)

Defined in Text.Tokenizer.Types

Methods

showsPrec :: Int -> Alt a -> ShowS

show :: Alt a -> String

showList :: [Alt a] -> ShowS

data Count

Number of symbols acceptable by Repeatable

Constructors

One

Some

Instances

Eq Count

Defined in Text.Tokenizer.Types

Methods

(==) :: Count -> Count -> Bool

(/=) :: Count -> Count -> Bool

Ord Count

Defined in Text.Tokenizer.Types

Methods

compare :: Count -> Count -> Ordering

(<) :: Count -> Count -> Bool

(<=) :: Count -> Count -> Bool

(>) :: Count -> Count -> Bool

(>=) :: Count -> Count -> Bool

max :: Count -> Count -> Count

min :: Count -> Count -> Count

Show Count

Defined in Text.Tokenizer.Types

Methods

showsPrec :: Int -> Count -> ShowS

show :: Count -> String

showList :: [Count] -> ShowS

data Repeatable c

BlackWhiteSet that can be repeated.

Constructors

Repeatable

Fields

• getCnt :: Count
• getBWS :: BlackWhiteSet c

Instances

Eq c => Eq (Repeatable c)

Defined in Text.Tokenizer.Types

Methods

(==) :: Repeatable c -> Repeatable c -> Bool

(/=) :: Repeatable c -> Repeatable c -> Bool

Ord c => Ord (Repeatable c)

Defined in Text.Tokenizer.Types

Methods

compare :: Repeatable c -> Repeatable c -> Ordering

(<) :: Repeatable c -> Repeatable c -> Bool

(<=) :: Repeatable c -> Repeatable c -> Bool

(>) :: Repeatable c -> Repeatable c -> Bool

(>=) :: Repeatable c -> Repeatable c -> Bool

max :: Repeatable c -> Repeatable c -> Repeatable c

min :: Repeatable c -> Repeatable c -> Repeatable c

Show c => Show (Repeatable c)

Defined in Text.Tokenizer.Types

Methods

showsPrec :: Int -> Repeatable c -> ShowS

show :: Repeatable c -> String

showList :: [Repeatable c] -> ShowS

type TokId = Int

Token id type synonym.

data Token k c

Token with name of type k (used for uniqueness error messages and tokenizing output) over char type c.

Constructors

Token

Fields

• name :: k

  the name of token

• behind, ahead :: [BlackWhiteSet c]

  restrictions on symbols before/after matchable part

  NB! they are assumed to be satisfied if there are no symbols before/after matched part respectively

• body :: [Repeatable c]

  matchable sequences of char sets with possible repetitions

Instances

(Show k, Show c) => Show (Token k c)

Defined in Text.Tokenizer.Types

Methods

showsPrec :: Int -> Token k c -> ShowS

show :: Token k c -> String

showList :: [Token k c] -> ShowS

data RToken c

Type for internal needs. Contains autogenerated $sel:tokId:RToken and restrictions behind token are reversed

Constructors

RToken

Fields

• tokId :: TokId

  unique token's id (generated automatically)

• rbehind, ahead :: [Repeatable c]

  constraints on symbols behind/ahead of matchable part

• body :: [Repeatable c]

  matchable part of string

Instances

Eq (RToken c)

Compares by token's id

Defined in Text.Tokenizer.Types

Methods

(==) :: RToken c -> RToken c -> Bool

(/=) :: RToken c -> RToken c -> Bool

Ord (RToken c)

Compares by token's id

Defined in Text.Tokenizer.Types

Methods

compare :: RToken c -> RToken c -> Ordering

(<) :: RToken c -> RToken c -> Bool

(<=) :: RToken c -> RToken c -> Bool

(>) :: RToken c -> RToken c -> Bool

(>=) :: RToken c -> RToken c -> Bool

max :: RToken c -> RToken c -> RToken c

min :: RToken c -> RToken c -> RToken c

Show c => Show (RToken c)

Defined in Text.Tokenizer.Types

Methods

showsPrec :: Int -> RToken c -> ShowS

show :: RToken c -> String

showList :: [RToken c] -> ShowS

makeRToken :: TokId -> Token k c -> RToken c

Construct an RToken from Token and its id