Text.Tokenizer.Uniqueness

This module contains implementation of uniqueness checking algorithm based on Sardinas-Patterson's algorithm

Documentation

data Rem c

Constructors

Rem1 [Repeatable c]

First list reminder. May be empty if there is no rem

Rem2 [Repeatable c]

Second list reminder. Always is nonempty

data MergeRes c

Constructors

MergeRes

Fields

• merged :: [Repeatable c]
• mergeRem :: Rem c

mergeReps :: Ord c => [Repeatable c] -> [Repeatable c] -> Alt (MergeRes c)

mergedList :: MergeRes c -> [Repeatable c]

remList :: MergeRes c -> [Repeatable c]

rem1 :: MergeRes c -> [Repeatable c]

rem2 :: MergeRes c -> [Repeatable c]

data Suff c

Dangling suffix

Constructors

Suff

Fields

• srbeh :: [Repeatable c]

  Symbols behind suffix. Note that only maxBehind symbols are preserved

• scur :: [Repeatable c]

  Symbols from suffix' body

• sahead :: [Repeatable c]

  Symbols ahead suffix

Instances

Eq c => Eq (Suff c)

Defined in Text.Tokenizer.Uniqueness

Methods

(==) :: Suff c -> Suff c -> Bool

(/=) :: Suff c -> Suff c -> Bool

Ord c => Ord (Suff c)

Defined in Text.Tokenizer.Uniqueness

Methods

compare :: Suff c -> Suff c -> Ordering

(<) :: Suff c -> Suff c -> Bool

(<=) :: Suff c -> Suff c -> Bool

(>) :: Suff c -> Suff c -> Bool

(>=) :: Suff c -> Suff c -> Bool

max :: Suff c -> Suff c -> Suff c

min :: Suff c -> Suff c -> Suff c

Show c => Show (Suff c)

Defined in Text.Tokenizer.Uniqueness

Methods

showsPrec :: Int -> Suff c -> ShowS

show :: Suff c -> String

showList :: [Suff c] -> ShowS

data Div c

Result of division.

It looks like

      rtoks       |       lastTok
--------|---------|-----------------------|~~~~~
    rprefToks        |
-----|-----|---------|
suff (remained part):
              behind |     current        | ahead
              -------|====================|~~~~~

Constructors

Div

Fields

• rtoks :: [(TokId, Int)]

  Tokens in main sequence, except last one

• lastTok :: (TokId, Int)

  Last token in main sequence

• rprefToks :: [(TokId, Int)]

  Tokens in alter sequence

• processed :: [Repeatable c]

  Processed symbols

• suff :: Suff c

  Remained suffix

Instances

Eq c => Eq (Div c)

Defined in Text.Tokenizer.Uniqueness

Methods

(==) :: Div c -> Div c -> Bool

(/=) :: Div c -> Div c -> Bool

Ord c => Ord (Div c)

Defined in Text.Tokenizer.Uniqueness

Methods

compare :: Div c -> Div c -> Ordering

(<) :: Div c -> Div c -> Bool

(<=) :: Div c -> Div c -> Bool

(>) :: Div c -> Div c -> Bool

(>=) :: Div c -> Div c -> Bool

max :: Div c -> Div c -> Div c

min :: Div c -> Div c -> Div c

Show c => Show (Div c)

Defined in Text.Tokenizer.Uniqueness

Methods

showsPrec :: Int -> Div c -> ShowS

show :: Div c -> String

showList :: [Div c] -> ShowS

initDiv :: RToken c -> Div c

stepDiv :: Ord c => Int -> Div c -> RToken c -> Alt (Div c)

data ConflictTokens k c

Two ways of tokenizing a string, demonstrating non-uniqueness

Constructors

ConflictTokens

Fields

• tokList1, tokList2 :: [(k, [BlackWhiteSet c])]

Instances

(Eq k, Eq c) => Eq (ConflictTokens k c)

Defined in Text.Tokenizer.Uniqueness

Methods

(==) :: ConflictTokens k c -> ConflictTokens k c -> Bool

(/=) :: ConflictTokens k c -> ConflictTokens k c -> Bool

(Ord k, Ord c) => Ord (ConflictTokens k c)

Defined in Text.Tokenizer.Uniqueness

Methods

compare :: ConflictTokens k c -> ConflictTokens k c -> Ordering

(<) :: ConflictTokens k c -> ConflictTokens k c -> Bool

(<=) :: ConflictTokens k c -> ConflictTokens k c -> Bool

(>) :: ConflictTokens k c -> ConflictTokens k c -> Bool

(>=) :: ConflictTokens k c -> ConflictTokens k c -> Bool

max :: ConflictTokens k c -> ConflictTokens k c -> ConflictTokens k c

min :: ConflictTokens k c -> ConflictTokens k c -> ConflictTokens k c

(Show k, Show c) => Show (ConflictTokens k c)

Defined in Text.Tokenizer.Uniqueness

Methods

showsPrec :: Int -> ConflictTokens k c -> ShowS

show :: ConflictTokens k c -> String

showList :: [ConflictTokens k c] -> ShowS

checkUniqueTokenizing :: forall k c. Ord c => [Token k c] -> Either (ConflictTokens k c) ()

Check that there is no list of symbols, that can be decomposed to ways on the tokens from given list