NLP.Scoring.Unigram.Import

TODO normalization of characters! (though it might be better to do this not in the importer, but a normalization function)

Documentation

data Env

Constructors

Env

Fields

• _warnings :: !(Seq Text)
• _defaults :: !(HashMap Text Double)
• _charGroups :: !(HashMap Text (HashSet Text))
• _matchScores :: !(HashMap (Text, Text) Double)
• _ignoredScoresFstK :: !(HashMap Text Double)
• _ignoredScoresSndL :: !(HashMap Text Double)

Instances

Show Env

Defined in NLP.Scoring.Unigram.Import

Methods

showsPrec :: Int -> Env -> ShowS

show :: Env -> String

showList :: [Env] -> ShowS

DeltaParsing (Unlined UnigramParser)

Defined in NLP.Scoring.Unigram.Import

Methods

line :: Unlined UnigramParser ByteString

position :: Unlined UnigramParser Delta

slicedWith :: (a -> ByteString -> r) -> Unlined UnigramParser a -> Unlined UnigramParser r

rend :: Unlined UnigramParser Rendering

restOfLine :: Unlined UnigramParser ByteString

warnings :: Lens' Env (Seq Text)

matchScores :: Lens' Env (HashMap (Text, Text) Double)

ignoredScoresSndL :: Lens' Env (HashMap Text Double)

ignoredScoresFstK :: Lens' Env (HashMap Text Double)

defaults :: Lens' Env (HashMap Text Double)

charGroups :: Lens' Env (HashMap Text (HashSet Text))

defaultEnv :: Env

test :: IO (Either ErrInfo (UnigramScoring k3 l))

prettyErrorAndExit :: MonadIO m => ErrInfo -> m ()

This will prettyprint the error message and ungracefully exit

errorToString :: ErrInfo -> String

Returns the error message, but will not exit.

fromByteString :: ByteString -> String -> Except ErrInfo (UnigramScoring k l)

fromFile :: Bool -> FilePath -> ExceptT ErrInfo IO (UnigramScoring k l)

pUnigram :: UnigramParser (UnigramScoring k l)

pDefaults :: UnigramParser ()

Defaults are key-value pairs, of which there is only a small set.

pCharGroup :: UnigramParser ()

Gives a name to a set of characters we want to work with later on.

pSimilarity :: UnigramParser ()

Parses a similarity line and updates the scores for the pairs of characters.

pEquality :: UnigramParser ()

Parses an equality line and updates the scores for the pairs of characters.

data FstKSndL

Constructors

FstK

SndL

Instances

Eq FstKSndL

Defined in NLP.Scoring.Unigram.Import

Methods

(==) :: FstKSndL -> FstKSndL -> Bool

(/=) :: FstKSndL -> FstKSndL -> Bool

Ord FstKSndL

Defined in NLP.Scoring.Unigram.Import

Methods

compare :: FstKSndL -> FstKSndL -> Ordering

(<) :: FstKSndL -> FstKSndL -> Bool

(<=) :: FstKSndL -> FstKSndL -> Bool

(>) :: FstKSndL -> FstKSndL -> Bool

(>=) :: FstKSndL -> FstKSndL -> Bool

max :: FstKSndL -> FstKSndL -> FstKSndL

min :: FstKSndL -> FstKSndL -> FstKSndL

pIgnored :: UnigramParser ()

pGrapheme :: (CharParsing p, TokenParsing p) => p Text

Defines what a grapheme is. Basically, don't be a whitespace and don't start with $.

TODO we probably want to allow $ to stand for $.

pKnownCharGroup :: Unlined UnigramParser (HashSet Text)

Returns the set of characters from a known character group

pExpansionOptions :: UnigramParser Text

How we can expand a group with special functions.

specialFunctions :: [(Text, Text -> Text)]

applySpecialFunctions :: Foldable t => t Text -> HashSet Text -> HashSet Text

setIdent :: HashSet Text -> Unlined UnigramParser Text

TODO only insert warning, not error, after seeing a character again!

reserved :: TokenParsing m => IdentifierStyle m

newtype P a

This is just the trifecta parser, but with haskell-style comments enabled.

Constructors

P

Fields

• runP :: Parser a

Instances

Monad P

Defined in NLP.Scoring.Unigram.Import

Methods

(>>=) :: P a -> (a -> P b) -> P b

(>>) :: P a -> P b -> P b

return :: a -> P a

fail :: String -> P a

Functor P

Defined in NLP.Scoring.Unigram.Import

Methods

fmap :: (a -> b) -> P a -> P b

(<$) :: a -> P b -> P a

Applicative P

Defined in NLP.Scoring.Unigram.Import

Methods

pure :: a -> P a

(<*>) :: P (a -> b) -> P a -> P b

liftA2 :: (a -> b -> c) -> P a -> P b -> P c

(*>) :: P a -> P b -> P b

(<*) :: P a -> P b -> P a

Alternative P

Defined in NLP.Scoring.Unigram.Import

Methods

empty :: P a

(<|>) :: P a -> P a -> P a

some :: P a -> P [a]

many :: P a -> P [a]

MonadPlus P

Defined in NLP.Scoring.Unigram.Import

Methods

mzero :: P a

mplus :: P a -> P a -> P a

TokenParsing P

This enables the haskell-style comments.

Defined in NLP.Scoring.Unigram.Import

Methods

someSpace :: P ()

nesting :: P a -> P a

semi :: P Char

highlight :: Highlight -> P a -> P a

token :: P a -> P a

CharParsing P

Defined in NLP.Scoring.Unigram.Import

Methods

satisfy :: (Char -> Bool) -> P Char

char :: Char -> P Char

notChar :: Char -> P Char

anyChar :: P Char

string :: String -> P String

text :: Text -> P Text

Parsing P

Defined in NLP.Scoring.Unigram.Import

Methods

try :: P a -> P a

(<?>) :: P a -> String -> P a

skipMany :: P a -> P ()

skipSome :: P a -> P ()

unexpected :: String -> P a

eof :: P ()

notFollowedBy :: Show a => P a -> P ()

DeltaParsing P

Defined in NLP.Scoring.Unigram.Import

Methods

line :: P ByteString

position :: P Delta

slicedWith :: (a -> ByteString -> r) -> P a -> P r

rend :: P Rendering

restOfLine :: P ByteString

DeltaParsing (Unlined UnigramParser)

Defined in NLP.Scoring.Unigram.Import

Methods

line :: Unlined UnigramParser ByteString

position :: Unlined UnigramParser Delta

slicedWith :: (a -> ByteString -> r) -> Unlined UnigramParser a -> Unlined UnigramParser r

rend :: Unlined UnigramParser Rendering

restOfLine :: Unlined UnigramParser ByteString

type UnigramParser = StateT Env P