hpp-0.4.0: A Haskell pre-processor

Safe HaskellSafe
LanguageHaskell2010

Hpp.Tokens

Description

Tokenization breaks a String into pieces of whitespace, constants, symbols, and identifiers.

Synopsis

Documentation

data Token s Source #

Tokenization is words except the white space is tagged rather than discarded.

Constructors

Important s

Identifiers, symbols, and constants

Other s

White space, etc.

Instances

Functor Token Source # 

Methods

fmap :: (a -> b) -> Token a -> Token b #

(<$) :: a -> Token b -> Token a #

Eq s => Eq (Token s) Source # 

Methods

(==) :: Token s -> Token s -> Bool #

(/=) :: Token s -> Token s -> Bool #

Ord s => Ord (Token s) Source # 

Methods

compare :: Token s -> Token s -> Ordering #

(<) :: Token s -> Token s -> Bool #

(<=) :: Token s -> Token s -> Bool #

(>) :: Token s -> Token s -> Bool #

(>=) :: Token s -> Token s -> Bool #

max :: Token s -> Token s -> Token s #

min :: Token s -> Token s -> Token s #

Show s => Show (Token s) Source # 

Methods

showsPrec :: Int -> Token s -> ShowS #

show :: Token s -> String #

showList :: [Token s] -> ShowS #

detok :: Token s -> s Source #

Extract the contents of a Token.

isImportant :: Token s -> Bool Source #

True if the given Token is Important; False otherwise.

notImportant :: Token s -> Bool Source #

True if the given Token is not Important; False otherwise.

importants :: [Token s] -> [s] Source #

Return the contents of only Important (non-space) tokens.

trimUnimportant :: [Token s] -> [Token s] Source #

Trim Other Tokens from both ends of a list of Tokens.

detokenize :: Monoid s => [Token s] -> s Source #

Collapse a sequence of Tokens back into a String. detokenize . tokenize == id.

tokenize :: Stringy s => s -> [Token s] Source #

Break an input String into a sequence of Tokens. Warning: This may not exactly correspond to your target language's definition of a valid identifier!

newLine :: (Eq s, IsString s) => Token s -> Bool Source #

Is a Token a newline character?

skipLiteral :: Stringy s => s -> (s, s) Source #