Hpp.Tokens

Tokenization breaks a String into pieces of whitespace, constants, symbols, and identifiers.

Documentation

data Token s

Tokenization is words except the white space is tagged rather than discarded.

Constructors

Important s

Identifiers, symbols, and constants

Other s

White space, etc.

Instances

Functor Token

Defined in Hpp.Tokens

Methods

fmap :: (a -> b) -> Token a -> Token b

(<$) :: a -> Token b -> Token a

Eq s => Eq (Token s)

Defined in Hpp.Tokens

Methods

(==) :: Token s -> Token s -> Bool

(/=) :: Token s -> Token s -> Bool

Ord s => Ord (Token s)

Defined in Hpp.Tokens

Methods

compare :: Token s -> Token s -> Ordering

(<) :: Token s -> Token s -> Bool

(<=) :: Token s -> Token s -> Bool

(>) :: Token s -> Token s -> Bool

(>=) :: Token s -> Token s -> Bool

max :: Token s -> Token s -> Token s

min :: Token s -> Token s -> Token s

Show s => Show (Token s)

Defined in Hpp.Tokens

Methods

showsPrec :: Int -> Token s -> ShowS

show :: Token s -> String

showList :: [Token s] -> ShowS

detok :: Token s -> s

Extract the contents of a Token.

isImportant :: Token s -> Bool

True if the given Token is Important; False otherwise.

notImportant :: Token s -> Bool

True if the given Token is not Important; False otherwise.

importants :: [Token s] -> [s]

Return the contents of only Important (non-space) tokens.

trimUnimportant :: [Token s] -> [Token s]

Trim Other Tokens from both ends of a list of Tokens.

detokenize :: Monoid s => [Token s] -> s

Collapse a sequence of Tokens back into a String. detokenize . tokenize == id.

tokenize :: Stringy s => s -> [Token s]

Break an input String into a sequence of Tokens. Warning: This may not exactly correspond to your target language's definition of a valid identifier!

newLine :: (Eq s, IsString s) => Token s -> Bool

Is a Token a newline character?

skipLiteral :: Stringy s => s -> (s, s)

Skip over a string or character literal returning the literal and the remaining the input.