module Language.Haskell.Lexer
( PosToken
, Token(..)
, lexerPass0
, lexerPass0'
, lexerPass1
, rmSpace
, layoutPre
, module Language.Haskell.Lexer.Position
) where
import Language.Haskell.Lexer.Lex(haskellLex)
import Language.Haskell.Lexer.Utils
import Language.Haskell.Lexer.Layout(layoutPre,PosToken)
import Language.Haskell.Lexer.Position
import Data.List(mapAccumL)
default(Int)
lexerPass1 :: String -> [PosToken]
lexerPass1 = lexerPass1Only . lexerPass0
lexerPass1Only :: [PosToken] -> [PosToken]
lexerPass1Only = layoutPre . rmSpace
rmSpace :: [PosToken] -> [PosToken]
rmSpace = filter (notWhite.fst)
notWhite :: Token -> Bool
notWhite t = t/=Whitespace &&
t/=Commentstart && t/=Comment &&
t/=NestedComment
lexerPass0 :: String -> [PosToken]
lexerPass0 = lexerPass0' startPos
lexerPass0' :: Pos -> String -> [PosToken]
lexerPass0' pos0 = addPos . haskellLex . rmcr
where
addPos = snd . mapAccumL pos pos0
pos p (t,s) = (p',(t,(p,s)))
where p' = nextPos p s
rmcr :: String -> String
rmcr ('\CR':'\LF':s) = '\LF':rmcr s
rmcr (c:s) = c:rmcr s
rmcr "" = ""