Safe Haskell | None |
---|
- type Robot = ([([UserAgent], [Directive])], [Unparsable])
- type Unparsable = ByteString
- data UserAgent
- type Path = ByteString
- data Directive
- parseRobots :: ByteString -> Either String Robot
- robotP :: Parser Robot
- unparsableP :: Parser ByteString ByteString
- agentDirectiveP :: Parser ([UserAgent], [Directive])
- skipSpace :: Parser ()
- directiveP :: Parser Directive
- agentP :: Parser UserAgent
- commentsP :: Parser ()
- tokenP :: Parser ByteString
- canAccess :: ByteString -> Robot -> Path -> Bool
Documentation
type Robot = ([([UserAgent], [Directive])], [Unparsable])Source
type Unparsable = ByteStringSource
type Path = ByteStringSource
parseRobots :: ByteString -> Either String RobotSource
parseRobots is the main entry point for parsing a robots.txt file.
agentDirectiveP :: Parser ([UserAgent], [Directive])Source