Network.HTTP.Robots

Documentation

type Robot = ([([UserAgent], [Directive])], [Unparsable])

type Unparsable = ByteString

data UserAgent

Constructors

Wildcard

Literal ByteString

Instances

Eq UserAgent

Show UserAgent

type Path = ByteString

data Directive

Constructors

Allow Path

Disallow Path

CrawlDelay Int

Instances

Eq Directive

Show Directive

strip :: ByteString -> ByteString

parseRobots :: ByteString -> Either String Robot

parseRobots is the main entry point for parsing a robots.txt file.

robotP :: Parser Robot

unparsableP :: Parser ByteString ByteString

agentDirectiveP :: Parser ([UserAgent], [Directive])

skipSpace :: Parser ()

directiveP :: Parser Directive

agentP :: Parser UserAgent

commentsP :: Parser ()

tokenP :: Parser ByteString

canAccess :: ByteString -> Robot -> Path -> Bool