robots-txt-0.1.0.0: Parser for robots.txt

Safe Haskell	None

Network.HTTP.Robots

Documentation

type Robot = [(UserAgent, [Directive])]Source

data UserAgent Source

Constructors

Wildcard
Literal ByteString

Instances

Eq UserAgent
Show UserAgent

type Path = ByteString Source

data Directive Source

Constructors

Allow Path
Disallow Path
CrawlDelay Int

Instances

Eq Directive
Show Directive

parseRobots :: ByteString -> Either String Robot Source

data Hole Source

Constructors

robotP :: Parser Robot Source

directiveP :: Parser Directive Source

agentP :: Parser UserAgent Source

tokenP :: Parser ByteString Source