Network.HTTP.Robots

Documentation

type Robot = ([([UserAgent], [Directive])], [Unparsable])

type Unparsable = ByteString

data UserAgent

Constructors

Wildcard

Literal ByteString

Instances

Eq UserAgent

Show UserAgent

type Path = ByteString

type TimeInterval = (DiffTime, DiffTime)

data Directive

Constructors

Allow Path

Disallow Path

CrawlDelay

Fields

crawlDelay :: Rational

timeInterval :: TimeInterval

NoArchive Path

NoSnippet Path

NoTranslate Path

NoIndex Path

Instances

Eq Directive

Show Directive

subParser :: Parser a -> ByteString -> Parser a

safeParseRational :: Parser Rational

dropUTF8BOM :: ByteString -> ByteString

parseHourMinute :: Parser (Integer, Integer)

parseTimeInterval :: Parser TimeInterval

allDay :: TimeInterval

parseRequestRate :: Parser Directive

parseVisitTime :: Parser Directive

parseCrawlDelay :: Parser Directive

strip :: ByteString -> ByteString

parseRobots :: ByteString -> Either String Robot

parseRobots is the main entry point for parsing a robots.txt file.

robotP :: Parser Robot

unparsableP :: Parser ByteString

agentDirectiveP :: Parser ([UserAgent], [Directive])

skipSpace :: Parser ()

directiveP :: Parser Directive

agentP :: Parser UserAgent

commentsP :: Parser ()

tokenP :: Parser ByteString

tokenWithSpacesP :: Parser ByteString

canAccess :: ByteString -> Robot -> Path -> Bool