Network.HTTP.Robots

Documentation

type Robot = ([([UserAgent], [Directive])], [Unparsable])

type Unparsable = ByteString

data UserAgent

Constructors

Wildcard

Literal ByteString

Instances

Show UserAgent

Defined in Network.HTTP.Robots

Methods

showsPrec :: Int -> UserAgent -> ShowS

show :: UserAgent -> String

showList :: [UserAgent] -> ShowS

Eq UserAgent

Defined in Network.HTTP.Robots

Methods

(==) :: UserAgent -> UserAgent -> Bool

(/=) :: UserAgent -> UserAgent -> Bool

type Path = ByteString

type TimeInterval = (DiffTime, DiffTime)

data Directive

Constructors

Allow Path

Disallow Path

CrawlDelay

Fields

• crawlDelay :: Rational
• timeInterval :: TimeInterval

NoArchive Path

NoSnippet Path

NoTranslate Path

NoIndex Path

Instances

Show Directive

Defined in Network.HTTP.Robots

Methods

showsPrec :: Int -> Directive -> ShowS

show :: Directive -> String

showList :: [Directive] -> ShowS

Eq Directive

Defined in Network.HTTP.Robots

Methods

(==) :: Directive -> Directive -> Bool

(/=) :: Directive -> Directive -> Bool

subParser :: Parser a -> ByteString -> Parser a

safeParseRational :: Parser Rational

dropUTF8BOM :: ByteString -> ByteString

parseHourMinute :: Parser (Integer, Integer)

parseTimeInterval :: Parser TimeInterval

allDay :: TimeInterval

parseRequestRate :: Parser Directive

parseVisitTime :: Parser Directive

parseCrawlDelay :: Parser Directive

strip :: ByteString -> ByteString

parseRobots :: ByteString -> Either String Robot

parseRobots is the main entry point for parsing a robots.txt file.

robotP :: Parser Robot

unparsableP :: Parser ByteString

agentDirectiveP :: Parser ([UserAgent], [Directive])

skipSpace :: Parser ()

directiveP :: Parser Directive

agentP :: Parser UserAgent

commentsP :: Parser ()

tokenP :: Parser ByteString

tokenWithSpacesP :: Parser ByteString

canAccess :: ByteString -> Robot -> Path -> Bool