| Safe Haskell | None |
|---|
Holumbus.Crawler.RobotTypes
- type Robots = Map URI RobotRestriction
- type RobotRestriction = [RobotSpec]
- type RobotSpec = (URI, RobotAction)
- data RobotAction
- type AddRobotsAction = URI -> Robots -> IO Robots
- xpRobots :: PU Robots
- xpRobotRestriction :: PU RobotRestriction
- emptyRobots :: Robots
- robotsExtend :: String -> AddRobotsAction
- robotsIndex :: URI -> Robots -> Bool
- robotsFollow :: URI -> Robots -> Bool
- robotsNo :: String -> LA XmlTree XmlTree
- robotsNoIndex :: ArrowXml a => a XmlTree XmlTree
- robotsNoFollow :: ArrowXml a => a XmlTree XmlTree
Documentation
type Robots = Map URI RobotRestrictionSource
type RobotRestriction = [RobotSpec]Source
type RobotSpec = (URI, RobotAction)Source
data RobotAction Source
robotsIndex :: URI -> Robots -> BoolSource
robotsFollow :: URI -> Robots -> BoolSource
robotsNoIndex :: ArrowXml a => a XmlTree XmlTreeSource
robots no index filter. This filter checks HTML documents for a <meta name="robots" content="noindex"> in the head of the document
robotsNoFollow :: ArrowXml a => a XmlTree XmlTreeSource
robots no follow filter. This filter checks HTML documents for a <meta name="robots" content="nofollow"> in the head of the document