Holumbus.Crawler.Types

Documentation

type AccumulateDocResult a r = (URI, a) -> r -> IO r

The action to combine the result of a single document with the accumulator for the overall crawler result. This combining function runs in the IO monad to enable storing parts of the result externally but it is not a CrawlerAction, else parallel crawling with forkIO is not longer applicable

type MergeDocResults r = r -> r -> IO r

The folding operator for merging partial results when working with mapFold and parallel crawling

type SavePartialResults r = FilePath -> r -> IO r

The operator for saving intermediate results

type ProcessDocument a = IOSArrow XmlTree a

The extractor function for a single document

type CrawlerAction a r = ReaderStateIO (CrawlerConfig a r) (CrawlerState r)

The crawler action monad

data CrawlerConfig a r

The crawler configuration record

Constructors

CrawlerConfig

Fields

cc_sysConfig :: SysConfig

cc_preRefsFilter :: IOSArrow XmlTree XmlTree

cc_processRefs :: IOSArrow XmlTree URI

cc_preDocFilter :: IOSArrow XmlTree XmlTree

cc_processDoc :: ProcessDocument a

cc_accumulate :: AccumulateDocResult a r

cc_fold :: MergeDocResults r

cc_followRef :: URI -> Bool

cc_addRobotsTxt :: CrawlerConfig a r -> AddRobotsAction

cc_clickLevel :: !Int

cc_maxNoOfDocs :: !Int

cc_maxParDocs :: !Int

cc_maxParThreads :: !Int

cc_saveIntervall :: !Int

cc_savePathPrefix :: !String

cc_savePreAction :: FilePath -> CrawlerAction a r ()

cc_traceLevel :: !Priority

cc_traceLevelHxt :: !Priority

data CrawlerState r

The crawler state record

Constructors

CrawlerState

Fields

cs_toBeProcessed :: !URIsWithLevel

cs_alreadyProcessed :: !URIs

cs_robots :: !Robots

cs_noOfDocs :: !Int

cs_noOfDocsSaved :: !Int

cs_listOfDocsSaved :: ![Int]

cs_resultAccu :: !r

cs_resultInit :: !r

Instances

Show r => Show (CrawlerState r)

Binary r => Binary (CrawlerState r)

NFData r => NFData (CrawlerState r)

XmlPickler r => XmlPickler (CrawlerState r)

theToBeProcessed :: Selector (CrawlerState r) URIsWithLevel

selector functions for CrawlerState

theAlreadyProcessed :: Selector (CrawlerState r) URIs

theRobots :: Selector (CrawlerState r) Robots

theNoOfDocs :: Selector (CrawlerState r) Int

theNoOfDocsSaved :: Selector (CrawlerState r) Int

theListOfDocsSaved :: Selector (CrawlerState r) [Int]

theResultAccu :: Selector (CrawlerState r) r

theResultInit :: Selector (CrawlerState r) r

theSysConfig :: Selector (CrawlerConfig a r) SysConfig

selector functions for CrawlerConfig

theTraceLevel :: Selector (CrawlerConfig a r) Priority

theTraceLevelHxt :: Selector (CrawlerConfig a r) Priority

theClickLevel :: Selector (CrawlerConfig a r) Int

theMaxNoOfDocs :: Selector (CrawlerConfig a r) Int

theMaxParDocs :: Selector (CrawlerConfig a r) Int

theMaxParThreads :: Selector (CrawlerConfig a r) Int

theSaveIntervall :: Selector (CrawlerConfig a r) Int

theSavePathPrefix :: Selector (CrawlerConfig a r) String

theSavePreAction :: Selector (CrawlerConfig a r) (FilePath -> CrawlerAction a r ())

theFollowRef :: Selector (CrawlerConfig a r) (URI -> Bool)

theAddRobotsAction :: Selector (CrawlerConfig a r) (CrawlerConfig a r -> AddRobotsAction)

theAccumulateOp :: Selector (CrawlerConfig a r) (AccumulateDocResult a r)

theFoldOp :: Selector (CrawlerConfig a r) (MergeDocResults r)

thePreRefsFilter :: Selector (CrawlerConfig a r) (IOSArrow XmlTree XmlTree)

theProcessRefs :: Selector (CrawlerConfig a r) (IOSArrow XmlTree URI)

thePreDocFilter :: Selector (CrawlerConfig a r) (IOSArrow XmlTree XmlTree)

theProcessDoc :: Selector (CrawlerConfig a r) (IOSArrow XmlTree a)

defaultCrawlerConfig :: AccumulateDocResult a r -> MergeDocResults r -> CrawlerConfig a r

theInputOptions :: Selector (CrawlerConfig a r) Attributes

theCrawlerName :: Selector (CrawlerConfig a r) String

theMaxTime :: Selector (CrawlerConfig a r) Int

theConnectTimeout :: Selector (CrawlerConfig a r) Int

addSysConfig :: SysConfig -> CrawlerConfig a r -> CrawlerConfig a r

Add attributes for accessing documents

addRobotsNoFollow :: CrawlerConfig a r -> CrawlerConfig a r

Insert a robots no follow filter before thePreRefsFilter

addRobotsNoIndex :: CrawlerConfig a r -> CrawlerConfig a r

Insert a robots no follow filter before thePreRefsFilter

setCrawlerTraceLevel :: Priority -> Priority -> CrawlerConfig a r -> CrawlerConfig a r

Set the log level

setCrawlerSaveConf :: Int -> String -> CrawlerConfig a r -> CrawlerConfig a r

Set save intervall in config

setCrawlerSaveAction :: (FilePath -> CrawlerAction a r ()) -> CrawlerConfig a r -> CrawlerConfig a r

Set action performed before saving crawler state

setCrawlerClickLevel :: Int -> CrawlerConfig a r -> CrawlerConfig a r

Set max # of steps (clicks) to reach a document

setCrawlerMaxDocs :: Int -> Int -> Int -> CrawlerConfig a r -> CrawlerConfig a r

Set max # of documents to be crawled and max # of documents crawled in parallel

setCrawlerPreRefsFilter :: IOSArrow XmlTree XmlTree -> CrawlerConfig a r -> CrawlerConfig a r

Set the pre hook filter executed before the hrefs are collected

putCrawlerState :: Binary r => CrawlerState r -> Put

getCrawlerState :: Binary r => Get (CrawlerState r)

initCrawlerState :: r -> CrawlerState r

getConf :: Selector (CrawlerConfig a r) v -> CrawlerAction a r v

Load a component from the crawler configuration

getState :: Selector (CrawlerState r) v -> CrawlerAction a r v

putState :: Selector (CrawlerState r) v -> v -> CrawlerAction a r ()

modifyState :: Selector (CrawlerState r) v -> (v -> v) -> CrawlerAction a r ()

modifyStateIO :: Selector (CrawlerState r) v -> (v -> IO v) -> CrawlerAction a r ()