Holumbus.Crawler.CacheCore

Documentation

type CacheCrawlerConfig = CrawlerConfig () CacheState

type CacheCrawlerState = CrawlerState CacheState

newtype CacheState

Constructors

CS ()

Instances

Binary CacheState

NFData CacheState

XmlPickler CacheState

emptyCacheState :: CacheState

unionCacheStatesM :: Monad m => CacheState -> CacheState -> m CacheState

insertCacheM :: Monad m => (URI, ()) -> CacheState -> m CacheState

cacheCrawlerConfig

Arguments

:: SysConfig

document read options

-> (URI -> Bool)

the filter for deciding, whether the URI shall be processed

-> CacheCrawlerConfig

result is a crawler config

stdCacher

Arguments

:: (Int, Int, Int)

the parameters for parallel crawling

-> (Int, String)

the save intervall and file path

-> (Priority, Priority)

the log levels for the crawler and hxt

-> SysConfig

the read attributes

-> (CacheCrawlerConfig -> CacheCrawlerConfig)

further configuration settings

-> Maybe String

resume from interrupted index run with state stored in file

-> [URI]

start caching with this set of uris

-> (URI -> Bool)

-> IO CacheCrawlerState