Data.Warc

Documentation

data Record m r

A WARC record

Constructors

Record

Fields

• recHeader :: RecordHeader
• recContent :: Producer ByteString m r

Instances

Monad m => Functor (Record m)

Methods

fmap :: (a -> b) -> Record m a -> Record m b

(<$) :: a -> Record m b -> Record m a

type Warc m a = FreeT (Record m) m (Producer ByteString m a)

A WARC archive

Parsing

parseWarc :: (Functor m, Monad m) => Producer ByteString m a -> Warc m a

Parse a WARC archive.

iterRecords :: forall m a. Monad m => (forall b. Record m b -> m b) -> Warc m a -> m (Producer ByteString m a)

Iterate over the Records in a WARC archive

produceRecords

Arguments

:: Monad m

=> (forall b. RecordHeader -> Producer ByteString m b -> Producer o m b)

consume the record producing some output

-> Warc m a

a WARC archive (see parseWarc)

-> Producer o m (Producer ByteString m a)

returns any leftovers

Encoding

encodeRecord :: Monad m => Record m a -> Producer ByteString m a

Headers

module Data.Warc.Header