Network.HTTP.Conduit.Downloader

Download operations

urlGetContents :: String -> IO ByteString

Download single URL with default DownloaderSettings. Fails if result is not DROK.

urlGetContentsPost :: String -> ByteString -> IO ByteString

Post data and download single URL with default DownloaderSettings. Fails if result is not DROK.

download

Arguments

:: Downloader

-> String

URL

-> Maybe HostAddress

Optional resolved HostAddress

-> DownloadOptions

-> IO DownloadResult

Perform download

post :: Downloader -> String -> Maybe HostAddress -> ByteString -> IO DownloadResult

Perform HTTP POST.

downloadG

Arguments

:: (Request -> ResourceT IO Request)

Function to modify Request (e.g. sign or make postRequest)

-> Downloader

-> String

URL

-> Maybe HostAddress

Optional resolved HostAddress

-> DownloadOptions

-> IO DownloadResult

Generic version of download with ability to modify http-conduit Request.

data DownloadResult

Result of download operation.

Constructors

DROK ByteString DownloadOptions

Successful download with data and options for next download.

DRRedirect String

Redirect URL

DRError String

Error

DRNotModified

HTTP 304 Not Modified

Instances

Eq DownloadResult

Read DownloadResult

Show DownloadResult

type DownloadOptions = [String]

If-None-Match and/or If-Modified-Since headers.

Downloader

data DownloaderSettings

Settings used in downloader.

Constructors

DownloaderSettings

Fields

dsUserAgent :: ByteString

User agent string. Default: "Mozilla/5.0 (compatible; HttpConduitDownloader/1.0; +http://hackage.haskell.org/package/http-conduit-downloader)".

Be a good crawler. Provide your User-Agent please.

dsTimeout :: Int

Download timeout. Default: 30 seconds.

dsManagerSettings :: ManagerSettings

Conduit Manager settings. Default: ManagerSettings with SSL certificate checks removed.

dsMaxDownloadSize :: Int

Download size limit in bytes. Default: 10MB.

Instances

Default DownloaderSettings

data Downloader

Keeps http-conduit Manager and DownloaderSettings.

withDownloader :: (Downloader -> IO a) -> IO a

Create a new Downloader, use it in the provided function, and then release it.

withDownloaderSettings :: DownloaderSettings -> (Downloader -> IO a) -> IO a

Create a new Downloader with provided settings, use it in the provided function, and then release it.

newDownloader :: DownloaderSettings -> IO Downloader

Create a Downloader with settings.

Utils

postRequest :: ByteString -> Request -> Request

Make HTTP POST request.

sinkByteString :: MonadIO m => Int -> Sink ByteString m (Maybe ByteString)

Sink data using 32k buffers to reduce memory fragmentation. Returns Nothing if downloaded too much data.