Codec.Compression.LZ4.Conduit

Help Wanted / TODOs

Please feel free to send me a pull request for any of the following items:

• TODO Block checksumming
• TODO Dictionary support
• TODO Performance: Write a version of compress that emits ByteStrings of known constant length. That will allow us to do compression in a zero-copy fashion, writing compressed bytes directly into a the ByteStrings (e.g using unsafePackMallocCString or equivalent). We currently don't do that (instead, use allocaBytes + copying packCStringLen) to ensure that the ByteStrings generated are as compact as possible (for the case that `written < size`), since the current compress conduit directly yields the outputs of LZ4F_compressUpdate() (unless they are of 0 length when they are buffered in the context tmp buffer).
• TODO Try enabling checksums, then corrupt a bit and see if lz4c detects it.
• TODO Add `with*` style bracketed functions for creating the LZ4F_createCompressionContext and Lz4FramePreferencesPtr for prompt resource release, in addition to the GC'd variants below. This would replace our use of finalizeForeignPtr in the conduit. finalizeForeignPtr seems almost as good, but note that it doesn't guarantee prompt resource release on exceptions; a `with*` style function that uses bracket does. However, it isn't clear yet which one would be faster (what the cost of mask is compared to foreign pointer finalizers). Also note that prompt freeing has side benefits, such as reduced malloc() fragmentation (the closer malloc() and free() are to each other, the smaller is the chance to have malloc()s on top of the our malloc() in the heap, thus the smaller the chance that we cannot decrease the heap pointer upon free() (because "mallocs on top" render heap memory unreturnable to the OS; memory fragmentation).

Documentation

data Lz4FrameException

Constructors

Lz4FormatException String

Instances

Exception Lz4FrameException

Defined in Codec.Compression.LZ4.CTypes

Methods

toException :: Lz4FrameException -> SomeException

fromException :: SomeException -> Maybe Lz4FrameException

displayException :: Lz4FrameException -> String

Show Lz4FrameException

Defined in Codec.Compression.LZ4.CTypes

Methods

showsPrec :: Int -> Lz4FrameException -> ShowS

show :: Lz4FrameException -> String

showList :: [Lz4FrameException] -> ShowS

Eq Lz4FrameException

Defined in Codec.Compression.LZ4.CTypes

Methods

(==) :: Lz4FrameException -> Lz4FrameException -> Bool

(/=) :: Lz4FrameException -> Lz4FrameException -> Bool

Ord Lz4FrameException

Defined in Codec.Compression.LZ4.CTypes

Methods

compare :: Lz4FrameException -> Lz4FrameException -> Ordering

(<) :: Lz4FrameException -> Lz4FrameException -> Bool

(<=) :: Lz4FrameException -> Lz4FrameException -> Bool

(>) :: Lz4FrameException -> Lz4FrameException -> Bool

(>=) :: Lz4FrameException -> Lz4FrameException -> Bool

max :: Lz4FrameException -> Lz4FrameException -> Lz4FrameException

min :: Lz4FrameException -> Lz4FrameException -> Lz4FrameException

data BlockSizeID

Constructors

LZ4F_default

LZ4F_max64KB

LZ4F_max256KB

LZ4F_max1MB

LZ4F_max4MB

Instances

Storable BlockSizeID

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: BlockSizeID -> Int

alignment :: BlockSizeID -> Int

peekElemOff :: Ptr BlockSizeID -> Int -> IO BlockSizeID

pokeElemOff :: Ptr BlockSizeID -> Int -> BlockSizeID -> IO ()

peekByteOff :: Ptr b -> Int -> IO BlockSizeID

pokeByteOff :: Ptr b -> Int -> BlockSizeID -> IO ()

peek :: Ptr BlockSizeID -> IO BlockSizeID

poke :: Ptr BlockSizeID -> BlockSizeID -> IO ()

Show BlockSizeID

Defined in Codec.Compression.LZ4.CTypes

Methods

showsPrec :: Int -> BlockSizeID -> ShowS

show :: BlockSizeID -> String

showList :: [BlockSizeID] -> ShowS

Eq BlockSizeID

Defined in Codec.Compression.LZ4.CTypes

Methods

(==) :: BlockSizeID -> BlockSizeID -> Bool

(/=) :: BlockSizeID -> BlockSizeID -> Bool

Ord BlockSizeID

Defined in Codec.Compression.LZ4.CTypes

Methods

compare :: BlockSizeID -> BlockSizeID -> Ordering

(<) :: BlockSizeID -> BlockSizeID -> Bool

(<=) :: BlockSizeID -> BlockSizeID -> Bool

(>) :: BlockSizeID -> BlockSizeID -> Bool

(>=) :: BlockSizeID -> BlockSizeID -> Bool

max :: BlockSizeID -> BlockSizeID -> BlockSizeID

min :: BlockSizeID -> BlockSizeID -> BlockSizeID

data BlockMode

Constructors

LZ4F_blockLinked

LZ4F_blockIndependent

Instances

Storable BlockMode

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: BlockMode -> Int

alignment :: BlockMode -> Int

peekElemOff :: Ptr BlockMode -> Int -> IO BlockMode

pokeElemOff :: Ptr BlockMode -> Int -> BlockMode -> IO ()

peekByteOff :: Ptr b -> Int -> IO BlockMode

pokeByteOff :: Ptr b -> Int -> BlockMode -> IO ()

peek :: Ptr BlockMode -> IO BlockMode

poke :: Ptr BlockMode -> BlockMode -> IO ()

Show BlockMode

Defined in Codec.Compression.LZ4.CTypes

Methods

showsPrec :: Int -> BlockMode -> ShowS

show :: BlockMode -> String

showList :: [BlockMode] -> ShowS

Eq BlockMode

Defined in Codec.Compression.LZ4.CTypes

Methods

(==) :: BlockMode -> BlockMode -> Bool

(/=) :: BlockMode -> BlockMode -> Bool

Ord BlockMode

Defined in Codec.Compression.LZ4.CTypes

Methods

compare :: BlockMode -> BlockMode -> Ordering

(<) :: BlockMode -> BlockMode -> Bool

(<=) :: BlockMode -> BlockMode -> Bool

(>) :: BlockMode -> BlockMode -> Bool

(>=) :: BlockMode -> BlockMode -> Bool

max :: BlockMode -> BlockMode -> BlockMode

min :: BlockMode -> BlockMode -> BlockMode

data ContentChecksum

Constructors

LZ4F_noContentChecksum

LZ4F_contentChecksumEnabled

Instances

Storable ContentChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: ContentChecksum -> Int

alignment :: ContentChecksum -> Int

peekElemOff :: Ptr ContentChecksum -> Int -> IO ContentChecksum

pokeElemOff :: Ptr ContentChecksum -> Int -> ContentChecksum -> IO ()

peekByteOff :: Ptr b -> Int -> IO ContentChecksum

pokeByteOff :: Ptr b -> Int -> ContentChecksum -> IO ()

peek :: Ptr ContentChecksum -> IO ContentChecksum

poke :: Ptr ContentChecksum -> ContentChecksum -> IO ()

Show ContentChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

showsPrec :: Int -> ContentChecksum -> ShowS

show :: ContentChecksum -> String

showList :: [ContentChecksum] -> ShowS

Eq ContentChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

(==) :: ContentChecksum -> ContentChecksum -> Bool

(/=) :: ContentChecksum -> ContentChecksum -> Bool

Ord ContentChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

compare :: ContentChecksum -> ContentChecksum -> Ordering

(<) :: ContentChecksum -> ContentChecksum -> Bool

(<=) :: ContentChecksum -> ContentChecksum -> Bool

(>) :: ContentChecksum -> ContentChecksum -> Bool

(>=) :: ContentChecksum -> ContentChecksum -> Bool

max :: ContentChecksum -> ContentChecksum -> ContentChecksum

min :: ContentChecksum -> ContentChecksum -> ContentChecksum

data BlockChecksum

Constructors

LZ4F_noBlockChecksum

LZ4F_blockChecksumEnabled

Instances

Storable BlockChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: BlockChecksum -> Int

alignment :: BlockChecksum -> Int

peekElemOff :: Ptr BlockChecksum -> Int -> IO BlockChecksum

pokeElemOff :: Ptr BlockChecksum -> Int -> BlockChecksum -> IO ()

peekByteOff :: Ptr b -> Int -> IO BlockChecksum

pokeByteOff :: Ptr b -> Int -> BlockChecksum -> IO ()

peek :: Ptr BlockChecksum -> IO BlockChecksum

poke :: Ptr BlockChecksum -> BlockChecksum -> IO ()

Show BlockChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

showsPrec :: Int -> BlockChecksum -> ShowS

show :: BlockChecksum -> String

showList :: [BlockChecksum] -> ShowS

Eq BlockChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

(==) :: BlockChecksum -> BlockChecksum -> Bool

(/=) :: BlockChecksum -> BlockChecksum -> Bool

Ord BlockChecksum

Defined in Codec.Compression.LZ4.CTypes

Methods

compare :: BlockChecksum -> BlockChecksum -> Ordering

(<) :: BlockChecksum -> BlockChecksum -> Bool

(<=) :: BlockChecksum -> BlockChecksum -> Bool

(>) :: BlockChecksum -> BlockChecksum -> Bool

(>=) :: BlockChecksum -> BlockChecksum -> Bool

max :: BlockChecksum -> BlockChecksum -> BlockChecksum

min :: BlockChecksum -> BlockChecksum -> BlockChecksum

data FrameType

Constructors

LZ4F_frame

LZ4F_skippableFrame

Instances

Storable FrameType

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: FrameType -> Int

alignment :: FrameType -> Int

peekElemOff :: Ptr FrameType -> Int -> IO FrameType

pokeElemOff :: Ptr FrameType -> Int -> FrameType -> IO ()

peekByteOff :: Ptr b -> Int -> IO FrameType

pokeByteOff :: Ptr b -> Int -> FrameType -> IO ()

peek :: Ptr FrameType -> IO FrameType

poke :: Ptr FrameType -> FrameType -> IO ()

Show FrameType

Defined in Codec.Compression.LZ4.CTypes

Methods

showsPrec :: Int -> FrameType -> ShowS

show :: FrameType -> String

showList :: [FrameType] -> ShowS

Eq FrameType

Defined in Codec.Compression.LZ4.CTypes

Methods

(==) :: FrameType -> FrameType -> Bool

(/=) :: FrameType -> FrameType -> Bool

Ord FrameType

Defined in Codec.Compression.LZ4.CTypes

Methods

compare :: FrameType -> FrameType -> Ordering

(<) :: FrameType -> FrameType -> Bool

(<=) :: FrameType -> FrameType -> Bool

(>) :: FrameType -> FrameType -> Bool

(>=) :: FrameType -> FrameType -> Bool

max :: FrameType -> FrameType -> FrameType

min :: FrameType -> FrameType -> FrameType

data FrameInfo

Constructors

FrameInfo

Fields

• blockSizeID :: BlockSizeID
• blockMode :: BlockMode
• contentChecksumFlag :: ContentChecksum
• frameType :: FrameType
• contentSize :: Word64
• dictID :: Word32

  unsigned int in lz4frame.h, which can be 16 or 32 bits; AFAIK GHC does not run on archs where it is 16-bit, so there's a compile-time check for it.

• blockChecksumFlag :: BlockChecksum

Instances

Storable FrameInfo

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: FrameInfo -> Int

alignment :: FrameInfo -> Int

peekElemOff :: Ptr FrameInfo -> Int -> IO FrameInfo

pokeElemOff :: Ptr FrameInfo -> Int -> FrameInfo -> IO ()

peekByteOff :: Ptr b -> Int -> IO FrameInfo

pokeByteOff :: Ptr b -> Int -> FrameInfo -> IO ()

peek :: Ptr FrameInfo -> IO FrameInfo

poke :: Ptr FrameInfo -> FrameInfo -> IO ()

data Preferences

Constructors

Preferences

Fields

• frameInfo :: FrameInfo
• compressionLevel :: Int
• autoFlush :: Bool
• favorDecSpeed :: Bool

Instances

Storable Preferences

Defined in Codec.Compression.LZ4.CTypes

Methods

sizeOf :: Preferences -> Int

alignment :: Preferences -> Int

peekElemOff :: Ptr Preferences -> Int -> IO Preferences

pokeElemOff :: Ptr Preferences -> Int -> Preferences -> IO ()

peekByteOff :: Ptr b -> Int -> IO Preferences

pokeByteOff :: Ptr b -> Int -> Preferences -> IO ()

peek :: Ptr Preferences -> IO Preferences

poke :: Ptr Preferences -> Preferences -> IO ()

lz4DefaultPreferences :: Preferences

compress :: (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m ()

Note [Single call to LZ4F_compressUpdate() can create multiple blocks] A single call to LZ4F_compressUpdate() can create multiple blocks, and handles buffers > 32-bit sizes; see: https://github.com/lz4/lz4/blob/52cac9a97342641315c76cfb861206d6acd631a8/lib/lz4frame.c#L601 So we don't need to loop around LZ4F_compressUpdate() to compress an arbitrarily large amount of input data, as long as the destination buffer is large enough.

compressYieldImmediately :: (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m ()

Compresses the incoming stream of ByteStrings with the lz4 frame format.

Yields every LZ4 output as a ByteString as soon as the lz4 frame library produces it.

Note that this does not imply ZL4 frame autoFlush (which affects when the lz4 frame library produces outputs).

compressWithOutBufferSize :: forall m. (MonadUnliftIO m, MonadResource m) => CSize -> ConduitT ByteString ByteString m ()

Compresses the incoming stream of ByteStrings with the lz4 frame format.

This function implements two optimisations to reduce unnecessary allocations:

• Incoming ByteStrings are processed in blocks of 16 KB, allowing us to use a single intermediate output buffer through the lifetime of the conduit.
• The bufferSize of the output buffer can controlled by the caller via the bufferSize argument, to reduce the number of small ByteStrings being yielded (especially in the case that the input data compresses very well, e.g. a stream of zeros).

Note that the given bufferSize is not a hard limit, it can only be used to *increase* the amount of output buffer we're allowed to use: The function will choose `max(bufferSize, minBufferSizeNeededByLz4)` as the eventual output buffer size.

Setting `bufferSize = 0` is the legitimate way to set the output buffer size to be the minimum required to compress 16 KB inputs and is still a fast default.

decompress :: forall m. (MonadUnliftIO m, MonadResource m) => ConduitT ByteString ByteString m ()

TODO check why decompressSizeHint is always 4

bsChunksOf :: Int -> ByteString -> [ByteString]

Internals

newtype Lz4FrameCompressionContext

Constructors

Lz4FrameCompressionContext

Fields

• unLz4FrameCompressionContext :: ForeignPtr (Ptr LZ4F_cctx)

Instances

Show Lz4FrameCompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

showsPrec :: Int -> Lz4FrameCompressionContext -> ShowS

show :: Lz4FrameCompressionContext -> String

showList :: [Lz4FrameCompressionContext] -> ShowS

Eq Lz4FrameCompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

(==) :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Bool

(/=) :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Bool

Ord Lz4FrameCompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

compare :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Ordering

(<) :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Bool

(<=) :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Bool

(>) :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Bool

(>=) :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Bool

max :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Lz4FrameCompressionContext

min :: Lz4FrameCompressionContext -> Lz4FrameCompressionContext -> Lz4FrameCompressionContext

newtype ScopedLz4FrameCompressionContext

Constructors

ScopedLz4FrameCompressionContext

Fields

• unScopedLz4FrameCompressionContext :: Ptr LZ4F_cctx

Instances

Show ScopedLz4FrameCompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

showsPrec :: Int -> ScopedLz4FrameCompressionContext -> ShowS

show :: ScopedLz4FrameCompressionContext -> String

showList :: [ScopedLz4FrameCompressionContext] -> ShowS

Eq ScopedLz4FrameCompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

(==) :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Bool

(/=) :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Bool

Ord ScopedLz4FrameCompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

compare :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Ordering

(<) :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Bool

(<=) :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Bool

(>) :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Bool

(>=) :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> Bool

max :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext

min :: ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext -> ScopedLz4FrameCompressionContext

newtype ScopedLz4FramePreferencesPtr

Constructors

ScopedLz4FramePreferencesPtr

Fields

• unScopedLz4FramePreferencesPtr :: Ptr Preferences

Instances

Show ScopedLz4FramePreferencesPtr

Defined in Codec.Compression.LZ4.Conduit

Methods

showsPrec :: Int -> ScopedLz4FramePreferencesPtr -> ShowS

show :: ScopedLz4FramePreferencesPtr -> String

showList :: [ScopedLz4FramePreferencesPtr] -> ShowS

Eq ScopedLz4FramePreferencesPtr

Defined in Codec.Compression.LZ4.Conduit

Methods

(==) :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Bool

(/=) :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Bool

Ord ScopedLz4FramePreferencesPtr

Defined in Codec.Compression.LZ4.Conduit

Methods

compare :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Ordering

(<) :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Bool

(<=) :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Bool

(>) :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Bool

(>=) :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> Bool

max :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr

min :: ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr -> ScopedLz4FramePreferencesPtr

newtype Lz4FramePreferencesPtr

Constructors

Lz4FramePreferencesPtr

Fields

• unLz4FramePreferencesPtr :: ForeignPtr Preferences

Instances

Show Lz4FramePreferencesPtr

Defined in Codec.Compression.LZ4.Conduit

Methods

showsPrec :: Int -> Lz4FramePreferencesPtr -> ShowS

show :: Lz4FramePreferencesPtr -> String

showList :: [Lz4FramePreferencesPtr] -> ShowS

Eq Lz4FramePreferencesPtr

Defined in Codec.Compression.LZ4.Conduit

Methods

(==) :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Bool

(/=) :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Bool

Ord Lz4FramePreferencesPtr

Defined in Codec.Compression.LZ4.Conduit

Methods

compare :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Ordering

(<) :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Bool

(<=) :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Bool

(>) :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Bool

(>=) :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Bool

max :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr

min :: Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr -> Lz4FramePreferencesPtr

newtype Lz4FrameDecompressionContext

Constructors

Lz4FrameDecompressionContext

Fields

• unLz4FrameDecompressionContext :: ForeignPtr (Ptr LZ4F_dctx)

Instances

Show Lz4FrameDecompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

showsPrec :: Int -> Lz4FrameDecompressionContext -> ShowS

show :: Lz4FrameDecompressionContext -> String

showList :: [Lz4FrameDecompressionContext] -> ShowS

Eq Lz4FrameDecompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

(==) :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Bool

(/=) :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Bool

Ord Lz4FrameDecompressionContext

Defined in Codec.Compression.LZ4.Conduit

Methods

compare :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Ordering

(<) :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Bool

(<=) :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Bool

(>) :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Bool

(>=) :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Bool

max :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext

min :: Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext -> Lz4FrameDecompressionContext

lz4fCreatePreferences :: IO Lz4FramePreferencesPtr

lz4fCreateCompressonContext :: HasCallStack => IO Lz4FrameCompressionContext

lz4fCreateDecompressionContext :: HasCallStack => IO Lz4FrameDecompressionContext

All notes that apply to lz4fCreateCompressonContext apply here as well.

withScopedLz4fPreferences :: HasCallStack => (ScopedLz4FramePreferencesPtr -> IO a) -> IO a

withScopedLz4fCompressionContext :: HasCallStack => (ScopedLz4FrameCompressionContext -> IO a) -> IO a