Data.Avro.Internal.Container

Documentation

data ContainerHeader

Constructors

ContainerHeader

Fields

• syncBytes :: ByteString
• decompress :: forall a. Decompress a
• containedSchema :: Schema

nrSyncBytes :: Integral sb => sb

newSyncBytes :: IO ByteString

Generates a new synchronization marker for encoding Avro containers

getContainerHeader :: Get ContainerHeader

decodeRawBlocks :: ByteString -> Either String (Schema, [Either String (Int, ByteString)])

Reads the container as a list of blocks without decoding them into actual values.

This can be useful for streaming splitting merging Avro containers without paying the cost for Avro encoding/decoding.

Each block is returned as a raw ByteString annotated with the number of Avro values that are contained in this block.

The "outer" error represents the error in opening the container itself (including problems like reading schemas embedded into the container.)

getNextBlock :: ByteString -> Decompress ByteString -> ByteString -> Either String (Maybe (Int, ByteString, ByteString))

extractContainerValuesBytes :: forall a schema. (Schema -> Either String schema) -> (schema -> Get a) -> ByteString -> Either String (Schema, [Either String (a, ByteString)])

Splits container into a list of individual avro-encoded values. This version provides both encoded and decoded values.

This is particularly useful when slicing up containers into one or more smaller files. By extracting the original bytestring it is possible to avoid re-encoding data.

extractContainerValues :: forall a schema. (Schema -> Either String schema) -> (schema -> Get a) -> ByteString -> Either String (Schema, [Either String a])

packContainerValues :: Codec -> Schema -> [[ByteString]] -> IO ByteString

Packs a container from a given list of already encoded Avro values Each bytestring should represent exactly one one value serialised to Avro.

packContainerValuesWithSync :: Codec -> Schema -> ByteString -> [[ByteString]] -> ByteString

Packs a container from a given list of already encoded Avro values Each bytestring should represent exactly one one value serialised to Avro.

packContainerValuesWithSync' :: (Schema -> a -> Builder) -> Codec -> Schema -> ByteString -> [[a]] -> ByteString

Packs a container from a given list of already encoded Avro values Each bytestring should represent exactly one one value serialised to Avro.

packContainerBlocks :: Codec -> Schema -> [(Int, ByteString)] -> IO ByteString

Packs a new container from a list of already encoded Avro blocks. Each block is denoted as a pair of a number of objects within that block and the block content.

packContainerBlocksWithSync :: Codec -> Schema -> ByteString -> [(Int, ByteString)] -> ByteString

Packs a new container from a list of already encoded Avro blocks. Each block is denoted as a pair of a number of objects within that block and the block content.

containerHeaderWithSync :: Codec -> Schema -> ByteString -> Builder

Creates an Avro container header for a given schema.

consumeN :: Int64 -> (a -> (a, b)) -> a -> (a, [b])

parseCodec :: Monad m => Maybe ByteString -> m Codec

takeWhileInclusive :: (a -> Bool) -> [a] -> [a]