Data.Avro.Internal.Container

Documentation

data ContainerHeader

Constructors

ContainerHeader

Fields

• syncBytes :: ByteString
• decompress :: forall a. Decompress a
• containedSchema :: Schema

nrSyncBytes :: Integral sb => sb

newSyncBytes :: IO ByteString

Generates a new synchronization marker for encoding Avro containers

getContainerHeader :: Get ContainerHeader

decodeRawBlocks :: ByteString -> Either String (Schema, [Either String (Int, ByteString)])

Reads the container as a list of blocks without decoding them into actual values.

This can be useful for streaming splitting merging Avro containers without paying the cost for Avro encoding/decoding.

Each block is returned as a raw ByteString annotated with the number of Avro values that are contained in this block.

The "outer" error represents the error in opening the container itself (including problems like reading schemas embedded into the container.)

data Blocks a

Constructors

Block a (Blocks a)

More (ByteString -> Blocks a)

Feed more bytes. Pass the empty ByteString to signal end of input.

Error

Fields

• String

  Error message

• ByteString

  Leftover bytes

Done ByteString

Leftover bytes

Instances

Functor Blocks

Defined in Data.Avro.Internal.Container

Methods

fmap :: (a -> b) -> Blocks a -> Blocks b

(<$) :: a -> Blocks b -> Blocks a

foldrBlocks :: (a -> b -> b) -> (String -> b) -> b -> ByteString -> Blocks a -> b

Feeds a ByteString to the Blocks until exhausted. Consumes the ByteString lazily.

decodeRawBlocksIncremental :: ContainerHeader -> Blocks (Int, ByteString)

extractContainerValuesBytes :: forall a schema. (Schema -> Either String schema) -> (schema -> Get a) -> ByteString -> Either String (Schema, [Either String (a, ByteString)])

Splits container into a list of individual avro-encoded values. This version provides both encoded and decoded values.

This is particularly useful when slicing up containers into one or more smaller files. By extracting the original bytestring it is possible to avoid re-encoding data.

extractContainerValues :: forall a schema. (Schema -> Either String schema) -> (schema -> Get a) -> ByteString -> Either String (Schema, [Either String a])

extractContainerValuesIncremental :: (Schema -> Either String schema) -> (schema -> Get a) -> ContainerHeader -> Either String (Schema, Blocks [Either String a])

packContainerValues :: Codec -> Schema -> [[ByteString]] -> IO ByteString

Packs a container from a given list of already encoded Avro values Each bytestring should represent exactly one one value serialised to Avro.

packContainerValuesWithSync :: Codec -> Schema -> ByteString -> [[ByteString]] -> ByteString

Packs a container from a given list of already encoded Avro values Each bytestring should represent exactly one one value serialised to Avro.

packContainerValuesWithSync' :: (Schema -> a -> Builder) -> Codec -> Schema -> ByteString -> [[a]] -> ByteString

Packs a container from a given list of already encoded Avro values Each bytestring should represent exactly one one value serialised to Avro.

packContainerBlocks :: Codec -> Schema -> [(Int, ByteString)] -> IO ByteString

Packs a new container from a list of already encoded Avro blocks. Each block is denoted as a pair of a number of objects within that block and the block content.

packContainerBlocksWithSync :: Codec -> Schema -> ByteString -> [(Int, ByteString)] -> ByteString

Packs a new container from a list of already encoded Avro blocks. Each block is denoted as a pair of a number of objects within that block and the block content.

containerHeaderWithSync :: Codec -> Schema -> ByteString -> Builder

Creates an Avro container header for a given schema.

consumeN :: Int64 -> (a -> (a, b)) -> a -> (a, [b])

parseCodec :: Monad m => Maybe ByteString -> m Codec

takeWhileInclusive :: (a -> Bool) -> [a] -> [a]