Data.Avro

Avro encoding and decoding routines.

This library provides a high level interface for encoding (and decoding) Haskell values in Apache's Avro serialization format.

The goal is to match Aeson's API whenever reasonable, meaning user experience with one effectively translate to the other.

Avro RPC is not currently supported.

Library Structure

The library structure includes:

• This module, Data.Avro, providing a high-level interface via classes of FromAvro and ToAvro for decoding and encoding values.
• Data.Avro.Schema: Defines the type for Avro schema's and its JSON encoding/decoding.
• Data.Avro.Encode and Data.Avro.Decode: More efficient conversion capable of avoiding the intermediate representation. Also, the implementation of the en/decoding of the intermediate representation.
• Data.Avro.Decode.Lazy: Lazy/Streaming decoding for Avro containers.
• Data.Avro.Deconflict: translate decoded data from an encoder schema to the (potentially different) decoder's schema.

Schema

type Schema = Type

An Avro schema is either

• A "JSON object in the form {"type":"typeName" ...}
• A "JSON string, naming a defined type" (basic type without free variables)
• A "JSON array, representing a union"

N.B. It is possible to create a Haskell value (of Schema type) that is not a valid Avro schema by violating one of the above or one of the conditions called out in validateSchema.

Encoding and decoding

data Result a

Constructors

Success a

Error String

Instances

Monad Result

Defined in Data.Avro.Schema

Methods

(>>=) :: Result a -> (a -> Result b) -> Result b

(>>) :: Result a -> Result b -> Result b

return :: a -> Result a

fail :: String -> Result a

Functor Result

Defined in Data.Avro.Schema

Methods

fmap :: (a -> b) -> Result a -> Result b

(<$) :: a -> Result b -> Result a

MonadFail Result

Defined in Data.Avro.Schema

Methods

fail :: String -> Result a

Applicative Result

Defined in Data.Avro.Schema

Methods

pure :: a -> Result a

(<*>) :: Result (a -> b) -> Result a -> Result b

liftA2 :: (a -> b -> c) -> Result a -> Result b -> Result c

(*>) :: Result a -> Result b -> Result b

(<*) :: Result a -> Result b -> Result a

Foldable Result

Defined in Data.Avro.Schema

Methods

fold :: Monoid m => Result m -> m

foldMap :: Monoid m => (a -> m) -> Result a -> m

foldr :: (a -> b -> b) -> b -> Result a -> b

foldr' :: (a -> b -> b) -> b -> Result a -> b

foldl :: (b -> a -> b) -> b -> Result a -> b

foldl' :: (b -> a -> b) -> b -> Result a -> b

foldr1 :: (a -> a -> a) -> Result a -> a

foldl1 :: (a -> a -> a) -> Result a -> a

toList :: Result a -> [a]

null :: Result a -> Bool

length :: Result a -> Int

elem :: Eq a => a -> Result a -> Bool

maximum :: Ord a => Result a -> a

minimum :: Ord a => Result a -> a

sum :: Num a => Result a -> a

product :: Num a => Result a -> a

Traversable Result

Defined in Data.Avro.Schema

Methods

traverse :: Applicative f => (a -> f b) -> Result a -> f (Result b)

sequenceA :: Applicative f => Result (f a) -> f (Result a)

mapM :: Monad m => (a -> m b) -> Result a -> m (Result b)

sequence :: Monad m => Result (m a) -> m (Result a)

Alternative Result

Defined in Data.Avro.Schema

Methods

empty :: Result a

(<|>) :: Result a -> Result a -> Result a

some :: Result a -> Result [a]

many :: Result a -> Result [a]

MonadPlus Result

Defined in Data.Avro.Schema

Methods

mzero :: Result a

mplus :: Result a -> Result a -> Result a

MonadError String Result

Defined in Data.Avro.Schema

Methods

throwError :: String -> Result a

catchError :: Result a -> (String -> Result a) -> Result a

Eq a => Eq (Result a)

Defined in Data.Avro.Schema

Methods

(==) :: Result a -> Result a -> Bool

(/=) :: Result a -> Result a -> Bool

Ord a => Ord (Result a)

Defined in Data.Avro.Schema

Methods

compare :: Result a -> Result a -> Ordering

(<) :: Result a -> Result a -> Bool

(<=) :: Result a -> Result a -> Bool

(>) :: Result a -> Result a -> Bool

(>=) :: Result a -> Result a -> Bool

max :: Result a -> Result a -> Result a

min :: Result a -> Result a -> Result a

Show a => Show (Result a)

Defined in Data.Avro.Schema

Methods

showsPrec :: Int -> Result a -> ShowS

show :: Result a -> String

showList :: [Result a] -> ShowS

Semigroup (Result a)

Defined in Data.Avro.Schema

Methods

(<>) :: Result a -> Result a -> Result a

sconcat :: NonEmpty (Result a) -> Result a

stimes :: Integral b => b -> Result a -> Result a

Monoid (Result a)

Defined in Data.Avro.Schema

Methods

mempty :: Result a

mappend :: Result a -> Result a -> Result a

mconcat :: [Result a] -> Result a

badValue :: Show t => t -> String -> Result a

encode :: ToAvro a => a -> ByteString

Encodes a value to a lazy ByteString

decode :: forall a. FromAvro a => ByteString -> Result a

Decode a lazy bytestring using a Schema of the return type.

(.:) :: FromAvro a => HashMap Text (Value Type) -> Text -> Result a

(.=) :: ToAvro a => Text -> a -> (Text, Value Type)

record :: Foldable f => Type -> f (Text, Value Type) -> Value Type

fixed :: Type -> ByteString -> Value Type

Working with containers

Decoding containers

decodeWithSchema :: FromAvro a => Schema -> ByteString -> Result a

Decode a lazy bytestring using a provided schema

decodeContainer :: forall a. FromAvro a => ByteString -> [[a]]

Decode a container and de-conflict the writer schema with a reader schema for a return type. Like in decodeContainerWithSchema exceptions are thrown instead of a Result type to allow this function to be read lazy (to be done in some later version).

decodeContainerWithSchema :: FromAvro a => Schema -> ByteString -> [[a]]

Decode a container and de-conflict the writer schema with a given reader-schema. Exceptions are thrown instead of a Result type to allow this function to be read lazy (to be done in some later version).

decodeContainerBytes :: ByteString -> [[ByteString]]

Like decodeContainer but returns the avro-encoded bytes for each object in the container instead of the Haskell type.

This is particularly useful when slicing up containers into one or more smaller files. By extracting the original bytestring it is possible to avoid re-encoding data.

Encoding containers

encodeContainer :: forall a. ToAvro a => [[a]] -> IO ByteString

Encode chunks of objects into a container, using 16 random bytes for the synchronization markers.

encodeContainer' :: forall a. ToAvro a => Codec -> [[a]] -> IO ByteString

encodeContainerWithSync :: forall a. ToAvro a => (Word64, Word64, Word64, Word64) -> [[a]] -> ByteString

Encode chunks of objects into a container, using the provided ByteString as the synchronization markers.

encodeContainerWithSync' :: forall a. ToAvro a => Codec -> (Word64, Word64, Word64, Word64) -> [[a]] -> ByteString

Encode chunks of objects into a container, using the provided ByteString as the synchronization markers.

Classes and instances

class HasAvroSchema a => FromAvro a where

Methods

fromAvro :: Value Type -> Result a

Instances

FromAvro Bool

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Bool

FromAvro Double

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Double

FromAvro Float

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Float

FromAvro Int

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Int

FromAvro Int32

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Int32

FromAvro Int64

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Int64

FromAvro ByteString

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result ByteString

FromAvro ByteString

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result ByteString

FromAvro Text

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Text

FromAvro Text

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result Text

FromAvro a => FromAvro [a]

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result [a]

FromAvro a => FromAvro (Maybe a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result (Maybe a)

(Unbox a, FromAvro a) => FromAvro (Vector a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result (Vector a)

FromAvro a => FromAvro (Vector a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result (Vector a)

(FromAvro a, FromAvro b) => FromAvro (Either a b)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result (Either a b)

FromAvro a => FromAvro (HashMap Text a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result (HashMap Text a)

FromAvro a => FromAvro (Map Text a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Type -> Result (Map Text a)

(FromAvro a, FromAvro b, FromAvro c) => FromAvro (Either3 a b c)

Defined in Data.Avro.EitherN

Methods

fromAvro :: Value Type -> Result (Either3 a b c)

(FromAvro a, FromAvro b, FromAvro c, FromAvro d) => FromAvro (Either4 a b c d)

Defined in Data.Avro.EitherN

Methods

fromAvro :: Value Type -> Result (Either4 a b c d)

(FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e) => FromAvro (Either5 a b c d e)

Defined in Data.Avro.EitherN

Methods

fromAvro :: Value Type -> Result (Either5 a b c d e)

class HasAvroSchema a => ToAvro a where

Methods

toAvro :: a -> Value Type

Instances

ToAvro Bool

Defined in Data.Avro.ToAvro

Methods

toAvro :: Bool -> Value Type

ToAvro Double

Defined in Data.Avro.ToAvro

Methods

toAvro :: Double -> Value Type

ToAvro Float

Defined in Data.Avro.ToAvro

Methods

toAvro :: Float -> Value Type

ToAvro Int

Defined in Data.Avro.ToAvro

Methods

toAvro :: Int -> Value Type

ToAvro Int32

Defined in Data.Avro.ToAvro

Methods

toAvro :: Int32 -> Value Type

ToAvro Int64

Defined in Data.Avro.ToAvro

Methods

toAvro :: Int64 -> Value Type

ToAvro ()

Defined in Data.Avro.ToAvro

Methods

toAvro :: () -> Value Type

ToAvro ByteString

Defined in Data.Avro.ToAvro

Methods

toAvro :: ByteString -> Value Type

ToAvro ByteString

Defined in Data.Avro.ToAvro

Methods

toAvro :: ByteString -> Value Type

ToAvro Text

Defined in Data.Avro.ToAvro

Methods

toAvro :: Text -> Value Type

ToAvro Text

Defined in Data.Avro.ToAvro

Methods

toAvro :: Text -> Value Type

ToAvro a => ToAvro [a]

Defined in Data.Avro.ToAvro

Methods

toAvro :: [a] -> Value Type

ToAvro a => ToAvro (Maybe a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Maybe a -> Value Type

(Unbox a, ToAvro a) => ToAvro (Vector a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Vector a -> Value Type

ToAvro a => ToAvro (Vector a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Vector a -> Value Type

(ToAvro a, ToAvro b) => ToAvro (Either a b)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Either a b -> Value Type

ToAvro a => ToAvro (HashMap Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: HashMap Text a -> Value Type

ToAvro a => ToAvro (HashMap Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: HashMap Text a -> Value Type

ToAvro a => ToAvro (HashMap String a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: HashMap String a -> Value Type

ToAvro a => ToAvro (Map Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Map Text a -> Value Type

ToAvro a => ToAvro (Map Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Map Text a -> Value Type

ToAvro a => ToAvro (Map String a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Map String a -> Value Type

(ToAvro a, ToAvro b, ToAvro c) => ToAvro (Either3 a b c)

Defined in Data.Avro.EitherN

Methods

toAvro :: Either3 a b c -> Value Type

(ToAvro a, ToAvro b, ToAvro c, ToAvro d) => ToAvro (Either4 a b c d)

Defined in Data.Avro.EitherN

Methods

toAvro :: Either4 a b c d -> Value Type

(ToAvro a, ToAvro b, ToAvro c, ToAvro d, ToAvro e) => ToAvro (Either5 a b c d e)

Defined in Data.Avro.EitherN

Methods

toAvro :: Either5 a b c d e -> Value Type

class HasAvroSchema a where

Methods

schema :: Tagged a Type

Instances

HasAvroSchema Bool

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Bool Type

HasAvroSchema Double

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Double Type

HasAvroSchema Float

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Float Type

HasAvroSchema Int

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int Type

HasAvroSchema Int8

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int8 Type

HasAvroSchema Int16

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int16 Type

HasAvroSchema Int32

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int32 Type

HasAvroSchema Int64

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int64 Type

HasAvroSchema Word8

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word8 Type

HasAvroSchema Word16

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word16 Type

HasAvroSchema Word32

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word32 Type

HasAvroSchema Word64

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word64 Type

HasAvroSchema ()

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged () Type

HasAvroSchema ByteString

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged ByteString Type

HasAvroSchema ByteString

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged ByteString Type

HasAvroSchema Text

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Text Type

HasAvroSchema Text

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Text Type

HasAvroSchema a => HasAvroSchema [a]

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged [a] Type

HasAvroSchema a => HasAvroSchema (Maybe a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Maybe a) Type

HasAvroSchema a => HasAvroSchema (Set a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Set a) Type

HasAvroSchema a => HasAvroSchema (Vector a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Vector a) Type

HasAvroSchema a => HasAvroSchema (Vector a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Vector a) Type

(HasAvroSchema a, HasAvroSchema b) => HasAvroSchema (Either a b)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Either a b) Type

HasAvroSchema a => HasAvroSchema (HashMap Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (HashMap Text a) Type

HasAvroSchema a => HasAvroSchema (HashMap Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (HashMap Text a) Type

HasAvroSchema a => HasAvroSchema (HashMap String a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (HashMap String a) Type

HasAvroSchema a => HasAvroSchema (Map Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Map Text a) Type

HasAvroSchema a => HasAvroSchema (Map Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Map Text a) Type

HasAvroSchema a => HasAvroSchema (Map String a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Map String a) Type

(HasAvroSchema a, Ix i) => HasAvroSchema (Array i a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Array i a) Type

(HasAvroSchema a, HasAvroSchema b, HasAvroSchema c) => HasAvroSchema (Either3 a b c)

Defined in Data.Avro.EitherN

Methods

schema :: Tagged (Either3 a b c) Type

(HasAvroSchema a, HasAvroSchema b, HasAvroSchema c, HasAvroSchema d) => HasAvroSchema (Either4 a b c d)

Defined in Data.Avro.EitherN

Methods

schema :: Tagged (Either4 a b c d) Type

(HasAvroSchema a, HasAvroSchema b, HasAvroSchema c, HasAvroSchema d, HasAvroSchema e) => HasAvroSchema (Either5 a b c d e)

Defined in Data.Avro.EitherN

Methods

schema :: Tagged (Either5 a b c d e) Type

schemaOf :: HasAvroSchema a => a -> Type

Misc

type Avro a = (FromAvro a, ToAvro a)