Data.Avro

Avro encoding and decoding routines.

This library provides a high level interface for encoding (and decoding) Haskell values in Apache's Avro serialization format.

The goal is to match Aeson's API whenever reasonable, meaning user experience with one effectively translate to the other.

Avro RPC is not currently supported.

Library Structure

The library structure includes:

• This module, Data.Avro, providing a high-level interface via classes of FromAvro and ToAvro for decoding and encoding values.
• Data.Avro.Schema: Defines the type for Avro schema's and its JSON encoding/decoding.
• Data.Avro.Encode and Data.Avro.Decode: More efficient conversion capable of avoiding the intermediate representation. Also, the implementation of the en/decoding of the intermediate representation.
• Data.Avro.Decode.Lazy: Lazy/Streaming decoding for Avro containers.
• Data.Avro.Deconflict: translate decoded data from an encoder schema to the (potentially different) decoder's schema.

Schema

data Schema

N.B. It is possible to create a Haskell value (of Schema type) that is not a valid Avro schema by violating one of the above or one of the conditions called out in validateSchema.

Instances

Eq Schema

Defined in Data.Avro.Schema

Methods

(==) :: Schema -> Schema -> Bool

(/=) :: Schema -> Schema -> Bool

Show Schema

Defined in Data.Avro.Schema

Methods

showsPrec :: Int -> Schema -> ShowS

show :: Schema -> String

showList :: [Schema] -> ShowS

Generic Schema

Defined in Data.Avro.Schema

Associated Types

type Rep Schema :: Type -> Type

Methods

from :: Schema -> Rep Schema x

to :: Rep Schema x -> Schema

Lift Schema

Defined in Data.Avro.Deriving.Lift

Methods

lift :: Schema -> Q Exp

ToJSON Schema

Defined in Data.Avro.Schema

Methods

toJSON :: Schema -> Value

toEncoding :: Schema -> Encoding

toJSONList :: [Schema] -> Value

toEncodingList :: [Schema] -> Encoding

FromJSON Schema

Defined in Data.Avro.Schema

Methods

parseJSON :: Value -> Parser Schema

parseJSONList :: Value -> Parser [Schema]

NFData Schema

Defined in Data.Avro.Schema

Methods

rnf :: Schema -> ()

ToJSON (Value Schema)

Defined in Data.Avro.Schema

Methods

toJSON :: Value Schema -> Value0

toEncoding :: Value Schema -> Encoding

toJSONList :: [Value Schema] -> Value0

toEncodingList :: [Value Schema] -> Encoding

EncodeAvro (Value Schema)

Defined in Data.Avro.Encode

Methods

avro :: Value Schema -> AvroM

type Rep Schema

Defined in Data.Avro.Schema

type Rep Schema = D1 (MetaData "Schema" "Data.Avro.Schema" "avro-0.4.6.0-JVPQW2rQlvVJJXmyvcVSKi" False) (((C1 (MetaCons "Null" PrefixI False) (U1 :: Type -> Type) :+: (C1 (MetaCons "Boolean" PrefixI False) (U1 :: Type -> Type) :+: C1 (MetaCons "Int" PrefixI False) (U1 :: Type -> Type))) :+: ((C1 (MetaCons "Long" PrefixI False) (U1 :: Type -> Type) :+: C1 (MetaCons "Float" PrefixI False) (U1 :: Type -> Type)) :+: (C1 (MetaCons "Double" PrefixI False) (U1 :: Type -> Type) :+: C1 (MetaCons "Bytes" PrefixI False) (U1 :: Type -> Type)))) :+: (((C1 (MetaCons "String" PrefixI False) (U1 :: Type -> Type) :+: C1 (MetaCons "Array" PrefixI True) (S1 (MetaSel (Just "item") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Schema))) :+: (C1 (MetaCons "Map" PrefixI True) (S1 (MetaSel (Just "values") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Schema)) :+: C1 (MetaCons "NamedType" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 TypeName)))) :+: ((C1 (MetaCons "Record" PrefixI True) ((S1 (MetaSel (Just "name") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 TypeName) :*: S1 (MetaSel (Just "aliases") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 [TypeName])) :*: (S1 (MetaSel (Just "doc") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Maybe Text)) :*: (S1 (MetaSel (Just "order") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Maybe Order)) :*: S1 (MetaSel (Just "fields") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 [Field])))) :+: C1 (MetaCons "Enum" PrefixI True) ((S1 (MetaSel (Just "name") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 TypeName) :*: S1 (MetaSel (Just "aliases") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 [TypeName])) :*: (S1 (MetaSel (Just "doc") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Maybe Text)) :*: S1 (MetaSel (Just "symbols") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Vector Text))))) :+: (C1 (MetaCons "Union" PrefixI True) (S1 (MetaSel (Just "options") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 (Vector Schema))) :+: C1 (MetaCons "Fixed" PrefixI True) (S1 (MetaSel (Just "name") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 TypeName) :*: (S1 (MetaSel (Just "aliases") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 [TypeName]) :*: S1 (MetaSel (Just "size") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Int)))))))

Encoding and decoding

data Result a

Constructors

Success a

Error String

Instances

Monad Result

Defined in Data.Avro.Schema

Methods

(>>=) :: Result a -> (a -> Result b) -> Result b

(>>) :: Result a -> Result b -> Result b

return :: a -> Result a

fail :: String -> Result a

Functor Result

Defined in Data.Avro.Schema

Methods

fmap :: (a -> b) -> Result a -> Result b

(<$) :: a -> Result b -> Result a

MonadFail Result

Defined in Data.Avro.Schema

Methods

fail :: String -> Result a

Applicative Result

Defined in Data.Avro.Schema

Methods

pure :: a -> Result a

(<*>) :: Result (a -> b) -> Result a -> Result b

liftA2 :: (a -> b -> c) -> Result a -> Result b -> Result c

(*>) :: Result a -> Result b -> Result b

(<*) :: Result a -> Result b -> Result a

Foldable Result

Defined in Data.Avro.Schema

Methods

fold :: Monoid m => Result m -> m

foldMap :: Monoid m => (a -> m) -> Result a -> m

foldr :: (a -> b -> b) -> b -> Result a -> b

foldr' :: (a -> b -> b) -> b -> Result a -> b

foldl :: (b -> a -> b) -> b -> Result a -> b

foldl' :: (b -> a -> b) -> b -> Result a -> b

foldr1 :: (a -> a -> a) -> Result a -> a

foldl1 :: (a -> a -> a) -> Result a -> a

toList :: Result a -> [a]

null :: Result a -> Bool

length :: Result a -> Int

elem :: Eq a => a -> Result a -> Bool

maximum :: Ord a => Result a -> a

minimum :: Ord a => Result a -> a

sum :: Num a => Result a -> a

product :: Num a => Result a -> a

Traversable Result

Defined in Data.Avro.Schema

Methods

traverse :: Applicative f => (a -> f b) -> Result a -> f (Result b)

sequenceA :: Applicative f => Result (f a) -> f (Result a)

mapM :: Monad m => (a -> m b) -> Result a -> m (Result b)

sequence :: Monad m => Result (m a) -> m (Result a)

Alternative Result

Defined in Data.Avro.Schema

Methods

empty :: Result a

(<|>) :: Result a -> Result a -> Result a

some :: Result a -> Result [a]

many :: Result a -> Result [a]

MonadPlus Result

Defined in Data.Avro.Schema

Methods

mzero :: Result a

mplus :: Result a -> Result a -> Result a

MonadError String Result

Defined in Data.Avro.Schema

Methods

throwError :: String -> Result a

catchError :: Result a -> (String -> Result a) -> Result a

Eq a => Eq (Result a)

Defined in Data.Avro.Schema

Methods

(==) :: Result a -> Result a -> Bool

(/=) :: Result a -> Result a -> Bool

Ord a => Ord (Result a)

Defined in Data.Avro.Schema

Methods

compare :: Result a -> Result a -> Ordering

(<) :: Result a -> Result a -> Bool

(<=) :: Result a -> Result a -> Bool

(>) :: Result a -> Result a -> Bool

(>=) :: Result a -> Result a -> Bool

max :: Result a -> Result a -> Result a

min :: Result a -> Result a -> Result a

Show a => Show (Result a)

Defined in Data.Avro.Schema

Methods

showsPrec :: Int -> Result a -> ShowS

show :: Result a -> String

showList :: [Result a] -> ShowS

Semigroup (Result a)

Defined in Data.Avro.Schema

Methods

(<>) :: Result a -> Result a -> Result a

sconcat :: NonEmpty (Result a) -> Result a

stimes :: Integral b => b -> Result a -> Result a

Monoid (Result a)

Defined in Data.Avro.Schema

Methods

mempty :: Result a

mappend :: Result a -> Result a -> Result a

mconcat :: [Result a] -> Result a

badValue :: Show t => t -> String -> Result a

encode :: ToAvro a => a -> ByteString

Encodes a value to a lazy ByteString

decode :: forall a. FromAvro a => ByteString -> Result a

Decode a lazy bytestring using a Schema of the return type.

(.:) :: FromAvro a => HashMap Text (Value Schema) -> Text -> Result a

(.=) :: ToAvro a => Text -> a -> (Text, Value Schema)

record :: Foldable f => Schema -> f (Text, Value Schema) -> Value Schema

fixed :: Schema -> ByteString -> Value Schema

Working with containers

Decoding containers

decodeWithSchema :: FromAvro a => Schema -> ByteString -> Result a

Decode a lazy bytestring using a provided schema

decodeContainer :: forall a. FromAvro a => ByteString -> [[a]]

Decode a container and de-conflict the writer schema with a reader schema for a return type. Like in decodeContainerWithSchema exceptions are thrown instead of a Result type to allow this function to be read lazy (to be done in some later version).

decodeContainerWithSchema :: FromAvro a => Schema -> ByteString -> [[a]]

Decode a container and de-conflict the writer schema with a given reader-schema. Exceptions are thrown instead of a Result type to allow this function to be read lazy (to be done in some later version).

decodeContainerBytes :: ByteString -> [[ByteString]]

Like decodeContainer but returns the avro-encoded bytes for each object in the container instead of the Haskell type.

This is particularly useful when slicing up containers into one or more smaller files. By extracting the original bytestring it is possible to avoid re-encoding data.

Encoding containers

encodeContainer :: forall a. ToAvro a => [[a]] -> IO ByteString

Encode chunks of objects into a container, using 16 random bytes for the synchronization markers.

encodeContainer' :: forall a. ToAvro a => Codec -> [[a]] -> IO ByteString

encodeContainerWithSync :: forall a. ToAvro a => (Word64, Word64, Word64, Word64) -> [[a]] -> ByteString

Encode chunks of objects into a container, using the provided ByteString as the synchronization markers.

encodeContainerWithSync' :: forall a. ToAvro a => Codec -> (Word64, Word64, Word64, Word64) -> [[a]] -> ByteString

Encode chunks of objects into a container, using the provided ByteString as the synchronization markers.

Classes and instances

class HasAvroSchema a => FromAvro a where

Methods

fromAvro :: Value Schema -> Result a

Instances

FromAvro Bool

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Bool

FromAvro Double

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Double

FromAvro Float

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Float

FromAvro Int

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Int

FromAvro Int32

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Int32

FromAvro Int64

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Int64

FromAvro ByteString

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result ByteString

FromAvro ByteString

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result ByteString

FromAvro Text

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Text

FromAvro Text

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result Text

FromAvro a => FromAvro [a]

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result [a]

FromAvro a => FromAvro (Maybe a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result (Maybe a)

(Unbox a, FromAvro a) => FromAvro (Vector a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result (Vector a)

FromAvro a => FromAvro (Vector a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result (Vector a)

(FromAvro a, FromAvro b) => FromAvro (Either a b)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result (Either a b)

FromAvro a => FromAvro (HashMap Text a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result (HashMap Text a)

FromAvro a => FromAvro (Map Text a)

Defined in Data.Avro.FromAvro

Methods

fromAvro :: Value Schema -> Result (Map Text a)

(FromAvro a, FromAvro b, FromAvro c) => FromAvro (Either3 a b c)

Defined in Data.Avro.EitherN

Methods

fromAvro :: Value Schema -> Result (Either3 a b c)

(FromAvro a, FromAvro b, FromAvro c, FromAvro d) => FromAvro (Either4 a b c d)

Defined in Data.Avro.EitherN

Methods

fromAvro :: Value Schema -> Result (Either4 a b c d)

(FromAvro a, FromAvro b, FromAvro c, FromAvro d, FromAvro e) => FromAvro (Either5 a b c d e)

Defined in Data.Avro.EitherN

Methods

fromAvro :: Value Schema -> Result (Either5 a b c d e)

class HasAvroSchema a => ToAvro a where

Methods

toAvro :: a -> Value Schema

Instances

ToAvro Bool

Defined in Data.Avro.ToAvro

Methods

toAvro :: Bool -> Value Schema

ToAvro Double

Defined in Data.Avro.ToAvro

Methods

toAvro :: Double -> Value Schema

ToAvro Float

Defined in Data.Avro.ToAvro

Methods

toAvro :: Float -> Value Schema

ToAvro Int

Defined in Data.Avro.ToAvro

Methods

toAvro :: Int -> Value Schema

ToAvro Int32

Defined in Data.Avro.ToAvro

Methods

toAvro :: Int32 -> Value Schema

ToAvro Int64

Defined in Data.Avro.ToAvro

Methods

toAvro :: Int64 -> Value Schema

ToAvro ()

Defined in Data.Avro.ToAvro

Methods

toAvro :: () -> Value Schema

ToAvro ByteString

Defined in Data.Avro.ToAvro

Methods

toAvro :: ByteString -> Value Schema

ToAvro ByteString

Defined in Data.Avro.ToAvro

Methods

toAvro :: ByteString -> Value Schema

ToAvro Text

Defined in Data.Avro.ToAvro

Methods

toAvro :: Text -> Value Schema

ToAvro Text

Defined in Data.Avro.ToAvro

Methods

toAvro :: Text -> Value Schema

ToAvro a => ToAvro [a]

Defined in Data.Avro.ToAvro

Methods

toAvro :: [a] -> Value Schema

ToAvro a => ToAvro (Maybe a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Maybe a -> Value Schema

(Unbox a, ToAvro a) => ToAvro (Vector a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Vector a -> Value Schema

ToAvro a => ToAvro (Vector a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Vector a -> Value Schema

(ToAvro a, ToAvro b) => ToAvro (Either a b)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Either a b -> Value Schema

ToAvro a => ToAvro (HashMap Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: HashMap Text a -> Value Schema

ToAvro a => ToAvro (HashMap Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: HashMap Text a -> Value Schema

ToAvro a => ToAvro (HashMap String a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: HashMap String a -> Value Schema

ToAvro a => ToAvro (Map Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Map Text a -> Value Schema

ToAvro a => ToAvro (Map Text a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Map Text a -> Value Schema

ToAvro a => ToAvro (Map String a)

Defined in Data.Avro.ToAvro

Methods

toAvro :: Map String a -> Value Schema

(ToAvro a, ToAvro b, ToAvro c) => ToAvro (Either3 a b c)

Defined in Data.Avro.EitherN

Methods

toAvro :: Either3 a b c -> Value Schema

(ToAvro a, ToAvro b, ToAvro c, ToAvro d) => ToAvro (Either4 a b c d)

Defined in Data.Avro.EitherN

Methods

toAvro :: Either4 a b c d -> Value Schema

(ToAvro a, ToAvro b, ToAvro c, ToAvro d, ToAvro e) => ToAvro (Either5 a b c d e)

Defined in Data.Avro.EitherN

Methods

toAvro :: Either5 a b c d e -> Value Schema

class HasAvroSchema a where

Methods

schema :: Tagged a Schema

Instances

HasAvroSchema Bool

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Bool Schema

HasAvroSchema Double

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Double Schema

HasAvroSchema Float

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Float Schema

HasAvroSchema Int

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int Schema

HasAvroSchema Int8

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int8 Schema

HasAvroSchema Int16

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int16 Schema

HasAvroSchema Int32

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int32 Schema

HasAvroSchema Int64

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Int64 Schema

HasAvroSchema Word8

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word8 Schema

HasAvroSchema Word16

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word16 Schema

HasAvroSchema Word32

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word32 Schema

HasAvroSchema Word64

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Word64 Schema

HasAvroSchema ()

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged () Schema

HasAvroSchema ByteString

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged ByteString Schema

HasAvroSchema ByteString

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged ByteString Schema

HasAvroSchema Text

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Text Schema

HasAvroSchema Text

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged Text Schema

HasAvroSchema a => HasAvroSchema [a]

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged [a] Schema

HasAvroSchema a => HasAvroSchema (Maybe a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Maybe a) Schema

HasAvroSchema a => HasAvroSchema (Set a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Set a) Schema

HasAvroSchema a => HasAvroSchema (Vector a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Vector a) Schema

HasAvroSchema a => HasAvroSchema (Vector a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Vector a) Schema

(HasAvroSchema a, HasAvroSchema b) => HasAvroSchema (Either a b)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Either a b) Schema

HasAvroSchema a => HasAvroSchema (HashMap Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (HashMap Text a) Schema

HasAvroSchema a => HasAvroSchema (HashMap Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (HashMap Text a) Schema

HasAvroSchema a => HasAvroSchema (HashMap String a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (HashMap String a) Schema

HasAvroSchema a => HasAvroSchema (Map Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Map Text a) Schema

HasAvroSchema a => HasAvroSchema (Map Text a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Map Text a) Schema

HasAvroSchema a => HasAvroSchema (Map String a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Map String a) Schema

(HasAvroSchema a, Ix i) => HasAvroSchema (Array i a)

Defined in Data.Avro.HasAvroSchema

Methods

schema :: Tagged (Array i a) Schema

(HasAvroSchema a, HasAvroSchema b, HasAvroSchema c) => HasAvroSchema (Either3 a b c)

Defined in Data.Avro.EitherN

Methods

schema :: Tagged (Either3 a b c) Schema

(HasAvroSchema a, HasAvroSchema b, HasAvroSchema c, HasAvroSchema d) => HasAvroSchema (Either4 a b c d)

Defined in Data.Avro.EitherN

Methods

schema :: Tagged (Either4 a b c d) Schema

(HasAvroSchema a, HasAvroSchema b, HasAvroSchema c, HasAvroSchema d, HasAvroSchema e) => HasAvroSchema (Either5 a b c d e)

Defined in Data.Avro.EitherN

Methods

schema :: Tagged (Either5 a b c d e) Schema

schemaOf :: HasAvroSchema a => a -> Schema

Misc

type Avro a = (FromAvro a, ToAvro a)