Transient.MapReduce

Documentation

class (Foldable c, Typeable c, Typeable a, Monoid (c a), Loggable (c a)) => Distributable c a where

Minimal complete definition

singleton, splitAt, fromList

Methods

singleton :: a -> c a

splitAt :: Int -> c a -> (c a, c a)

fromList :: [a] -> c a

Instances

Loggable a => Distributable Vector a

Methods

singleton :: a -> Vector a

splitAt :: Int -> Vector a -> (Vector a, Vector a)

fromList :: [a] -> Vector a

(Loggable a, Unbox a) => Distributable Vector a

Methods

singleton :: a -> Vector a

splitAt :: Int -> Vector a -> (Vector a, Vector a)

fromList :: [a] -> Vector a

distribute :: (Loggable a, Distributable vector a) => vector a -> DDS (vector a)

distribute a vector of values among many nodes. If the vector is static and sharable, better use the get* primitives since each node will load the data independently.

getText :: (Loggable a, Distributable vector a) => (String -> [a]) -> String -> DDS (vector a)

input data from a text that must be static and shared by all the nodes. The function parameter partition the text in words

getUrl :: (Loggable a, Distributable vector a) => (String -> [a]) -> String -> DDS (vector a)

generate a DDS from the content of a URL. The first parameter is a function that divide the text in words

getFile :: (Loggable a, Distributable vector a) => (String -> [a]) -> String -> DDS (vector a)

generate a DDS from a file. All the nodes must access the file with the same path the first parameter is the parser that generates elements from the content

textUrl :: String -> DDS (Vector Text)

get the worlds of an URL

textFile :: String -> DDS (Vector Text)

get the words of a file

mapKeyB :: (Loggable a, Loggable b, Loggable k, Ord k) => (a -> (k, b)) -> DDS (Vector a) -> DDS (Map k (Vector b))

perform a map and partition the result with different keys using boxed vectors The final result will be used by reduce.

mapKeyU :: (Loggable a, Unbox a, Loggable b, Unbox b, Loggable k, Ord k) => (a -> (k, b)) -> DDS (Vector a) -> DDS (Map k (Vector b))

perform a map and partition the result with different keys using unboxed vectors The final result will be used by reduce.

reduce :: (Hashable k, Ord k, Distributable vector a, Loggable k, Loggable a) => (a -> a -> a) -> DDS (Map k (vector a)) -> Cloud (Map k a)

eval :: DDS a -> Cloud (PartRef a)

data PartRef a

Instances

Read (PartRef a)

Methods

readsPrec :: Int -> ReadS (PartRef a)

readList :: ReadS [PartRef a]

readPrec :: ReadPrec (PartRef a)

readListPrec :: ReadPrec [PartRef a]

Show (PartRef a)

Methods

showsPrec :: Int -> PartRef a -> ShowS

show :: PartRef a -> String

showList :: [PartRef a] -> ShowS

Orphan instances

Foldable Vector

Methods

fold :: Monoid m => Vector m -> m

foldMap :: Monoid m => (a -> m) -> Vector a -> m

foldr :: (a -> b -> b) -> b -> Vector a -> b

foldr' :: (a -> b -> b) -> b -> Vector a -> b

foldl :: (b -> a -> b) -> b -> Vector a -> b

foldl' :: (b -> a -> b) -> b -> Vector a -> b

foldr1 :: (a -> a -> a) -> Vector a -> a

foldl1 :: (a -> a -> a) -> Vector a -> a

toList :: Vector a -> [a]

null :: Vector a -> Bool

length :: Vector a -> Int

elem :: Eq a => a -> Vector a -> Bool

maximum :: Ord a => Vector a -> a

minimum :: Ord a => Vector a -> a

sum :: Num a => Vector a -> a

product :: Num a => Vector a -> a