|
Holumbus.MapReduce.Types | Portability | portable | Stability | experimental | Maintainer | Stefan Schmidt (stefanschmidt@web.de) |
|
|
|
|
|
Description |
Version : 0.1
|
|
Synopsis |
|
class Hash a where | | | hashedPartition :: (Hash k2, Binary k2, Binary v2, NFData k2, NFData v2) => MapPartition a k2 v2 | | | | type TaskId = Integer | | | | | | getNextTaskState :: TaskState -> TaskState | | | | data TaskData = TaskData {} | | type JobId = Integer | | | | getNextJobState :: JobState -> JobState | | getPrevJobState :: JobState -> JobState | | fromJobStatetoTaskType :: JobState -> Maybe TaskType | | type OutputMap = Map JobState (AccuMap Int FunctionData) | | data JobAction = JobAction {} | | data JobInfo = JobInfo {} | | data JobData = JobData {} | | data JobResultContainer = JobResultContainer (MVar JobResult) | | data JobResult = JobResult {} | | type ActionName = String | | type ActionInfo = String | | data ActionEnvironment = ActionEnvironment {} | | mkActionEnvironment :: TaskData -> FileSystem -> ActionEnvironment | | type InputReader k1 v1 = ByteString -> IO [(k1, v1)] | | type OutputWriter k2 v2 = [(k2, v2)] -> IO ByteString | | type OptionsDecoder a = ByteString -> a | | defaultInputReader :: (NFData v1, NFData k1, Binary k1, Binary v1) => InputReader k1 v1 | | defaultOutputWriter :: (NFData v2, NFData k2, Binary k2, Binary v2) => OutputWriter k2 v2 | | defaultSplit :: (NFData k1, NFData v1) => SplitFunction a k1 v1 | | defaultPartition :: (Binary k2, Binary v2) => MapPartition a k2 v2 | | defaultMerge :: (Ord k2, Binary k2, Binary v2) => ReduceMerge a k2 v2 | | readConnector :: (NFData k1, NFData v1, Binary k1, Binary v1) => InputReader k1 v1 -> ActionEnvironment -> [FunctionData] -> IO [(k1, v1)] | | writeConnector :: (Binary k2, Binary v2) => OutputWriter k2 v2 -> ActionEnvironment -> [(Int, [(k2, v2)])] -> IO [(Int, [FunctionData])] | | data ActionConfiguration a k1 v1 k2 v2 v3 v4 = ActionConfiguration {} | | data SplitConfiguration a k1 v1 = SplitConfiguration {} | | data MapConfiguration a k1 v1 k2 v2 = MapConfiguration {} | | data ReduceConfiguration a k2 v3 v4 = ReduceConfiguration {} | | defaultActionConfiguration :: (NFData v1, NFData k1, Binary a, Binary k1, Binary v1, Binary k2, Binary v4) => ActionName -> ActionConfiguration a k1 v1 k2 v2 v3 v4 | | defaultSplitConfiguration :: (NFData v1, NFData k1, Binary a, Binary k1, Binary v1) => SplitConfiguration a k1 v1 | | defaultMapConfiguration :: (NFData v1, NFData k1, NFData v2, NFData k2, Ord k2, Binary a, Binary k1, Binary v1, Binary k2, Binary v2) => MapFunction a k1 v1 k2 v2 -> MapConfiguration a k1 v1 k2 v2 | | defaultReduceConfiguration :: (NFData v2, NFData k2, NFData v3, Ord k2, Binary a, Binary k2, Binary v2, Binary v3) => ReduceFunction a k2 v2 v3 -> ReduceConfiguration a k2 v2 v3 | | readActionConfiguration :: (Ord k2, Binary a, Show k1, Show v1, Show k2, Show v2, Show v3, Show v4, NFData k1, NFData v1, NFData k2, NFData v2, NFData v3, Binary k1, Binary v1, Binary k2, Binary v2, Binary v3, Binary v4) => ActionConfiguration a k1 v1 k2 v2 v3 v4 -> ActionData | | createJobInfoFromConfiguration :: ActionConfiguration a k1 v1 k2 v2 v3 v4 -> a -> [(k1, v1)] -> [FileId] -> Int -> Int -> Int -> Int -> TaskOutputType -> JobInfo | | createListsFromJobResult :: ActionConfiguration a k1 v1 k2 v2 v3 v4 -> JobResult -> ([(k2, v4)], [FileId]) | | data ActionData = ActionData {} | | getActionForTaskType :: TaskType -> ActionData -> Maybe BinaryReduceAction | | type ActionMap = KeyMap ActionData | | type MapAction a k1 v1 k2 v2 = ActionEnvironment -> a -> Int -> [(k1, v1)] -> IO [(Int, [(k2, v2)])] | | type MapFunction a k1 v1 k2 v2 = ActionEnvironment -> a -> k1 -> v1 -> IO [(k2, v2)] | | type MapPartition a k2 v2 = ActionEnvironment -> a -> Int -> [(k2, v2)] -> IO [(Int, [(k2, v2)])] | | type ReduceAction a k2 v2 v3 = ActionEnvironment -> a -> Int -> [(k2, v2)] -> IO [(Int, [(k2, v3)])] | | type ReduceMerge a k2 v2 = ActionEnvironment -> a -> [(k2, v2)] -> IO [(k2, [v2])] | | type ReduceFunction a k2 v2 v3 = ActionEnvironment -> a -> k2 -> [v2] -> IO (Maybe v3) | | type ReducePartition a k2 v3 = ActionEnvironment -> a -> Int -> [(k2, v3)] -> IO [(Int, [(k2, v3)])] | | type SplitFunction a k1 v1 = SplitAction a k1 v1 | | type SplitAction a k1 v1 = ActionEnvironment -> a -> Int -> [(k1, v1)] -> IO [(Int, [(k1, v1)])] |
|
|
Documentation |
|
|
| Methods | | | Instances | |
|
|
|
|
|
Constructors | | Instances | |
|
|
TaskData
|
|
|
the task id (should be unique in the system)
|
|
|
which type (map, combine, reduce)
| Constructors | TTSplit | | TTMap | | TTCombine | | TTReduce | | TTError | |
| Instances | |
|
|
|
the task state
| Constructors | TSIdle | | TSSending | | TSInProgress | | TSCompleted | | TSFinished | | TSError | |
| Instances | |
|
|
|
|
|
Constructors | | Instances | |
|
|
|
the TaskData, contains all information to do the task
| Constructors | | Instances | |
|
|
JobData
|
|
|
the job id (should be unique in the system)
|
|
|
the job state
| Constructors | JSPlanned | | JSIdle | | JSSplit | | JSMap | | JSCombine | | JSReduce | | JSCompleted | | JSFinished | | JSError | |
| Instances | |
|
|
|
|
|
|
|
|
|
|
|
Constructors | | Instances | |
|
|
|
defines a job, this is all data the user has to give to run a job
| Constructors | | Instances | |
|
|
|
the job data, include the user-input and some additional control-data
| Constructors | | Instances | |
|
|
data JobResultContainer | Source |
|
Constructors | | Instances | |
|
|
|
the result of the job, given by the master
| Constructors | | Instances | |
|
|
TaskAction
|
|
|
|
|
|
|
the ActionEnvironment contains all data that might be needed
during an action. So far, it only keeps the current task data and
a reference to the global filesystem and the options.
This is a good place to implement counters for the map-reduce-system
or other stuff...
| Constructors | |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data ActionConfiguration a k1 v1 k2 v2 v3 v4 | Source |
|
|
|
data SplitConfiguration a k1 v1 | Source |
|
|
|
data MapConfiguration a k1 v1 k2 v2 | Source |
|
|
|
data ReduceConfiguration a k2 v3 v4 | Source |
|
|
|
|
|
|
|
|
|
|
|
readActionConfiguration :: (Ord k2, Binary a, Show k1, Show v1, Show k2, Show v2, Show v3, Show v4, NFData k1, NFData v1, NFData k2, NFData v2, NFData v3, Binary k1, Binary v1, Binary k2, Binary v2, Binary v3, Binary v4) => ActionConfiguration a k1 v1 k2 v2 v3 v4 -> ActionData | Source |
|
|
createJobInfoFromConfiguration | Source |
|
|
|
|
|
|
Constructors | | Instances | |
|
|
|
|
|
|
MapAction
|
|
|
general MapAction
|
|
|
|
|
|
Combine-Reduce-Action
|
|
|
general MapAction
|
|
|
|
|
|
|
|
|
|
|
general SplitAction
|
|
Produced by Haddock version 2.6.1 |