Scientific.Workflow

SciFlow is a DSL for building scientific workflows. Workflows built with SciFlow can be run either on desktop computers or in grid computing environments that support DRMAA.

Features:

1. Easy to use and safe: Provide a simple and flexible way to design type safe computational pipelines in Haskell.
2. Automatic Checkpointing: The states of intermediate steps are automatically logged, allowing easy restart upon failures.
3. Parallelism and grid computing support.

Example:

import           Control.Lens             ((.=))
import           Scientific.Workflow

f :: Int -> Int
f = (+1)

defaultMain $ do
    nodeS "step0" [| return . const [1..10] :: () -> WorkflowConfig () [Int] |] $ return ()
    nodeP' 2 "step1" 'f $ note .= "run in parallel with batch size 2"
    nodeP' 4 "step2" 'f $ note .= "run in parallel with batch size 4"
    node' "step3" [| \(x, y) -> x ++ y |] $ return ()

    ["step0"] ~> "step1"
    ["step0"] ~> "step2"
    ["step1", "step2"] ~> "step3"

Documentation

defaultMain :: Builder () -> Q [Dec]

mainWith :: MainOpts -> Builder () -> Q [Dec]

defaultMainOpts :: MainOpts

data MainOpts

Constructors

MainOpts

Fields

• preAction :: Name

  An action to be execute before the workflow. The action should have type: IO () -> IO (). e.g., some initialization processes.

• programHeader :: String

Instances

Lift MainOpts

Methods

lift :: MainOpts -> Q Exp

type Builder = State ([Node], [Edge])

namespace :: Text -> Builder () -> Builder ()

Add a prefix to IDs of nodes for a given builder, i.e., id becomes prefix_id.

node

Arguments

:: ToExpQ fun

=> PID

Node id

-> fun

Template Haskell expression representing functions with type a -> IO b.

-> State Attribute ()

Attribues

-> Builder ()

Declare an IO computational step.

node'

Arguments

:: ToExpQ fun

=> PID

-> fun

Template Haskell expression representing functions with type a -> b.

-> State Attribute ()

-> Builder ()

Declare a pure computational step.

nodeS

Arguments

:: ToExpQ fun

=> PID

-> fun

Template Haskell expression representing functions with type "a -> WorkflowConfig st b".

-> State Attribute ()

-> Builder ()

Declare a stateful computational step.

nodeP

Arguments

:: ToExpQ fun

=> Int

Batch size for parallel execution.

-> PID

-> fun

-> State Attribute ()

-> Builder ()

Declare an IO and parallel computational step. This will turn functions with type "a -> IO b" into functions with type "[a] -> IO [b]". And [a] will be processed in parallel with provided batch size. Note: Currently, parallelism is available only when "--remote" flag is on.

nodeP' :: ToExpQ fun => Int -> PID -> fun -> State Attribute () -> Builder ()

Same as nodeP but work with pure functions.

nodePS :: ToExpQ fun => Int -> PID -> fun -> State Attribute () -> Builder ()

Same as nodeP but work with stateful functions.

nodeSharedP

Arguments

:: ToExpQ fun

=> Int

-> PID

-> fun

Template Haskell expression representing functions with type ContextData context a -> IO b.

-> State Attribute ()

-> Builder ()

Similar to nodeP but work with inputs that are associated with a shared context. Turn ContextData context a -> IO b into ContextData context [a] -> IO [b].

nodeSharedP' :: ToExpQ fun => Int -> PID -> fun -> State Attribute () -> Builder ()

nodeSharedPS :: ToExpQ fun => Int -> PID -> fun -> State Attribute () -> Builder ()

link :: [PID] -> PID -> Builder ()

Declare the dependency between nodes. Example:

node' "step1" [| \() -> 1 :: Int |] $ return ()
node' "step2" [| \() -> 2 :: Int |] $ return ()
node' "step3" [| \(x, y) -> x * y |] $ return ()
link ["step1", "step2"] "step3"

(~>) :: [PID] -> PID -> Builder ()

(~>) = link.

path :: [PID] -> Builder ()

"path [a, b, c]" is equivalent to "link a b >> link b c"

label :: Lens' Attribute Text

note :: Lens' Attribute Text

submitToRemote :: Lens' Attribute (Maybe Bool)

remoteParam :: Lens' Attribute String

data ContextData context dat

Data and its environment.

Constructors

ContextData

Fields

• _context :: context
• _data :: dat

Instances

Generic (ContextData context dat)

Associated Types

type Rep (ContextData context dat) :: * -> *

Methods

from :: ContextData context dat -> Rep (ContextData context dat) x

to :: Rep (ContextData context dat) x -> ContextData context dat

(ToJSON c, ToJSON d) => ToJSON (ContextData c d)

Methods

toJSON :: ContextData c d -> Value

toEncoding :: ContextData c d -> Encoding

toJSONList :: [ContextData c d] -> Value

toEncodingList :: [ContextData c d] -> Encoding

(FromJSON c, FromJSON d) => FromJSON (ContextData c d)

Methods

parseJSON :: Value -> Parser (ContextData c d)

parseJSONList :: Value -> Parser [ContextData c d]

(Serialize c, Serialize d) => Serialize (ContextData c d)

Methods

put :: Putter (ContextData c d)

get :: Get (ContextData c d)

type Rep (ContextData context dat)

type Rep (ContextData context dat) = D1 * (MetaData "ContextData" "Scientific.Workflow.Internal.Builder.Types" "SciFlow-0.6.1-CdrNBSgkFzOFQ5Zr3LWD5G" False) (C1 * (MetaCons "ContextData" PrefixI True) ((:*:) * (S1 * (MetaSel (Just Symbol "_context") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * context)) (S1 * (MetaSel (Just Symbol "_data") NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 * dat))))

type WorkflowConfig config = ReaderT config IO