Development.Benchmark.Rules

This module provides a bunch of Shake rules to build multiple revisions of a project and analyse their performance.

It assumes a project bench suite composed of examples that runs a fixed set of experiments on every example

Your code must implement all of the GetFoo oracles and the IsExample class, instantiate the Shake rules, and probably want a set of targets.

The results of the benchmarks and the analysis are recorded in the file system, using the following structure:

build-folder ├── binaries │ └── git-reference │  ├── ghc.path - path to ghc used to build the executable │  └── executable - binary for this version │  └── commitid - Git commit id for this reference ├─ example │ ├── results.csv - aggregated results for all the versions │ └── git-reference │   ├── experiment.gcStats.log - RTS -s output │   ├── experiment.csv - stats for the experiment │   ├── experiment.svg - Graph of bytes over elapsed time │   ├── experiment.diff.svg - idem, including the previous version │   ├── experiment.heap.svg - Heap profile │   ├── experiment.log - bench stdout │   └── results.csv - results of all the experiments for the example ├── results.csv - aggregated results of all the experiments and versions └── experiment.svg - graph of bytes over elapsed time, for all the included versions

For diff graphs, the "previous version" is the preceding entry in the list of versions in the config file. A possible improvement is to obtain this info via `git rev-list`.

Documentation

buildRules :: FilePattern -> MkBuildRules BuildSystem -> Rules ()

Rules that drive a build system to build various revisions of a project

data MkBuildRules buildSystem

Constructors

MkBuildRules

Fields

• findGhc :: buildSystem -> FilePath -> IO FilePath

  Return the path to the GHC executable to use for the project found in the cwd

• executableName :: String

  Name of the binary produced by $sel:buildProject:MkBuildRules

• projectDepends :: Action ()

  An action that captures the source dependencies, used for the HEAD build

• buildProject :: buildSystem -> [CmdOption] -> OutputFolder -> Action ()

  Build the project found in the cwd and save the build artifacts in the output folder

benchRules :: RuleResultForExample example => FilePattern -> MkBenchRules BuildSystem example -> Rules ()

data MkBenchRules buildSystem example

Constructors

forall setup. MkBenchRules

Fields

• setupProject :: Action setup

  Workaround for Shake not allowing to call askOracle from 'benchProject

• benchProject :: setup -> buildSystem -> [CmdOption] -> BenchProject example -> Action ()

  An action that invokes the executable to run the benchmark

• warmupProject :: buildSystem -> FilePath -> [CmdOption] -> example -> Action ()

  An action that performs any necessary warmup. Will only be invoked once

• executableName :: String

  Name of the executable to benchmark. Should match the one used to MkBuildRules

data BenchProject example

Constructors

BenchProject

Fields

• outcsv :: FilePath

  where to save the CSV output

• exePath :: FilePath

  where to find the executable for benchmarking

• exeExtraArgs :: [String]

  extra args for the executable

• example :: example

  example to benchmark

• experiment :: Escaped String

  experiment to run

data ProfilingMode

Constructors

NoProfiling

CheapHeapProfiling Seconds

Instances

Eq ProfilingMode

Defined in Development.Benchmark.Rules

Methods

(==) :: ProfilingMode -> ProfilingMode -> Bool

(/=) :: ProfilingMode -> ProfilingMode -> Bool

csvRules :: forall example. RuleResultForExample example => FilePattern -> Rules ()

Rules to aggregate the CSV output of individual experiments

svgRules :: FilePattern -> Rules ()

Rules to produce charts for the GC stats

heapProfileRules :: FilePattern -> Rules ()

phonyRules

Arguments

:: (Traversable t, IsExample e)

=> String

prefix

-> String

Executable name

-> ProfilingMode

-> FilePath

-> t e

-> Rules ()

Generate a set of phony rules: * prefixall * prefixexample for each example

allTargetsForExample :: IsExample e => ProfilingMode -> FilePath -> e -> Action [FilePath]

newtype GetExample

Constructors

GetExample String

Instances

Eq GetExample

Defined in Development.Benchmark.Rules

Methods

(==) :: GetExample -> GetExample -> Bool

(/=) :: GetExample -> GetExample -> Bool

Show GetExample

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetExample -> ShowS

show :: GetExample -> String

showList :: [GetExample] -> ShowS

Hashable GetExample

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetExample -> Int

hash :: GetExample -> Int

Binary GetExample

Defined in Development.Benchmark.Rules

Methods

put :: GetExample -> Put

get :: Get GetExample

putList :: [GetExample] -> Put

NFData GetExample

Defined in Development.Benchmark.Rules

Methods

rnf :: GetExample -> ()

newtype GetExamples

Constructors

GetExamples ()

Instances

Eq GetExamples

Defined in Development.Benchmark.Rules

Methods

(==) :: GetExamples -> GetExamples -> Bool

(/=) :: GetExamples -> GetExamples -> Bool

Show GetExamples

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetExamples -> ShowS

show :: GetExamples -> String

showList :: [GetExamples] -> ShowS

Hashable GetExamples

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetExamples -> Int

hash :: GetExamples -> Int

Binary GetExamples

Defined in Development.Benchmark.Rules

Methods

put :: GetExamples -> Put

get :: Get GetExamples

putList :: [GetExamples] -> Put

NFData GetExamples

Defined in Development.Benchmark.Rules

Methods

rnf :: GetExamples -> ()

class (Binary e, Eq e, Hashable e, NFData e, Show e, Typeable e) => IsExample e where

Knowledge needed to run an example

Methods

getExampleName :: e -> String

type RuleResultForExample e = (RuleResult GetExample ~ Maybe e, RuleResult GetExamples ~ [e], IsExample e)

newtype GetExperiments

Constructors

GetExperiments ()

Instances

Eq GetExperiments

Defined in Development.Benchmark.Rules

Methods

(==) :: GetExperiments -> GetExperiments -> Bool

(/=) :: GetExperiments -> GetExperiments -> Bool

Show GetExperiments

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetExperiments -> ShowS

show :: GetExperiments -> String

showList :: [GetExperiments] -> ShowS

Hashable GetExperiments

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetExperiments -> Int

hash :: GetExperiments -> Int

Binary GetExperiments

Defined in Development.Benchmark.Rules

Methods

put :: GetExperiments -> Put

get :: Get GetExperiments

putList :: [GetExperiments] -> Put

NFData GetExperiments

Defined in Development.Benchmark.Rules

Methods

rnf :: GetExperiments -> ()

type RuleResult GetExperiments

Defined in Development.Benchmark.Rules

type RuleResult GetExperiments = [Unescaped String]

newtype GetVersions

Constructors

GetVersions ()

Instances

Eq GetVersions

Defined in Development.Benchmark.Rules

Methods

(==) :: GetVersions -> GetVersions -> Bool

(/=) :: GetVersions -> GetVersions -> Bool

Show GetVersions

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetVersions -> ShowS

show :: GetVersions -> String

showList :: [GetVersions] -> ShowS

Hashable GetVersions

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetVersions -> Int

hash :: GetVersions -> Int

Binary GetVersions

Defined in Development.Benchmark.Rules

Methods

put :: GetVersions -> Put

get :: Get GetVersions

putList :: [GetVersions] -> Put

NFData GetVersions

Defined in Development.Benchmark.Rules

Methods

rnf :: GetVersions -> ()

type RuleResult GetVersions

Defined in Development.Benchmark.Rules

type RuleResult GetVersions = [GitCommit]

newtype GetCommitId

Constructors

GetCommitId String

Instances

Eq GetCommitId

Defined in Development.Benchmark.Rules

Methods

(==) :: GetCommitId -> GetCommitId -> Bool

(/=) :: GetCommitId -> GetCommitId -> Bool

Show GetCommitId

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetCommitId -> ShowS

show :: GetCommitId -> String

showList :: [GetCommitId] -> ShowS

Hashable GetCommitId

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetCommitId -> Int

hash :: GetCommitId -> Int

Binary GetCommitId

Defined in Development.Benchmark.Rules

Methods

put :: GetCommitId -> Put

get :: Get GetCommitId

putList :: [GetCommitId] -> Put

NFData GetCommitId

Defined in Development.Benchmark.Rules

Methods

rnf :: GetCommitId -> ()

type RuleResult GetCommitId

Defined in Development.Benchmark.Rules

type RuleResult GetCommitId = String

newtype GetBuildSystem

Constructors

GetBuildSystem ()

Instances

Eq GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

(==) :: GetBuildSystem -> GetBuildSystem -> Bool

(/=) :: GetBuildSystem -> GetBuildSystem -> Bool

Show GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetBuildSystem -> ShowS

show :: GetBuildSystem -> String

showList :: [GetBuildSystem] -> ShowS

Hashable GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetBuildSystem -> Int

hash :: GetBuildSystem -> Int

Binary GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

put :: GetBuildSystem -> Put

get :: Get GetBuildSystem

putList :: [GetBuildSystem] -> Put

NFData GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

rnf :: GetBuildSystem -> ()

type RuleResult GetBuildSystem

Defined in Development.Benchmark.Rules

type RuleResult GetBuildSystem = BuildSystem

data BuildSystem

Default build system that handles Cabal and Stack

Constructors

Cabal

Stack

Instances

Eq BuildSystem

Defined in Development.Benchmark.Rules

Methods

(==) :: BuildSystem -> BuildSystem -> Bool

(/=) :: BuildSystem -> BuildSystem -> Bool

Read BuildSystem

Defined in Development.Benchmark.Rules

Methods

readsPrec :: Int -> ReadS BuildSystem

readList :: ReadS [BuildSystem]

readPrec :: ReadPrec BuildSystem

readListPrec :: ReadPrec [BuildSystem]

Show BuildSystem

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> BuildSystem -> ShowS

show :: BuildSystem -> String

showList :: [BuildSystem] -> ShowS

Generic BuildSystem

Defined in Development.Benchmark.Rules

Associated Types

type Rep BuildSystem :: Type -> Type

Methods

from :: BuildSystem -> Rep BuildSystem x

to :: Rep BuildSystem x -> BuildSystem

Hashable BuildSystem

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> BuildSystem -> Int

hash :: BuildSystem -> Int

ToJSON BuildSystem

Defined in Development.Benchmark.Rules

Methods

toJSON :: BuildSystem -> Value

toEncoding :: BuildSystem -> Encoding

toJSONList :: [BuildSystem] -> Value

toEncodingList :: [BuildSystem] -> Encoding

FromJSON BuildSystem

Defined in Development.Benchmark.Rules

Methods

parseJSON :: Value -> Parser BuildSystem

parseJSONList :: Value -> Parser [BuildSystem]

Binary BuildSystem

Defined in Development.Benchmark.Rules

Methods

put :: BuildSystem -> Put

get :: Get BuildSystem

putList :: [BuildSystem] -> Put

NFData BuildSystem

Defined in Development.Benchmark.Rules

Methods

rnf :: BuildSystem -> ()

type Rep BuildSystem

Defined in Development.Benchmark.Rules

type Rep BuildSystem = D1 ('MetaData "BuildSystem" "Development.Benchmark.Rules" "shake-bench-0.1.0.3-9iZtOhtMvLmBAUDRjvlAYZ" 'False) (C1 ('MetaCons "Cabal" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Stack" 'PrefixI 'False) (U1 :: Type -> Type))

findGhcForBuildSystem :: BuildSystem -> FilePath -> IO FilePath

newtype Escaped a

Constructors

Escaped

Fields

• escaped :: a

newtype Unescaped a

Constructors

Unescaped

Fields

• unescaped :: a

Instances

Eq a => Eq (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

(==) :: Unescaped a -> Unescaped a -> Bool

(/=) :: Unescaped a -> Unescaped a -> Bool

Show a => Show (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> Unescaped a -> ShowS

show :: Unescaped a -> String

showList :: [Unescaped a] -> ShowS

Hashable a => Hashable (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> Unescaped a -> Int

hash :: Unescaped a -> Int

ToJSON a => ToJSON (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

toJSON :: Unescaped a -> Value

toEncoding :: Unescaped a -> Encoding

toJSONList :: [Unescaped a] -> Value

toEncodingList :: [Unescaped a] -> Encoding

FromJSON a => FromJSON (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

parseJSON :: Value -> Parser (Unescaped a)

parseJSONList :: Value -> Parser [Unescaped a]

Binary a => Binary (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

put :: Unescaped a -> Put

get :: Get (Unescaped a)

putList :: [Unescaped a] -> Put

NFData a => NFData (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

rnf :: Unescaped a -> ()

escapeExperiment :: Unescaped String -> Escaped String

unescapeExperiment :: Escaped String -> Unescaped String

data GitCommit

Instances

Eq GitCommit

Defined in Development.Benchmark.Rules

Methods

(==) :: GitCommit -> GitCommit -> Bool

(/=) :: GitCommit -> GitCommit -> Bool

Show GitCommit

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GitCommit -> ShowS

show :: GitCommit -> String

showList :: [GitCommit] -> ShowS

Generic GitCommit

Defined in Development.Benchmark.Rules

Associated Types

type Rep GitCommit :: Type -> Type

Methods

from :: GitCommit -> Rep GitCommit x

to :: Rep GitCommit x -> GitCommit

Hashable GitCommit

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GitCommit -> Int

hash :: GitCommit -> Int

ToJSON GitCommit

Defined in Development.Benchmark.Rules

Methods

toJSON :: GitCommit -> Value

toEncoding :: GitCommit -> Encoding

toJSONList :: [GitCommit] -> Value

toEncodingList :: [GitCommit] -> Encoding

FromJSON GitCommit

Defined in Development.Benchmark.Rules

Methods

parseJSON :: Value -> Parser GitCommit

parseJSONList :: Value -> Parser [GitCommit]

Binary GitCommit

Defined in Development.Benchmark.Rules

Methods

put :: GitCommit -> Put

get :: Get GitCommit

putList :: [GitCommit] -> Put

NFData GitCommit

Defined in Development.Benchmark.Rules

Methods

rnf :: GitCommit -> ()

type Rep GitCommit

Defined in Development.Benchmark.Rules

type Rep GitCommit = D1 ('MetaData "GitCommit" "Development.Benchmark.Rules" "shake-bench-0.1.0.3-9iZtOhtMvLmBAUDRjvlAYZ" 'False) (C1 ('MetaCons "GitCommit" 'PrefixI 'True) ((S1 ('MetaSel ('Just "gitName") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text) :*: S1 ('MetaSel ('Just "name") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text))) :*: (S1 ('MetaSel ('Just "parent") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text)) :*: S1 ('MetaSel ('Just "include") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Bool))))