Development.Benchmark.Rules

This module provides a bunch of Shake rules to build multiple revisions of a project and analyse their performance.

It assumes a project bench suite composed of examples that runs a fixed set of experiments on every example

Your code must implement all of the GetFoo oracles and the IsExample class, instantiate the Shake rules, and probably want a set of targets.

The results of the benchmarks and the analysis are recorded in the file system, using the following structure:

build-folder ├── binaries │ └── git-reference │  ├── ghc.path - path to ghc used to build the executable │  ├── executable - binary for this version │  └── commitid - Git commit id for this reference ├─ example │ ├── results.csv - aggregated results for all the versions and configurations │ ├── experiment.svg - graph of bytes over elapsed time, for all the versions and configurations | └── git-reference │ └── configuration │   ├── experiment.gcStats.log - RTS -s output │   ├── experiment.csv - stats for the experiment │   ├── experiment.svg - Graph of bytes over elapsed time │   ├── experiment.diff.svg - idem, including the previous version │   ├── experiment.heap.svg - Heap profile │   ├── experiment.log - bench stdout │   └── results.csv - results of all the experiments for the example ├── results.csv - aggregated results of all the examples, experiments, versions and configurations └── experiment.svg - graph of bytes over elapsed time, for all the examples, experiments, versions and configurations

For diff graphs, the "previous version" is the preceding entry in the list of versions in the config file. A possible improvement is to obtain this info via `git rev-list`.

Documentation

buildRules :: FilePattern -> MkBuildRules BuildSystem -> Rules ()

Rules that drive a build system to build various revisions of a project

data MkBuildRules buildSystem

Constructors

MkBuildRules

Fields

• findGhc :: buildSystem -> FilePath -> IO FilePath

  Return the path to the GHC executable to use for the project found in the cwd

• executableName :: String

  Name of the binary produced by $sel:buildProject:MkBuildRules

• projectDepends :: Action ()

  An action that captures the source dependencies, used for the HEAD build

• buildProject :: buildSystem -> ProjectRoot -> OutputFolder -> Action ()

  Build the project found in the given path and save the build artifacts in the output folder

type OutputFolder = FilePath

type ProjectRoot = FilePath

benchRules :: RuleResultForExample example => FilePattern -> MkBenchRules BuildSystem example -> Rules ()

data MkBenchRules buildSystem example

Constructors

forall setup. MkBenchRules

Fields

• setupProject :: Action setup

  Workaround for Shake not allowing to call askOracle from 'benchProject

• benchProject :: setup -> buildSystem -> [CmdOption] -> BenchProject example -> Action ()

  An action that invokes the executable to run the benchmark

• warmupProject :: buildSystem -> FilePath -> [CmdOption] -> example -> Action ()

  An action that performs any necessary warmup. Will only be invoked once

• executableName :: String

  Name of the executable to benchmark. Should match the one used to MkBuildRules

• parallelism :: Natural

  Number of concurrent benchmarks to run

data BenchProject example

Constructors

BenchProject

Fields

• outcsv :: FilePath

  where to save the CSV output

• exePath :: FilePath

  where to find the executable for benchmarking

• exeExtraArgs :: [String]

  extra args for the executable

• example :: example

  example to benchmark

• experiment :: Escaped String

  experiment to run

• configuration :: ByteString

  configuration to use

data ProfilingMode

Constructors

NoProfiling

CheapHeapProfiling Seconds

Instances

Eq ProfilingMode

Defined in Development.Benchmark.Rules

Methods

(==) :: ProfilingMode -> ProfilingMode -> Bool

(/=) :: ProfilingMode -> ProfilingMode -> Bool

csvRules :: forall example. RuleResultForExample example => FilePattern -> Rules ()

Rules to aggregate the CSV output of individual experiments

svgRules :: FilePattern -> Rules ()

Rules to produce charts for the GC stats

heapProfileRules :: FilePattern -> Rules ()

phonyRules

Arguments

:: (Traversable t, IsExample e)

=> String

prefix

-> String

Executable name

-> ProfilingMode

-> FilePath

-> t e

-> Rules ()

Generate a set of phony rules: * prefixall * prefixexample for each example

allTargetsForExample :: IsExample e => ProfilingMode -> FilePath -> e -> Action [FilePath]

newtype GetExample

Constructors

GetExample String

Instances

Show GetExample

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetExample -> ShowS

show :: GetExample -> String

showList :: [GetExample] -> ShowS

Binary GetExample

Defined in Development.Benchmark.Rules

Methods

put :: GetExample -> Put

get :: Get GetExample

putList :: [GetExample] -> Put

NFData GetExample

Defined in Development.Benchmark.Rules

Methods

rnf :: GetExample -> ()

Eq GetExample

Defined in Development.Benchmark.Rules

Methods

(==) :: GetExample -> GetExample -> Bool

(/=) :: GetExample -> GetExample -> Bool

Hashable GetExample

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetExample -> Int

hash :: GetExample -> Int

newtype GetExamples

Constructors

GetExamples ()

Instances

Show GetExamples

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetExamples -> ShowS

show :: GetExamples -> String

showList :: [GetExamples] -> ShowS

Binary GetExamples

Defined in Development.Benchmark.Rules

Methods

put :: GetExamples -> Put

get :: Get GetExamples

putList :: [GetExamples] -> Put

NFData GetExamples

Defined in Development.Benchmark.Rules

Methods

rnf :: GetExamples -> ()

Eq GetExamples

Defined in Development.Benchmark.Rules

Methods

(==) :: GetExamples -> GetExamples -> Bool

(/=) :: GetExamples -> GetExamples -> Bool

Hashable GetExamples

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetExamples -> Int

hash :: GetExamples -> Int

class (Binary e, Eq e, Hashable e, NFData e, Show e, Typeable e) => IsExample e where

Knowledge needed to run an example

Methods

getExampleName :: e -> String

type RuleResultForExample e = (RuleResult GetExample ~ Maybe e, RuleResult GetExamples ~ [e], IsExample e)

newtype GetExperiments

Constructors

GetExperiments ()

Instances

Show GetExperiments

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetExperiments -> ShowS

show :: GetExperiments -> String

showList :: [GetExperiments] -> ShowS

Binary GetExperiments

Defined in Development.Benchmark.Rules

Methods

put :: GetExperiments -> Put

get :: Get GetExperiments

putList :: [GetExperiments] -> Put

NFData GetExperiments

Defined in Development.Benchmark.Rules

Methods

rnf :: GetExperiments -> ()

Eq GetExperiments

Defined in Development.Benchmark.Rules

Methods

(==) :: GetExperiments -> GetExperiments -> Bool

(/=) :: GetExperiments -> GetExperiments -> Bool

Hashable GetExperiments

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetExperiments -> Int

hash :: GetExperiments -> Int

type RuleResult GetExperiments

Defined in Development.Benchmark.Rules

type RuleResult GetExperiments = [Unescaped String]

newtype GetVersions

Constructors

GetVersions ()

Instances

Show GetVersions

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetVersions -> ShowS

show :: GetVersions -> String

showList :: [GetVersions] -> ShowS

Binary GetVersions

Defined in Development.Benchmark.Rules

Methods

put :: GetVersions -> Put

get :: Get GetVersions

putList :: [GetVersions] -> Put

NFData GetVersions

Defined in Development.Benchmark.Rules

Methods

rnf :: GetVersions -> ()

Eq GetVersions

Defined in Development.Benchmark.Rules

Methods

(==) :: GetVersions -> GetVersions -> Bool

(/=) :: GetVersions -> GetVersions -> Bool

Hashable GetVersions

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetVersions -> Int

hash :: GetVersions -> Int

type RuleResult GetVersions

Defined in Development.Benchmark.Rules

type RuleResult GetVersions = [GitCommit]

newtype GetCommitId

Constructors

GetCommitId String

Instances

Show GetCommitId

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetCommitId -> ShowS

show :: GetCommitId -> String

showList :: [GetCommitId] -> ShowS

Binary GetCommitId

Defined in Development.Benchmark.Rules

Methods

put :: GetCommitId -> Put

get :: Get GetCommitId

putList :: [GetCommitId] -> Put

NFData GetCommitId

Defined in Development.Benchmark.Rules

Methods

rnf :: GetCommitId -> ()

Eq GetCommitId

Defined in Development.Benchmark.Rules

Methods

(==) :: GetCommitId -> GetCommitId -> Bool

(/=) :: GetCommitId -> GetCommitId -> Bool

Hashable GetCommitId

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetCommitId -> Int

hash :: GetCommitId -> Int

type RuleResult GetCommitId

Defined in Development.Benchmark.Rules

type RuleResult GetCommitId = String

newtype GetBuildSystem

Constructors

GetBuildSystem ()

Instances

Show GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetBuildSystem -> ShowS

show :: GetBuildSystem -> String

showList :: [GetBuildSystem] -> ShowS

Binary GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

put :: GetBuildSystem -> Put

get :: Get GetBuildSystem

putList :: [GetBuildSystem] -> Put

NFData GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

rnf :: GetBuildSystem -> ()

Eq GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

(==) :: GetBuildSystem -> GetBuildSystem -> Bool

(/=) :: GetBuildSystem -> GetBuildSystem -> Bool

Hashable GetBuildSystem

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetBuildSystem -> Int

hash :: GetBuildSystem -> Int

type RuleResult GetBuildSystem

Defined in Development.Benchmark.Rules

type RuleResult GetBuildSystem = BuildSystem

newtype GetConfigurations

Constructors

GetConfigurations ()

Instances

Show GetConfigurations

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GetConfigurations -> ShowS

show :: GetConfigurations -> String

showList :: [GetConfigurations] -> ShowS

Binary GetConfigurations

Defined in Development.Benchmark.Rules

Methods

put :: GetConfigurations -> Put

get :: Get GetConfigurations

putList :: [GetConfigurations] -> Put

NFData GetConfigurations

Defined in Development.Benchmark.Rules

Methods

rnf :: GetConfigurations -> ()

Eq GetConfigurations

Defined in Development.Benchmark.Rules

Methods

(==) :: GetConfigurations -> GetConfigurations -> Bool

(/=) :: GetConfigurations -> GetConfigurations -> Bool

Hashable GetConfigurations

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GetConfigurations -> Int

hash :: GetConfigurations -> Int

type RuleResult GetConfigurations

Defined in Development.Benchmark.Rules

type RuleResult GetConfigurations = [Configuration]

data Configuration

Constructors

Configuration

Fields

• confName :: String
• confValue :: ByteString

Instances

Generic Configuration

Defined in Development.Benchmark.Rules

Associated Types

type Rep Configuration :: Type -> Type

Methods

from :: Configuration -> Rep Configuration x

to :: Rep Configuration x -> Configuration

Show Configuration

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> Configuration -> ShowS

show :: Configuration -> String

showList :: [Configuration] -> ShowS

Binary Configuration

Defined in Development.Benchmark.Rules

Methods

put :: Configuration -> Put

get :: Get Configuration

putList :: [Configuration] -> Put

NFData Configuration

Defined in Development.Benchmark.Rules

Methods

rnf :: Configuration -> ()

Eq Configuration

Defined in Development.Benchmark.Rules

Methods

(==) :: Configuration -> Configuration -> Bool

(/=) :: Configuration -> Configuration -> Bool

Hashable Configuration

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> Configuration -> Int

hash :: Configuration -> Int

type Rep Configuration

Defined in Development.Benchmark.Rules

type Rep Configuration = D1 ('MetaData "Configuration" "Development.Benchmark.Rules" "shake-bench-0.2.0.0-GBfwpyOoQh65kxC6vx003f" 'False) (C1 ('MetaCons "Configuration" 'PrefixI 'True) (S1 ('MetaSel ('Just "confName") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 String) :*: S1 ('MetaSel ('Just "confValue") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 ByteString)))

data BuildSystem

Default build system that handles Cabal and Stack

Constructors

Cabal

Stack

Instances

FromJSON BuildSystem

Defined in Development.Benchmark.Rules

Methods

parseJSON :: Value -> Parser BuildSystem

parseJSONList :: Value -> Parser [BuildSystem]

ToJSON BuildSystem

Defined in Development.Benchmark.Rules

Methods

toJSON :: BuildSystem -> Value

toEncoding :: BuildSystem -> Encoding

toJSONList :: [BuildSystem] -> Value

toEncodingList :: [BuildSystem] -> Encoding

Generic BuildSystem

Defined in Development.Benchmark.Rules

Associated Types

type Rep BuildSystem :: Type -> Type

Methods

from :: BuildSystem -> Rep BuildSystem x

to :: Rep BuildSystem x -> BuildSystem

Read BuildSystem

Defined in Development.Benchmark.Rules

Methods

readsPrec :: Int -> ReadS BuildSystem

readList :: ReadS [BuildSystem]

readPrec :: ReadPrec BuildSystem

readListPrec :: ReadPrec [BuildSystem]

Show BuildSystem

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> BuildSystem -> ShowS

show :: BuildSystem -> String

showList :: [BuildSystem] -> ShowS

Binary BuildSystem

Defined in Development.Benchmark.Rules

Methods

put :: BuildSystem -> Put

get :: Get BuildSystem

putList :: [BuildSystem] -> Put

NFData BuildSystem

Defined in Development.Benchmark.Rules

Methods

rnf :: BuildSystem -> ()

Eq BuildSystem

Defined in Development.Benchmark.Rules

Methods

(==) :: BuildSystem -> BuildSystem -> Bool

(/=) :: BuildSystem -> BuildSystem -> Bool

Hashable BuildSystem

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> BuildSystem -> Int

hash :: BuildSystem -> Int

type Rep BuildSystem

Defined in Development.Benchmark.Rules

type Rep BuildSystem = D1 ('MetaData "BuildSystem" "Development.Benchmark.Rules" "shake-bench-0.2.0.0-GBfwpyOoQh65kxC6vx003f" 'False) (C1 ('MetaCons "Cabal" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Stack" 'PrefixI 'False) (U1 :: Type -> Type))

findGhcForBuildSystem :: BuildSystem -> FilePath -> IO FilePath

newtype Escaped a

Constructors

Escaped

Fields

• escaped :: a

newtype Unescaped a

Constructors

Unescaped

Fields

• unescaped :: a

Instances

FromJSON a => FromJSON (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

parseJSON :: Value -> Parser (Unescaped a)

parseJSONList :: Value -> Parser [Unescaped a]

ToJSON a => ToJSON (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

toJSON :: Unescaped a -> Value

toEncoding :: Unescaped a -> Encoding

toJSONList :: [Unescaped a] -> Value

toEncodingList :: [Unescaped a] -> Encoding

Show a => Show (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> Unescaped a -> ShowS

show :: Unescaped a -> String

showList :: [Unescaped a] -> ShowS

Binary a => Binary (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

put :: Unescaped a -> Put

get :: Get (Unescaped a)

putList :: [Unescaped a] -> Put

NFData a => NFData (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

rnf :: Unescaped a -> ()

Eq a => Eq (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

(==) :: Unescaped a -> Unescaped a -> Bool

(/=) :: Unescaped a -> Unescaped a -> Bool

Hashable a => Hashable (Unescaped a)

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> Unescaped a -> Int

hash :: Unescaped a -> Int

escapeExperiment :: Unescaped String -> Escaped String

unescapeExperiment :: Escaped String -> Unescaped String

data GitCommit

Instances

FromJSON GitCommit

Defined in Development.Benchmark.Rules

Methods

parseJSON :: Value -> Parser GitCommit

parseJSONList :: Value -> Parser [GitCommit]

ToJSON GitCommit

Defined in Development.Benchmark.Rules

Methods

toJSON :: GitCommit -> Value

toEncoding :: GitCommit -> Encoding

toJSONList :: [GitCommit] -> Value

toEncodingList :: [GitCommit] -> Encoding

Generic GitCommit

Defined in Development.Benchmark.Rules

Associated Types

type Rep GitCommit :: Type -> Type

Methods

from :: GitCommit -> Rep GitCommit x

to :: Rep GitCommit x -> GitCommit

Show GitCommit

Defined in Development.Benchmark.Rules

Methods

showsPrec :: Int -> GitCommit -> ShowS

show :: GitCommit -> String

showList :: [GitCommit] -> ShowS

Binary GitCommit

Defined in Development.Benchmark.Rules

Methods

put :: GitCommit -> Put

get :: Get GitCommit

putList :: [GitCommit] -> Put

NFData GitCommit

Defined in Development.Benchmark.Rules

Methods

rnf :: GitCommit -> ()

Eq GitCommit

Defined in Development.Benchmark.Rules

Methods

(==) :: GitCommit -> GitCommit -> Bool

(/=) :: GitCommit -> GitCommit -> Bool

Hashable GitCommit

Defined in Development.Benchmark.Rules

Methods

hashWithSalt :: Int -> GitCommit -> Int

hash :: GitCommit -> Int

type Rep GitCommit

Defined in Development.Benchmark.Rules

type Rep GitCommit = D1 ('MetaData "GitCommit" "Development.Benchmark.Rules" "shake-bench-0.2.0.0-GBfwpyOoQh65kxC6vx003f" 'False) (C1 ('MetaCons "GitCommit" 'PrefixI 'True) ((S1 ('MetaSel ('Just "gitName") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text) :*: S1 ('MetaSel ('Just "name") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text))) :*: (S1 ('MetaSel ('Just "parent") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (Maybe Text)) :*: S1 ('MetaSel ('Just "include") 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Bool))))