-- Hoogle documentation, generated by Haddock -- See Hoogle, http://www.haskell.org/hoogle/ -- | Create tests and benchmarks together -- -- Test your benchmarks! Benchmark your tests! -- -- It's too easy to accidentally try and benchmark apples and oranges -- together. Wouldn't it be nice if you could somehow guarantee that your -- benchmarks satisfy some simple tests (e.g. a group of comparisons all -- return the same value)? -- -- Furthermore, trying to compare multiple inputs/functions against each -- other requires a lot of boilerplate, making it even easier to -- accidentally compare the wrong things (e.g. using whnf instead -- of nf). -- -- testbench aims to help solve these problems and more by making -- it easier to write unit tests and benchmarks together by stating -- up-front what requirements are needed and then using simple functions -- to state the next parameter to be tested/benchmarked. @package testbench @version 0.2.1.2 -- | Make it easier to compare benchmarks and to test that benchmarks are -- indeed valid. -- -- At the top level you will probably run the testBench function, -- and create comparisons using compareFunc or the list-based -- variants. -- -- For example: -- -- 
--   main :: IO ()
--   main = testBench $ do
--     -- Compare how long it takes to make a list of the specified length.
--     compareFunc "List length"
--                 (\n -> length (replicate n ()) == n)
--                 [testWith (@? "Not as long as specified"), benchNormalForm]
--                 (mapM_ (\n -> comp ("len == " ++ show n) n) [1..5])
--
-- -- When run, the output will look something like: -- -- 
--   Cases: 7  Tried: 7  Errors: 0  Failures: 0
--                             Mean    MeanLB    MeanUB    Stddev  StddevLB  StddevUB  OutlierVariance
--   List length
--     len == 1            323.8 ns  318.6 ns  335.9 ns  23.86 ns  5.834 ns  40.90 ns              83%
--     len == 2            352.8 ns  349.1 ns  358.1 ns  15.05 ns  11.76 ns  19.62 ns              61%
--     len == 3            372.4 ns  358.4 ns  393.8 ns  62.50 ns  39.83 ns  90.85 ns              96%
--     len == 4            396.3 ns  378.4 ns  419.2 ns  67.83 ns  46.71 ns  94.74 ns              96%
--     len == 5            426.0 ns  407.0 ns  459.5 ns  82.23 ns  53.37 ns  110.2 ns              97%
--
module TestBench -- | An environment for combining testing and benchmarking. type TestBench = TestBenchM () -- | Run the specified benchmarks if and only if all tests pass, using a -- comparison-based format for benchmarking output. -- -- For more control, use getTestBenches. testBench :: TestBench -> IO () -- | As with testBench but allow specifying a custom default -- Config parameter rather than testBenchConfig. testBenchWith :: Config -> TestBench -> IO () -- | This is the same as defaultConfig from criterion but with the -- verbosity set to Quiet to avoid unnecessary noise on stdout. testBenchConfig :: Config -- | Label a sub-part of a TestBench. collection :: String -> TestBench -> TestBench -- | Compare how various input values (of the same type) behave for a -- specific function. -- -- By default: -- -- compareFunc :: (ProvideParams params a b) => String -> (a -> b) -> params -> Comparison a b -> TestBench -- | As with compareFunc but use the provided list of values to base -- the benchmarking off of. -- -- This is useful in situations where you create an enumeration type to -- describe everything you're benchmarking and a function that takes one -- of these values and evaluates it. -- -- baseline is used on the first value (if non-empty); the -- Show instance is used to provide labels. compareFuncList :: (ProvideParams params a b, Show a, Eq b, Show b) => String -> (a -> b) -> params -> [a] -> TestBench -- | A variant of compareFuncList that allows for the function to -- return an IO value. compareFuncListIO :: (ProvideParams params a (IO b), Show a, Eq b, Show b) => String -> (a -> IO b) -> params -> [a] -> TestBench -- | A variant of compareFuncList where you provide your own -- equivalent to baseline. -- -- Most useful with baselineWith. compareFuncListWith :: (ProvideParams params a b, Show a) => (String -> a -> CompParams a b) -> String -> (a -> b) -> params -> [a] -> TestBench -- | A variant of compareFuncList that doesn't use baseline -- (allowing you to specify your own test). compareFuncList' :: (ProvideParams params a b, Show a) => String -> (a -> b) -> params -> [a] -> TestBench -- | An extension to compareFuncList that uses the Bounded -- and Enum instances to generate the list of all values. compareFuncAll :: (ProvideParams params a b, Show a, Enum a, Bounded a, Eq b, Show b) => String -> (a -> b) -> params -> TestBench -- | An extension to compareFuncListIO that uses the Bounded -- and Enum instances to generate the list of all values. compareFuncAllIO :: (ProvideParams params a (IO b), Show a, Enum a, Bounded a, Eq b, Show b) => String -> (a -> IO b) -> params -> TestBench -- | An extension to compareFuncListWith that uses the -- Bounded and Enum instances to generate the list of all -- values. compareFuncAllWith :: (ProvideParams params a b, Show a, Enum a, Bounded a, Eq b, Show b) => (String -> a -> CompParams a b) -> String -> (a -> b) -> params -> TestBench -- | A variant of comapreFuncAll that doesn't use baseline -- (allowing you to specify your own test). compareFuncAll' :: (ProvideParams params a b, Show a, Enum a, Bounded a) => String -> (a -> b) -> params -> TestBench -- | Monoidally build up the parameters used to control a Comparison -- environment. -- -- This will typically be a combination of benchNormalForm with -- either baseline or testWith. data CompParams a b -- | A convenience class to make it easier to provide CompParams -- values. -- -- You can either: -- -- class ProvideParams cp a b | cp -> a b toParams :: ProvideParams cp a b => cp -> CompParams a b -- | A combination of benchNormalForm and weigh, taking into -- account the common case that you want to consider a value that can - -- and should - be evaluated to normal form. normalForm :: (NFData b) => CompParams a b -- | A variant of normalForm where the results are within -- IO. normalFormIO :: (NFData b) => CompParams a (IO b) -- | Evaluate all benchmarks to normal form. benchNormalForm :: (NFData b) => CompParams a b -- | Evaluate all IO-based benchmarks to weak head normal form. benchIO :: CompParams a (IO b) -- | Evaluate all IO-based benchmarks to normal form. benchNormalFormIO :: (NFData b) => CompParams a (IO b) -- | Allow specifying how benchmarks should be evaluated. This may allow -- usage of methods such as nfIO, but this has not been tested -- as yet. withBenchMode :: ((a -> b) -> a -> Benchmarkable) -> CompParams a b -- | Don't run any benchmarks. I'm not sure why you'd want to do this as -- there's surely easier/better testing environments available, but this -- way it's possible. noBenchmarks :: CompParams a b -- | Specify a sample baseline value to benchmark and test against (such -- that the result of applying the function to this a is what -- everything should match). -- -- You shouldn't specify this more than once, nor mix it with -- noTests or testWith. baseline :: (Eq b, Show b) => String -> a -> CompParams a b -- | A variant of baseline where the function returns an IO -- value. baselineIO :: (Eq b, Show b) => String -> a -> CompParams a (IO b) -- | A variant of baseline that lets you specify how to test for -- equality. -- -- The first argument to the provided function will be the "baseline" -- value; the second will be the value being tested. baselineWith :: (b -> b -> Assertion) -> String -> a -> CompParams a b -- | Specify a predicate that all results should satisfy. -- -- Note that the last statement between testWith, baseline -- and noTests "wins" in specifying which testing (if any) to do. testWith :: (b -> Assertion) -> CompParams a b -- | Don't run any tests. This isn't recommended, but could be useful if -- all you want to do is run comparisons (potentially because no -- meaningful tests are possible). noTests :: CompParams a b -- | Calculate memory usage of the various parameters. -- -- Note that to achieve this, testBench and associated functions -- will run copies of itself to be able to calculate memory usage in a -- pristine environment (i.e. without influence of caching from testing -- and benchmarking). As such, you may wish to use the -threaded -- GHC option when building your benchmarking executable. weigh :: (NFData b) => CompParams a b -- | An IO-based equivalent to weigh weighIO :: (NFData b) => CompParams a (IO b) -- | The results from measuring memory usage. data GetWeight -- | How to weigh a function. getWeight :: (NFData b) => (a -> b) -> a -> GetWeight -- | An IO-based variant of getWeight. getWeightIO :: (NFData b) => (a -> IO b) -> a -> GetWeight -- | A specialised monad used solely for running comparisons. -- -- No lifting is permitted; the only operations permitted are -- comp, compBench and compTest. type Comparison a b = ComparisonM a b () -- | Benchmark and test (if specified) this value against the specified -- function. comp :: String -> a -> Comparison a b -- | Only benchmark and possibly weigh (but do not test) this value against -- the specified function. compBench :: String -> a -> Comparison a b -- | Only test (but do not benchmark or weigh) this value against the -- specified function. compTest :: String -> a -> Comparison a b data ComparisonM a b r -- | Obtain the resulting tests and benchmarks from the specified -- TestBench. getTestBenches :: TestBench -> IO (Test, EvalForest) data Eval Eval :: !String -> !(Maybe Benchmarkable) -> !(Maybe GetWeight) -> Eval [eName] :: Eval -> !String [eBench] :: Eval -> !(Maybe Benchmarkable) [eWeigh] :: Eval -> !(Maybe GetWeight) -- | A more explicit tree-like structure for benchmarks than using -- Criterion's Benchmark type. type EvalTree = LabelTree Eval type EvalForest = [EvalTree] -- | Remove the explicit tree-like structure into the implicit one used by -- Criterion. -- -- Useful for embedding the results into an existing benchmark suite. flattenBenchForest :: EvalForest -> [Benchmark] -- | Run the specified benchmarks, printing the results (once they're all -- complete) to stdout in a tabular format for easier comparisons. evalForest :: Config -> EvalForest -> IO () -- | Create a single benchmark evaluated to normal form, where the results -- should equal the value specified. -- -- Will also weigh the function. nfEq :: (NFData b, Show b, Eq b) => b -> (a -> b) -> String -> a -> TestBench -- | Create a single benchmark evaluated to weak head normal form, where -- the results should equal the value specified. whnfEq :: (Show b, Eq b) => b -> (a -> b) -> String -> a -> TestBench -- | A way of writing custom testing/benchmarking statements. You will -- probably want to use one of the pre-defined versions instead. mkTestBench :: ((a -> b) -> a -> Maybe Benchmarkable) -> ((a -> b) -> a -> Maybe GetWeight) -> (b -> Maybe Assertion) -> (a -> b) -> String -> a -> TestBench data TestBenchM r -- | A tree of operations. type OpTree = LabelTree Operation -- | An individual operation potentially consisting of a benchmark and/or -- test. data Operation -- | A simple labelled rose-tree data structure also containing the depth. data LabelTree a Leaf :: !Depth -> a -> LabelTree a Branch :: !Depth -> String -> [LabelTree a] -> LabelTree a type Depth = Int instance Control.Monad.IO.Class.MonadIO (TestBench.ComparisonM a b) instance GHC.Base.Monad (TestBench.ComparisonM a b) instance GHC.Base.Applicative (TestBench.ComparisonM a b) instance GHC.Base.Functor (TestBench.ComparisonM a b) instance Control.Monad.IO.Class.MonadIO TestBench.TestBenchM instance GHC.Base.Monad TestBench.TestBenchM instance GHC.Base.Applicative TestBench.TestBenchM instance GHC.Base.Functor TestBench.TestBenchM instance TestBench.ProvideParams (TestBench.CompParams a b) a b instance TestBench.ProvideParams [TestBench.CompParams a b] a b instance Data.Semigroup.Semigroup (TestBench.CompParams a b) instance GHC.Base.Monoid (TestBench.CompParams a b)