{-# LANGUAGE Trustworthy #-}
{-# LANGUAGE BangPatterns, DeriveDataTypeable, RecordWildCards #-}

-- |
-- Module      : Criterion.Analysis
-- Copyright   : (c) 2009-2014 Bryan O'Sullivan
--
-- License     : BSD-style
-- Maintainer  : bos@serpentine.com
-- Stability   : experimental
-- Portability : GHC
--
-- Analysis code for benchmarks.

module Criterion.Analysis
    (
      Outliers(..)
    , OutlierEffect(..)
    , OutlierVariance(..)
    , SampleAnalysis(..)
    , analyseSample
    , scale
    , analyseMean
    , countOutliers
    , classifyOutliers
    , noteOutliers
    , outlierVariance
    , resolveAccessors
    , validateAccessors
    , regress
    ) where

import Control.Arrow (second)
import Control.Monad (unless, when)
import Control.Monad.Reader (ask)
import Control.Monad.Trans
import Control.Monad.Trans.Except
import Criterion.IO.Printf (note, prolix)
import Criterion.Measurement (secs, threshold)
import Criterion.Monad (Criterion, getGen)
import Criterion.Types
import Data.Int (Int64)
import Data.List.NonEmpty (NonEmpty(..))
import Data.Maybe (fromJust)
import Prelude ()
import Prelude.Compat
import Statistics.Function (sort)
import Statistics.Quantile (weightedAvg)
import Statistics.Regression (bootstrapRegress, olsRegress)
import Statistics.Resampling (Estimator(..),resample)
import Statistics.Sample (mean)
import Statistics.Sample.KernelDensity (kde)
import Statistics.Types (Sample)
import System.Random.MWC (GenIO)
import qualified Data.List as List
import qualified Data.List.NonEmpty as NE
import qualified Data.Map as Map
import qualified Data.Vector as V
import qualified Data.Vector.Generic as G
import qualified Data.Vector.Unboxed as U
import qualified Statistics.Resampling.Bootstrap as B
import qualified Statistics.Types                as B

-- | Classify outliers in a data set, using the boxplot technique.
classifyOutliers :: Sample -> Outliers
classifyOutliers :: Sample -> Outliers
classifyOutliers Sample
sa = forall b a. Unbox b => (a -> b -> a) -> a -> Vector b -> a
U.foldl' ((forall b c a. (b -> c) -> (a -> b) -> a -> c
. Double -> Outliers
outlier) forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall a. Monoid a => a -> a -> a
mappend) forall a. Monoid a => a
mempty Sample
ssa
    where outlier :: Double -> Outliers
outlier Double
e = Outliers {
                        samplesSeen :: Int64
samplesSeen = Int64
1
                      , lowSevere :: Int64
lowSevere = if Double
e forall a. Ord a => a -> a -> Bool
<= Double
loS Bool -> Bool -> Bool
&& Double
e forall a. Ord a => a -> a -> Bool
< Double
hiM then Int64
1 else Int64
0
                      , lowMild :: Int64
lowMild = if Double
e forall a. Ord a => a -> a -> Bool
> Double
loS Bool -> Bool -> Bool
&& Double
e forall a. Ord a => a -> a -> Bool
<= Double
loM then Int64
1 else Int64
0
                      , highMild :: Int64
highMild = if Double
e forall a. Ord a => a -> a -> Bool
>= Double
hiM Bool -> Bool -> Bool
&& Double
e forall a. Ord a => a -> a -> Bool
< Double
hiS then Int64
1 else Int64
0
                      , highSevere :: Int64
highSevere = if Double
e forall a. Ord a => a -> a -> Bool
>= Double
hiS Bool -> Bool -> Bool
&& Double
e forall a. Ord a => a -> a -> Bool
> Double
loM then Int64
1 else Int64
0
                      }
          !loS :: Double
loS = Double
q1 forall a. Num a => a -> a -> a
- (Double
iqr forall a. Num a => a -> a -> a
* Double
3)
          !loM :: Double
loM = Double
q1 forall a. Num a => a -> a -> a
- (Double
iqr forall a. Num a => a -> a -> a
* Double
1.5)
          !hiM :: Double
hiM = Double
q3 forall a. Num a => a -> a -> a
+ (Double
iqr forall a. Num a => a -> a -> a
* Double
1.5)
          !hiS :: Double
hiS = Double
q3 forall a. Num a => a -> a -> a
+ (Double
iqr forall a. Num a => a -> a -> a
* Double
3)
          q1 :: Double
q1   = forall (v :: * -> *).
Vector v Double =>
Int -> Int -> v Double -> Double
weightedAvg Int
1 Int
4 Sample
ssa
          q3 :: Double
q3   = forall (v :: * -> *).
Vector v Double =>
Int -> Int -> v Double -> Double
weightedAvg Int
3 Int
4 Sample
ssa
          ssa :: Sample
ssa  = Sample -> Sample
sort Sample
sa
          iqr :: Double
iqr  = Double
q3 forall a. Num a => a -> a -> a
- Double
q1

-- | Compute the extent to which outliers in the sample data affect
-- the sample mean and standard deviation.
outlierVariance
  :: B.Estimate B.ConfInt Double -- ^ Bootstrap estimate of sample mean.
  -> B.Estimate B.ConfInt Double -- ^ Bootstrap estimate of sample
                                 --   standard deviation.
  -> Double                      -- ^ Number of original iterations.
  -> OutlierVariance
outlierVariance :: Estimate ConfInt Double
-> Estimate ConfInt Double -> Double -> OutlierVariance
outlierVariance Estimate ConfInt Double
µ Estimate ConfInt Double
σ Double
a = OutlierEffect -> String -> Double -> OutlierVariance
OutlierVariance OutlierEffect
effect String
desc Double
varOutMin
  where
    ( OutlierEffect
effect, String
desc ) | Double
varOutMin forall a. Ord a => a -> a -> Bool
< Double
0.01 = (OutlierEffect
Unaffected, String
"no")
                     | Double
varOutMin forall a. Ord a => a -> a -> Bool
< Double
0.1  = (OutlierEffect
Slight,     String
"a slight")
                     | Double
varOutMin forall a. Ord a => a -> a -> Bool
< Double
0.5  = (OutlierEffect
Moderate,   String
"a moderate")
                     | Bool
otherwise        = (OutlierEffect
Severe,     String
"a severe")
    varOutMin :: Double
varOutMin = (forall {a} {t}. Ord a => (t -> a) -> t -> t -> a
minBy Double -> Double
varOut Double
1 (forall {a} {t}. Ord a => (t -> a) -> t -> t -> a
minBy forall {b}. Num b => Double -> b
cMax Double
0 Double
µgMin)) forall a. Fractional a => a -> a -> a
/ Double
σb2
    varOut :: Double -> Double
varOut Double
c  = (Double
ac forall a. Fractional a => a -> a -> a
/ Double
a) forall a. Num a => a -> a -> a
* (Double
σb2 forall a. Num a => a -> a -> a
- Double
ac forall a. Num a => a -> a -> a
* Double
σg2) where ac :: Double
ac = Double
a forall a. Num a => a -> a -> a
- Double
c
    σb :: Double
σb        = forall (e :: * -> *) a. Estimate e a -> a
B.estPoint Estimate ConfInt Double
σ
    µa :: Double
µa        = forall (e :: * -> *) a. Estimate e a -> a
B.estPoint Estimate ConfInt Double
µ forall a. Fractional a => a -> a -> a
/ Double
a
    µgMin :: Double
µgMin     = Double
µa forall a. Fractional a => a -> a -> a
/ Double
2
    σg :: Double
σg        = forall a. Ord a => a -> a -> a
min (Double
µgMin forall a. Fractional a => a -> a -> a
/ Double
4) (Double
σb forall a. Fractional a => a -> a -> a
/ forall a. Floating a => a -> a
sqrt Double
a)
    σg2 :: Double
σg2       = Double
σg forall a. Num a => a -> a -> a
* Double
σg
    σb2 :: Double
σb2       = Double
σb forall a. Num a => a -> a -> a
* Double
σb
    minBy :: (t -> a) -> t -> t -> a
minBy t -> a
f t
q t
r = forall a. Ord a => a -> a -> a
min (t -> a
f t
q) (t -> a
f t
r)
    cMax :: Double -> b
cMax Double
x    = forall a b. (Integral a, Num b) => a -> b
fromIntegral (forall a b. (RealFrac a, Integral b) => a -> b
floor (-Double
2 forall a. Num a => a -> a -> a
* Double
k0 forall a. Fractional a => a -> a -> a
/ (Double
k1 forall a. Num a => a -> a -> a
+ forall a. Floating a => a -> a
sqrt Double
det)) :: Int)
      where
        k1 :: Double
k1    = Double
σb2 forall a. Num a => a -> a -> a
- Double
a forall a. Num a => a -> a -> a
* Double
σg2 forall a. Num a => a -> a -> a
+ Double
ad
        k0 :: Double
k0    = -Double
a forall a. Num a => a -> a -> a
* Double
ad
        ad :: Double
ad    = Double
a forall a. Num a => a -> a -> a
* Double
d
        d :: Double
d     = Double
k forall a. Num a => a -> a -> a
* Double
k where k :: Double
k = Double
µa forall a. Num a => a -> a -> a
- Double
x
        det :: Double
det   = Double
k1 forall a. Num a => a -> a -> a
* Double
k1 forall a. Num a => a -> a -> a
- Double
4 forall a. Num a => a -> a -> a
* Double
σg2 forall a. Num a => a -> a -> a
* Double
k0

-- | Count the total number of outliers in a sample.
countOutliers :: Outliers -> Int64
countOutliers :: Outliers -> Int64
countOutliers (Outliers Int64
_ Int64
a Int64
b Int64
c Int64
d) = Int64
a forall a. Num a => a -> a -> a
+ Int64
b forall a. Num a => a -> a -> a
+ Int64
c forall a. Num a => a -> a -> a
+ Int64
d
{-# INLINE countOutliers #-}

-- | Display the mean of a 'Sample', and characterise the outliers
-- present in the sample.
analyseMean :: Sample
            -> Int              -- ^ Number of iterations used to
                                -- compute the sample.
            -> Criterion Double
analyseMean :: Sample -> Int -> Criterion Double
analyseMean Sample
a Int
iters = do
  let µ :: Double
µ = forall (v :: * -> *). Vector v Double => v Double -> Double
mean Sample
a
  Any
_ <- forall r. CritHPrintfType r => String -> r
note String
"mean is %s (%d iterations)\n" (Double -> String
secs Double
µ) Int
iters
  Outliers -> Criterion ()
noteOutliers forall b c a. (b -> c) -> (a -> b) -> a -> c
. Sample -> Outliers
classifyOutliers forall a b. (a -> b) -> a -> b
$ Sample
a
  forall (m :: * -> *) a. Monad m => a -> m a
return Double
µ

-- | Multiply the 'Estimate's in an analysis by the given value, using
-- 'B.scale'.
scale :: Double                 -- ^ Value to multiply by.
      -> SampleAnalysis -> SampleAnalysis
scale :: Double -> SampleAnalysis -> SampleAnalysis
scale Double
f s :: SampleAnalysis
s@SampleAnalysis{[Regression]
Estimate ConfInt Double
OutlierVariance
anOutlierVar :: SampleAnalysis -> OutlierVariance
anStdDev :: SampleAnalysis -> Estimate ConfInt Double
anMean :: SampleAnalysis -> Estimate ConfInt Double
anRegress :: SampleAnalysis -> [Regression]
anOutlierVar :: OutlierVariance
anStdDev :: Estimate ConfInt Double
anMean :: Estimate ConfInt Double
anRegress :: [Regression]
..} = SampleAnalysis
s {
                                 anMean :: Estimate ConfInt Double
anMean = forall (e :: * -> *) a. (Scale e, Ord a, Num a) => a -> e a -> e a
B.scale Double
f Estimate ConfInt Double
anMean
                               , anStdDev :: Estimate ConfInt Double
anStdDev = forall (e :: * -> *) a. (Scale e, Ord a, Num a) => a -> e a -> e a
B.scale Double
f Estimate ConfInt Double
anStdDev
                               }

-- | Perform an analysis of a measurement.
analyseSample :: Int            -- ^ Experiment number.
              -> String         -- ^ Experiment name.
              -> V.Vector Measured -- ^ Sample data.
              -> ExceptT String Criterion Report
analyseSample :: Int -> String -> Vector Measured -> ExceptT String Criterion Report
analyseSample Int
i String
name Vector Measured
meas = do
  Config{Double
Int
String
[([String], String)]
Maybe String
CL Double
Verbosity
template :: Config -> String
verbosity :: Config -> Verbosity
junitFile :: Config -> Maybe String
jsonFile :: Config -> Maybe String
csvFile :: Config -> Maybe String
reportFile :: Config -> Maybe String
rawDataFile :: Config -> Maybe String
regressions :: Config -> [([String], String)]
resamples :: Config -> Int
timeLimit :: Config -> Double
confInterval :: Config -> CL Double
template :: String
verbosity :: Verbosity
junitFile :: Maybe String
jsonFile :: Maybe String
csvFile :: Maybe String
reportFile :: Maybe String
rawDataFile :: Maybe String
regressions :: [([String], String)]
resamples :: Int
timeLimit :: Double
confInterval :: CL Double
..} <- forall r (m :: * -> *). MonadReader r m => m r
ask
  let ests :: [Estimator]
ests      = [Estimator
Mean,Estimator
StdDev]
      -- The use of filter here throws away very-low-quality
      -- measurements when bootstrapping the mean and standard
      -- deviations.  Without this, the numbers look nonsensical when
      -- very brief actions are measured.
      stime :: Sample
stime     = forall a. Unbox a => (Measured -> a) -> Vector Measured -> Vector a
measure (Measured -> Double
measTime forall b c a. (b -> c) -> (a -> b) -> a -> c
. Measured -> Measured
rescale) forall b c a. (b -> c) -> (a -> b) -> a -> c
.
                  forall (v :: * -> *) a. Vector v a => (a -> Bool) -> v a -> v a
G.filter ((forall a. Ord a => a -> a -> Bool
>= Double
threshold) forall b c a. (b -> c) -> (a -> b) -> a -> c
. Measured -> Double
measTime) forall a b. (a -> b) -> a -> b
$ Vector Measured
meas
      n :: Int
n         = forall (v :: * -> *) a. Vector v a => v a -> Int
G.length Vector Measured
meas
      s :: Int
s         = forall (v :: * -> *) a. Vector v a => v a -> Int
G.length Sample
stime
  Any
_ <- forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift forall a b. (a -> b) -> a -> b
$ forall r. CritHPrintfType r => String -> r
prolix String
"bootstrapping with %d of %d samples (%d%%)\n"
              Int
s Int
n ((Int
s forall a. Num a => a -> a -> a
* Int
100) forall a. Integral a => a -> a -> a
`quot` Int
n)
  Gen RealWorld
gen <- forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift Criterion GenIO
getGen
  [Regression]
rs <- forall (t :: * -> *) (m :: * -> *) a b.
(Traversable t, Monad m) =>
(a -> m b) -> t a -> m (t b)
mapM (\([String]
ps,String
r) -> GenIO
-> [String]
-> String
-> Vector Measured
-> ExceptT String Criterion Regression
regress Gen RealWorld
gen [String]
ps String
r Vector Measured
meas) forall a b. (a -> b) -> a -> b
$
        (([String
"iters"],String
"time")forall a. a -> [a] -> [a]
:[([String], String)]
regressions)
  [(Estimator, Bootstrap Vector Double)]
resamps <- forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO forall a b. (a -> b) -> a -> b
$ GenIO
-> [Estimator]
-> Int
-> Sample
-> IO [(Estimator, Bootstrap Vector Double)]
resample Gen RealWorld
gen [Estimator]
ests Int
resamples Sample
stime
  (Estimate ConfInt Double
estMean,Estimate ConfInt Double
estStdDev) <- case CL Double
-> Sample
-> [(Estimator, Bootstrap Vector Double)]
-> [Estimate ConfInt Double]
B.bootstrapBCA CL Double
confInterval Sample
stime [(Estimator, Bootstrap Vector Double)]
resamps of
    [Estimate ConfInt Double
estMean',Estimate ConfInt Double
estStdDev'] -> forall (m :: * -> *) a. Monad m => a -> m a
return (Estimate ConfInt Double
estMean',Estimate ConfInt Double
estStdDev')
    [Estimate ConfInt Double]
ests' -> forall (m :: * -> *) e a. Monad m => e -> ExceptT e m a
throwE forall a b. (a -> b) -> a -> b
$ String
"analyseSample: Expected two estimation functions, received: " forall a. [a] -> [a] -> [a]
++ forall a. Show a => a -> String
show [Estimate ConfInt Double]
ests'
  let ov :: OutlierVariance
ov = Estimate ConfInt Double
-> Estimate ConfInt Double -> Double -> OutlierVariance
outlierVariance Estimate ConfInt Double
estMean Estimate ConfInt Double
estStdDev (forall a b. (Integral a, Num b) => a -> b
fromIntegral Int
n)
      an :: SampleAnalysis
an = SampleAnalysis {
               anRegress :: [Regression]
anRegress    = [Regression]
rs
             , anMean :: Estimate ConfInt Double
anMean       = Estimate ConfInt Double
estMean
             , anStdDev :: Estimate ConfInt Double
anStdDev     = Estimate ConfInt Double
estStdDev
             , anOutlierVar :: OutlierVariance
anOutlierVar = OutlierVariance
ov
             }
  forall (m :: * -> *) a. Monad m => a -> m a
return Report {
      reportNumber :: Int
reportNumber   = Int
i
    , reportName :: String
reportName     = String
name
    , reportKeys :: [String]
reportKeys     = [String]
measureKeys
    , reportMeasured :: Vector Measured
reportMeasured = Vector Measured
meas
    , reportAnalysis :: SampleAnalysis
reportAnalysis = SampleAnalysis
an
    , reportOutliers :: Outliers
reportOutliers = Sample -> Outliers
classifyOutliers Sample
stime
    , reportKDEs :: [KDE]
reportKDEs     = [forall a b c. (a -> b -> c) -> (a, b) -> c
uncurry (String -> Sample -> Sample -> KDE
KDE String
"time") (forall (v :: * -> *).
(Vector v CD, Vector v Double, Vector v Int) =>
Int -> v Double -> (v Double, v Double)
kde Int
128 Sample
stime)]
    }

-- | Regress the given predictors against the responder.
--
-- Errors may be returned under various circumstances, such as invalid
-- names or lack of needed data.
--
-- See 'olsRegress' for details of the regression performed.
regress :: GenIO
        -> [String]             -- ^ Predictor names.
        -> String               -- ^ Responder name.
        -> V.Vector Measured
        -> ExceptT String Criterion Regression
regress :: GenIO
-> [String]
-> String
-> Vector Measured
-> ExceptT String Criterion Regression
regress GenIO
gen [String]
predNames String
respName Vector Measured
meas = do
  forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (forall (v :: * -> *) a. Vector v a => v a -> Bool
G.null Vector Measured
meas) forall a b. (a -> b) -> a -> b
$
    forall (m :: * -> *) e a. Monad m => e -> ExceptT e m a
throwE String
"no measurements"
  [(String, Measured -> Maybe Double)]
accs <- forall e (m :: * -> *) a. m (Either e a) -> ExceptT e m a
ExceptT forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall (m :: * -> *) a. Monad m => a -> m a
return forall a b. (a -> b) -> a -> b
$ [String]
-> String -> Either String [(String, Measured -> Maybe Double)]
validateAccessors [String]
predNames String
respName
  let unmeasured :: [String]
unmeasured = [String
n | (String
n, Maybe Double
Nothing) <- forall a b. (a -> b) -> [a] -> [b]
map (forall (a :: * -> * -> *) b c d.
Arrow a =>
a b c -> a (d, b) (d, c)
second (forall a b. (a -> b) -> a -> b
$ forall (v :: * -> *) a. Vector v a => v a -> a
G.head Vector Measured
meas)) [(String, Measured -> Maybe Double)]
accs]
  forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (forall (t :: * -> *) a. Foldable t => t a -> Bool
null [String]
unmeasured) forall a b. (a -> b) -> a -> b
$
    forall (m :: * -> *) e a. Monad m => e -> ExceptT e m a
throwE forall a b. (a -> b) -> a -> b
$ String
"no data available for " forall a. [a] -> [a] -> [a]
++ [String] -> String
renderNames [String]
unmeasured
  (Sample
r,[Sample]
ps) <- case forall a b. (a -> b) -> [a] -> [b]
map ((forall a. Unbox a => (Measured -> a) -> Vector Measured -> Vector a
`measure` Vector Measured
meas) forall b c a. (b -> c) -> (a -> b) -> a -> c
. (forall a. HasCallStack => Maybe a -> a
fromJust forall b c a. (b -> c) -> (a -> b) -> a -> c
.) forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall a b. (a, b) -> b
snd) [(String, Measured -> Maybe Double)]
accs of
    (Sample
r':[Sample]
ps') -> forall (m :: * -> *) a. Monad m => a -> m a
return (Sample
r',[Sample]
ps')
    []       -> forall (m :: * -> *) e a. Monad m => e -> ExceptT e m a
throwE String
"regress: Expected at least one accessor"
  Config{Double
Int
String
[([String], String)]
Maybe String
CL Double
Verbosity
template :: String
verbosity :: Verbosity
junitFile :: Maybe String
jsonFile :: Maybe String
csvFile :: Maybe String
reportFile :: Maybe String
rawDataFile :: Maybe String
regressions :: [([String], String)]
resamples :: Int
timeLimit :: Double
confInterval :: CL Double
template :: Config -> String
verbosity :: Config -> Verbosity
junitFile :: Config -> Maybe String
jsonFile :: Config -> Maybe String
csvFile :: Config -> Maybe String
reportFile :: Config -> Maybe String
rawDataFile :: Config -> Maybe String
regressions :: Config -> [([String], String)]
resamples :: Config -> Int
timeLimit :: Config -> Double
confInterval :: Config -> CL Double
..} <- forall r (m :: * -> *). MonadReader r m => m r
ask
  (Vector (Estimate ConfInt Double)
coeffs,Estimate ConfInt Double
r2) <- forall (m :: * -> *) a. MonadIO m => IO a -> m a
liftIO forall a b. (a -> b) -> a -> b
$
                 GenIO
-> Int
-> CL Double
-> ([Sample] -> Sample -> (Sample, Double))
-> [Sample]
-> Sample
-> IO (Vector (Estimate ConfInt Double), Estimate ConfInt Double)
bootstrapRegress GenIO
gen Int
resamples CL Double
confInterval [Sample] -> Sample -> (Sample, Double)
olsRegress [Sample]
ps Sample
r
  forall (m :: * -> *) a. Monad m => a -> m a
return Regression {
      regResponder :: String
regResponder = String
respName
    , regCoeffs :: Map String (Estimate ConfInt Double)
regCoeffs    = forall k a. Ord k => [(k, a)] -> Map k a
Map.fromList (forall a b. [a] -> [b] -> [(a, b)]
zip ([String]
predNames forall a. [a] -> [a] -> [a]
++ [String
"y"]) (forall (v :: * -> *) a. Vector v a => v a -> [a]
G.toList Vector (Estimate ConfInt Double)
coeffs))
    , regRSquare :: Estimate ConfInt Double
regRSquare   = Estimate ConfInt Double
r2
    }

singleton :: NonEmpty a -> Bool
singleton :: forall a. NonEmpty a -> Bool
singleton (a
_ :| []) = Bool
True
singleton NonEmpty a
_         = Bool
False

-- | Given a list of accessor names (see 'measureKeys'), return either
-- a mapping from accessor name to function or an error message if
-- any names are wrong.
resolveAccessors :: [String]
                 -> Either String [(String, Measured -> Maybe Double)]
resolveAccessors :: [String] -> Either String [(String, Measured -> Maybe Double)]
resolveAccessors [String]
names =
  case [String]
unresolved of
    [] -> forall a b. b -> Either a b
Right [(String
n, Measured -> Maybe Double
a) | (String
n, Just (Measured -> Maybe Double
a,String
_)) <- [(String, Maybe (Measured -> Maybe Double, String))]
accessors]
    [String]
_  -> forall a b. a -> Either a b
Left forall a b. (a -> b) -> a -> b
$ String
"unknown metric " forall a. [a] -> [a] -> [a]
++ [String] -> String
renderNames [String]
unresolved
  where
    unresolved :: [String]
unresolved = [String
n | (String
n, Maybe (Measured -> Maybe Double, String)
Nothing) <- [(String, Maybe (Measured -> Maybe Double, String))]
accessors]
    accessors :: [(String, Maybe (Measured -> Maybe Double, String))]
accessors = forall a b c. (a -> b -> c) -> b -> a -> c
flip forall a b. (a -> b) -> [a] -> [b]
map [String]
names forall a b. (a -> b) -> a -> b
$ \String
n -> (String
n, forall k a. Ord k => k -> Map k a -> Maybe a
Map.lookup String
n Map String (Measured -> Maybe Double, String)
measureAccessors)

-- | Given predictor and responder names, do some basic validation,
-- then hand back the relevant accessors.
validateAccessors :: [String]   -- ^ Predictor names.
                  -> String     -- ^ Responder name.
                  -> Either String [(String, Measured -> Maybe Double)]
validateAccessors :: [String]
-> String -> Either String [(String, Measured -> Maybe Double)]
validateAccessors [String]
predNames String
respName = do
  forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (forall (t :: * -> *) a. Foldable t => t a -> Bool
null [String]
predNames) forall a b. (a -> b) -> a -> b
$
    forall a b. a -> Either a b
Left String
"no predictors specified"
  let names :: [String]
names = String
respNameforall a. a -> [a] -> [a]
:[String]
predNames
      dups :: [String]
dups = forall a b. (a -> b) -> [a] -> [b]
map forall a. NonEmpty a -> a
NE.head forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall a. (a -> Bool) -> [a] -> [a]
List.filter (Bool -> Bool
not forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall a. NonEmpty a -> Bool
singleton) forall b c a. (b -> c) -> (a -> b) -> a -> c
.
             forall (f :: * -> *) a. (Foldable f, Eq a) => f a -> [NonEmpty a]
NE.group forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall a. Ord a => [a] -> [a]
List.sort forall a b. (a -> b) -> a -> b
$ [String]
names
  forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
unless (forall (t :: * -> *) a. Foldable t => t a -> Bool
null [String]
dups) forall a b. (a -> b) -> a -> b
$
    forall a b. a -> Either a b
Left forall a b. (a -> b) -> a -> b
$ String
"duplicated metric " forall a. [a] -> [a] -> [a]
++ [String] -> String
renderNames [String]
dups
  [String] -> Either String [(String, Measured -> Maybe Double)]
resolveAccessors [String]
names

renderNames :: [String] -> String
renderNames :: [String] -> String
renderNames = forall a. [a] -> [[a]] -> [a]
List.intercalate String
", " forall b c a. (b -> c) -> (a -> b) -> a -> c
. forall a b. (a -> b) -> [a] -> [b]
map forall a. Show a => a -> String
show

-- | Display a report of the 'Outliers' present in a 'Sample'.
noteOutliers :: Outliers -> Criterion ()
noteOutliers :: Outliers -> Criterion ()
noteOutliers Outliers
o = do
  let frac :: a -> Double
frac a
n = (Double
100::Double) forall a. Num a => a -> a -> a
* forall a b. (Integral a, Num b) => a -> b
fromIntegral a
n forall a. Fractional a => a -> a -> a
/ forall a b. (Integral a, Num b) => a -> b
fromIntegral (Outliers -> Int64
samplesSeen Outliers
o)
      check :: Int64 -> Double -> String -> Criterion ()
      check :: Int64 -> Double -> String -> Criterion ()
check Int64
k Double
t String
d = forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (forall {a}. Integral a => a -> Double
frac Int64
k forall a. Ord a => a -> a -> Bool
> Double
t) forall a b. (a -> b) -> a -> b
$
                    forall r. CritHPrintfType r => String -> r
note String
"  %d (%.1g%%) %s\n" Int64
k (forall {a}. Integral a => a -> Double
frac Int64
k) String
d
      outCount :: Int64
outCount = Outliers -> Int64
countOutliers Outliers
o
  forall (f :: * -> *). Applicative f => Bool -> f () -> f ()
when (Int64
outCount forall a. Ord a => a -> a -> Bool
> Int64
0) forall a b. (a -> b) -> a -> b
$ do
    Any
_ <- forall r. CritHPrintfType r => String -> r
note String
"found %d outliers among %d samples (%.1g%%)\n"
         Int64
outCount (Outliers -> Int64
samplesSeen Outliers
o) (forall {a}. Integral a => a -> Double
frac Int64
outCount)
    Int64 -> Double -> String -> Criterion ()
check (Outliers -> Int64
lowSevere Outliers
o) Double
0 String
"low severe"
    Int64 -> Double -> String -> Criterion ()
check (Outliers -> Int64
lowMild Outliers
o) Double
1 String
"low mild"
    Int64 -> Double -> String -> Criterion ()
check (Outliers -> Int64
highMild Outliers
o) Double
1 String
"high mild"
    Int64 -> Double -> String -> Criterion ()
check (Outliers -> Int64
highSevere Outliers
o) Double
0 String
"high severe"