{-# LANGUAGE CPP #-} -- | -- Stability: provisional module Test.Hspec.Core.Runner ( -- * Running a spec hspec , runSpec -- * Config , Config (..) , ColorMode (..) , Path , defaultConfig , configAddFilter , readConfig -- * Summary , Summary (..) , isSuccess , evaluateSummary -- * Legacy -- | The following primitives are deprecated. Use `runSpec` instead. , hspecWith , hspecResult , hspecWithResult #ifdef TEST , rerunAll #endif ) where import Prelude () import Test.Hspec.Core.Compat import Data.Maybe import System.IO import System.Environment (getArgs, withArgs) import System.Exit import qualified Control.Exception as E import System.Random import Control.Monad.ST import Data.STRef import System.Console.ANSI (hHideCursor, hShowCursor) import qualified Test.QuickCheck as QC import Test.Hspec.Core.Util (Path) import Test.Hspec.Core.Spec import Test.Hspec.Core.Config import Test.Hspec.Core.Formatters import Test.Hspec.Core.Formatters.Internal import Test.Hspec.Core.FailureReport import Test.Hspec.Core.QuickCheckUtil import Test.Hspec.Core.Shuffle import Test.Hspec.Core.Runner.Eval -- | Filter specs by given predicate. -- -- The predicate takes a list of "describe" labels and a "requirement". filterSpecs :: Config -> [EvalTree] -> [EvalTree] filterSpecs c = go [] where p :: Path -> Bool p path = (fromMaybe (const True) (configFilterPredicate c) path) && not (fromMaybe (const False) (configSkipPredicate c) path) go :: [String] -> [EvalTree] -> [EvalTree] go groups = mapMaybe (goSpec groups) goSpecs :: [String] -> [EvalTree] -> ([EvalTree] -> b) -> Maybe b goSpecs groups specs ctor = case go groups specs of [] -> Nothing xs -> Just (ctor xs) goSpec :: [String] -> EvalTree -> Maybe (EvalTree) goSpec groups spec = case spec of Leaf item -> guard (p (groups, evalItemDescription item)) >> return spec Node group specs -> goSpecs (groups ++ [group]) specs (Node group) NodeWithCleanup action specs -> goSpecs groups specs (NodeWithCleanup action) applyDryRun :: Config -> [SpecTree ()] -> [SpecTree ()] applyDryRun c | configDryRun c = map (removeCleanup . fmap markSuccess) | otherwise = id where markSuccess :: Item () -> Item () markSuccess item = item {itemExample = safeEvaluateExample (Result "" Success)} removeCleanup :: SpecTree () -> SpecTree () removeCleanup spec = case spec of Node x xs -> Node x (map removeCleanup xs) NodeWithCleanup _ xs -> NodeWithCleanup (\() -> return ()) (map removeCleanup xs) leaf@(Leaf _) -> leaf -- | Run a given spec and write a report to `stdout`. -- Exit with `exitFailure` if at least one spec item fails. -- -- /Note/: `hspec` handles command-line options and reads config files. This -- is not always desired. Use `runSpec` if you need more control over these -- aspects. hspec :: Spec -> IO () hspec spec = getArgs >>= readConfig defaultConfig >>= doNotLeakCommandLineArgumentsToExamples . runSpec spec >>= evaluateSummary -- Add a seed to given config if there is none. That way the same seed is used -- for all properties. This helps with --seed and --rerun. ensureSeed :: Config -> IO Config ensureSeed c = case configQuickCheckSeed c of Nothing -> do seed <- newSeed return c {configQuickCheckSeed = Just (fromIntegral seed)} _ -> return c -- | Run given spec with custom options. -- This is similar to `hspec`, but more flexible. hspecWith :: Config -> Spec -> IO () hspecWith config spec = getArgs >>= readConfig config >>= doNotLeakCommandLineArgumentsToExamples . runSpec spec >>= evaluateSummary -- | `True` if the given `Summary` indicates that there were no -- failures, `False` otherwise. isSuccess :: Summary -> Bool isSuccess summary = summaryFailures summary == 0 -- | Exit with `exitFailure` if the given `Summary` indicates that there was at -- least one failure. evaluateSummary :: Summary -> IO () evaluateSummary summary = unless (isSuccess summary) exitFailure -- | Run given spec and returns a summary of the test run. -- -- /Note/: `hspecResult` does not exit with `exitFailure` on failing spec -- items. If you need this, you have to check the `Summary` yourself and act -- accordingly. hspecResult :: Spec -> IO Summary hspecResult spec = getArgs >>= readConfig defaultConfig >>= doNotLeakCommandLineArgumentsToExamples . runSpec spec -- | Run given spec with custom options and returns a summary of the test run. -- -- /Note/: `hspecWithResult` does not exit with `exitFailure` on failing spec -- items. If you need this, you have to check the `Summary` yourself and act -- accordingly. hspecWithResult :: Config -> Spec -> IO Summary hspecWithResult config spec = getArgs >>= readConfig config >>= doNotLeakCommandLineArgumentsToExamples . runSpec spec -- | -- `runSpec` is the most basic primitive to run a spec. `hspec` is defined in -- terms of @runSpec@: -- -- @ -- hspec spec = -- `getArgs` -- >>= `readConfig` `defaultConfig` -- >>= `withArgs` [] . runSpec spec -- >>= `evaluateSummary` -- @ runSpec :: Spec -> Config -> IO Summary runSpec spec c_ = do oldFailureReport <- readFailureReportOnRerun c_ c <- ensureSeed (applyFailureReport oldFailureReport c_) if configRerunAllOnSuccess c -- With --rerun-all we may run the spec twice. For that reason GHC can not -- optimize away the spec tree. That means that the whole spec tree has to -- be constructed in memory and we loose constant space behavior. -- -- By separating between rerunAllMode and normalMode here, we retain -- constant space behavior in normalMode. -- -- see: https://github.com/hspec/hspec/issues/169 then rerunAllMode c oldFailureReport else normalMode c where normalMode c = runSpec_ c spec rerunAllMode c oldFailureReport = do summary <- runSpec_ c spec if rerunAll c oldFailureReport summary then runSpec spec c_ else return summary failFocused :: Item a -> Item a failFocused item = item {itemExample = example} where failure = Failure Nothing (Reason "item is focused; failing due to --fail-on-focused") example | itemIsFocused item = \ params hook p -> do Result info status <- itemExample item params hook p return $ Result info $ case status of Success -> failure Pending _ _ -> failure Failure{} -> status | otherwise = itemExample item failFocusedItems :: Config -> Spec -> Spec failFocusedItems config spec | configFailOnFocused config = mapSpecItem_ failFocused spec | otherwise = spec focusSpec :: Config -> Spec -> Spec focusSpec config spec | configFocusedOnly config = spec | otherwise = focus spec runSpec_ :: Config -> Spec -> IO Summary runSpec_ config spec = do withHandle config $ \h -> do let formatter = fromMaybe specdoc (configFormatter config) seed = (fromJust . configQuickCheckSeed) config qcArgs = configQuickCheckArgs config concurrentJobs <- case configConcurrentJobs config of Nothing -> getDefaultConcurrentJobs Just n -> return n useColor <- doesUseColor h config let focusedSpec = focusSpec config (failFocusedItems config spec) params = Params (configQuickCheckArgs config) (configSmallCheckDepth config) randomize | configRandomize config = randomizeForest seed | otherwise = id filteredSpec <- randomize . filterSpecs config . mapMaybe (toEvalTree params) . applyDryRun config <$> runSpecM focusedSpec (total, failures) <- withHiddenCursor useColor h $ do let formatConfig = FormatConfig { formatConfigHandle = h , formatConfigUseColor = useColor , formatConfigUseDiff = configDiff config , formatConfigHtmlOutput = configHtmlOutput config , formatConfigPrintCpuTime = configPrintCpuTime config , formatConfigUsedSeed = seed } evalConfig = EvalConfig { evalConfigFormat = formatterToFormat formatter formatConfig , evalConfigConcurrentJobs = concurrentJobs , evalConfigFastFail = configFastFail config } runFormatter evalConfig filteredSpec dumpFailureReport config seed qcArgs failures return (Summary total (length failures)) toEvalTree :: Params -> SpecTree () -> Maybe EvalTree toEvalTree params = go where go :: Tree (() -> c) (Item ()) -> Maybe (Tree c EvalItem) go t = case t of Node s xs -> Just $ Node s (mapMaybe go xs) NodeWithCleanup c xs -> Just $ NodeWithCleanup (c ()) (mapMaybe go xs) Leaf (Item requirement loc isParallelizable isFocused e) -> guard isFocused >> return (Leaf (EvalItem requirement loc (fromMaybe False isParallelizable) (e params $ ($ ())))) dumpFailureReport :: Config -> Integer -> QC.Args -> [Path] -> IO () dumpFailureReport config seed qcArgs xs = do writeFailureReport config FailureReport { failureReportSeed = seed , failureReportMaxSuccess = QC.maxSuccess qcArgs , failureReportMaxSize = QC.maxSize qcArgs , failureReportMaxDiscardRatio = QC.maxDiscardRatio qcArgs , failureReportPaths = xs } doNotLeakCommandLineArgumentsToExamples :: IO a -> IO a doNotLeakCommandLineArgumentsToExamples = withArgs [] withHiddenCursor :: Bool -> Handle -> IO a -> IO a withHiddenCursor useColor h | useColor = E.bracket_ (hHideCursor h) (hShowCursor h) | otherwise = id doesUseColor :: Handle -> Config -> IO Bool doesUseColor h c = case configColorMode c of ColorAuto -> (&&) <$> hIsTerminalDevice h <*> (not <$> isDumb) ColorNever -> return False ColorAlways -> return True withHandle :: Config -> (Handle -> IO a) -> IO a withHandle c action = case configOutputFile c of Left h -> action h Right path -> withFile path WriteMode action rerunAll :: Config -> Maybe FailureReport -> Summary -> Bool rerunAll _ Nothing _ = False rerunAll config (Just oldFailureReport) summary = configRerunAllOnSuccess config && configRerun config && isSuccess summary && (not . null) (failureReportPaths oldFailureReport) isDumb :: IO Bool isDumb = maybe False (== "dumb") <$> lookupEnv "TERM" -- | Summary of a test run. data Summary = Summary { summaryExamples :: Int , summaryFailures :: Int } deriving (Eq, Show) instance Monoid Summary where mempty = Summary 0 0 #if !MIN_VERSION_base(4,11,0) (Summary x1 x2) `mappend` (Summary y1 y2) = Summary (x1 + y1) (x2 + y2) #else instance Semigroup Summary where (Summary x1 x2) <> (Summary y1 y2) = Summary (x1 + y1) (x2 + y2) #endif randomizeForest :: Integer -> [Tree c a] -> [Tree c a] randomizeForest seed t = runST $ do ref <- newSTRef (mkStdGen $ fromIntegral seed) shuffleForest ref t