{-| Module : TLT Description : Testing in a monad transformer layer Copyright : (c) John Maraist, 2022 License : GPL3 Maintainer : haskell-tlt@maraist.org Stability : experimental Portability : POSIX TLT is a small unit test system oriented towards examining intermediate results of computations in monad transformers. It is intended to be lightweight for the programmer, and does not require tests to be specified in some sort of formal list of tests. Rather, tests are simply commands in a monad stack which includes the transformer layer @Test.TLT@. This Haddock page is the main piece of documentation; or see also the GitHub repository . -} {-# LANGUAGE RankNTypes #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE FunctionalDependencies #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} module Test.TLT ( -- * The TLT transformer TLT, tlt, MonadTLT, liftTLT, -- ** Session options reportAllTestResults, setExitAfterFailDisplay, -- * Writing tests Assertion, -- ** `TLT` commands (~:), (~::), (~::-), tltFail, inGroup, -- ** Assertions -- *** About the values of pure expressions of `Eq`- and `Ord`-type (@==), (@/=), (@<), (@>), (@<=), (@>=), -- *** About monadic computations returing `Eq`s and `Ord`s (@==-), (@/=-), (@<-), (@>-), (@<=-), (@>=-), -- *** About list values empty, nonempty, emptyP, nonemptyP, -- *** About `Maybe` values nothing, nothingP, assertFailed, assertSuccess, -- ** Building new assertions -- *** Unary assertions liftAssertionPure, assertionPtoM, liftAssertionM, -- *** Binary assertions liftAssertion2Pure, assertion2PtoM, liftAssertion2M ) where import Data.Maybe import Control.Exception import Control.Monad import Control.Monad.IO.Class import Control.Monad.ST.Trans import Control.Monad.Trans.Class -- import Control.Monad.Trans.Either import Control.Monad.Trans.Free import Control.Monad.Trans.Identity import Control.Monad.Trans.Maybe import Control.Monad.Trans.Reader import Control.Monad.Trans.Resource import Control.Monad.Trans.State.Strict import qualified Control.Monad.Trans.State.Lazy as SL import qualified Control.Monad.Trans.Writer.Lazy as WL import qualified Control.Monad.Trans.Writer.Strict as WS import System.Console.ANSI import System.Exit -- * Results of tests -- |Reasons why a test might fail. data TestFail = Asserted String -- ^ A failure arising from an `Assertion` which is not met. | Erred String -- ^ A failure associated with a call to a Haskell -- function triggering an error. formatFail :: TestFail -> String formatFail (Asserted s) = s formatFail (Erred s) = "Assertion raised exception: " ++ s -- |An assertion is a computation (typically in the monad wrapped by -- `TLT`) which returns a list of zero of more reasons for the failure -- of the assertion. A successful computation returns an empty list: -- no reasons for failure, hence success. type Assertion m = m [TestFail] -- |Hierarchical structure holding the result of running tests, -- possibly grouped into tests. data TestResult = Test String [TestFail] | Group String Int Int [TestResult] -- ^ The `Int`s are respectively the total number of -- tests executed, and total number of failures -- detected. -- |Return the number of failed tests reported in a `TestResult`. failCount :: TestResult -> Int failCount (Test _ []) = 0 failCount (Test _ _) = 1 failCount (Group _ _ n _) = n testCount :: TestResult -> Int testCount (Test _ _) = 1 testCount (Group _ n _ _) = n totalFailCount :: [TestResult] -> Int totalFailCount = foldr (+) 0 . map failCount totalTestCount :: [TestResult] -> Int totalTestCount = foldr (+) 0 . map testCount -- |Report the results of tests. report :: TLTopts -> [TestResult] -> IO () report (TLTopts showPasses exitAfterFailDisplay) trs = let fails = totalFailCount trs tests = totalTestCount trs in do report' "" trs if fails > 0 then do boldRed putStrLn $ "Found " ++ show fails ++ " error" ++ (if fails > 1 then "s" else "") ++ " in " ++ show tests ++ " tests; exiting" mediumBlack when exitAfterFailDisplay exitFailure else do boldGreen putStrLn $ show tests ++ " test" ++ (if tests > 1 then "s" else "") ++ " passing." mediumBlack where report' ind trs = forM_ trs $ \ tr -> when (failCount tr > 0 || showPasses) $ case tr of Test s r -> do putStr $ ind ++ "- " ++ s ++ ": " case r of [] -> do greenPass putStrLn "" x : [] -> do redFail putStrLn $ " " ++ formatFail x _ -> do redFail putStrLn ":" forM_ r $ \ f -> putStrLn $ ind ++ "- " ++ formatFail f Group s _ _ trs' -> do putStrLn $ ind ++ "- " ++ s ++ ":" report' (" " ++ ind) trs' boldBlack = setSGR [ SetColor Foreground Vivid Black, SetConsoleIntensity BoldIntensity ] boldRed = setSGR [ SetColor Foreground Vivid Red, SetConsoleIntensity BoldIntensity ] boldGreen = setSGR [ SetColor Foreground Vivid Green, SetConsoleIntensity BoldIntensity ] mediumRed = setSGR [ SetColor Foreground Vivid Red, SetConsoleIntensity NormalIntensity ] mediumGreen = setSGR [ SetColor Foreground Vivid Green, SetConsoleIntensity NormalIntensity ] mediumBlue = setSGR [ SetColor Foreground Vivid Blue, SetConsoleIntensity NormalIntensity ] mediumBlack = setSGR [ SetColor Foreground Vivid Black, SetConsoleIntensity NormalIntensity ] greenPass = do mediumBlue putStr "Pass" mediumBlack redFail = do boldRed putStr "FAIL" mediumBlack -- |Accumulator for test results, in the style of a simplified Huet's -- zipper which only ever adds to the end of the structure. data TRBuf = Buf TRBuf Int Int String [TestResult] | Top Int Int [TestResult] -- |Add a single test result to a `TRBuf`. addResult :: TRBuf -> TestResult -> TRBuf addResult (Top tc fc trs) tr = Top (tc + testCount tr) (fc + failCount tr) $ tr : trs addResult (Buf up tc fc s trs) tr = Buf up (tc + testCount tr) (fc + failCount tr) s $ tr : trs -- |Convert the topmost group of a bottom-up `TRBuf` into a completed -- top-down report about the group. currentGroup :: TRBuf -> TestResult currentGroup (Buf up tc fc s trs) = Group s tc fc (reverse trs) -- |Derive a new `TRBuf` corresponding to finishing the current group -- and continuing to accumulate results into its enclosure. popGroup :: TRBuf -> TRBuf popGroup trb@(Buf acc _ _ _ _) = addResult acc $ currentGroup trb -- |Convert a `TRBuf` into a list of top-down `TestResult`s. closeTRBuf :: TRBuf -> [TestResult] closeTRBuf (Top _ _ ts) = reverse ts closeTRBuf b = closeTRBuf $ popGroup b -- |Record of options which may be specified for running and reporting -- TLT tests. data TLTopts = TLTopts { optShowPasses :: Bool, optQuitAfterFailReport :: Bool } -- |Default initial options defaultOpts = TLTopts False True -- |Update the display of showing passes in a `TLTopts` record. withShowPasses :: TLTopts -> Bool -> TLTopts withShowPasses (TLTopts _ f) b = TLTopts b f -- |Update the display of showing passes in a `TLTopts` record. withExitAfterFail :: TLTopts -> Bool -> TLTopts withExitAfterFail (TLTopts p _) b = TLTopts p b -- |Synonym for the elements of the `TLT` state. type TLTstate = (TLTopts, TRBuf) -- |Monad transformer for TLT tests. This layer stores the results -- from tests as they are executed. newtype Monad m => TLT m r = TLT { unwrap :: StateT TLTstate m r } deriving (Functor, Applicative, Monad, MonadTrans, MonadIO) {- ------------------------------------------------------------ -} -- |Extending `TLT` operations across other monad transformers. For -- easiest and most flexible testing, declare the monad transformers -- of your application as instances of this class. class (Monad m, Monad n) => MonadTLT m n | m -> n where -- |Lift TLT operations within a monad transformer stack. Note that -- with enough transformer types included in this class, the -- @liftTLT@ function should usually be unnecessary: the commands in -- this module which actually configure testing, or specify a test, -- already @liftTLT@ their own result. So they will all act as -- top-level transformers in @MonadTLT@. liftTLT :: TLT n a -> m a instance Monad m => MonadTLT (TLT m) m where liftTLT = id instance (MonadTLT m n, Functor f) => MonadTLT (FreeT f m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (IdentityT m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (MaybeT m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (ReaderT r m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (ResourceT m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (StateT s m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (SL.StateT s m) n where liftTLT = lift . liftTLT instance MonadTLT m n => MonadTLT (STT s m) n where liftTLT = lift . liftTLT instance (MonadTLT m n, Monoid w) => MonadTLT (WL.WriterT w m) n where liftTLT = lift . liftTLT instance (MonadTLT m n, Monoid w) => MonadTLT (WS.WriterT w m) n where liftTLT = lift . liftTLT {- ------------------------------------------------------------ -} -- |Execute the tests specified in a `TLT` monad, and report the -- results. tlt :: MonadIO m => TLT m r -> m () tlt (TLT t) = do liftIO $ putStrLn "Running tests:" (_, (opts, resultsBuf)) <- runStateT t $ (defaultOpts, Top 0 0 []) liftIO $ report opts $ closeTRBuf resultsBuf -- |This function controls whether `tlt` will report only tests which -- fail, suppressing any display of tests which pass, or else report -- the results of all tests. The default is the former: the idea is -- that no news should be good news, with the programmer bothered only -- with problems which need fixing. reportAllTestResults :: MonadTLT m n => Bool -> m () reportAllTestResults b = liftTLT $ TLT $ do (opts, tr) <- get put $ (opts `withShowPasses` b, tr) -- |This function controls whether `tlt` will exit after displaying -- test results which include at least one failing test. By default, -- it will exit in this situation. The idea is that a test suite can -- be broken into parts when it makes sense to run the latter parts -- only when the former parts all pass. setExitAfterFailDisplay :: MonadTLT m n => Bool -> m () setExitAfterFailDisplay b = liftTLT $ TLT $ do (opts, tr) <- get put $ (opts `withExitAfterFail` b, tr) -- |Report a failure. Useful in pattern-matching cases which are -- entirely not expected. tltFail :: MonadTLT m n => String -> String -> m () desc `tltFail` detail = liftTLT $ TLT $ do (opts, before) <- get let after = addResult before $ Test desc [Asserted detail] put (opts, after) -- |Organize the tests in the given subcomputation as a separate group -- within the test results we will report. inGroup :: MonadTLT m n => String -> m a -> m a inGroup name group = do (opts, before) <- liftTLT $ TLT get liftTLT $ TLT $ put $ (opts, Buf before 0 0 name []) result <- group (opts', after) <- liftTLT $ TLT $ get liftTLT $ TLT $ put $ (opts', popGroup after) return result -- * Specifying individual tests infix 0 ~:, ~::, ~::- -- |Label and perform a test of an `Assertion`. -- -- ===== Example -- -- > test :: Monad m => TLT m () -- > test = do -- > "2 is 2 as result" ~: 2 @== return 2 -- This test passes. -- > "2 not 3" ~: 2 @/=- 3 -- This test fails. (~:) :: MonadTLT m n => String -> Assertion m -> m () s ~: a = do (opts, oldState) <- liftTLT $ TLT $ get assessment <- a liftTLT $ TLT $ put (opts, addResult oldState $ Test s assessment) -- |Label and perform a test of a (pure) boolean value. -- -- ===== Example -- -- > test :: Monad m => TLT m () -- > test = do -- > "True passes" ~::- return True -- This test passes. -- > "2 is 2 as single Bool" ~::- return (2 == 2) -- This test passes. -- > "2 is 3!?" ~::- myFn 4 "Hammer" -- Passes if myFn (which -- > -- must be monadic) -- > -- returns True. (~::-) :: MonadTLT m n => String -> Bool -> m () s ~::- b = do (opts, oldState) <- liftTLT $ TLT $ get liftTLT $ TLT $ put (opts, addResult oldState $ Test s $ if b then [] else [Asserted $ "Expected True but got False"]) -- |Label and perform a test of a boolean value returned by a -- computation in the wrapped monad @m@. -- -- ===== Example -- -- > test :: Monad m => TLT m () -- > test = do -- > "True passes" ~::- True -- This test passes. -- > "2 is 2 as single Bool" ~::- 2 == 2 -- This test passes. -- > "2 is 3!?" ~::- 2 == 2 -- This test fails. (~::) :: MonadTLT m n => String -> m Bool -> m () s ~:: bM = do b <- bM (opts, oldState) <- liftTLT $ TLT $ get liftTLT $ TLT $ put (opts, addResult oldState $ Test s $ if b then [] else [Asserted $ "Expected True but got False"]) infix 1 @==, @/=, @<, @>, @<=, @>= infix 1 @==-, @/=-, @<-, @>-, @<=-, @>=- -- |Transform a binary function on an expected and an actual value -- (plus a binary generator of a failure message) into an `Assertion` -- for a pure given actual value. -- -- ===== Example -- -- TLT's scalar-testing operators like @\@==-@ are defined with this -- function: -- -- > (@==-) :: (Monad m, Eq a, Show a) => a -> a -> Assertion m -- > (@==-) = liftAssertion2Pure (==) $ -- > \ exp actual -> "Expected " ++ show exp ++ " but got " ++ show actual -- -- The `(==)` operator tests equality, and the result here allows the -- assertion that a value should be exactly equal to a target. The -- second argument formats the detail reported when the assertion -- fails. liftAssertion2Pure :: (Monad m) => (a -> a -> Bool) -> (a -> a -> String) -> a -> a -> Assertion m liftAssertion2Pure tester explainer exp actual = return $ if (tester exp actual) then [] else [Asserted $ explainer exp actual] -- |Given an `Assertion` for two pure values (expected and actual), -- lift it to an `Assertion` expecting the actual value to be returned -- from a computation. -- -- ===== Examples -- -- The TLT assertion `(@==)` lifts `(@==-)` from expecting a pure -- actual result to expecting a computation returning a value to test. -- -- > (@==) :: (Monad m, Eq a, Show a) => a -> m a -> Assertion m -- > (@==) = assertion2PtoM (@==-) assertion2PtoM :: (Monad m) => (a -> a -> Assertion m) -> a -> m a -> Assertion m assertion2PtoM pa exp actualM = do actual <- actualM pa exp actual -- |Transform a binary function on expected and actual values (plus -- a generator of a failure message) into an `Assertion` where the -- actual value is to be returned from a subcomputation. liftAssertion2M :: (Monad m) => (a -> a -> Bool) -> (a -> a -> String) -> a -> m a -> Assertion m liftAssertion2M tester explainer exp actualM = let assertPure = liftAssertion2Pure tester explainer exp in do actual <- actualM assertPure actual -- |Assert that two values are equal. This assertion takes an -- expected and an actual /value/; see `(@==)` to compare the result -- of a /monadic computation/ to an expected value. -- -- ===== Examples -- -- > test :: Monad m => TLT m () -- > test = do -- > "Make sure that 2 is still equal to itself" ~: 2 @==- 2 -- > "Make sure that there are four lights" ~: 4 @==- length lights (@==-) :: (Monad m, Eq a, Show a) => a -> a -> Assertion m (@==-) = liftAssertion2Pure (==) $ \ exp actual -> "Expected " ++ show exp ++ " but got " ++ show actual -- |Assert that a calculated value is as expected. This assertion -- compare the result of a /monadic computation/ to an expected value; -- see `(@==-)` to compare an /actual value/ to the expected value. -- -- ===== Examples -- -- > test :: Monad m => TLT m () -- > test = do -- > "Make sure that 2 is still equal to itself" ~: 2 @== return 2 -- > "Make sure that there are four lights" ~: 4 @== countLights -- > -- where countLights :: m Int (@==) :: (Monad m, Eq a, Show a) => a -> m a -> Assertion m (@==) = assertion2PtoM (@==-) -- |Assert that two values are not equal. This assertion takes an -- expected and an actual /value/; see `(@/=)` to compare the result -- of a /monadic computation/ to an expected value. (@/=-) :: (Monad m, Eq a, Show a) => a -> a -> Assertion m (@/=-) = liftAssertion2Pure (/=) $ \ exp actual -> "Expected other than " ++ show exp ++ " but got " ++ show actual -- |Assert that a calculated value differs from some known value. -- This assertion compares the result of a /monadic computation/ to an -- expected value; see `(@/=-)` to compare an /actual value/ to the -- expected value. (@/=) :: (Monad m, Eq a, Show a) => a -> m a -> Assertion m (@/=) = assertion2PtoM (@/=-) -- |Assert that a given boundary is strictly less than some value. -- This assertion takes an expected and an actual /value/; see `(@<)` -- to compare the result of a /monadic computation/ to an expected -- value. (@<-) :: (Monad m, Ord a, Show a) => a -> a -> Assertion m (@<-) = liftAssertion2Pure (<) $ \ exp actual -> "Lower bound (open) is " ++ show exp ++ " but got " ++ show actual -- |Assert that a given, constant boundary is strictly less than some -- calculated value. This assertion compares the result of a /monadic -- computation/ to an expected value; see `(@<-)` to compare an -- /actual value/ to the expected value. (@<) :: (Monad m, Ord a, Show a) => a -> m a -> Assertion m (@<) = assertion2PtoM (@<-) -- |Assert that a given boundary is strictly less than some value. -- This assertion takes an expected and an actual /value/; see `(@>)` -- to compare the result of a /monadic computation/ to an expected -- value. (@>-) :: (Monad m, Ord a, Show a) => a -> a -> Assertion m (@>-) = liftAssertion2Pure (>) $ \ exp actual -> "Upper bound (open) is " ++ show exp ++ " but got " ++ show actual -- |Assert that a given, constant boundary is strictly less than some -- calculated value. This assertion compares the result of a /monadic -- computation/ to an expected value; see `(@>-)` to compare an -- /actual value/ to the expected value. (@>) :: (Monad m, Ord a, Show a) => a -> m a -> Assertion m (@>) = assertion2PtoM (@>-) -- |Assert that a given boundary is strictly less than some value. -- This assertion takes an expected and an actual /value/; see `(@<=)` -- to compare the result of a /monadic computation/ to an expected -- value. (@<=-) :: (Monad m, Ord a, Show a) => a -> a -> Assertion m (@<=-) = liftAssertion2Pure (<=) $ \ exp actual -> "Lower bound (closed) is " ++ show exp ++ " but got " ++ show actual -- |Assert that a given, constant boundary is strictly less than some -- calculated value. This assertion compares the result of a /monadic -- computation/ to an expected value; see `(@<=-)` to compare an -- /actual value/ to the expected value. (@<=) :: (Monad m, Ord a, Show a) => a -> m a -> Assertion m (@<=) = assertion2PtoM (@<=-) -- |Assert that a given boundary is strictly less than some value. -- This assertion takes an expected and an actual /value/; see `(@>=)` -- to compare the result of a /monadic computation/ to an expected -- value. (@>=-) :: (Monad m, Ord a, Show a) => a -> a -> Assertion m (@>=-) = liftAssertion2Pure (>=) $ \ exp actual -> "Upper bound (closed) is " ++ show exp ++ " but got " ++ show actual -- |Assert that a given, constant boundary is strictly less than some -- calculated value. This assertion compares the result of a /monadic -- computation/ to an expected value; see `(@>=-)` to compare an -- /actual value/ to the expected value. (@>=) :: (Monad m, Ord a, Show a) => a -> m a -> Assertion m (@>=) = assertion2PtoM (@>=-) -- |This assertion always fails with the given message. assertFailed :: Monad m => String -> Assertion m assertFailed msg = return [Asserted msg] -- |This assertion always succeeds. assertSuccess :: Monad m => Assertion m assertSuccess = return [] -- |Transform a unary function on a value (plus a generator of a -- failure message) into a unary function returning an `Assertion` for -- a pure given actual value. -- -- ===== Example -- -- The TLT assertion `emptyP` is built from the `Traversable` predicate -- `null` -- -- > emptyP :: (Monad m, Traversable t) => t a -> Assertion m -- > emptyP = liftAssertionPure null -- > (\ _ -> "Expected empty structure but got non-empty") liftAssertionPure :: (Monad m) => (a -> Bool) -> (a -> String) -> a -> Assertion m liftAssertionPure tester explainer actual = return $ if (tester actual) then [] else [Asserted $ explainer actual] -- |Given an `Assertion` for a pure (actual) value, lift it to an -- `Assertion` expecting the value to be returned from a computation. -- -- ===== Example -- -- The TLT assertion `empty` on monadic computations returning lists -- is defined in terms of the corresponging assertion on pure -- list-valued expressions. -- -- > empty :: (Monad m, Traversable t) => m (t a) -> Assertion m -- > empty = assertionPtoM emptyP assertionPtoM :: (Monad m) => (a -> Assertion m) -> m a -> Assertion m assertionPtoM pa actualM = do actual <- actualM pa actual -- |Transform a unary function on an actual value (plus a generator of -- a failure message) into an `Assertion` where the value is to be -- returned from a subcomputation. liftAssertionM :: (Monad m) => (a -> Bool) -> (a -> String) -> m a -> Assertion m liftAssertionM tester explainer actualM = let assertPure = liftAssertionPure tester explainer in do actual <- actualM assertPure actual -- |Assert that a pure traversable structure (such as a list) is -- empty. emptyP :: (Monad m, Traversable t) => t a -> Assertion m emptyP = liftAssertionPure null (\ _ -> "Expected empty structure but got non-empty") -- |Assert that a traversable structure (such as a list) returned from -- a computation is empty. empty :: (Monad m, Traversable t) => m (t a) -> Assertion m empty = assertionPtoM emptyP -- |Assert that a pure traversable structure (such as a list) is -- nonempty. nonemptyP :: (Monad m, Traversable t) => t a -> Assertion m nonemptyP = liftAssertionPure (not . null) (\ _ -> "Expected non-empty structure but got empty") -- |Assert that a traversable structure (such as a list) returned from -- a computation is non-empty. nonempty :: (Monad m, Traversable t) => m (t a) -> Assertion m nonempty = assertionPtoM nonemptyP -- |Assert that a `Maybe` value is `Nothing`. nothingP :: Monad m => Maybe a -> Assertion m nothingP = liftAssertionPure isNothing (\ _ -> "Expected empty Maybe value but got non-Nothing") -- |Assert that a `Maybe` result ofa computation is `Nothing`. nothing :: Monad m => m (Maybe a) -> Assertion m nothing = assertionPtoM nothingP