-- Hoogle documentation, generated by Haddock
-- See Hoogle, http://www.haskell.org/hoogle/
-- | Heterogeneous, type-safe automatic backpropagation in Haskell
--
-- See README.md
--
-- At the moment, this project is in pre-alpha, and is published and put
-- up on Hackage with 100% documentation coverage as a call for comments
-- and thoughts. See TODO.md section in the README for more
-- information on what's missing and potential avenues for contribution.
@package backprop
@version 0.0.3.0
-- | A poor substitute for the Control.Lens.Iso module in
-- lens, providing the Iso type synonym and some sample
-- useful Isos for usage with backprop, without incuring a
-- lens dependency.
--
-- If you also import lens, you should only use this module for the
-- Isos it exports, and not import the redefined Iso type
-- synonym or from / iso / review.
module Numeric.Backprop.Iso
-- | A family of isomorphisms. See Iso'.
type Iso s t a b = forall p f. (Profunctor p, Functor f) => p a (f b) -> p s (f t)
-- | An Iso' s a encodes an isomorphism between an
-- s and an a. It basically lets you go from s
-- -> a and back (from a -> s) while preserving
-- structure. You can basically imagine an Iso' s a to be
-- an (s -> a, a -> s) tuple.
--
-- You can get the "forward" direction of an Iso' with
-- view:
--
--
-- view :: Iso'' s a -> (s -> a)
--
--
-- And the "backwards" direction with review:
--
--
-- review :: Iso'' s a -> (a -> s)
--
--
-- You can construct an Iso' using iso, giving the forward
-- and backwards functions:
--
--
-- >>> myIso :: Iso' (Identity a) a
-- myIso = iso runIdentity Identity
--
-- >>> view myIso (Identity "hello")
-- "hello"
--
-- >>> review myIso "hello"
-- Identity "hello"
--
--
-- One powerful thing about Iso's is that they're
-- composable using .:
--
--
-- (.) :: Iso' c b -> Iso' b a -> Iso' c a
--
--
-- This is basically provided here so that this package doesn't incurr a
-- lens dependecy, but if you already depend on lens, you
-- should use the version from Control.Lens.Iso instead.
type Iso' s a = Iso s s a a
-- | Construct an Iso by giving the "forward" and "backward"
-- direction functions:
--
--
-- >>> myIso :: Iso' (Identity a) a
-- myIso = iso runIdentity Identity
--
-- >>> view myIso (Identity "hello")
-- "hello"
--
-- >>> review myIso "hello"
-- Identity "hello"
--
--
-- This is basically provided here so that this package doesn't incurr a
-- lens dependecy, but if you already depend on lens, you
-- should use the version from Control.Lens.Iso instead.
iso :: (s -> a) -> (b -> t) -> Iso s t a b
-- | Reverse an Iso'. The forward function becomes the backwards
-- function, and the backwards function becomes the forward function.
--
-- This is basically provided here so that this package doesn't incurr a
-- lens dependecy, but if you already depend on lens, you
-- should use the version from Control.Lens.Review instead.
from :: Iso' s a -> Iso' a s
-- | Get the "reverse" direction function from an Iso.
--
-- This is basically provided here so that this package doesn't incurr a
-- lens dependecy, but if you already depend on lens, you
-- should use the version from Control.Lens.Review instead.
review :: Iso s t a b -> b -> t
-- | view is a synonym for (^.):
--
--
-- >>> view _1 (1, 2)
-- 1
--
--
-- The reason it's not in Lens.Micro is that view in lens
-- has a more general signature:
--
--
-- view :: MonadReader s m => Getting a s a -> m a
--
--
-- So, you would be able to use this view with functions, but not
-- in various reader monads. For most people this shouldn't be an issue;
-- if it is for you, use view from microlens-mtl.
view :: Getting a s a -> s -> a
-- | A useful Iso between two types with the same runtime
-- representation.
coerced :: Coercible s a => Iso' s a
-- | An Iso between a type that is a product type, and a tuple that
-- contains all of its components. Uses Generics.SOP and the
-- Generic typeclass.
--
--
-- >>> import qualified Generics.SOP as SOP
--
-- >>> data Foo = A Int Bool deriving Generic
--
-- >>> instance SOP.Generic Foo
--
-- >>> view gTuple (A 10 True)
-- 10 ::< True ::< Ø
--
-- >>> review gTuple (15 ::< False ::< Ø)
-- A 15 False
--
gTuple :: (Generic a, Code a ~ '[as]) => Iso' a (Tuple as)
-- | An Iso between a sum type whose constructors are products, and
-- a sum (Sum) of products (Tuple). Uses
-- Generics.SOP and the Generic typeclass.
--
--
-- >>> import qualified Generics.SOP as SOP
--
-- >>> data Bar = A Int Bool | B String Double
--
-- >>> instance SOP.Generic Bar
--
-- >>> 'view' 'gSOP' (A 10 True)
-- 'InL' (10 ::< True ::< Ø)
--
-- >>> 'view' 'gSOP' (B "hello" 3.4)
-- 'InR' ('InL' ("hello" ::< 3.4 ::< Ø))
--
-- >>> 'review' 'gTuple' ('InL' (15 ::< False ::< Ø))
-- A 15 False
--
-- >>> 'review' 'gTuple' ('InR' ('InL' ("bye" ::< 9.8 ::< Ø)))
-- B "bye" 9.8
--
gSOP :: Generic a => Iso' a (Sum Tuple (Code a))
-- | An iso between a single-type Sum and the single type.
sum1 :: Iso' (Sum f '[a]) (f a)
-- | An iso between a single type and a single-type Sum.
resum1 :: Iso' (f a) (Sum f '[a])
data Prod k (f :: k -> *) (a :: [k]) :: forall k. (k -> *) -> [k] -> *
[Ø] :: Prod k f ([] k)
[:<] :: Prod k f ((:) k a1 as)
-- | A Prod of simple Haskell types.
type Tuple = Prod * I
data Sum k (f :: k -> *) (a :: [k]) :: forall k. (k -> *) -> [k] -> *
[InL] :: Sum k f ((:) k a1 as)
[InR] :: Sum k f ((:) k a1 as)
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Provides the Op (and OpM) type and combinators, which
-- represent differentiable functions/operations on values, and are used
-- by the library to perform back-propagation.
--
-- Note that Op is a subset or subtype of
-- OpM, and so, any function that expects an OpM m as
-- a (or an OpB s as a) can be given an
-- Op as a and it'll work just fine.
module Numeric.Backprop.Op
-- | An Op as a describes a differentiable function from
-- as to a.
--
-- For example, a value of type
--
--
-- Op '[Int, Bool] Double
--
--
-- is a function from an Int and a Bool, returning a
-- Double. It can be differentiated to give a gradient of
-- an Int and a Bool if given a total derivative for the
-- Double. If we call Bool <math>, then,
-- mathematically, it is akin to a:
--
-- <math>
--
-- See runOp, gradOp, and gradOpWith for examples on
-- how to run it, and Op for instructions on creating it.
--
-- This type is abstracted over using the pattern synonym with
-- constructor Op, so you can create one from scratch with it.
-- However, it's simplest to create it using op2', op1',
-- op2', and op3' helper smart constructors And, if your
-- function is a numeric function, they can even be created automatically
-- using op1, op2, op3, and opN with a little
-- help from Numeric.AD from the ad library.
--
-- Note that this type is a subset or subtype of OpM
-- (and also of OpB). So, if a function ever expects an
-- OpM m as a (or a OpB), you can always provide
-- an Op as a instead.
--
-- Many functions in this library will expect an OpM m as
-- a (or an OpB s as a), and in all of these cases,
-- you can provide an Op as a.
type Op as a = forall m. Monad m => OpM m as a
-- | Construct an Op by giving a function creating the result, and
-- also a continuation on how to create the gradient, given the total
-- derivative of a.
--
-- See the module documentation for Numeric.Backprop.Op for more
-- details on the function that this constructor and OpM expect.
-- | An OpM m as a represents a differentiable
-- (monadic) function from as to a, in the context of a
-- Monad m.
--
-- For example, an
--
--
-- OpM IO '[Int, Bool] Double
--
--
-- would be a function that takes an Int and a Bool and
-- returns a Double (in IO). It can be differentiated to
-- give a gradient of an Int and a Bool (also in
-- IO) if given the total derivative for the Double.
--
-- Note that an OpM is a superclass of Op, so any
-- function that expects an OpM m as a can also accept an
-- Op as a.
--
-- See runOpM, gradOpM, and gradOpWithM for examples
-- on how to run it.
newtype OpM m as a
-- | Construct an OpM by giving a (monadic) function creating the
-- result, and also a continuation on how to create the gradient, given
-- the total derivative of a.
--
-- See the module documentation for Numeric.Backprop.Op for more
-- details on the function that this constructor and Op expect.
OpM :: (Tuple as -> m (a, Maybe a -> m (Tuple as))) -> OpM m as a
data Prod k (f :: k -> *) (a :: [k]) :: forall k. (k -> *) -> [k] -> *
[Ø] :: Prod k f ([] k)
[:<] :: Prod k f ((:) k a1 as)
-- | A Prod of simple Haskell types.
type Tuple = Prod * I
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Run the function that an Op encodes, to get the result.
--
--
-- >>> runOp (op2 (*)) (3 ::< 5 ::< Ø)
-- 15
--
runOp :: Op as a -> Tuple as -> a
-- | Run the function that an Op encodes, and get the gradient of
-- the output with respect to the inputs.
--
--
-- >>> gradOp (op2 (*)) (3 ::< 5 ::< Ø)
-- 5 ::< 3 ::< Ø
-- -- the gradient of x*y is (y, x)
--
gradOp :: Op as a -> Tuple as -> Tuple as
-- | Run the function that an Op encodes, to get the resulting
-- output and also its gradient with respect to the inputs.
--
--
-- >>> gradOpM' (op2 (*)) (3 ::< 5 ::< Ø) :: IO (Int, Tuple '[Int, Int])
-- (15, 5 ::< 3 ::< Ø)
--
gradOp' :: Op as a -> Tuple as -> (a, Tuple as)
-- | Run the function that an Op encodes, and get the gradient of a
-- "final result" with respect to the inputs, given the total derivative
-- of the output with the final result.
--
-- See gradOp and the module documentaiton for
-- Numeric.Backprop.Op for more information.
gradOpWith :: Op as a -> Tuple as -> a -> Tuple as
-- | A combination of gradOp and gradOpWith. The third
-- argument is (optionally) the total derivative the result. Give
-- Nothing and it is assumed that the result is the final result
-- (and the total derivative is 1), and this behaves the same as
-- gradOp. Give Just d and it uses the d
-- as the total derivative of the result, and this behaves like
-- gradOpWith.
--
-- See gradOp and the module documentaiton for
-- Numeric.Backprop.Op for more information.
gradOpWith' :: Op as a -> Tuple as -> Maybe a -> Tuple as
-- | A combination of runOp and gradOpWith'. Given an
-- Op and inputs, returns the result of the Op and a
-- continuation that gives its gradient.
--
-- The continuation takes the total derivative of the result as input.
-- See documenation for gradOpWith' and module documentation for
-- Numeric.Backprop.Op for more information.
runOp' :: Op as a -> Tuple as -> (a, Maybe a -> Tuple as)
-- | The monadic version of runOp, for OpMs.
--
--
-- >>> runOpM (op2 (*)) (3 ::< 5 ::< Ø) :: IO Int
-- 15
--
runOpM :: Functor m => OpM m as a -> Tuple as -> m a
-- | The monadic version of gradOp, for OpMs.
gradOpM :: Monad m => OpM m as a -> Tuple as -> m (Tuple as)
-- | The monadic version of gradOp', for OpMs.
gradOpM' :: Monad m => OpM m as a -> Tuple as -> m (a, Tuple as)
-- | The monadic version of gradOpWith, for OpMs.
gradOpWithM :: Monad m => OpM m as a -> Tuple as -> a -> m (Tuple as)
-- | The monadic version of gradOpWith', for OpMs.
gradOpWithM' :: Monad m => OpM m as a -> Tuple as -> Maybe a -> m (Tuple as)
-- | A combination of runOpM and gradOpWithM'. Given an
-- OpM and inputs, returns the result of the OpM and a
-- continuation that gives its gradient.
--
-- The continuation takes the total derivative of the result as input.
-- See documenation for gradOpWithM' and module documentation for
-- Numeric.Backprop.Op for more information.
runOpM' :: OpM m as a -> Tuple as -> m (a, Maybe a -> m (Tuple as))
-- | Compose OpMs together, similar to .. But, because all
-- OpMs are <math>, this is more like sequence for
-- functions, or liftAN.
--
-- That is, given an OpM m as b1, an OpM m as
-- b2, and an OpM m as b3, it can compose them with
-- an OpM m '[b1,b2,b3] c to create an OpM m
-- as c.
composeOp :: (Monad m, Every Num as, Known Length as) => Prod (OpM m as) bs -> OpM m bs c -> OpM m as c
-- | Convenient wrapper over composeOp for the case where the second
-- function only takes one input, so the two OpMs can be directly
-- piped together, like for ..
composeOp1 :: (Monad m, Every Num as, Known Length as) => OpM m as b -> OpM m '[b] c -> OpM m as c
-- | Convenient infix synonym for (flipped) composeOp1. Meant to be
-- used just like .:
--
--
-- op1 negate :: Op '[a] a
-- op2 (+) :: Op '[a,a] a
--
-- op1 negate ~. op2 (+) :: Op '[a, a] a
--
(~.) :: (Monad m, Known Length as, Every Num as) => OpM m '[b] c -> OpM m as b -> OpM m as c
infixr 9 ~.
-- | A version of composeOp taking explicit Length,
-- indicating the number of inputs expected and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the the expected input tuple. If you ever actually
-- explicitly write down as as a list of types, you should be
-- able to just use composeOp.
composeOp' :: forall m as bs c. (Monad m, Every Num as) => Length as -> Prod (OpM m as) bs -> OpM m bs c -> OpM m as c
-- | A version of composeOp1 taking explicit Length,
-- indicating the number of inputs expected and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the the expected input tuple. If you ever actually
-- explicitly write down as as a list of types, you should be
-- able to just use composeOp1.
composeOp1' :: (Monad m, Every Num as) => Length as -> OpM m as b -> OpM m '[b] c -> OpM m as c
-- | Create an Op that takes no inputs and always returns the given
-- value.
--
-- There is no gradient, of course (using gradOp will give you an
-- empty tuple), because there is no input to have a gradient of.
--
--
-- >>> gradOp' (op0 10) Ø
-- (10, Ø)
--
--
-- For a constant Op that takes input and ignores it, see
-- opConst and opConst'.
--
-- Note that because this returns an Op, it can be used with any
-- function that expects an OpM or OpB, as well.
op0 :: a -> Op '[] a
-- | An Op that ignores all of its inputs and returns a given
-- constant value.
--
--
-- >>> gradOp' (opConst 10) (1 ::< 2 ::< 3 ::< Ø)
-- (10, 0 ::< 0 ::< 0 ::< Ø)
--
opConst :: forall as a. (Every Num as, Known Length as) => a -> Op as a
-- | A version of opConst taking explicit Length, indicating
-- the number of inputs and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the the expected input tuple. If you ever actually
-- explicitly write down as as a list of types, you should be
-- able to just use opConst.
opConst' :: forall as a. Every Num as => Length as -> a -> Op as a
-- | Automatically create an Op of a numerical function taking one
-- argument. Uses diff, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op1 (recip . negate)) (5 ::< Ø)
-- (-0.2, 0.04 ::< Ø)
--
op1 :: Num a => (forall s. AD s (Forward a) -> AD s (Forward a)) -> Op '[a] a
-- | Automatically create an Op of a numerical function taking two
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op2 (\x y -> x * sqrt y)) (3 ::< 4 ::< Ø)
-- (6.0, 2.0 ::< 0.75 ::< Ø)
--
op2 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a) -> Op '[a, a] a
-- | Automatically create an Op of a numerical function taking three
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op3 (\x y z -> (x * sqrt y)**z)) (3 ::< 4 ::< 2 ::< Ø)
-- (36.0, 24.0 ::< 9.0 ::< 64.503 ::< Ø)
--
op3 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a -> Reverse s a) -> Op '[a, a, a] a
-- | Automatically create an Op of a numerical function taking
-- multiple arguments. Uses grad, and so can take any numerical
-- function polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (opN (\(x :+ y :+ Ø) -> x * sqrt y)) (3 ::< 4 ::< Ø)
-- (6.0, 2.0 ::< 0.75 ::< Ø)
--
opN :: (Num a, Known Nat n) => (forall s. Reifies s Tape => Vec n (Reverse s a) -> Reverse s a) -> Op (Replicate n a) a
-- | Replicate n a is a list of as repeated
-- n times.
--
--
-- >>> :kind! Replicate N3 Int
-- '[Int, Int, Int]
--
-- >>> :kind! Replicate N5 Double
-- '[Double, Double, Double, Double, Double]
--
-- | Create an Op of a function taking one input, by giving its
-- explicit derivative. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the derivative <math>, it would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that squares its input:
--
--
-- square :: Num a => Op '[a] a
-- square = op1' $ \x -> (x*x, \case Nothing -> 2 * x
-- Just d -> 2 * d * x
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, single-input Ops can be generated
-- automatically using op1.
op1' :: (a -> (b, Maybe b -> a)) -> Op '[a] b
-- | Create an Op of a function taking two inputs, by giving its
-- explicit gradient. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the gradient <math> would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that multiplies its inputs:
--
--
-- mul :: Num a => Op '[a, a] a
-- mul = op2' $ \x y -> (x*y, \case Nothing -> (y , x )
-- Just d -> (d*y, x*d)
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, two-input Ops can be generated
-- automatically using op2.
op2' :: (a -> b -> (c, Maybe c -> (a, b))) -> Op '[a, b] c
-- | Create an Op of a function taking three inputs, by giving its
-- explicit gradient. See documentation for op2' for more details.
op3' :: (a -> b -> c -> (d, Maybe d -> (a, b, c))) -> Op '[a, b, c] d
-- | An Op that coerces an item into another item whose type has the
-- same runtime representation. Requires the input to be an instance of
-- Num.
--
--
-- >>> gradOp' opCoerce (Identity 5) :: (Int, Identity Int)
-- (5, Identity 1)
--
--
--
-- opCoerce = opIso coerced
--
opCoerce :: Num a => Coercible a b => Op '[a] b
-- | An Op that takes as and returns exactly the input
-- tuple.
--
--
-- >>> gradOp' opTup (1 ::< 2 ::< 3 ::< Ø)
-- (1 ::< 2 ::< 3 ::< Ø, 1 ::< 1 ::< 1 ::< Ø)
--
opTup :: (Every Num as, Known Length as) => Length as -> Op as (Tuple as)
-- | An Op that runs the input value through the isomorphism encoded
-- in the Iso. Requires the input to be an instance of Num.
--
-- Warning: This is unsafe! It assumes that the isomorphisms themselves
-- have derivative 1, so will break for things like
-- exponentiating. Basically, don't use this for any "numeric"
-- isomorphisms.
opIso :: Num a => Iso' a b -> Op '[a] b
-- | A version of opTup taking explicit Length, indicating
-- the number of inputs expected and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the the expected input tuple. If you ever actually
-- explicitly write down as as a list of types, you should be
-- able to just use opTup.
opTup' :: Every Num as => Length as -> Op as (Tuple as)
-- | Construct a two element Prod. Since the precedence of (:>) is
-- higher than (:<), we can conveniently write lists like:
--
--
-- >>> a :< b :> c
--
--
-- Which is identical to:
--
--
-- >>> a :< b :< c :< Ø
--
infix 6 :>
-- | Build a singleton Prod.
only :: f a -> Prod k f ((:) k a ([] k))
head' :: Prod k f ((:<) k a as) -> f a
-- | Cons onto a Tuple.
infixr 5 ::<
-- | Singleton Tuple.
only_ :: a -> Tuple ((:) * a ([] *))
-- | Optimized version of op1 (+).
(+.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 (-).
(-.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 (*).
(*.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 negate.
negateOp :: Num a => Op '[a] a
-- | Optimized version of op1 abs.
absOp :: Num a => Op '[a] a
-- | Optimized version of op1 signum.
signumOp :: Num a => Op '[a] a
-- | Optimized version of op1 (/).
(/.) :: Fractional a => Op '[a, a] a
-- | Optimized version of op1 recip.
recipOp :: Fractional a => Op '[a] a
-- | Optimized version of op1 exp.
expOp :: Floating a => Op '[a] a
-- | Optimized version of op1 log.
logOp :: Floating a => Op '[a] a
-- | Optimized version of op1 sqrt.
sqrtOp :: Floating a => Op '[a] a
-- | Optimized version of op1 (**).
(**.) :: Floating a => Op '[a, a] a
-- | Optimized version of op2 logBase.
logBaseOp :: Floating a => Op '[a, a] a
-- | Optimized version of op1 sin.
sinOp :: Floating a => Op '[a] a
-- | Optimized version of op1 cos.
cosOp :: Floating a => Op '[a] a
-- | Optimized version of op1 tan.
tanOp :: Floating a => Op '[a] a
-- | Optimized version of op1 asin.
asinOp :: Floating a => Op '[a] a
-- | Optimized version of op1 acos.
acosOp :: Floating a => Op '[a] a
-- | Optimized version of op1 atan.
atanOp :: Floating a => Op '[a] a
-- | Optimized version of op1 sinh.
sinhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 cosh.
coshOp :: Floating a => Op '[a] a
-- | Optimized version of op1 tanh.
tanhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 asinh.
asinhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 acosh.
acoshOp :: Floating a => Op '[a] a
-- | Optimized version of op1 atanh.
atanhOp :: Floating a => Op '[a] a
instance (GHC.Base.Monad m, Type.Class.Known.Known Data.Type.Length.Length as, Data.Type.Index.Every GHC.Num.Num as, GHC.Num.Num a) => GHC.Num.Num (Numeric.Backprop.Op.OpM m as a)
instance (GHC.Base.Monad m, Type.Class.Known.Known Data.Type.Length.Length as, Data.Type.Index.Every GHC.Real.Fractional as, Data.Type.Index.Every GHC.Num.Num as, GHC.Real.Fractional a) => GHC.Real.Fractional (Numeric.Backprop.Op.OpM m as a)
instance (GHC.Base.Monad m, Type.Class.Known.Known Data.Type.Length.Length as, Data.Type.Index.Every GHC.Float.Floating as, Data.Type.Index.Every GHC.Real.Fractional as, Data.Type.Index.Every GHC.Num.Num as, GHC.Float.Floating a) => GHC.Float.Floating (Numeric.Backprop.Op.OpM m as a)
-- | Provides the BP monad and the BVar type; after
-- manipulating BVars (inputs to your function) to produce a
-- result, the library tracks internal data dependencies, which are used
-- to perform back-propagation (reverse-mode automatic differentiation)
-- to calculate the gradient of the output with respect to the inputs.
--
-- Similar to automatic differentiation from the ad library and
-- Numeric.AD.Mode.Reverse, except for a few key differences:
--
--
-- - Most importantly, this library implements heterogeneous
-- back-propagation, so you can manipulate values of different types
-- (like different matrix and vector types, and product and sum types).
-- This is essential for things like back-propagation for neural
-- networks.
-- - This module allows you to explicitly build your data
-- dependency graph if you wish, which allows the library to perform
-- optimizations and reduce extra allocation, which may or may not
-- provide advantages over Numeric.AD.Mode.Reverse's
-- unsafePerformIO-based implicit graph building.
--
--
-- See the README for more information and links to demonstrations
-- and tutorials. If you want to plunge right in, you can also look
-- directly at the main types, BP, BPOp, BVar,
-- Op, and the main functions, backprop and opVar.
--
-- Note that every type involved has to be an instance of Num.
-- This is because gradients all need to be "summable" (which is
-- implemented using sum and +), and we also need to able
-- to generate gradients of '1' and '0'.
module Numeric.Backprop
-- | A Monad allowing you to explicitly build hetereogeneous data
-- dependency graphs and that the library can perform back-propagation
-- on.
--
-- A BP s rs a is a BP action that uses an
-- environment of rs returning a a. When "run", it will
-- compute a gradient that is a tuple of rs. (The phantom
-- parameter s is used to ensure that any BVars aren't
-- leaked out of the monad)
--
-- Note that you can only "run" a BP s rs that produces a
-- BVar -- that is, things of the form
--
--
-- BP s rs (BVar s rs a)
--
--
-- The above is a BP action that returns a BVar containing
-- an a. When this is run, it'll produce a result of type
-- a and a gradient of that is a tuple of rs. (This
-- form has a type synonym, BPOp, for convenience)
--
-- For example, a BP s '[ Int, Double, Double ] is a
-- monad that represents a computation with an Int, Double,
-- and Double as inputs. And, if you ran a
--
--
-- BP s '[ Int, Double, Double ] (BVar s '[ Int, Double, Double ] Double)
--
--
-- Or, using the BPOp type synonym:
--
--
-- BPOp s '[ Int, Double, Double ] Double
--
--
-- with backprop or gradBPOp, it'll return a gradient on
-- the inputs (Int, Double, and Double) and produce
-- a value of type Double.
--
-- Now, one powerful thing about this type is that a BP is itself
-- an Op (or more precisely, an OpB, which is a subtype of
-- OpM). So, once you create your fancy BP computation, you
-- can transform it into an OpM using bpOp.
data BP s rs a
-- | A handy type synonym representing a BP action that returns a
-- BVar. This is handy because this is the form of BP
-- actions that backprop and gradBPOp (etc.) expects.
--
-- A value of type:
--
--
-- BPOp s rs a
--
--
-- is an action that takes an input environment of rs and
-- produces a BVar containing a value of type a. Because
-- it returns a BVar, the library can track the data dependencies
-- between the BVar and the input environment and perform
-- back-propagation.
--
-- See documentation for BP for an explanation of the phantom type
-- parameter s.
type BPOp s rs a = BP s rs (BVar s rs a)
-- | An "implicit" operation on BVars that can be backpropagated. A
-- value of type:
--
--
-- BPOpI s rs a
--
--
-- takes a bunch of BVars containg rs and uses them to
-- (purely) produce a BVar containing an a.
--
--
-- foo :: BPOpI s '[ Double, Double ] Double
-- foo (x :< y :< Ø) = x + sqrt y
--
--
-- If you are exclusively doing implicit back-propagation by combining
-- BVars and using BPOpIs, you are probably better off just
-- importing Numeric.Backprop.Implicit, which provides better
-- tools. This type synonym exists in Numeric.Backprop just for
-- the implicitly function, which can convert "implicit" backprop
-- functions like a BPOpI s rs a into an "explicit" graph
-- backprop function, a BPOp s rs a.
type BPOpI s rs a = Prod (BVar s rs) rs -> BVar s rs a
-- | The basic unit of manipulation inside BP (or inside an
-- implicit-graph backprop function). Instead of directly working with
-- values, you work with BVars contating those values. When you
-- work with a BVar, the backprop library can keep track of
-- what values refer to which other values, and so can perform
-- back-propagation to compute gradients.
--
-- A BVar s rs a refers to a value of type a,
-- with an environment of values of the types rs. The phantom
-- parameter s is used to ensure that stray BVars don't
-- leak outside of the backprop process.
--
-- (That is, if you're using implicit backprop, it ensures that you
-- interact with BVars in a polymorphic way. And, if you're using
-- explicit backprop, it ensures that a BVar s rs a never
-- leaves the BP s rs that it was created in.)
--
-- BVars have Num, Fractional, Floating, etc.
-- instances, so they can be manipulated using polymorphic functions and
-- numeric functions in Haskell. You can add them, subtract them, etc.,
-- in "implicit" backprop style.
--
-- (However, note that if you directly manipulate BVars using
-- those instances or using liftB, it delays evaluation, so every
-- usage site has to re-compute the result/create a new node. If you want
-- to re-use a BVar you created using + or - or
-- liftB, use bindVar to force it first. See documentation
-- for bindVar for more details.)
data BVar :: Type -> [Type] -> Type -> Type
-- | An Op as a describes a differentiable function from
-- as to a.
--
-- For example, a value of type
--
--
-- Op '[Int, Bool] Double
--
--
-- is a function from an Int and a Bool, returning a
-- Double. It can be differentiated to give a gradient of
-- an Int and a Bool if given a total derivative for the
-- Double. If we call Bool <math>, then,
-- mathematically, it is akin to a:
--
-- <math>
--
-- See runOp, gradOp, and gradOpWith for examples on
-- how to run it, and Op for instructions on creating it.
--
-- This type is abstracted over using the pattern synonym with
-- constructor Op, so you can create one from scratch with it.
-- However, it's simplest to create it using op2', op1',
-- op2', and op3' helper smart constructors And, if your
-- function is a numeric function, they can even be created automatically
-- using op1, op2, op3, and opN with a little
-- help from Numeric.AD from the ad library.
--
-- Note that this type is a subset or subtype of OpM
-- (and also of OpB). So, if a function ever expects an
-- OpM m as a (or a OpB), you can always provide
-- an Op as a instead.
--
-- Many functions in this library will expect an OpM m as
-- a (or an OpB s as a), and in all of these cases,
-- you can provide an Op as a.
type Op as a = forall m. Monad m => OpM m as a
-- | A subclass of OpM (and superclass of Op), representing
-- Ops that the backprop library uses to perform
-- backpropation.
--
-- An
--
--
-- OpB s rs a
--
--
-- represents a differentiable function that takes a tuple of rs
-- and produces an a a, which can be run on BVar
-- ss and also inside BP ss. For example, an
-- OpB s '[ Int, Double ] Bool takes an Int and a
-- Double and produces a Bool, and does it in a
-- differentiable way.
--
-- OpB is a superset of Op, so, if you see any
-- function that expects an OpB (like opVar' and ~$,
-- for example), you can give them an Op, as well.
--
-- You can think of OpB as a superclass/parent class of Op
-- in this sense, and of Op as a subclass of OpB.
type OpB s as a = OpM (ST s) as a
data Prod k (f :: k -> *) (a :: [k]) :: forall k. (k -> *) -> [k] -> *
[Ø] :: Prod k f ([] k)
[:<] :: Prod k f ((:) k a1 as)
-- | A Prod of simple Haskell types.
type Tuple = Prod * I
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Perform back-propagation on the given BPOp. Returns the result
-- of the operation it represents, as well as the gradient of the result
-- with respect to its inputs. See module header for
-- Numeric.Backprop and package documentation for examples and
-- usages.
backprop :: Every Num rs => (forall s. BPOp s rs a) -> Tuple rs -> (a, Tuple rs)
-- | Simply run the BPOp on an input tuple, getting the result
-- without bothering with the gradient or with back-propagation.
evalBPOp :: (forall s. BPOp s rs a) -> Tuple rs -> a
-- | Run the BPOp on an input tuple and return the gradient of the
-- result with respect to the input tuple.
gradBPOp :: Every Num rs => (forall s. BPOp s rs a) -> Tuple rs -> Tuple rs
-- | Runs a continuation on a Prod of all of the input BVars.
--
-- Handy for bringing the environment into scope and doing stuff with it:
--
--
-- foo :: BPOp '[Double, Int] a
-- foo = withInps $ \(x :< y :< Ø) -> do
-- -- do stuff with inputs
--
--
-- Looks kinda like foo (x :< y :< Ø) = -- ..., don't it?
--
-- Note that the above is the same as
--
--
-- foo :: BPOp '[Double, Int] a
-- foo = do
-- case inpVars of
-- x ::< Ø - do
-- -- do stuff with inputs
--
--
-- But just a little nicer!
withInps :: Known Length rs => (Prod (BVar s rs) rs -> BP s rs a) -> BP s rs a
-- | Convert a BPOpI into a BPOp. That is, convert a function
-- on a bundle of BVars (generating an implicit graph) into a
-- fully fledged BPOp that you can run backprop on. See
-- BPOpI for more information.
--
-- If you are going to write exclusively using implicit BVar
-- operations, it might be more convenient to use
-- Numeric.Backprop.Implicit instead, which is geared around that
-- use case.
implicitly :: (Known Length rs, Num a) => BPOpI s rs a -> BPOp s rs a
-- | A version of withInps taking explicit Length, indicating
-- the number of inputs required and their types.
--
-- Mostly useful for rare "extremely polymorphic" situations, where GHC
-- can't infer the type and length of the list of inputs. If you ever
-- actually explicitly write down rs as a list of types, you
-- should be able to just use withInps.
withInps' :: Length rs -> (Prod (BVar s rs) rs -> BP s rs a) -> BP s rs a
-- | A version of implicitly taking explicit Length,
-- indicating the number of inputs required and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the list of inputs. If you ever actually explicitly write
-- down rs as a list of types, you should be able to just use
-- implicitly.
implicitly' :: Num a => Length rs -> BPOpI s rs a -> BPOp s rs a
-- | Create a BVar that represents just a specific value, that
-- doesn't depend on any other BVars.
constVar :: a -> BVar s rs a
-- | Create a BVar given an index into the input environment. For an
-- example,
--
--
-- inpVar IZ
--
--
-- would refer to the first input variable (the Int in a
-- BP s '[Int, Bool]), and
--
--
-- inpVar (IS IZ)
--
--
-- Would refer to the second input variable (the Bool in a
-- BP s '[Int, Bool])
--
-- Typically, there shouldn't be any reason to use inpVar
-- directly. It's cleaner to get all of your input BVars together
-- using withInps or inpVars.
inpVar :: Index rs a -> BVar s rs a
-- | Get a Prod (tupling) of BVars for all of the input
-- environment (rs) of the BP s rs
--
-- For example, if your BP has an Int and Double in
-- its input environment (a BP s '[Int, Double]), this
-- would return a BVar pointing to the Int and a
-- BVar pointing to the Double.
--
--
-- case (inpVars :: Prod (BVar s '[Int, Double]) '[Int, Double]) of
-- x :< y :< Ø -> do
-- -- the first item, x, is a var to the input Int
-- -- x :: BVar s '[Int, Double] Int
-- -- the second item, y, is a var to the input Double
-- -- y :: BVar s '[Int, Double] Double
--
inpVars :: Known Length rs => Prod (BVar s rs) rs
-- | Turn a BPOp into an OpB. Basically converts a BP
-- taking an rs and producing an a into an Op
-- taking an rs and returning an a, with all of the
-- powers and utility of an Op, including all of its
-- gradient-finding glory.
--
-- Really just reveals the fact that any BPOp s rs a is
-- itself an Op, an OpB s rs a, which makes it a
-- differentiable function.
--
-- Handy because an OpB can be used with almost all of the
-- Op-related functions in this moduel, including opVar,
-- ~$, etc.
bpOp :: Every Num rs => BPOp s rs a -> OpB s rs a
-- | Concretizes a delayed BVar. If you build up a BVar using
-- numeric functions like + or * or using liftB,
-- it'll defer the evaluation, and all of its usage sites will create a
-- separate graph node.
--
-- Use bindVar if you ever intend to use a BVar in more
-- than one location.
--
--
-- -- bad
-- errSquared :: Num a => BP s '[a, a] a
-- errSquared = withInp $ \(r :< t :< Ø) -> do
-- let err = r - t
-- return (err * err) -- err is used twice!
--
-- -- good
-- errSquared :: Num a => BP s '[a, a] a
-- errSquared = withInps $ \(r :< t :< Ø) -> do
-- let err = r - t
-- e <- bindVar err -- force e, so that it's safe to use twice!
-- return (e * e)
--
-- -- better
-- errSquared :: Num a => BP s '[a, a] a
-- errSquared = withInps $ \(r :< t :< Ø) -> do
-- let err = r - t
-- e <- bindVar err
-- bindVar (e * e) -- result is forced so user doesn't have to worry
--
--
-- Note the relation to opVar ~$ liftB /
-- .$:
--
--
-- opVar o xs = bindVar (liftB o xs)
-- o ~$ xs = bindVar (o .$ xs)
-- op2 (*) ~$ (x :< y :< Ø) = bindVar (x * y)
--
--
-- So you can avoid bindVar altogether if you use the explicitly
-- binding ~$ and opVar etc.
--
-- Note that bindVar on BVars that are already forced is a
-- no-op.
bindVar :: Num a => BVar s rs a -> BP s rs (BVar s rs a)
-- | A version of inpVars taking explicit Length, indicating
-- the number of inputs required and their types.
--
-- Mostly useful for rare "extremely polymorphic" situations, where GHC
-- can't infer the type and length of the list of inputs. If you ever
-- actually explicitly write down rs as a list of types, you
-- should be able to just use inpVars.
inpVars' :: Length rs -> Prod (BVar s rs) rs
-- | Apply an OpB to a Prod (tupling) of BVars.
--
-- If you had an OpB s '[a, b, c] d, this function will
-- expect a 3-Prod of a BVar s rs a, a BVar s
-- rs b, and a BVar s rs c, and the result will be a
-- BVar s rs d:
--
--
-- myOp :: OpB s '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- x :< y :< z :< Ø :: Prod (BVar s rs) '[a, b, c]
-- opVar myOp (x :< y :< z :< Ø) :: BP s rs (BVar s rs d)
--
--
-- Note that OpB is a superclass of Op, so you can provide
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- opVar has an infix alias, ~$, so the above example can
-- also be written as:
--
--
-- myOp ~$ (x :< y :< z :< Ø) :: BP s rs (BVar s rs d)
--
--
-- to let you pretend that you're applying the myOp function to
-- three inputs.
--
-- Also note the relation between opVar and liftB and
-- bindVar:
--
--
-- opVar o xs = bindVar (liftB o xs)
--
--
-- opVar can be thought of as a "binding" version of liftB.
opVar :: forall s rs as a. Num a => OpB s as a -> Prod (BVar s rs) as -> BP s rs (BVar s rs a)
-- | Infix synonym for opVar, which lets you pretend that you're
-- applying OpBs as if they were functions:
--
--
-- myOp :: OpB s '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- x :< y :< z :< Ø :: Prod (BVar s rs) '[a, b, c]
-- myOp ~$ (x :< y :< z :< Ø) :: BP s rs (BVar s rs d)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- ~$ can also be thought of as a "binding" version of .$:
--
--
-- o ~$ xs = bindVar (o .$ xs)
--
(~$) :: Num a => OpB s as a -> Prod (BVar s rs) as -> BP s rs (BVar s rs a)
infixr 5 ~$
-- | Convenient wrapper over opVar that takes an OpB with one
-- argument and a single BVar argument. Lets you not have to type
-- out the entire Prod.
--
--
-- opVar1 o x = opVar o (x :< 'Ø')
--
-- myOp :: Op '[a] b
-- x :: BVar s rs a
--
-- opVar1 myOp x :: BP s rs (BVar s rs b)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op1) as well.
opVar1 :: Num b => OpB s '[a] b -> BVar s rs a -> BP s rs (BVar s rs b)
-- | Convenient wrapper over opVar that takes an OpB with two
-- arguments and two BVar arguments. Lets you not have to type out
-- the entire Prod.
--
--
-- opVar2 o x y = opVar o (x :< y :< 'Ø')
--
-- myOp :: Op '[a, b] c
-- x :: BVar s rs a
-- y :: BVar s rs b
--
-- opVar2 myOp x y :: BP s rs (BVar s rs c)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op2) as well.
opVar2 :: Num c => OpB s '[a, b] c -> BVar s rs a -> BVar s rs b -> BP s rs (BVar s rs c)
-- | Convenient wrapper over opVar that takes an OpB with
-- three arguments and three BVar arguments. Lets you not have to
-- type out the entire Prod.
--
--
-- opVar3 o x y z = opVar o (x :< y :< z :< 'Ø')
--
-- myOp :: Op '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- opVar3 myOp x y z :: BP s rs (BVar s rs d)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op3) as well.
opVar3 :: Num d => OpB s '[a, b, c] d -> BVar s rs a -> BVar s rs b -> BVar s rs c -> BP s rs (BVar s rs d)
-- | Lets you treat a BPOp s as b as an Op as
-- b, and "apply" arguments to it just like you would with an
-- Op and ~$ / opVar.
--
-- Basically a convenient wrapper over bpOp and ~$:
--
--
-- o -$ xs = bpOp o ~$ xs
--
--
-- So for a BPOp s as b, you can "plug in" BVars
-- to as, and get a b as a result.
--
-- Useful for running a BPOp s as b that you got from a
-- different function, and "plugging in" its as inputs with
-- BVars from your current environment.
(-$) :: (Every Num as, Known Length as, Num a) => BPOp s as a -> Prod (BVar s rs) as -> BPOp s rs a
infixr 5 -$
-- | Use an Iso (or compatible Iso from the lens library) to
-- "pull out" the parts of a data type and work with each part as a
-- BVar.
--
-- If there is an isomorphism between a b and a Tuple
-- as (that is, if an a is just a container for a bunch of
-- as), then it lets you break out the as inside and
-- work with those.
--
--
-- data Foo = F Int Bool
--
-- fooIso :: Iso' Foo (Tuple '[Int, Bool])
-- fooIso = iso (\(F i b) -> i ::< b ::< Ø)
-- (\(i ::< b ::< Ø) -> F i b )
--
-- partsVar fooIso :: BVar rs Foo -> BP s rs (Prod (BVar s rs) '[Int, Bool])
--
-- stuff :: BP s '[Foo] a
-- stuff = withInps $ \(foo :< Ø) -> do
-- i :< b :< Ø <- partsVar fooIso foo
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- You can use this to pass in product types as the environment to a
-- BP, and then break out the type into its constituent products.
--
-- Note that for a type like Foo, fooIso can be
-- generated automatically with Generic from GHC.Generics
-- and Generic from Generics.SOP and generics-sop,
-- using the gTuple iso. See gSplit for more information.
--
-- Also, if you are literally passing a tuple (like BP s
-- '[Tuple '[Int, Bool]) then you can give in the identity
-- isomorphism (id) or use splitVars.
partsVar :: forall s rs bs b. Every Num bs => Iso' b (Tuple bs) -> BVar s rs b -> BP s rs (Prod (BVar s rs) bs)
-- | A useful infix alias for partsVar.
--
-- Building on the example from partsVar:
--
--
-- data Foo = F Int Bool
--
-- fooIso :: Iso' Foo (Tuple '[Int, Bool])
-- fooIso = iso (\(F i b) -> i ::< b ::< Ø)
-- (\(i ::< b ::< Ø) -> F i b )
--
-- stuff :: BP s '[Foo] a
-- stuff = withInps $ \(foo :< Ø) -> do
-- i :< b :< Ø <- fooIso #<~ foo
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- See gSplit for an example usage of splitting up an arbitrary
-- product type (like Foo) using GHC.Geneics and
-- Generics.SOP.
(#<~) :: (Every Num bs, Known Length bs) => Iso' b (Tuple bs) -> BVar s rs b -> BP s rs (Prod (BVar s rs) bs)
infixr 1 #<~
-- | A continuation-based version of partsVar. Instead of binding
-- the parts and using it in the rest of the block, provide a
-- continuation to handle do stuff with the parts inside.
--
-- Building on the example from partsVar:
--
--
-- data Foo = F Int Bool
--
-- fooIso :: Iso' Foo (Tuple '[Int, Bool])
-- fooIso = iso (\(F i b) -> i ::< b ::< Ø)
-- (\(i ::< b ::< Ø) -> F i b )
--
-- stuff :: BP s '[Foo] a
-- stuff = withInps $ \(foo :< Ø) -> do
-- withParts fooIso foo $ \(i :< b :< Ø) -> do
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- Useful so that you can work with the internal parts of the data type
-- in a closure, so the parts don't leak out to the rest of your
-- BP. But, mostly just a stylistic choice.
withParts :: Every Num bs => Iso' b (Tuple bs) -> BVar s rs b -> (Prod (BVar s rs) bs -> BP s rs a) -> BP s rs a
-- | Split out a BVar of a tuple into a tuple (Prod) of
-- BVars.
--
--
-- -- the environment is a single Int-Bool tuple, tup
-- stuff :: BP s '[ Tuple '[Int, Bool] ] a
-- stuff = withInps $ \(tup :< Ø) -> do
-- i :< b :< Ø <- splitVars tup
-- -- now, i is a BVar pointing to the Int inside tup
-- -- and b is a BVar pointing to the Bool inside tup
-- -- you can do stuff with the i and b here
--
--
-- Note that
--
--
-- splitVars = partsVar id
--
splitVars :: forall s rs as. Every Num as => BVar s rs (Tuple as) -> BP s rs (Prod (BVar s rs) as)
-- | Using Generic from GHC.Generics and Generic from
-- Generics.SOP, split a BVar containing a product
-- type into a tuple (Prod) of BVars pointing to each value
-- inside.
--
-- Building on the example from partsVar:
--
--
-- import qualified Generics.SOP as SOP
--
-- data Foo = F Int Bool
-- deriving Generic
--
-- instance SOP.Generic Foo
--
-- gSplit :: BVar rs Foo -> BP s rs (Prod (BVar s rs) '[Int, Bool])
--
-- stuff :: BP s '[Foo] a
-- stuff = withInps $ \(foo :< Ø) -> do
-- i :< b :< Ø <- gSplit foo
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- Because Foo is a straight up product type, gSplit can
-- use GHC.Generics and take out the items inside.
--
-- Note that because
--
--
-- gSplit = splitVars gTuple
--
--
-- Then, you can also use gTuple with #<~:
--
--
-- stuff :: BP s '[Foo] a
-- stuff = withInps $ \(foo :< Ø) -> do
-- i :< b :< Ø <- gTuple #<~ foo
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
gSplit :: (Every Num bs, Generic b, Code b ~ '[bs]) => BVar s rs b -> BP s rs (Prod (BVar s rs) bs)
-- | An Iso between a type that is a product type, and a tuple that
-- contains all of its components. Uses Generics.SOP and the
-- Generic typeclass.
--
--
-- >>> import qualified Generics.SOP as SOP
--
-- >>> data Foo = A Int Bool deriving Generic
--
-- >>> instance SOP.Generic Foo
--
-- >>> view gTuple (A 10 True)
-- 10 ::< True ::< Ø
--
-- >>> review gTuple (15 ::< False ::< Ø)
-- A 15 False
--
gTuple :: (Generic a, Code a ~ '[as]) => Iso' a (Tuple as)
-- | Use an Iso (or compatible Iso from the lens library) to
-- "pull out" the different constructors of a sum type and return a
-- (choice) sum of BVars that you can pattern match on.
--
-- If there is an isomorphism between a b and a Sum
-- I as (that is, if an a is just a sum type for
-- every type in as), then it lets you branch on which
-- constructor is used inside the b.
--
-- Essentially implements pattern matching on BVar values.
--
--
-- data Bar = A Int | B Bool | C String
--
-- barIso :: Iso' Bar (Sum I '[Int, Bool, String])
-- barIso = iso (\case A i -> InL (I i)
-- B b -> InR (InL (I b))
-- C s -> InR (InR (InL (I s))
-- )
-- (\case InL (I i) -> A i
-- InR (InL (I b)) -> B b
-- InR (InR (InL (I s))) -> C s
-- )
--
-- choicesVar barIso :: BVar rs Bar -> BP s rs (Sum I (BVar s rs) '[Int, Bool, String])
--
-- stuff :: BP s '[Bar] a
-- stuff = withInps $ \(bar :< Ø) -> do
-- c <- choicesVar barIso bar
-- case c of
-- InL i -> do
-- -- in this branch, bar was made with the A constructor
-- -- i is the Int inside it
-- InR (InL b) -> do
-- -- in this branch, bar was made with the B constructor
-- -- b is the Bool inside it
-- InR (InR (InL s)) -> do
-- -- in this branch, bar was made with the B constructor
-- -- s is the String inside it
--
--
-- You can use this to pass in sum types as the environment to a
-- BP, and then branch on which constructor the value was made
-- with.
--
-- See Numeric.Backprop#sum for a mini-tutorial on Sum.
choicesVar :: forall s rs bs b. Every Num bs => Iso' b (Sum I bs) -> BVar s rs b -> BP s rs (Sum (BVar s rs) bs)
-- | A useful infix alias for choicesVar.
--
-- Building on the example from choicesVar:
--
--
-- data Bar = A Int | B Bool | C String
--
-- barIso :: Iso' Bar (Sum I '[Int, Bool, String])
-- barIso = iso (\case A i -> InL (I i)
-- B b -> InR (InL (I b))
-- C s -> InR (InR (InL (I s))
-- )
-- (\case InL (I i) -> A i
-- InR (InL (I b)) -> B b
-- InR (InR (InL (I s))) -> C s
-- )
--
-- stuff :: BP s '[Bar] a
-- stuff = withInps $ \(bar :< Ø) -> do
-- c <- barIso ?<~ bar
-- case c of
-- InL i -> do
-- -- in this branch, bar was made with the A constructor
-- -- i is the Int inside it
-- InR (InL b) -> do
-- -- in this branch, bar was made with the B constructor
-- -- b is the Bool inside it
-- InR (InR (InL s)) -> do
-- -- in this branch, bar was made with the B constructor
-- -- s is the String inside it
--
(?<~) :: (Every Num bs, Known Length bs) => Iso' b (Sum I bs) -> BVar s rs b -> BP s rs (Sum (BVar s rs) bs)
infixr 1 ?<~
-- | A continuation-based version of choicesVar. Instead of binding
-- the parts and using it in the rest of the block, provide a
-- continuation that will handle every possible constructor/case of the
-- type of the value the BVar points to.
--
-- Building on the example from choicesVar:
--
--
-- data Bar = A Int | B Bool | C String
--
-- barIso :: Iso' Bar (Sum I '[Int, Bool, String])
-- barIso = iso (\case A i -> InL (I i)
-- B b -> InR (InL (I b))
-- C s -> InR (InR (InL (I s))
-- )
-- (\case InL (I i) -> A i
-- InR (InL (I b)) -> B b
-- InR (InR (InL (I s))) -> C s
-- )
--
-- choicesVar barIso :: BVar rs Bar -> BP s rs (Sum I (BVar s rs) '[Int, Bool, String])
--
-- stuff :: BP s '[Bar] a
-- stuff = withInps $ \(bar :< Ø) -> do
-- withChoices barIso bar $ case
-- InL i -> do
-- -- in this branch, bar was made with the A constructor
-- -- i is the Int inside it
-- InR (InL b) -> do
-- -- in this branch, bar was made with the B constructor
-- -- b is the Bool inside it
-- InR (InR (InL s)) -> do
-- -- in this branch, bar was made with the B constructor
-- -- s is the String inside it
--
--
-- Nicer than choicesVar directly, because you don't have to give
-- the result a superfluous name before pattern matching on it. You can
-- just directly pattern match in the lambda, so there's a lot less
-- syntactical noise.
withChoices :: forall s rs bs b a. Every Num bs => Iso' b (Sum I bs) -> BVar s rs b -> (Sum (BVar s rs) bs -> BP s rs a) -> BP s rs a
data Sum k (f :: k -> *) (a :: [k]) :: forall k. (k -> *) -> [k] -> *
[InL] :: Sum k f ((:) k a1 as)
[InR] :: Sum k f ((:) k a1 as)
-- | A combination of partsVar and choicesVar, that lets you
-- split a type into a sum of products. Using an Iso (or
-- compatible Iso from the lens library), you can pull out a type
-- that is a sum of products into a sum of products of BVars.
--
-- Implements branching on the constructors of a value that a BVar
-- contains, and also splitting out the different items inside each
-- constructor.
--
--
-- data Baz = A Int Bool
-- | B String Double
--
--
-- bazIso :: Iso' Baz (Sum Tuple '[ '[Int, Bool], '[String, Double] ])
-- bazIso = iso (\case A i b -> InL (I (i ::< b ::< Ø))
-- B s d -> InR (InL (I (s ::< d ::< Ø)))
-- )
-- (\case InL (I (i ::::< Ø)) - A i b
-- InR (InL (I (s ::::< Ø))) - B s d
-- )
--
-- sopVar bazIso :: BVar rs Baz -> BP s rs (Sum (Prod (BVar s rs)) '[ '[Int, Bool], '[String, Double] ])
--
-- stuff :: BP s '[Baz] a
-- stuff = withInps $ \(baz :< Ø) -> do
-- c <- sopVar barIso baz
-- case c of
-- InL (i ::< Ø) - do
-- -- in this branch, baz was made with the A constructor
-- -- i and b are the Int and Bool inside it
-- InR (InL (s ::< Ø)) - do
-- -- in this branch, baz was made with the B constructor
-- -- s and d are the String and Double inside it
--
--
-- Essentially exists to implement "pattern matching" on multiple
-- constructors and fields for the value inside a BVar.
--
-- Note that for a type like Baz, bazIso can be
-- generated automatically with Generic from GHC.Generics
-- and Generic from Generics.SOP and generics-sop,
-- with gSOP. See gSplits for more information.
--
-- See Numeric.Backprop#sum for a mini-tutorial on Sum.
sopVar :: forall s rs bss b. Every (Every Num) bss => Iso' b (Sum Tuple bss) -> BVar s rs b -> BP s rs (Sum (Prod (BVar s rs)) bss)
-- | Using Generic from GHC.Generics and Generic from
-- Generics.SOP, split a BVar containing a sum of
-- products (any simple ADT, essentialy) into a Sum of each
-- constructor, each containing a tuple (Prod) of BVars
-- pointing to each value inside.
--
-- Building on the example from sopVar:
--
--
-- import qualified Generics.SOP as SOP
--
-- data Baz = A Int Bool
-- | B String Double
-- deriving Generic
--
-- instance SOP.Generic Baz
--
-- gSplits :: BVar rs Baz -> BP s rs (Sum (Prod (BVar s rs)) '[ '[Int, Bool], '[String, Double] ])
--
-- stuff :: BP s '[Baz] a
-- stuff = withInps $ \(baz :< Ø) -> do
-- c <- gSplits baz
-- case c of
-- InL (i ::< Ø) - do
-- -- in this branch, baz was made with the A constructor
-- -- i and b are the Int and Bool inside it
-- InR (InL (s ::< Ø)) - do
-- -- in this branch, baz was made with the B constructor
-- -- s and d are the String and Double inside it
--
--
-- Because Foo is a straight up sum-of-products type,
-- gSplits can use GHC.Generics and take out the items
-- inside.
--
-- Note:
--
--
-- gSplit = splitVars gSOP
--
--
-- See Numeric.Backprop#sum for a mini-tutorial on Sum.
gSplits :: forall s rs b. (Generic b, Every (Every Num) (Code b)) => BVar s rs b -> BP s rs (Sum (Prod (BVar s rs)) (Code b))
-- | An Iso between a sum type whose constructors are products, and
-- a sum (Sum) of products (Tuple). Uses
-- Generics.SOP and the Generic typeclass.
--
--
-- >>> import qualified Generics.SOP as SOP
--
-- >>> data Bar = A Int Bool | B String Double
--
-- >>> instance SOP.Generic Bar
--
-- >>> 'view' 'gSOP' (A 10 True)
-- 'InL' (10 ::< True ::< Ø)
--
-- >>> 'view' 'gSOP' (B "hello" 3.4)
-- 'InR' ('InL' ("hello" ::< 3.4 ::< Ø))
--
-- >>> 'review' 'gTuple' ('InL' (15 ::< False ::< Ø))
-- A 15 False
--
-- >>> 'review' 'gTuple' ('InR' ('InL' ("bye" ::< 9.8 ::< Ø)))
-- B "bye" 9.8
--
gSOP :: Generic a => Iso' a (Sum Tuple (Code a))
-- | Special unsafe combinator that lets you pattern match and work
-- on GADTs.
--
--
-- data MyGADT :: Bool -> Type where
-- A :: String -> Int -> MyGADT 'True
-- B :: Bool -> Double -> MyGADT 'False
--
--
-- foo :: BP s '[ MyGADT b ] a
-- foo = withInps $ \( gVar :< Ø ) ->
-- withGADT gVar $ \case
-- A s i -> BPC (s ::< i ::< Ø) (\(s' ::< i' ::< Ø) -> A s i) $
-- \(sVar :- do
-- -- .. in this BP action, sVar and iVar are BPVars that
-- -- refer to the String and Int inside the A constructor in
-- -- gVar
-- B b d -> BPC (b ::< d ::< Ø) (\(b' ::< d' ::< Ø) -> B b d) $
-- \(bVar :- do
-- -- .. in this BP action, bVar and dVar are BPVars that
-- -- refer to the Bool and DOuble inside the B constructor in
-- -- gVar
--
--
-- withGADT lets to directly pattern match on the GADT, but as
-- soon as you pattern match, you must handle the results with a
-- BPCont containing:
--
--
-- - All of the items inside the GADT constructor, in a
-- Tuple
-- - A function from a Tuple of items inside the GADT
-- constructor that assembles them back into the original same
-- constructor.
-- - A function from a Prod of BVars (that contain the
-- items inside the constructor) and doing whatever you wanted to do with
-- it, inside BP.
--
--
-- If you don't provide all of the items inside the GADT into the
-- BPC, or if your "re-assembling" function doesn't properly
-- reassemble things correctly or changes some of the values, this will
-- not work.
withGADT :: forall s rs a b. BVar s rs a -> (a -> BPCont s rs a b) -> BP s rs b
-- | For usage with withGADT, to handle constructors of a GADT. See
-- documentation for withGADT for more information.
data BPCont :: Type -> [Type] -> Type -> Type -> Type
[BPC] :: Every Num as => Tuple as -> (Tuple as -> a) -> (Prod (BVar s rs) as -> BP s rs b) -> BPCont s rs a b
-- | Apply OpB over a Prod of BVars, as inputs.
-- Provides "implicit-graph" back-propagation, with deferred evaluation.
--
-- If you had an OpB s '[a, b, c] d, this function will
-- expect a 3-Prod of a BVar s rs a, a BVar s
-- rs b, and a BVar s rs c, and the result will be a
-- BVar s rs d:
--
--
-- myOp :: OpB s '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- x :< y :< z :< Ø :: Prod (BVar s rs) '[a, b, c]
-- liftB myOp (x :< y :< z :< Ø) :: BVar s rs d
--
--
-- Note that OpB is a superclass of Op, so you can provide
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- liftB has an infix alias, .$, so the above example can
-- also be written as:
--
--
-- myOp .$ (x :< y :< z :< Ø) :: BVar s rs d
--
--
-- to let you pretend that you're applying the myOp function to
-- three inputs.
--
-- The result is a new deferred BVar. This should be fine
-- in most cases, unless you use the result in more than one location.
-- This will cause evaluation to be duplicated and multiple redundant
-- graph nodes to be created. If you need to use it in two locations, you
-- should use opVar instead of liftB, or use
-- bindVar:
--
--
-- opVar o xs = bindVar (liftB o xs)
--
--
-- liftB can be thought of as a "deferred evaluation" version of
-- opVar.
liftB :: OpB s as a -> Prod (BVar s rs) as -> BVar s rs a
-- | Infix synonym for liftB, which lets you pretend that you're
-- applying OpBs as if they were functions:
--
--
-- myOp :: OpB s '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- x :< y :< z :< Ø :: Prod (BVar s rs) '[a, b, c]
-- myOp .$ (x :< y :< z :< Ø) :: BVar s rs d
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- See the documentation for liftB for all the caveats of this
-- usage.
--
-- .$ can also be thought of as a "deferred evaluation" version of
-- ~$:
--
--
-- o ~$ xs = bindVar (o .$ xs)
--
(.$) :: OpB s as a -> Prod (BVar s rs) as -> BVar s rs a
infixr 5 .$
-- | Convenient wrapper over liftB that takes an OpB with one
-- argument and a single BVar argument. Lets you not have to type
-- out the entire Prod.
--
--
-- liftB1 o x = liftB o (x :< 'Ø')
--
-- myOp :: Op '[a] b
-- x :: BVar s rs a
--
-- liftB1 myOp x :: BVar s rs b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op1) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB1 :: OpB s '[a] b -> BVar s rs a -> BVar s rs b
-- | Convenient wrapper over liftB that takes an OpB with two
-- arguments and two BVar arguments. Lets you not have to type out
-- the entire Prod.
--
--
-- liftB2 o x y = liftB o (x :< y :< 'Ø')
--
-- myOp :: Op '[a, b] c
-- x :: BVar s rs a
-- y :: BVar s rs b
--
-- liftB2 myOp x y :: BVar s rs c
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op2) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB2 :: OpB s '[a, b] c -> BVar s rs a -> BVar s rs b -> BVar s rs c
-- | Convenient wrapper over liftB that takes an OpB with
-- three arguments and three BVar arguments. Lets you not have to
-- type out the entire Prod.
--
--
-- liftB3 o x y z = liftB o (x :< y :< z :< 'Ø')
--
-- myOp :: Op '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- liftB3 myOp x y z :: BVar s rs d
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op3) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB3 :: OpB s '[a, b, c] d -> BVar s rs a -> BVar s rs b -> BVar s rs c -> BVar s rs d
-- | Automatically create an Op of a numerical function taking one
-- argument. Uses diff, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op1 (recip . negate)) (5 ::< Ø)
-- (-0.2, 0.04 ::< Ø)
--
op1 :: Num a => (forall s. AD s (Forward a) -> AD s (Forward a)) -> Op '[a] a
-- | Automatically create an Op of a numerical function taking two
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op2 (\x y -> x * sqrt y)) (3 ::< 4 ::< Ø)
-- (6.0, 2.0 ::< 0.75 ::< Ø)
--
op2 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a) -> Op '[a, a] a
-- | Automatically create an Op of a numerical function taking three
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op3 (\x y z -> (x * sqrt y)**z)) (3 ::< 4 ::< 2 ::< Ø)
-- (36.0, 24.0 ::< 9.0 ::< 64.503 ::< Ø)
--
op3 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a -> Reverse s a) -> Op '[a, a, a] a
-- | Automatically create an Op of a numerical function taking
-- multiple arguments. Uses grad, and so can take any numerical
-- function polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (opN (\(x :+ y :+ Ø) -> x * sqrt y)) (3 ::< 4 ::< Ø)
-- (6.0, 2.0 ::< 0.75 ::< Ø)
--
opN :: (Num a, Known Nat n) => (forall s. Reifies s Tape => Vec n (Reverse s a) -> Reverse s a) -> Op (Replicate n a) a
-- | Compose OpMs together, similar to .. But, because all
-- OpMs are <math>, this is more like sequence for
-- functions, or liftAN.
--
-- That is, given an OpM m as b1, an OpM m as
-- b2, and an OpM m as b3, it can compose them with
-- an OpM m '[b1,b2,b3] c to create an OpM m
-- as c.
composeOp :: (Monad m, Every Num as, Known Length as) => Prod (OpM m as) bs -> OpM m bs c -> OpM m as c
-- | Convenient wrapper over composeOp for the case where the second
-- function only takes one input, so the two OpMs can be directly
-- piped together, like for ..
composeOp1 :: (Monad m, Every Num as, Known Length as) => OpM m as b -> OpM m '[b] c -> OpM m as c
-- | Convenient infix synonym for (flipped) composeOp1. Meant to be
-- used just like .:
--
--
-- op1 negate :: Op '[a] a
-- op2 (+) :: Op '[a,a] a
--
-- op1 negate ~. op2 (+) :: Op '[a, a] a
--
(~.) :: (Monad m, Known Length as, Every Num as) => OpM m '[b] c -> OpM m as b -> OpM m as c
infixr 9 ~.
-- | Create an Op of a function taking one input, by giving its
-- explicit derivative. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the derivative <math>, it would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that squares its input:
--
--
-- square :: Num a => Op '[a] a
-- square = op1' $ \x -> (x*x, \case Nothing -> 2 * x
-- Just d -> 2 * d * x
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, single-input Ops can be generated
-- automatically using op1.
op1' :: (a -> (b, Maybe b -> a)) -> Op '[a] b
-- | Create an Op of a function taking two inputs, by giving its
-- explicit gradient. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the gradient <math> would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that multiplies its inputs:
--
--
-- mul :: Num a => Op '[a, a] a
-- mul = op2' $ \x y -> (x*y, \case Nothing -> (y , x )
-- Just d -> (d*y, x*d)
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, two-input Ops can be generated
-- automatically using op2.
op2' :: (a -> b -> (c, Maybe c -> (a, b))) -> Op '[a, b] c
-- | Create an Op of a function taking three inputs, by giving its
-- explicit gradient. See documentation for op2' for more details.
op3' :: (a -> b -> c -> (d, Maybe d -> (a, b, c))) -> Op '[a, b, c] d
-- | Construct a two element Prod. Since the precedence of (:>) is
-- higher than (:<), we can conveniently write lists like:
--
--
-- >>> a :< b :> c
--
--
-- Which is identical to:
--
--
-- >>> a :< b :< c :< Ø
--
infix 6 :>
-- | Build a singleton Prod.
only :: f a -> Prod k f ((:) k a ([] k))
head' :: Prod k f ((:<) k a as) -> f a
-- | Cons onto a Tuple.
infixr 5 ::<
-- | Singleton Tuple.
only_ :: a -> Tuple ((:) * a ([] *))
-- | Optimized version of op1 (+).
(+.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 (-).
(-.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 (*).
(*.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 negate.
negateOp :: Num a => Op '[a] a
-- | Optimized version of op1 abs.
absOp :: Num a => Op '[a] a
-- | Optimized version of op1 signum.
signumOp :: Num a => Op '[a] a
-- | Optimized version of op1 (/).
(/.) :: Fractional a => Op '[a, a] a
-- | Optimized version of op1 recip.
recipOp :: Fractional a => Op '[a] a
-- | Optimized version of op1 exp.
expOp :: Floating a => Op '[a] a
-- | Optimized version of op1 log.
logOp :: Floating a => Op '[a] a
-- | Optimized version of op1 sqrt.
sqrtOp :: Floating a => Op '[a] a
-- | Optimized version of op1 (**).
(**.) :: Floating a => Op '[a, a] a
-- | Optimized version of op2 logBase.
logBaseOp :: Floating a => Op '[a, a] a
-- | Optimized version of op1 sin.
sinOp :: Floating a => Op '[a] a
-- | Optimized version of op1 cos.
cosOp :: Floating a => Op '[a] a
-- | Optimized version of op1 tan.
tanOp :: Floating a => Op '[a] a
-- | Optimized version of op1 asin.
asinOp :: Floating a => Op '[a] a
-- | Optimized version of op1 acos.
acosOp :: Floating a => Op '[a] a
-- | Optimized version of op1 atan.
atanOp :: Floating a => Op '[a] a
-- | Optimized version of op1 sinh.
sinhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 cosh.
coshOp :: Floating a => Op '[a] a
-- | Optimized version of op1 tanh.
tanhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 asinh.
asinhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 acosh.
acoshOp :: Floating a => Op '[a] a
-- | Optimized version of op1 atanh.
atanhOp :: Floating a => Op '[a] a
-- | Offers full functionality for implicit-graph back-propagation. The
-- intended usage is to write a BPOp, which is a normal Haskell
-- function from BVars to a result BVar. These BVars
-- can be manipulated using their Num / Fractional /
-- Floating instances.
--
-- The library can then perform back-propagation on the function (using
-- backprop or grad) by using an implicitly built graph.
--
-- This should actually be powerful enough for most use cases, but falls
-- short for a couple of situations:
--
--
-- - If the result of a function on BVars is used twice (like
-- z in let z = x * y in z + z), this will allocate a
-- new redundant graph node for every usage site of z. You can
-- explicitly force z, but only using an explicit graph
-- description using Numeric.Backprop.
-- - This can't handle sum types, like Numeric.Backprop can. You
-- can never pattern match on the constructors of a value inside a
-- BVar. I'm not sure if this is a fundamental limitation (I
-- suspect it might be) or if I just can't figure out how to implement
-- it. Suggestions welcome!
--
--
-- As a comparison, this module offers functionality and an API very
-- similar to Numeric.AD.Mode.Reverse from the ad library,
-- except for the fact that it can handle heterogeneous values.
--
-- Note that every type involved has to be an instance of Num.
-- This is because gradients all need to be "summable" (which is
-- implemented using sum and +), and we also need to able
-- to generate gradients of '1' and '0'.
module Numeric.Backprop.Implicit
-- | An operation on BVars that can be backpropagated. A value of
-- type:
--
--
-- BPOp rs a
--
--
-- takes a bunch of BVars containg rs and uses them to
-- (purely) produce a BVar containing an a.
--
--
-- foo :: BPOp '[ Double, Double ] Double
-- foo (x :< y :< 'Ø') = x + sqrt y
--
--
-- BPOp here is related to BPOpI from the normal
-- explicit-graph backprop module Numeric.Backprop.
type BPOp rs a = forall s. Prod (BVar s rs) rs -> BVar s rs a
-- | The basic unit of manipulation inside BP (or inside an
-- implicit-graph backprop function). Instead of directly working with
-- values, you work with BVars contating those values. When you
-- work with a BVar, the backprop library can keep track of
-- what values refer to which other values, and so can perform
-- back-propagation to compute gradients.
--
-- A BVar s rs a refers to a value of type a,
-- with an environment of values of the types rs. The phantom
-- parameter s is used to ensure that stray BVars don't
-- leak outside of the backprop process.
--
-- (That is, if you're using implicit backprop, it ensures that you
-- interact with BVars in a polymorphic way. And, if you're using
-- explicit backprop, it ensures that a BVar s rs a never
-- leaves the BP s rs that it was created in.)
--
-- BVars have Num, Fractional, Floating, etc.
-- instances, so they can be manipulated using polymorphic functions and
-- numeric functions in Haskell. You can add them, subtract them, etc.,
-- in "implicit" backprop style.
--
-- (However, note that if you directly manipulate BVars using
-- those instances or using liftB, it delays evaluation, so every
-- usage site has to re-compute the result/create a new node. If you want
-- to re-use a BVar you created using + or - or
-- liftB, use bindVar to force it first. See documentation
-- for bindVar for more details.)
data BVar :: Type -> [Type] -> Type -> Type
-- | An Op as a describes a differentiable function from
-- as to a.
--
-- For example, a value of type
--
--
-- Op '[Int, Bool] Double
--
--
-- is a function from an Int and a Bool, returning a
-- Double. It can be differentiated to give a gradient of
-- an Int and a Bool if given a total derivative for the
-- Double. If we call Bool <math>, then,
-- mathematically, it is akin to a:
--
-- <math>
--
-- See runOp, gradOp, and gradOpWith for examples on
-- how to run it, and Op for instructions on creating it.
--
-- This type is abstracted over using the pattern synonym with
-- constructor Op, so you can create one from scratch with it.
-- However, it's simplest to create it using op2', op1',
-- op2', and op3' helper smart constructors And, if your
-- function is a numeric function, they can even be created automatically
-- using op1, op2, op3, and opN with a little
-- help from Numeric.AD from the ad library.
--
-- Note that this type is a subset or subtype of OpM
-- (and also of OpB). So, if a function ever expects an
-- OpM m as a (or a OpB), you can always provide
-- an Op as a instead.
--
-- Many functions in this library will expect an OpM m as
-- a (or an OpB s as a), and in all of these cases,
-- you can provide an Op as a.
type Op as a = forall m. Monad m => OpM m as a
-- | A subclass of OpM (and superclass of Op), representing
-- Ops that the backprop library uses to perform
-- backpropation.
--
-- An
--
--
-- OpB s rs a
--
--
-- represents a differentiable function that takes a tuple of rs
-- and produces an a a, which can be run on BVar
-- ss and also inside BP ss. For example, an
-- OpB s '[ Int, Double ] Bool takes an Int and a
-- Double and produces a Bool, and does it in a
-- differentiable way.
--
-- OpB is a superset of Op, so, if you see any
-- function that expects an OpB (like opVar' and ~$,
-- for example), you can give them an Op, as well.
--
-- You can think of OpB as a superclass/parent class of Op
-- in this sense, and of Op as a subclass of OpB.
type OpB s as a = OpM (ST s) as a
data Prod k (f :: k -> *) (a :: [k]) :: forall k. (k -> *) -> [k] -> *
[Ø] :: Prod k f ([] k)
[:<] :: Prod k f ((:) k a1 as)
-- | A Prod of simple Haskell types.
type Tuple = Prod * I
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Run back-propagation on a BPOp function, getting both the
-- result and the gradient of the result with respect to the inputs.
--
--
-- foo :: BPOp '[Double, Double] Double
-- foo (x :< y :< Ø) =
-- let z = x * sqrt y
-- in z + x ** y
--
--
--
-- >>> 'backprop' foo (2 ::< 3 ::< Ø)
-- (11.46, 13.73 ::< 6.12 ::< Ø)
--
backprop :: Every Num rs => BPOp rs a -> Tuple rs -> (a, Tuple rs)
-- | Run the BPOp on an input tuple and return the gradient of the
-- result with respect to the input tuple.
--
--
-- foo :: BPOp '[Double, Double] Double
-- foo (x :< y :< Ø) =
-- let z = x * sqrt y
-- in z + x ** y
--
--
--
-- >>> grad foo (2 ::< 3 ::< Ø)
-- 13.73 ::< 6.12 ::< Ø
--
grad :: Every Num rs => BPOp rs a -> Tuple rs -> Tuple rs
-- | Simply run the BPOp on an input tuple, getting the result
-- without bothering with the gradient or with back-propagation.
--
--
-- foo :: BPOp '[Double, Double] Double
-- foo (x :< y :< Ø) =
-- let z = x * sqrt y
-- in z + x ** y
--
--
--
-- >>> eval foo (2 ::< 3 ::< Ø)
-- 11.46
--
eval :: (Known Length rs, Num a) => BPOp rs a -> Tuple rs -> a
-- | Create a BVar that represents just a specific value, that
-- doesn't depend on any other BVars.
constVar :: a -> BVar s rs a
-- | Apply OpB over a Prod of BVars, as inputs.
-- Provides "implicit-graph" back-propagation, with deferred evaluation.
--
-- If you had an OpB s '[a, b, c] d, this function will
-- expect a 3-Prod of a BVar s rs a, a BVar s
-- rs b, and a BVar s rs c, and the result will be a
-- BVar s rs d:
--
--
-- myOp :: OpB s '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- x :< y :< z :< Ø :: Prod (BVar s rs) '[a, b, c]
-- liftB myOp (x :< y :< z :< Ø) :: BVar s rs d
--
--
-- Note that OpB is a superclass of Op, so you can provide
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- liftB has an infix alias, .$, so the above example can
-- also be written as:
--
--
-- myOp .$ (x :< y :< z :< Ø) :: BVar s rs d
--
--
-- to let you pretend that you're applying the myOp function to
-- three inputs.
--
-- The result is a new deferred BVar. This should be fine
-- in most cases, unless you use the result in more than one location.
-- This will cause evaluation to be duplicated and multiple redundant
-- graph nodes to be created. If you need to use it in two locations, you
-- should use opVar instead of liftB, or use
-- bindVar:
--
--
-- opVar o xs = bindVar (liftB o xs)
--
--
-- liftB can be thought of as a "deferred evaluation" version of
-- opVar.
liftB :: OpB s as a -> Prod (BVar s rs) as -> BVar s rs a
-- | Infix synonym for liftB, which lets you pretend that you're
-- applying OpBs as if they were functions:
--
--
-- myOp :: OpB s '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- x :< y :< z :< Ø :: Prod (BVar s rs) '[a, b, c]
-- myOp .$ (x :< y :< z :< Ø) :: BVar s rs d
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- See the documentation for liftB for all the caveats of this
-- usage.
--
-- .$ can also be thought of as a "deferred evaluation" version of
-- ~$:
--
--
-- o ~$ xs = bindVar (o .$ xs)
--
(.$) :: OpB s as a -> Prod (BVar s rs) as -> BVar s rs a
infixr 5 .$
-- | Convenient wrapper over liftB that takes an OpB with one
-- argument and a single BVar argument. Lets you not have to type
-- out the entire Prod.
--
--
-- liftB1 o x = liftB o (x :< 'Ø')
--
-- myOp :: Op '[a] b
-- x :: BVar s rs a
--
-- liftB1 myOp x :: BVar s rs b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op1) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB1 :: OpB s '[a] b -> BVar s rs a -> BVar s rs b
-- | Convenient wrapper over liftB that takes an OpB with two
-- arguments and two BVar arguments. Lets you not have to type out
-- the entire Prod.
--
--
-- liftB2 o x y = liftB o (x :< y :< 'Ø')
--
-- myOp :: Op '[a, b] c
-- x :: BVar s rs a
-- y :: BVar s rs b
--
-- liftB2 myOp x y :: BVar s rs c
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op2) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB2 :: OpB s '[a, b] c -> BVar s rs a -> BVar s rs b -> BVar s rs c
-- | Convenient wrapper over liftB that takes an OpB with
-- three arguments and three BVar arguments. Lets you not have to
-- type out the entire Prod.
--
--
-- liftB3 o x y z = liftB o (x :< y :< z :< 'Ø')
--
-- myOp :: Op '[a, b, c] d
-- x :: BVar s rs a
-- y :: BVar s rs b
-- z :: BVar s rs c
--
-- liftB3 myOp x y z :: BVar s rs d
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op3) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB3 :: OpB s '[a, b, c] d -> BVar s rs a -> BVar s rs b -> BVar s rs c -> BVar s rs d
-- | Use an Iso (or compatible Iso from the lens library) to
-- "pull out" the parts of a data type and work with each part as a
-- BVar.
--
-- If there is an isomorphism between a b and a Tuple
-- as (that is, if an a is just a container for a bunch of
-- as), then it lets you break out the as inside and
-- work with those.
--
--
-- data Foo = F Int Bool
--
-- fooIso :: Iso' Foo (Tuple '[Int, Bool])
-- fooIso = iso (\(F i b) -> i ::< b ::< Ø)
-- (\(i ::< b ::< Ø) -> F i b )
--
-- partsVar fooIso :: BVar rs Foo -> Prod (BVar s rs) '[Int, Bool]
--
-- stuff :: BPOp s '[Foo] a
-- stuff (foo :< Ø) =
-- case partsVar fooIso foo of
-- i ::< Ø -
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- You can use this to pass in product types as the environment to a
-- BP, and then break out the type into its constituent products.
--
-- Note that for a type like Foo, fooIso can be
-- generated automatically with Generic from GHC.Generics
-- and Generic from Generics.SOP and generics-sop,
-- using the gTuple iso. See gSplit for more information.
--
-- Also, if you are literally passing a tuple (like BP s
-- '[Tuple '[Int, Bool]) then you can give in the identity
-- isomorphism (id) or use splitVars.
--
-- At the moment, this implicit partsVar is less efficient than
-- the explicit partsVar, but this might change in the future.
partsVar :: forall s rs bs a. (Every Num bs, Known Length bs) => Iso' a (Tuple bs) -> BVar s rs a -> Prod (BVar s rs) bs
-- | A continuation-based version of partsVar. Instead of binding
-- the parts and using it in the rest of the block, provide a
-- continuation to handle do stuff with the parts inside.
--
-- Building on the example from partsVar:
--
--
-- data Foo = F Int Bool
--
-- fooIso :: Iso' Foo (Tuple '[Int, Bool])
-- fooIso = iso (\(F i b) -> i ::< b ::< Ø)
-- (\(i ::< b ::< Ø) -> F i b )
--
-- stuff :: BPOp s '[Foo] a
-- stuff (foo :< Ø) = withParts fooIso foo $ \case
-- i :< b :< Ø ->
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- Mostly just a stylistic alternative to partsVar.
withParts :: forall s rs bs a r. (Every Num bs, Known Length bs) => Iso' a (Tuple bs) -> BVar s rs a -> (Prod (BVar s rs) bs -> r) -> r
-- | Split out a BVar of a tuple into a tuple (Prod) of
-- BVars.
--
--
-- -- the environment is a single Int-Bool tuple, tup
-- stuff :: BPOp s '[ Tuple '[Int, Bool] ] a
-- stuff (tup :< Ø) =
-- case splitVar tup of
-- i :< b :< Ø <- splitVars tup
-- -- now, i is a BVar pointing to the Int inside tup
-- -- and b is a BVar pointing to the Bool inside tup
-- -- you can do stuff with the i and b here
--
--
-- Note that
--
--
-- splitVars = partsVar id
--
splitVars :: forall s rs as. (Every Num as, Known Length as) => BVar s rs (Tuple as) -> Prod (BVar s rs) as
-- | Using Generic from GHC.Generics and Generic from
-- Generics.SOP, split a BVar containing a product
-- type into a tuple (Prod) of BVars pointing to each value
-- inside.
--
-- Building on the example from partsVar:
--
--
-- import qualified Generics.SOP as SOP
--
-- data Foo = F Int Bool
-- deriving Generic
--
-- instance SOP.Generic Foo
--
-- gSplit :: BVar rs Foo -> Prod (BVar s rs) '[Int, Bool]
--
-- stuff :: BPOp s '[Foo] a
-- stuff (foo :< Ø) =
-- case gSplit foo of
-- i ::< Ø -
-- -- now, i is a BVar pointing to the Int inside foo
-- -- and b is a BVar pointing to the Bool inside foo
-- -- you can do stuff with the i and b here
--
--
-- Because Foo is a straight up product type, gSplit can
-- use GHC.Generics and take out the items inside.
--
-- Note that
--
--
-- gSplit = splitVars gTuple
--
gSplit :: forall s rs as a. (Generic a, Code a ~ '[as], Every Num as, Known Length as) => BVar s rs a -> Prod (BVar s rs) as
-- | An Iso between a type that is a product type, and a tuple that
-- contains all of its components. Uses Generics.SOP and the
-- Generic typeclass.
--
--
-- >>> import qualified Generics.SOP as SOP
--
-- >>> data Foo = A Int Bool deriving Generic
--
-- >>> instance SOP.Generic Foo
--
-- >>> view gTuple (A 10 True)
-- 10 ::< True ::< Ø
--
-- >>> review gTuple (15 ::< False ::< Ø)
-- A 15 False
--
gTuple :: (Generic a, Code a ~ '[as]) => Iso' a (Tuple as)
-- | A version of partsVar taking explicit Length, indicating
-- the number of items in the input tuple and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the internal tuple. If you ever actually explicitly write
-- down bs as a list of types, you should be able to just use
-- partsVar.
partsVar' :: forall s rs bs a. Every Num bs => Length bs -> Iso' a (Tuple bs) -> BVar s rs a -> Prod (BVar s rs) bs
-- | A version of withParts taking explicit Length,
-- indicating the number of internal items and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the internal tuple. If you ever actually explicitly write
-- down bs as a list of types, you should be able to just use
-- withParts.
withParts' :: forall s rs bs a r. Every Num bs => Length bs -> Iso' a (Tuple bs) -> BVar s rs a -> (Prod (BVar s rs) bs -> r) -> r
-- | A version of splitVars taking explicit Length,
-- indicating the number of internal items and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the internal tuple. If you ever actually explicitly write
-- down as as a list of types, you should be able to just use
-- splitVars.
splitVars' :: forall s rs as. Every Num as => Length as -> BVar s rs (Tuple as) -> Prod (BVar s rs) as
-- | A version of gSplit taking explicit Length, indicating
-- the number of internal items and their types.
--
-- Requiring an explicit Length is mostly useful for rare
-- "extremely polymorphic" situations, where GHC can't infer the type and
-- length of the internal tuple. If you ever actually explicitly write
-- down as as a list of types, you should be able to just use
-- gSplit.
gSplit' :: forall s rs as a. (Generic a, Code a ~ '[as], Every Num as) => Length as -> BVar s rs a -> Prod (BVar s rs) as
-- | Automatically create an Op of a numerical function taking one
-- argument. Uses diff, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op1 (recip . negate)) (5 ::< Ø)
-- (-0.2, 0.04 ::< Ø)
--
op1 :: Num a => (forall s. AD s (Forward a) -> AD s (Forward a)) -> Op '[a] a
-- | Automatically create an Op of a numerical function taking two
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op2 (\x y -> x * sqrt y)) (3 ::< 4 ::< Ø)
-- (6.0, 2.0 ::< 0.75 ::< Ø)
--
op2 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a) -> Op '[a, a] a
-- | Automatically create an Op of a numerical function taking three
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op3 (\x y z -> (x * sqrt y)**z)) (3 ::< 4 ::< 2 ::< Ø)
-- (36.0, 24.0 ::< 9.0 ::< 64.503 ::< Ø)
--
op3 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a -> Reverse s a) -> Op '[a, a, a] a
-- | Automatically create an Op of a numerical function taking
-- multiple arguments. Uses grad, and so can take any numerical
-- function polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (opN (\(x :+ y :+ Ø) -> x * sqrt y)) (3 ::< 4 ::< Ø)
-- (6.0, 2.0 ::< 0.75 ::< Ø)
--
opN :: (Num a, Known Nat n) => (forall s. Reifies s Tape => Vec n (Reverse s a) -> Reverse s a) -> Op (Replicate n a) a
-- | Create an Op of a function taking one input, by giving its
-- explicit derivative. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the derivative <math>, it would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that squares its input:
--
--
-- square :: Num a => Op '[a] a
-- square = op1' $ \x -> (x*x, \case Nothing -> 2 * x
-- Just d -> 2 * d * x
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, single-input Ops can be generated
-- automatically using op1.
op1' :: (a -> (b, Maybe b -> a)) -> Op '[a] b
-- | Create an Op of a function taking two inputs, by giving its
-- explicit gradient. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the gradient <math> would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that multiplies its inputs:
--
--
-- mul :: Num a => Op '[a, a] a
-- mul = op2' $ \x y -> (x*y, \case Nothing -> (y , x )
-- Just d -> (d*y, x*d)
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, two-input Ops can be generated
-- automatically using op2.
op2' :: (a -> b -> (c, Maybe c -> (a, b))) -> Op '[a, b] c
-- | Create an Op of a function taking three inputs, by giving its
-- explicit gradient. See documentation for op2' for more details.
op3' :: (a -> b -> c -> (d, Maybe d -> (a, b, c))) -> Op '[a, b, c] d
-- | Construct a two element Prod. Since the precedence of (:>) is
-- higher than (:<), we can conveniently write lists like:
--
--
-- >>> a :< b :> c
--
--
-- Which is identical to:
--
--
-- >>> a :< b :< c :< Ø
--
infix 6 :>
-- | Build a singleton Prod.
only :: f a -> Prod k f ((:) k a ([] k))
head' :: Prod k f ((:<) k a as) -> f a
-- | Cons onto a Tuple.
infixr 5 ::<
-- | Singleton Tuple.
only_ :: a -> Tuple ((:) * a ([] *))
-- | Optimized version of op1 (+).
(+.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 (-).
(-.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 (*).
(*.) :: Num a => Op '[a, a] a
-- | Optimized version of op1 negate.
negateOp :: Num a => Op '[a] a
-- | Optimized version of op1 abs.
absOp :: Num a => Op '[a] a
-- | Optimized version of op1 signum.
signumOp :: Num a => Op '[a] a
-- | Optimized version of op1 (/).
(/.) :: Fractional a => Op '[a, a] a
-- | Optimized version of op1 recip.
recipOp :: Fractional a => Op '[a] a
-- | Optimized version of op1 exp.
expOp :: Floating a => Op '[a] a
-- | Optimized version of op1 log.
logOp :: Floating a => Op '[a] a
-- | Optimized version of op1 sqrt.
sqrtOp :: Floating a => Op '[a] a
-- | Optimized version of op1 (**).
(**.) :: Floating a => Op '[a, a] a
-- | Optimized version of op2 logBase.
logBaseOp :: Floating a => Op '[a, a] a
-- | Optimized version of op1 sin.
sinOp :: Floating a => Op '[a] a
-- | Optimized version of op1 cos.
cosOp :: Floating a => Op '[a] a
-- | Optimized version of op1 tan.
tanOp :: Floating a => Op '[a] a
-- | Optimized version of op1 asin.
asinOp :: Floating a => Op '[a] a
-- | Optimized version of op1 acos.
acosOp :: Floating a => Op '[a] a
-- | Optimized version of op1 atan.
atanOp :: Floating a => Op '[a] a
-- | Optimized version of op1 sinh.
sinhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 cosh.
coshOp :: Floating a => Op '[a] a
-- | Optimized version of op1 tanh.
tanhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 asinh.
asinhOp :: Floating a => Op '[a] a
-- | Optimized version of op1 acosh.
acoshOp :: Floating a => Op '[a] a
-- | Optimized version of op1 atanh.
atanhOp :: Floating a => Op '[a] a
-- | Provides monomorphic versions of the types and combinators in
-- Numeric.Backprop.Op, for usage with
-- Numeric.Backprop.Mono and
-- Numeric.Backprop.Mono.Implicit.
--
-- They are monomorphic in the sense that all of the inputs have
-- to be of the same type. So, something like
--
--
-- Op '[Double, Double, Double] Int
--
--
-- From Numeric.Backprop would, in this module, be:
--
--
-- Op N3 Double Int
--
--
-- See the module header for Numeric.Backprop.Op for more
-- explicitly details on how to encode an Op and how they are
-- implemented. For the most part, the same principles will apply.
--
-- Note that Op is a subset or subtype of
-- OpM, and so, any function that expects an OpM m as
-- a (or an OpB s as a) can be given an
-- Op as a and it'll work just fine.
module Numeric.Backprop.Op.Mono
-- | An Op n a b describes a differentiable function from
-- n values of type a to a value of type b.
--
-- For example, a value of type
--
--
-- Op N2 Int Double
--
--
-- is a function that takes two Ints and returns a Double.
-- It can be differentiated to give a gradient of two Ints,
-- if given a total derivative for the Double. Mathematically, it
-- is akin to a:
--
-- <math>
--
-- See runOp, gradOp, and gradOpWith for examples on
-- how to run it, and Op for instructions on creating it.
--
-- This type is abstracted over using the pattern synonym with
-- constructor Op, so you can create one from scratch with it.
-- However, it's simplest to create it using op2', op1',
-- op2', and op3' helper smart constructors And, if your
-- function is a numeric function, they can even be created automatically
-- using op1, op2, op3, and opN with a little
-- help from Numeric.AD from the ad library.
--
-- Note that this type is a subset or subtype of OpM
-- (and also of OpB). So, if a function ever expects an
-- OpM m as a (or a OpB), you can always provide
-- an Op as a instead.
--
-- Many functions in this library will expect an OpM m as
-- a (or an OpB s as a), and in all of these cases,
-- you can provide an Op as a.
type Op n a b = Op (Replicate n a) b
-- | Construct an Op by giving a function creating the result, and
-- also a continuation on how to create the gradient, given the total
-- derivative of a.
--
-- See the module documentation for Numeric.Backprop.Op for more
-- details on the function that this constructor and OpM expect.
-- | An OpM m n a b represents a differentiable (monadic)
-- function from n values of type a to a value of type
-- b.
--
-- For example, an
--
--
-- OpM IO N2 Int Double
--
--
-- would be a function that takes two Ints and returns a
-- Double (in IO). It can be differentiated to give a
-- gradient of the two input Ints (also in IO) if
-- given the total derivative for a.
--
-- Note that an OpM is a superclass of Op, so any
-- function that expects an OpM m as a can also accept an
-- Op as a.
--
-- See runOpM, gradOpM, and gradOpWithM for examples
-- on how to run it.
type OpM m n a = OpM m (Replicate n a)
-- | Construct an OpM by giving a (monadic) function creating the
-- result, and also a continuation on how to create the gradient, given
-- the total derivative of a.
--
-- See the module documentation for Numeric.Backprop.Op for more
-- details on the function that this constructor and Op expect.
data VecT k (n :: N) (f :: k -> *) (a :: k) :: forall k. N -> (k -> *) -> k -> *
[ØV] :: VecT k Z f a
[:*] :: VecT k (S n1) f a
type Vec (n :: N) = VecT * n I
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Run the function that an Op encodes, to get the result.
--
--
-- >>> runOp (op2 (*)) (3 :+ 5 :+ Ø)
-- 15
--
runOp :: Op n a b -> Vec n a -> b
-- | Run the function that an Op encodes, and get the gradient of
-- the output with respect to the inputs.
--
--
-- >>> gradOp (op2 (*)) (3 :+ 5 :+ ØV)
-- 5 :+ 3 :+ ØV
-- -- the gradient of x*y is (y, x)
--
gradOp :: Op n a b -> Vec n a -> Vec n a
-- | Run the function that an Op encodes, to get the resulting
-- output and also its gradient with respect to the inputs.
--
--
-- >>> gradOpM' (op2 (*)) (3 :+ 5 :+ ØV) :: IO (Int, Vec N2 Int)
-- (15, 5 :+ 3 :+ ØV)
--
gradOp' :: Op n a b -> Vec n a -> (b, Vec n a)
-- | Run the function that an Op encodes, and get the gradient of a
-- "final result" with respect to the inputs, given the total derivative
-- of the output with the final result.
--
-- See gradOp and the module documentaiton for
-- Numeric.Backprop.Op for more information.
gradOpWith :: Op n a b -> Vec n a -> b -> Vec n a
-- | A combination of gradOp and gradOpWith. The third
-- argument is (optionally) the total derivative the result. Give
-- Nothing and it is assumed that the result is the final result
-- (and the total derivative is 1), and this behaves the same as
-- gradOp. Give Just d and it uses the d
-- as the total derivative of the result, and this behaves like
-- gradOpWith.
--
-- See gradOp and the module documentaiton for
-- Numeric.Backprop.Op for more information.
gradOpWith' :: Op n a b -> Vec n a -> Maybe b -> Vec n a
-- | A combination of runOp and gradOpWith'. Given an
-- Op and inputs, returns the result of the Op and a
-- continuation that gives its gradient.
--
-- The continuation takes the total derivative of the result as input.
-- See documenation for gradOpWith' and module documentation for
-- Numeric.Backprop.Op for more information.
runOp' :: Op n a b -> Vec n a -> (b, Maybe b -> Vec n a)
-- | The monadic version of runOp, for OpMs.
--
--
-- >>> runOpM (op2 (*)) (3 :+ 5 :+ ØV) :: IO Int
-- 15
--
runOpM :: Functor m => OpM m n a b -> Vec n a -> m b
-- | The monadic version of gradOp, for OpMs.
gradOpM :: Monad m => OpM m n a b -> Vec n a -> m (Vec n a)
-- | The monadic version of gradOp', for OpMs.
gradOpM' :: Monad m => OpM m n a b -> Vec n a -> m (b, Vec n a)
-- | The monadic version of gradOpWith, for OpMs.
gradOpWithM :: Monad m => OpM m n a b -> Vec n a -> b -> m (Vec n a)
-- | The monadic version of gradOpWith', for OpMs.
gradOpWithM' :: Monad m => OpM m n a b -> Vec n a -> Maybe b -> m (Vec n a)
-- | The monadic version of runOp, for OpMs.
--
--
-- >>> runOpM (op2 (*)) (3 :+ 5 :+ ØV) :: IO Int
-- 15
--
runOpM' :: Functor m => OpM m n a b -> Vec n a -> m (b, Maybe b -> m (Vec n a))
-- | Create an Op that takes no inputs and always returns the given
-- value.
--
-- There is no gradient, of course (using gradOp will give you an
-- empty vector), because there is no input to have a gradient of.
--
--
-- >>> gradOp' (op0 10) ØV
-- (10, ØV)
--
--
-- For a constant Op that takes input and ignores it, see
-- opConst.
--
-- Note that because this returns an Op, it can be used with any
-- function that expects an OpM or OpB, as well.
op0 :: a -> Op N0 b a
-- | An Op that ignores all of its inputs and returns a given
-- constant value.
--
--
-- >>> gradOp' (opConst 10) (1 :+ 2 :+ 3 :+ ØV)
-- (10, 0 :+ 0 :+ 0 :+ ØV)
--
opConst :: forall n a b. (Known Nat n, Num b) => a -> Op n b a
-- | Compose OpMs together, similar to .. But, because all
-- OpMs are <math>, this is more like sequence for
-- functions, or liftAN.
--
-- That is, given an o of OpM m n a bs, it can
-- compose them with an OpM m o b c to create an
-- OpM m o a c.
composeOp :: forall m n o a b c. (Monad m, Num a, Known Nat n) => VecT o (OpM m n a) b -> OpM m o b c -> OpM m n a c
-- | Convenient wrappver over composeOp for the case where the
-- second function only takes one input, so the two OpMs can be
-- directly piped together, like for ..
composeOp1 :: forall m n a b c. (Monad m, Num a, Known Nat n) => OpM m n a b -> OpM m N1 b c -> OpM m n a c
-- | Convenient infix synonym for (flipped) composeOp1. Meant to be
-- used just like .:
--
--
-- op1 negate :: Op '[a] a
-- op2 (+) :: Op '[a,a] a
--
-- op1 negate ~. op2 (+) :: Op '[a, a] a
--
(~.) :: forall m n a b c. (Monad m, Num a, Known Nat n) => OpM m N1 b c -> OpM m n a b -> OpM m n a c
infixr 9 ~.
-- | A version of opConst taking explicit Nat, indicating the
-- number of inputs required.
--
-- Requiring an explicit Nat is mostly useful for rare "extremely
-- polymorphic" situations, where GHC can't infer the length of the the
-- expected input vector. If you ever actually explicitly write down the
-- size n, you should be able to just use opConst.
opConst' :: forall n a b. Num b => Nat n -> a -> Op n b a
-- | A version of composeOp taking explicit Nat, indicating
-- the number of inputs expected in the first Ops
--
-- Requiring an explicit Nat is mostly useful for rare "extremely
-- polymorphic" situations, where GHC can't infer the length of the the
-- expected input vector. If you ever actually explicitly write down the
-- size n, you should be able to just use composeOp.
composeOp' :: forall m n o a b c. (Monad m, Num a) => Nat n -> VecT o (OpM m n a) b -> OpM m o b c -> OpM m n a c
-- | Convenient wrappver over composeOp for the case where the
-- second function only takes one input, so the two OpMs can be
-- directly piped together, like for ..
composeOp1' :: forall m n a b c. (Monad m, Num a) => Nat n -> OpM m n a b -> OpM m N1 b c -> OpM m n a c
-- | Automatically create an Op of a numerical function taking one
-- argument. Uses diff, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op1 (recip . negate)) (5 :+ ØV)
-- (-0.2, 0.04 :+ ØV)
--
op1 :: Num a => (forall s. AD s (Forward a) -> AD s (Forward a)) -> Op N1 a a
-- | Automatically create an Op of a numerical function taking two
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op2 (\x y -> x * sqrt y)) (3 :+ 4 :+ ØV)
-- (6.0, 2.0 :+ 0.75 :+ ØV)
--
op2 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a) -> Op N2 a a
-- | Automatically create an Op of a numerical function taking three
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op3 (\x y z -> (x * sqrt y)**z)) (3 :+ 4 :+ 2 :+ ØV)
-- (36.0, 24.0 :+ 9.0 :+ 64.503 :+ ØV)
--
op3 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a -> Reverse s a) -> Op N3 a a
-- | Automatically create an Op of a numerical function taking
-- multiple arguments. Uses grad, and so can take any numerical
-- function polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (opN (\(x :+ y :+ Ø) -> x * sqrt y)) (3 :+ 4 :+ ØV)
-- (6.0, 2.0 :+ 0.75 :+ ØV)
--
opN :: (Num a, Known Nat n) => (forall s. Reifies s Tape => Vec n (Reverse s a) -> Reverse s a) -> Op n a a
-- | Replicate n a is a list of as repeated
-- n times.
--
--
-- >>> :kind! Replicate N3 Int
-- '[Int, Int, Int]
--
-- >>> :kind! Replicate N5 Double
-- '[Double, Double, Double, Double, Double]
--
-- | Create an Op of a function taking one input, by giving its
-- explicit derivative. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the derivative <math>, it would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that squares its input:
--
--
-- square :: Num a => Op N1 a a
-- square = op1' $ \x -> (x*x, \case Nothing -> 2 * x
-- Just d -> 2 * d * x
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, single-input Ops can be generated
-- automatically using op1.
op1' :: (a -> (b, Maybe b -> a)) -> Op N1 a b
-- | Create an Op of a function taking two inputs, by giving its
-- explicit gradient. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the gradient <math> would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that multiplies its inputs:
--
--
-- mul :: Num a => Op N2 a a
-- mul = op2' $ \x y -> (x*y, \case Nothing -> (y , x )
-- Just d -> (d*y, x*d)
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, two-input Ops can be generated
-- automatically using op2.
op2' :: (a -> a -> (b, Maybe b -> (a, a))) -> Op N2 a b
-- | Create an Op of a function taking three inputs, by giving its
-- explicit gradient. See documentation for op2' for more details.
op3' :: (a -> a -> a -> (b, Maybe b -> (a, a, a))) -> Op N3 a b
infixr 4 :+
(*:) :: f a -> f a -> VecT k (S (S Z)) f a
infix 5 *:
(+:) :: a -> a -> Vec (S (S Z)) a
infix 5 +:
head' :: VecT k (S n) f a -> f a
-- | Convenient aliases for low-value Peano numbers.
type N0 = Z
type N1 = S N0
type N2 = S N1
type N3 = S N2
type N4 = S N3
type N5 = S N4
type N6 = S N5
type N7 = S N6
type N8 = S N7
type N9 = S N8
type N10 = S N9
-- | Optimized version of op2 (+).
(+.) :: Num a => Op N2 a a
-- | Optimized version of op2 (-).
(-.) :: Num a => Op N2 a a
-- | Optimized version of op2 (*).
(*.) :: Num a => Op N2 a a
-- | Optimized version of op1 negate.
negateOp :: Num a => Op N1 a a
-- | Optimized version of op1 abs.
absOp :: Num a => Op N1 a a
-- | Optimized version of op1 signum.
signumOp :: Num a => Op N1 a a
-- | Optimized version of op2 (/).
(/.) :: Fractional a => Op N2 a a
-- | Optimized version of op1 recip.
recipOp :: Fractional a => Op N1 a a
-- | Optimized version of op1 exp.
expOp :: Floating a => Op N1 a a
-- | Optimized version of op1 log.
logOp :: Floating a => Op N1 a a
-- | Optimized version of op1 sqrt.
sqrtOp :: Floating a => Op N1 a a
-- | Optimized version of op2 (**).
(**.) :: Floating a => Op N2 a a
-- | Optimized version of op2 logBase.
logBaseOp :: Floating a => Op N2 a a
-- | Optimized version of op1 sin.
sinOp :: Floating a => Op N1 a a
-- | Optimized version of op1 cos.
cosOp :: Floating a => Op N1 a a
-- | Optimized version of op1 tan.
tanOp :: Floating a => Op N1 a a
-- | Optimized version of op1 asin.
asinOp :: Floating a => Op N1 a a
-- | Optimized version of op1 acos.
acosOp :: Floating a => Op N1 a a
-- | Optimized version of op1 atan.
atanOp :: Floating a => Op N1 a a
-- | Optimized version of op1 sinh.
sinhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 cosh.
coshOp :: Floating a => Op N1 a a
-- | Optimized version of op1 tanh.
tanhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 asinh.
asinhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 acosh.
acoshOp :: Floating a => Op N1 a a
-- | Optimized version of op1 atanh.
atanhOp :: Floating a => Op N1 a a
-- | Provides a monomorphic interface to the library and to the
-- Numeric.Backprop module.
--
-- They are monomorphic in the sense that all of the inputs have
-- to be of the same type. So, something like
--
--
-- BP s '[Double, Double, Double] Int
--
--
-- From Numeric.Backprop would, in this module, be:
--
--
-- BP s N3 Double Int
--
--
-- Instead of dealing with Prods and Tuples, this
-- module works with VecTs and Vecs, respectively. These
-- are fixed-length vectors whose length are encoded in their types,
-- constructed with :* (for VecT) or :+ (for
-- Vec).
--
-- Most of the concepts in normal heterogeneous backprop (for
-- Numeric.Backprop) should apply here as well, so you can look at
-- any of the tutorials or examples and repurpose them to work here. Just
-- remember to convert something like Op '[a, a] b to
-- Op N2 a b.
--
-- As a comparison, this implements something similar in functionality to
-- Numeric.AD and Numeric.AD.Mode.Reverse from the
-- ad package, in that they both offer monomorphic automatic
-- differentiation through back-propagation. This module doesn't allow
-- the computation of jacobians or generalized gradients for <math>
-- functions. This module only computs gradients for (mathbb{R}^N
-- rightarrow mathbb{R})-like functions. This is more of a conscious
-- design decision in the API of this module rather than a fundamental
-- limitation of the implementation.
--
-- This module also allows you to build explicit data dependency graphs
-- so the library can reduce duplication and perform optimizations, which
-- may or may not provide advantages over
-- Numeric.AD.Mode.Reverse's unsafePerformIO-based implicit
-- graph building.
module Numeric.Backprop.Mono
-- | A Monad allowing you to explicitly build hetereogeneous data
-- dependency graphs and that the library can perform back-propagation
-- on.
--
-- A BP s n r a is a BP action that uses an
-- environment n values of type r, and returns an
-- a. When "run", it will compute a gradient that is a vector
-- (Vec) of n rs. (The phantom parameter
-- s is used to ensure that any BVars aren't leaked out
-- of the monad)
--
-- Note that you can only "run" a BP s n r that produces
-- a BVar -- that is, things of the form
--
--
-- BP s n r (BVar n r a)
--
--
-- The above is a BP action that returns a BVar containing
-- an a. When this is run, it'll produce a result of type
-- a and a gradient of that is a vector of n values of
-- type r. (This form has a type synonym, BPOp, for
-- convenience)
--
-- For example, BP s N3 Double is a monad that
-- represents a computation with three Doubles as inputs. And, if
-- you ran a
--
--
-- BP s N3 Double (BVar N3 Double Int)
--
--
-- Or, using the BPOp type synonym:
--
--
-- BPOp s N3 Double Int
--
--
-- with backprop or gradBPOp, it'll return a gradient on
-- the inputs (a vector of three Doubles) and produce a value of
-- type Int.
--
-- Now, one powerful thing about this type is that a BP is itself
-- an Op (or more precisely, an OpM). So, once you create
-- your fancy BP computation, you can transform it into an
-- OpM using bpOp.
type BP s n r = BP s (Replicate n r)
-- | A handy type synonym representing a BP action that returns a
-- BVar. This is handy because this is the form of BP
-- actions that backprop and gradBPOp (etc.) expects.
--
-- A value of type:
--
--
-- BPOp s n r a
--
--
-- is an action that takes an input environment of n values of
-- type r and produces a BVar containing a value of type
-- a. Because it returns a BVar, the library can track
-- the data dependencies between the BVar and the input
-- environment and perform back-propagation.
--
-- See documentation for BP for an explanation of the phantom type
-- parameter s.
type BPOp s n r a = BP s n r (BVar s n r a)
-- | An "implicit" operation on BVars that can be backpropagated. A
-- value of type:
--
--
-- BPOpI s n r a
--
--
-- takes a vector (Vec) of n of BVars containg
-- rs and uses them to (purely) produce a BVar containing
-- an a.
--
--
-- foo :: BPOpI s N2 Double Double
-- foo (x :* y :* ØV) = x + sqrt y
--
--
-- If you are exclusively doing implicit back-propagation by combining
-- BVars and using BPOpIs, you are probably better off just
-- importing Numeric.Backprop.Mono.Implicit, which provides better
-- tools. This type synonym exists in Numeric.Backprop.Mono just
-- for the implicitly function, which can convert "implicit"
-- backprop functions like a BPOpI s rs a into an
-- "explicit" graph backprop function, a BPOp s rs a.
type BPOpI s n r a = VecT n (BVar s n r) r -> BVar s n r a
-- | The basic unit of manipulation inside BP (or inside an
-- implicit-graph backprop function). Instead of directly working with
-- values, you work with BVars contating those values. When you
-- work with a BVar, the backprop library can keep track of
-- what values refer to which other values, and so can perform
-- back-propagation to compute gradients.
--
-- A BVar s n r a refers to a value of type a,
-- with an environment of n values of type r. The
-- phantom parameter s is used to ensure that stray BVars
-- don't leak outside of the backprop process.
--
-- (That is, if you're using implicit backprop, it ensures that you
-- interact with BVars in a polymorphic way. And, if you're using
-- explicit backprop, it ensures that a BVar s n r a
-- never leaves the BP s n r that it was created in.)
--
-- BVars have Num, Fractional, Floating, etc.
-- instances, so they can be manipulated using polymorphic functions and
-- numeric functions in Haskell. You can add them, subtract them, etc.,
-- in "implicit" backprop style.
--
-- (However, note that if you directly manipulate BVars using
-- those instances or using liftB, it delays evaluation, so every
-- usage site has to re-compute the result/create a new node. If you want
-- to re-use a BVar you created using + or - or
-- liftB, use bindVar to force it first. See documentation
-- for bindVar for more details.)
type BVar s n a = BVar s (Replicate n a)
-- | An Op n a b describes a differentiable function from
-- n values of type a to a value of type b.
--
-- For example, a value of type
--
--
-- Op N2 Int Double
--
--
-- is a function that takes two Ints and returns a Double.
-- It can be differentiated to give a gradient of two Ints,
-- if given a total derivative for the Double. Mathematically, it
-- is akin to a:
--
-- <math>
--
-- See runOp, gradOp, and gradOpWith for examples on
-- how to run it, and Op for instructions on creating it.
--
-- This type is abstracted over using the pattern synonym with
-- constructor Op, so you can create one from scratch with it.
-- However, it's simplest to create it using op2', op1',
-- op2', and op3' helper smart constructors And, if your
-- function is a numeric function, they can even be created automatically
-- using op1, op2, op3, and opN with a little
-- help from Numeric.AD from the ad library.
--
-- Note that this type is a subset or subtype of OpM
-- (and also of OpB). So, if a function ever expects an
-- OpM m as a (or a OpB), you can always provide
-- an Op as a instead.
--
-- Many functions in this library will expect an OpM m as
-- a (or an OpB s as a), and in all of these cases,
-- you can provide an Op as a.
type Op n a b = Op (Replicate n a) b
-- | A subclass of OpM (and superclass of Op), representing
-- Ops that the backprop library uses to perform
-- backpropation.
--
-- An
--
--
-- OpB s n a b
--
--
-- represents a differentiable function that takes a n values of
-- type a produces an a b, which can be run on
-- BVar ss and also inside BP ss. For
-- example, an OpB s N2 Double Bool takes two
-- Doubles and produces a Bool, and does it in a
-- differentiable way.
--
-- OpB is a superset of Op, so, if you see any
-- function that expects an OpB (like opVar' and ~$,
-- for example), you can give them an Op, as well.
--
-- You can think of OpB as a superclass/parent class of Op
-- in this sense, and of Op as a subclass of OpB.
type OpB s n a b = OpB s (Replicate n a) b
data VecT k (n :: N) (f :: k -> *) (a :: k) :: forall k. N -> (k -> *) -> k -> *
[ØV] :: VecT k Z f a
[:*] :: VecT k (S n1) f a
type Vec (n :: N) = VecT * n I
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Perform back-propagation on the given BPOp. Returns the result
-- of the operation it represents, as well as the gradient of the result
-- with respect to its inputs. See module header for
-- Numeric.Backprop.Mono and package documentation for examples
-- and usages.
backprop :: forall n r a. Num r => (forall s. BPOp s n r a) -> Vec n r -> (a, Vec n r)
-- | Simply run the BPOp on an input vector, getting the result
-- without bothering with the gradient or with back-propagation.
evalBPOp :: forall n r a. (forall s. BPOp s n r a) -> Vec n r -> a
-- | Run the BPOp on an input vector and return the gradient of the
-- result with respect to the input vector
gradBPOp :: forall n r a. Num r => (forall s. BPOp s n r a) -> Vec n r -> Vec n r
-- | Runs a continuation on a Vec of all of the input BVars.
--
-- Handy for bringing the environment into scope and doing stuff with it:
--
--
-- foo :: BPOp N2 Double Int
-- foo = withInps $ \(x :* y :* ØV) -> do
-- -- do stuff with inputs
--
--
-- Looks kinda like foo (x :* y *+ ØV) = -- ..., don't it?
--
-- Note that the above is the same as
--
--
-- foo :: BPOp N2 Double Int
-- foo = do
-- case inpVars of
-- x :* y :* ØV -> do
-- -- do stuff with inputs
--
--
-- But just a little nicer!
withInps :: Known Nat n => (VecT n (BVar s n r) r -> BP s n r a) -> BP s n r a
-- | Convert a BPOpI into a BPOp. That is, convert a function
-- on a bundle of BVars (generating an implicit graph) into a
-- fully fledged BPOp that you can run backprop on. See
-- BPOpI for more information.
--
-- If you are going to write exclusively using implicit BVar
-- operations, it might be more convenient to use
-- Numeric.Backprop.Mono.Implicit instead, which is geared around
-- that use case.
implicitly :: Known Nat n => BPOpI s n r a -> BPOp s n r a
-- | Create a BVar that represents just a specific value, that
-- doesn't depend on any other BVars.
constVar :: a -> BVar s n r a
-- | Create a BVar given an index (Fin) into the input
-- environment. For an example,
--
--
-- inpVar FZ
--
--
-- would refer to the first input variable, Bool]@), and
--
--
-- inpVar (FS FZ)
--
--
-- Would refer to the second input variable.
--
-- Typically, there shouldn't be any reason to use inpVar
-- directly. It's cleaner to get all of your input BVars together
-- using withInps or inpVars.
inpVar :: Fin n -> BVar s n r r
-- | Get a VecT (vector) of BVars for all of the input
-- environment (the n rs) of the BP s n
-- r
--
-- For example, if your BP has two Doubles inside its input
-- environment (a BP s N2 Double), this would
-- return two BVars, pointing to each input Double.
--
--
-- case (inpVars :: VecT N2 (BVar s N2 Double) Double) of
-- x :* y :* ØV -> do
-- -- the first item, x, is a var to the first input
-- x :: BVar s N2 Double
-- -- the second item, y, is a var to the second input
-- y :: BVar s N2 Double
--
inpVars :: Known Nat n => VecT n (BVar s n r) r
-- | Turn a BPOp into an OpB. Basically converts a BP
-- taking n rs and producing an a into an
-- Op taking an n rs and returning an
-- a, with all of the powers and utility of an Op,
-- including all of its gradient-finding glory.
--
-- Really just reveals the fact that any BPOp s rs a is
-- itself an Op, an OpB s rs a, which makes it a
-- differentiable function.
--
-- Handy because an OpB can be used with almost all of the
-- Op-related functions in this moduel, including opVar,
-- ~$, etc.
bpOp :: forall s n r a. (Num r, Known Nat n) => BPOp s n r a -> OpB s n r a
-- | A version of bpOp' taking explicit Nat, indicating the
-- size of the input VecT.
--
-- Requiring an explicit Nat is mostly useful for rare "extremely
-- polymorphic" situations, where GHC can't infer the length of the the
-- expected input vector. If you ever actually explicitly write down the
-- size n, you should be able to just use opConst.
bpOp' :: forall s n r a. Num r => Nat n -> BPOp s n r a -> OpB s n r a
-- | Concretizes a delayed BVar. If you build up a BVar using
-- numeric functions like + or * or using liftB,
-- it'll defer the evaluation, and all of its usage sites will create a
-- separate graph node.
--
-- Use bindVar if you ever intend to use a BVar in more
-- than one location.
--
--
-- -- bad
-- errSquared :: Num a => BP s N2 a a
-- errSquared = withInp $ \(x :* y :* Ø) -> do
-- let err = r - t
-- return (err * err) -- err is used twice!
--
-- -- good
-- errSquared :: Num a => BP s N2 a a
-- errSquared = withInp $ \(x :* y :* Ø) -> do
-- let err = r - t
-- e <- bindVar err -- force e, so that it's safe to use twice!
-- return (e * e)
--
-- -- better
-- errSquared :: Num a => BP s N2 a a
-- errSquared = withInp $ \(x :* y :* Ø) -> do
-- let err = r - t
-- e <- bindVar err
-- bindVar (e * e) -- result is forced so user doesn't have to worry
--
--
-- Note the relation to opVar ~$ liftB /
-- .$:
--
--
-- opVar o xs = bindVar (liftB o xs)
-- o ~$ xs = bindVar (o .$ xs)
-- op2 (*) ~$ (x :< y :< Ø) = bindVar (x * y)
--
--
-- So you can avoid bindVar altogether if you use the explicitly
-- binding ~$ and opVar etc.
--
-- Note that bindVar on BVars that are already forced is a
-- no-op.
bindVar :: forall s n r a. Num a => BVar s n r a -> BP s n r (BVar s n r a)
-- | Apply an OpB to a VecT (vector) of BVars.
--
-- If you had an OpB s N3 a b, this function will expect
-- a vector of of three BVar s n r as, and the result
-- will be a BVar s n r b:
--
--
-- myOp :: OpB s N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- x :* y :* z :* 'ØV' :: VecT N3 (BVar s n r) a
-- opVar myOp (x :* y :* z :* ØV) :: BP s n r (BVar s n r b)
--
--
-- Note that OpB is a superclass of Op, so you can provide
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- opVar has an infix alias, ~$, so the above example can
-- also be written as:
--
--
-- myOp ~$ (x :* y :* z :* ØV) :: BP s n r (BVar s n r b)
--
--
-- to let you pretend that you're applying the myOp function to
-- three inputs.
--
-- Also note the relation between opVar and liftB and
-- bindVar:
--
--
-- opVar o xs = bindVar (liftB o xs)
--
--
-- opVar can be thought of as a "binding" version of liftB.
opVar :: forall s m n r a b. Num b => OpB s m a b -> VecT m (BVar s n r) a -> BP s n r (BVar s n r b)
-- | Infix synonym for opVar, which lets you pretend that you're
-- applying OpBs as if they were functions:
--
--
-- myOp :: OpB s N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- x :* y :* z :* 'ØV' :: VecT N3 (BVar s n r) a
-- myOp ~$ (x :* y :* z :* ØV) :: BP s n r (BVar s n r b)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- ~$ can also be thought of as a "binding" version of .$:
--
--
-- o ~$ xs = bindVar (o .$ xs)
--
(~$) :: forall s m n r a b. Num b => OpB s m a b -> VecT m (BVar s n r) a -> BP s n r (BVar s n r b)
infixr 5 ~$
-- | Convenient wrapper over opVar that takes an OpB with one
-- argument and a single BVar argument. Lets you not have to type
-- out the entire VecT.
--
--
-- opVar1 o x = opVar o (x :* 'ØV')
--
-- myOp :: Op N2 a b
-- x :: BVar s n r a
--
-- opVar1 myOp x :: BP s n r (BVar s n r b)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op1) as well.
opVar1 :: forall s n r a b. Num b => OpB s N1 a b -> BVar s n r a -> BP s n r (BVar s n r b)
-- | Convenient wrapper over opVar that takes an OpB with two
-- arguments and two BVar arguments. Lets you not have to type out
-- the entire VecT.
--
--
-- opVar2 o x y = opVar o (x :* y :* 'ØV')
--
-- myOp :: Op N2 a b
-- x :: BVar s n r a
-- y :: BVar s n r b
--
-- opVar2 myOp x y :: BP s n r (BVar s n r b)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op2) as well.
opVar2 :: forall s n r a b. Num b => OpB s N2 a b -> BVar s n r a -> BVar s n r a -> BP s n r (BVar s n r b)
-- | Convenient wrapper over opVar that takes an OpB with
-- three arguments and three BVar arguments. Lets you not have to
-- type out the entire VecT.
--
--
-- opVar3 o x y z = opVar o (x :* y :* z :* 'ØV')
--
-- myOp :: Op N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- opVar3 myOp x y z :: BP s n r (BVar s n r b)
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op3) as well.
opVar3 :: forall s n r a b. Num b => OpB s N3 a b -> BVar s n r a -> BVar s n r a -> BVar s n r a -> BP s n r (BVar s n r b)
-- | Lets you treat a BPOp s n a b as an Op n a
-- b, and "apply" arguments to it just like you would with an
-- Op and ~$ / opVar.
--
-- Basically a convenient wrapper over bpOp and ~$:
--
--
-- o -$ xs = bpOp o ~$ xs
--
--
-- So for a BPOp s n a b, you can "plug in" BVars
-- to each a, and get a b as a result.
--
-- Useful for running a BPOp s n a b that you got from a
-- different function, and "plugging in" its a inputs with
-- BVars from your current environment.
(-$) :: forall s m n r a b. (Num a, Num b, Known Nat m) => BPOp s m a b -> VecT m (BVar s n r) a -> BP s n r (BVar s n r b)
infixr 5 -$
-- | Apply OpB over a VecT of BVars, as inputs.
-- Provides "implicit" back-propagation, with deferred evaluation.
--
-- If you had an OpB s N3 a b, this function will expect
-- a vector of of three BVar s n r as, and the result
-- will be a BVar s n r b:
--
--
-- myOp :: OpB s N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- x :* y :* z :* 'ØV' :: VecT N3 (BVar s n r) a
-- liftB myOp (x :* y :* z :* ØV) :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can provide
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- liftB has an infix alias, .$, so the above example can
-- also be written as:
--
--
-- myOp .$ (x :* y :* z :* ØV) :: BVar s n r b
--
--
-- to let you pretend that you're applying the myOp function to
-- three inputs.
--
-- The result is a new deferred BVar. This should be fine
-- in most cases, unless you use the result in more than one location.
-- This will cause evaluation to be duplicated and multiple redundant
-- graph nodes to be created. If you need to use it in two locations, you
-- should use opVar instead of liftB, or use
-- bindVar:
--
--
-- opVar o xs = bindVar (liftB o xs)
--
--
-- liftB can be thought of as a "deferred evaluation" version of
-- opVar.
liftB :: forall s m n a b r. OpB s m a b -> VecT m (BVar s n r) a -> BVar s n r b
-- | Infix synonym for liftB, which lets you pretend that you're
-- applying OpBs as if they were functions:
--
--
-- myOp :: OpB s N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- x :* y :* z :* 'ØV' :: VecT N3 (BVar s n r) a
-- myOp .$ (x :* y :* z :* ØV) :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- See the documentation for liftB for all the caveats of this
-- usage.
--
-- .$ can also be thought of as a "deferred evaluation" version of
-- ~$:
--
--
-- o ~$ xs = bindVar (o .$ xs)
--
(.$) :: forall s m n a b r. OpB s m a b -> VecT m (BVar s n r) a -> BVar s n r b
-- | Convenient wrapper over liftB that takes an OpB with one
-- argument and a single BVar argument. Lets you not have to type
-- out the entire VecT.
--
--
-- liftB1 o x = liftB o (x :* 'ØV')
--
-- myOp :: Op N2 a b
-- x :: BVar s n r a
--
-- liftB1 myOp x :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op1) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB1 :: OpB s N1 a a -> BVar s n r a -> BVar s n r a
-- | Convenient wrapper over liftB that takes an OpB with two
-- arguments and two BVar arguments. Lets you not have to type out
-- the entire VecT.
--
--
-- liftB2 o x y = liftB o (x :* y :* 'ØV')
--
-- myOp :: Op N2 a b
-- x :: BVar s n r a
-- y :: BVar s n r b
--
-- liftB2 myOp x y :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op2) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB2 :: OpB s N2 a a -> BVar s n r a -> BVar s n r a -> BVar s n r a
-- | Convenient wrapper over liftB that takes an OpB with
-- three arguments and three BVar arguments. Lets you not have to
-- type out the entire Prod.
--
--
-- liftB3 o x y z = liftB o (x :* y :* z :* 'ØV')
--
-- myOp :: Op N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r b
-- z :: BVar s n r b
--
-- liftB3 myOp x y z :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op3) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB3 :: OpB s N3 a a -> BVar s n r a -> BVar s n r a -> BVar s n r a -> BVar s n r a
-- | Automatically create an Op of a numerical function taking one
-- argument. Uses diff, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op1 (recip . negate)) (5 :+ ØV)
-- (-0.2, 0.04 :+ ØV)
--
op1 :: Num a => (forall s. AD s (Forward a) -> AD s (Forward a)) -> Op N1 a a
-- | Automatically create an Op of a numerical function taking two
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op2 (\x y -> x * sqrt y)) (3 :+ 4 :+ ØV)
-- (6.0, 2.0 :+ 0.75 :+ ØV)
--
op2 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a) -> Op N2 a a
-- | Automatically create an Op of a numerical function taking three
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op3 (\x y z -> (x * sqrt y)**z)) (3 :+ 4 :+ 2 :+ ØV)
-- (36.0, 24.0 :+ 9.0 :+ 64.503 :+ ØV)
--
op3 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a -> Reverse s a) -> Op N3 a a
-- | Automatically create an Op of a numerical function taking
-- multiple arguments. Uses grad, and so can take any numerical
-- function polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (opN (\(x :+ y :+ Ø) -> x * sqrt y)) (3 :+ 4 :+ ØV)
-- (6.0, 2.0 :+ 0.75 :+ ØV)
--
opN :: (Num a, Known Nat n) => (forall s. Reifies s Tape => Vec n (Reverse s a) -> Reverse s a) -> Op n a a
-- | Compose OpMs together, similar to .. But, because all
-- OpMs are <math>, this is more like sequence for
-- functions, or liftAN.
--
-- That is, given an o of OpM m n a bs, it can
-- compose them with an OpM m o b c to create an
-- OpM m o a c.
composeOp :: forall m n o a b c. (Monad m, Num a, Known Nat n) => VecT o (OpM m n a) b -> OpM m o b c -> OpM m n a c
-- | Convenient wrappver over composeOp for the case where the
-- second function only takes one input, so the two OpMs can be
-- directly piped together, like for ..
composeOp1 :: forall m n a b c. (Monad m, Num a, Known Nat n) => OpM m n a b -> OpM m N1 b c -> OpM m n a c
-- | Convenient infix synonym for (flipped) composeOp1. Meant to be
-- used just like .:
--
--
-- op1 negate :: Op '[a] a
-- op2 (+) :: Op '[a,a] a
--
-- op1 negate ~. op2 (+) :: Op '[a, a] a
--
(~.) :: forall m n a b c. (Monad m, Num a, Known Nat n) => OpM m N1 b c -> OpM m n a b -> OpM m n a c
infixr 9 ~.
-- | Create an Op of a function taking one input, by giving its
-- explicit derivative. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the derivative <math>, it would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that squares its input:
--
--
-- square :: Num a => Op N1 a a
-- square = op1' $ \x -> (x*x, \case Nothing -> 2 * x
-- Just d -> 2 * d * x
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, single-input Ops can be generated
-- automatically using op1.
op1' :: (a -> (b, Maybe b -> a)) -> Op N1 a b
-- | Create an Op of a function taking two inputs, by giving its
-- explicit gradient. The function should return a tuple containing the
-- result of the function, and also a function taking the derivative of
-- the result and return the derivative of the input.
--
-- If we have
--
-- <math>
--
-- Then the gradient <math> would be:
--
-- <math>
--
-- If our Op represents <math>, then the second item in the
-- resulting tuple should be a function that takes <math> and
-- returns <math>.
--
-- If the input is Nothing, then <math> should be taken to
-- be <math>.
--
-- As an example, here is an Op that multiplies its inputs:
--
--
-- mul :: Num a => Op N2 a a
-- mul = op2' $ \x y -> (x*y, \case Nothing -> (y , x )
-- Just d -> (d*y, x*d)
-- )
--
--
-- Remember that, generally, end users shouldn't directly construct
-- Ops; they should be provided by libraries or generated
-- automatically.
--
-- For numeric functions, two-input Ops can be generated
-- automatically using op2.
op2' :: (a -> a -> (b, Maybe b -> (a, a))) -> Op N2 a b
-- | Create an Op of a function taking three inputs, by giving its
-- explicit gradient. See documentation for op2' for more details.
op3' :: (a -> a -> a -> (b, Maybe b -> (a, a, a))) -> Op N3 a b
infixr 4 :+
(*:) :: f a -> f a -> VecT k (S (S Z)) f a
infix 5 *:
(+:) :: a -> a -> Vec (S (S Z)) a
infix 5 +:
head' :: VecT k (S n) f a -> f a
-- | Convenient aliases for low-value Peano numbers.
type N0 = Z
type N1 = S N0
type N2 = S N1
type N3 = S N2
type N4 = S N3
type N5 = S N4
type N6 = S N5
type N7 = S N6
type N8 = S N7
type N9 = S N8
type N10 = S N9
-- | Optimized version of op2 (+).
(+.) :: Num a => Op N2 a a
-- | Optimized version of op2 (-).
(-.) :: Num a => Op N2 a a
-- | Optimized version of op2 (*).
(*.) :: Num a => Op N2 a a
-- | Optimized version of op1 negate.
negateOp :: Num a => Op N1 a a
-- | Optimized version of op1 abs.
absOp :: Num a => Op N1 a a
-- | Optimized version of op1 signum.
signumOp :: Num a => Op N1 a a
-- | Optimized version of op2 (/).
(/.) :: Fractional a => Op N2 a a
-- | Optimized version of op1 recip.
recipOp :: Fractional a => Op N1 a a
-- | Optimized version of op1 exp.
expOp :: Floating a => Op N1 a a
-- | Optimized version of op1 log.
logOp :: Floating a => Op N1 a a
-- | Optimized version of op1 sqrt.
sqrtOp :: Floating a => Op N1 a a
-- | Optimized version of op2 (**).
(**.) :: Floating a => Op N2 a a
-- | Optimized version of op2 logBase.
logBaseOp :: Floating a => Op N2 a a
-- | Optimized version of op1 sin.
sinOp :: Floating a => Op N1 a a
-- | Optimized version of op1 cos.
cosOp :: Floating a => Op N1 a a
-- | Optimized version of op1 tan.
tanOp :: Floating a => Op N1 a a
-- | Optimized version of op1 asin.
asinOp :: Floating a => Op N1 a a
-- | Optimized version of op1 acos.
acosOp :: Floating a => Op N1 a a
-- | Optimized version of op1 atan.
atanOp :: Floating a => Op N1 a a
-- | Optimized version of op1 sinh.
sinhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 cosh.
coshOp :: Floating a => Op N1 a a
-- | Optimized version of op1 tanh.
tanhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 asinh.
asinhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 acosh.
acoshOp :: Floating a => Op N1 a a
-- | Optimized version of op1 atanh.
atanhOp :: Floating a => Op N1 a a
-- | Offers full functionality for implicit-graph back-propagation with
-- monomorphic inputs. The intended usage is to write a BPOp,
-- which is a normal Haskell function from BVars to a result
-- BVar. These BVars can be manipulated using their
-- Num Fractional Floating instances.
--
-- The library can then perform back-propagation on the function (using
-- backprop or grad) by using an implicitly built graph.
--
-- This is an "implicit-only" version of Numeric.Backprop.Mono,
-- and a monomorphic version of Numeric.Backprop.Implicit,
-- monomorphic in the sense that all of the inputs are of the same type.
--
-- Like for Numeric.Backprop.Implicit, this should actually be
-- powerful enough for most use cases, but falls short because without
-- explicit graph capabilities, recomputation can sometimes be
-- inevitable. If the result of a function on BVars is used twice
-- (like z in let z = x * y in z + z), this will
-- allocate a new redundant graph node for every usage site of
-- z. You can explicitly force z, but only using
-- an explicit graph description using Numeric.Backprop.Mono.
--
-- Like Numeric.Backprop.Implicit, this can't handle sum types,
-- but neither can Numeric.Backprop.Mono, so no loss here :)
--
-- This module implements pretty much the same functionality as
-- Numeric.AD and Numeric.AD.Mode.Reverse from the
-- ad package, because it uses the same implicit-graph
-- back-propagation method. It can't compute jacobians/generalized
-- gradients, however. This isn't a fundamental limitation of the
-- implementaiton, though, but rather just a conscious design decision
-- for this module's API.
module Numeric.Backprop.Mono.Implicit
-- | The basic unit of manipulation inside BP (or inside an
-- implicit-graph backprop function). Instead of directly working with
-- values, you work with BVars contating those values. When you
-- work with a BVar, the backprop library can keep track of
-- what values refer to which other values, and so can perform
-- back-propagation to compute gradients.
--
-- A BVar s n r a refers to a value of type a,
-- with an environment of n values of type r. The
-- phantom parameter s is used to ensure that stray BVars
-- don't leak outside of the backprop process.
--
-- (That is, if you're using implicit backprop, it ensures that you
-- interact with BVars in a polymorphic way. And, if you're using
-- explicit backprop, it ensures that a BVar s n r a
-- never leaves the BP s n r that it was created in.)
--
-- BVars have Num, Fractional, Floating, etc.
-- instances, so they can be manipulated using polymorphic functions and
-- numeric functions in Haskell. You can add them, subtract them, etc.,
-- in "implicit" backprop style.
--
-- (However, note that if you directly manipulate BVars using
-- those instances or using liftB, it delays evaluation, so every
-- usage site has to re-compute the result/create a new node. If you want
-- to re-use a BVar you created using + or - or
-- liftB, use bindVar to force it first. See documentation
-- for bindVar for more details.)
type BVar s n a = BVar s (Replicate n a)
-- | An operation on BVars that can be backpropagated. A value of
-- type:
--
--
-- BPOp n r a
--
--
-- takes a vector (VecT) of BVars containg n
-- rs and uses them to (purely) produce a BVar containing
-- an a.
--
--
-- foo :: BPOp N2 Double Double
-- foo (x :* y :* 'ØV') = x + sqrt y
--
--
-- BPOp here is related to BPOpI from the normal
-- explicit-graph backprop module Numeric.Backprop.Mono.
type BPOp n a b = forall s. VecT n (BVar s n a) a -> BVar s n a b
-- | An Op n a b describes a differentiable function from
-- n values of type a to a value of type b.
--
-- For example, a value of type
--
--
-- Op N2 Int Double
--
--
-- is a function that takes two Ints and returns a Double.
-- It can be differentiated to give a gradient of two Ints,
-- if given a total derivative for the Double. Mathematically, it
-- is akin to a:
--
-- <math>
--
-- See runOp, gradOp, and gradOpWith for examples on
-- how to run it, and Op for instructions on creating it.
--
-- This type is abstracted over using the pattern synonym with
-- constructor Op, so you can create one from scratch with it.
-- However, it's simplest to create it using op2', op1',
-- op2', and op3' helper smart constructors And, if your
-- function is a numeric function, they can even be created automatically
-- using op1, op2, op3, and opN with a little
-- help from Numeric.AD from the ad library.
--
-- Note that this type is a subset or subtype of OpM
-- (and also of OpB). So, if a function ever expects an
-- OpM m as a (or a OpB), you can always provide
-- an Op as a instead.
--
-- Many functions in this library will expect an OpM m as
-- a (or an OpB s as a), and in all of these cases,
-- you can provide an Op as a.
type Op n a b = Op (Replicate n a) b
-- | A subclass of OpM (and superclass of Op), representing
-- Ops that the backprop library uses to perform
-- backpropation.
--
-- An
--
--
-- OpB s n a b
--
--
-- represents a differentiable function that takes a n values of
-- type a produces an a b, which can be run on
-- BVar ss and also inside BP ss. For
-- example, an OpB s N2 Double Bool takes two
-- Doubles and produces a Bool, and does it in a
-- differentiable way.
--
-- OpB is a superset of Op, so, if you see any
-- function that expects an OpB (like opVar' and ~$,
-- for example), you can give them an Op, as well.
--
-- You can think of OpB as a superclass/parent class of Op
-- in this sense, and of Op as a subclass of OpB.
type OpB s n a b = OpB s (Replicate n a) b
data VecT k (n :: N) (f :: k -> *) (a :: k) :: forall k. N -> (k -> *) -> k -> *
[ØV] :: VecT k Z f a
[:*] :: VecT k (S n1) f a
type Vec (n :: N) = VecT * n I
newtype I a :: * -> *
I :: a -> I a
[getI] :: I a -> a
-- | Run back-propagation on a BPOp function, getting both the
-- result and the gradient of the result with respect to the inputs.
--
--
-- foo :: BPOp N2 Double Double
-- foo (x :* y :* ØV) =
-- let z = x * sqrt y
-- in z + x ** y
--
--
--
-- >>> backprop foo (2 :+ 3 :+ ØV)
-- (11.46, 13.73 :+ 6.12 :+ ØV)
--
backprop :: forall n a b. (Num a, Known Nat n) => BPOp n a b -> Vec n a -> (b, Vec n a)
-- | Run the BPOp on an input tuple and return the gradient of the
-- result with respect to the input tuple.
--
--
-- foo :: BPOp N2 Double Double
-- foo (x :* y :* ØV) =
-- let z = x * sqrt y
-- in z + x ** y
--
--
--
-- >>> grad foo (2 :+ 3 :+ ØV)
-- 13.73 :+ 6.12 :+ ØV
--
grad :: forall n a b. (Num a, Known Nat n) => BPOp n a b -> Vec n a -> Vec n a
-- | Simply run the BPOp on an input tuple, getting the result
-- without bothering with the gradient or with back-propagation.
--
--
-- foo :: BPOp N2 Double Double
-- foo (x :* y :* ØV) =
-- let z = x * sqrt y
-- in z + x ** y
--
--
--
-- >>> eval foo (2 :+ 3 :+ ØV)
-- 11.46
--
eval :: forall n a b. (Num a, Known Nat n) => BPOp n a b -> Vec n a -> b
-- | Create a BVar that represents just a specific value, that
-- doesn't depend on any other BVars.
constVar :: a -> BVar s n r a
-- | Apply OpB over a VecT of BVars, as inputs.
-- Provides "implicit" back-propagation, with deferred evaluation.
--
-- If you had an OpB s N3 a b, this function will expect
-- a vector of of three BVar s n r as, and the result
-- will be a BVar s n r b:
--
--
-- myOp :: OpB s N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- x :* y :* z :* 'ØV' :: VecT N3 (BVar s n r) a
-- liftB myOp (x :* y :* z :* ØV) :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can provide
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- liftB has an infix alias, .$, so the above example can
-- also be written as:
--
--
-- myOp .$ (x :* y :* z :* ØV) :: BVar s n r b
--
--
-- to let you pretend that you're applying the myOp function to
-- three inputs.
--
-- The result is a new deferred BVar. This should be fine
-- in most cases, unless you use the result in more than one location.
-- This will cause evaluation to be duplicated and multiple redundant
-- graph nodes to be created. If you need to use it in two locations, you
-- should use opVar instead of liftB, or use
-- bindVar:
--
--
-- opVar o xs = bindVar (liftB o xs)
--
--
-- liftB can be thought of as a "deferred evaluation" version of
-- opVar.
liftB :: forall s m n a b r. OpB s m a b -> VecT m (BVar s n r) a -> BVar s n r b
-- | Infix synonym for liftB, which lets you pretend that you're
-- applying OpBs as if they were functions:
--
--
-- myOp :: OpB s N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r a
-- z :: BVar s n r a
--
-- x :* y :* z :* 'ØV' :: VecT N3 (BVar s n r) a
-- myOp .$ (x :* y :* z :* ØV) :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- any Op here, as well (like those created by op1,
-- op2, constOp, op0 etc.)
--
-- See the documentation for liftB for all the caveats of this
-- usage.
--
-- .$ can also be thought of as a "deferred evaluation" version of
-- ~$:
--
--
-- o ~$ xs = bindVar (o .$ xs)
--
(.$) :: forall s m n a b r. OpB s m a b -> VecT m (BVar s n r) a -> BVar s n r b
-- | Convenient wrapper over liftB that takes an OpB with one
-- argument and a single BVar argument. Lets you not have to type
-- out the entire VecT.
--
--
-- liftB1 o x = liftB o (x :* 'ØV')
--
-- myOp :: Op N2 a b
-- x :: BVar s n r a
--
-- liftB1 myOp x :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op1) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB1 :: OpB s N1 a a -> BVar s n r a -> BVar s n r a
-- | Convenient wrapper over liftB that takes an OpB with two
-- arguments and two BVar arguments. Lets you not have to type out
-- the entire VecT.
--
--
-- liftB2 o x y = liftB o (x :* y :* 'ØV')
--
-- myOp :: Op N2 a b
-- x :: BVar s n r a
-- y :: BVar s n r b
--
-- liftB2 myOp x y :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op2) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB2 :: OpB s N2 a a -> BVar s n r a -> BVar s n r a -> BVar s n r a
-- | Convenient wrapper over liftB that takes an OpB with
-- three arguments and three BVar arguments. Lets you not have to
-- type out the entire Prod.
--
--
-- liftB3 o x y z = liftB o (x :* y :* z :* 'ØV')
--
-- myOp :: Op N3 a b
-- x :: BVar s n r a
-- y :: BVar s n r b
-- z :: BVar s n r b
--
-- liftB3 myOp x y z :: BVar s n r b
--
--
-- Note that OpB is a superclass of Op, so you can pass in
-- an Op here (like one made with op3) as well.
--
-- See the documentation for liftB for caveats and potential
-- problematic situations with this.
liftB3 :: OpB s N3 a a -> BVar s n r a -> BVar s n r a -> BVar s n r a -> BVar s n r a
-- | Automatically create an Op of a numerical function taking one
-- argument. Uses diff, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op1 (recip . negate)) (5 :+ ØV)
-- (-0.2, 0.04 :+ ØV)
--
op1 :: Num a => (forall s. AD s (Forward a) -> AD s (Forward a)) -> Op N1 a a
-- | Automatically create an Op of a numerical function taking two
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op2 (\x y -> x * sqrt y)) (3 :+ 4 :+ ØV)
-- (6.0, 2.0 :+ 0.75 :+ ØV)
--
op2 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a) -> Op N2 a a
-- | Automatically create an Op of a numerical function taking three
-- arguments. Uses grad, and so can take any numerical function
-- polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (op3 (\x y z -> (x * sqrt y)**z)) (3 :+ 4 :+ 2 :+ ØV)
-- (36.0, 24.0 :+ 9.0 :+ 64.503 :+ ØV)
--
op3 :: Num a => (forall s. Reifies s Tape => Reverse s a -> Reverse s a -> Reverse s a -> Reverse s a) -> Op N3 a a
-- | Automatically create an Op of a numerical function taking
-- multiple arguments. Uses grad, and so can take any numerical
-- function polymorphic over the standard numeric types.
--
--
-- >>> gradOp' (opN (\(x :+ y :+ Ø) -> x * sqrt y)) (3 :+ 4 :+ ØV)
-- (6.0, 2.0 :+ 0.75 :+ ØV)
--
opN :: (Num a, Known Nat n) => (forall s. Reifies s Tape => Vec n (Reverse s a) -> Reverse s a) -> Op n a a
infixr 4 :+
(*:) :: f a -> f a -> VecT k (S (S Z)) f a
infix 5 *:
(+:) :: a -> a -> Vec (S (S Z)) a
infix 5 +:
head' :: VecT k (S n) f a -> f a
-- | Convenient aliases for low-value Peano numbers.
type N0 = Z
type N1 = S N0
type N2 = S N1
type N3 = S N2
type N4 = S N3
type N5 = S N4
type N6 = S N5
type N7 = S N6
type N8 = S N7
type N9 = S N8
type N10 = S N9
-- | Optimized version of op2 (+).
(+.) :: Num a => Op N2 a a
-- | Optimized version of op2 (-).
(-.) :: Num a => Op N2 a a
-- | Optimized version of op2 (*).
(*.) :: Num a => Op N2 a a
-- | Optimized version of op1 negate.
negateOp :: Num a => Op N1 a a
-- | Optimized version of op1 abs.
absOp :: Num a => Op N1 a a
-- | Optimized version of op1 signum.
signumOp :: Num a => Op N1 a a
-- | Optimized version of op2 (/).
(/.) :: Fractional a => Op N2 a a
-- | Optimized version of op1 recip.
recipOp :: Fractional a => Op N1 a a
-- | Optimized version of op1 exp.
expOp :: Floating a => Op N1 a a
-- | Optimized version of op1 log.
logOp :: Floating a => Op N1 a a
-- | Optimized version of op1 sqrt.
sqrtOp :: Floating a => Op N1 a a
-- | Optimized version of op2 (**).
(**.) :: Floating a => Op N2 a a
-- | Optimized version of op2 logBase.
logBaseOp :: Floating a => Op N2 a a
-- | Optimized version of op1 sin.
sinOp :: Floating a => Op N1 a a
-- | Optimized version of op1 cos.
cosOp :: Floating a => Op N1 a a
-- | Optimized version of op1 tan.
tanOp :: Floating a => Op N1 a a
-- | Optimized version of op1 asin.
asinOp :: Floating a => Op N1 a a
-- | Optimized version of op1 acos.
acosOp :: Floating a => Op N1 a a
-- | Optimized version of op1 atan.
atanOp :: Floating a => Op N1 a a
-- | Optimized version of op1 sinh.
sinhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 cosh.
coshOp :: Floating a => Op N1 a a
-- | Optimized version of op1 tanh.
tanhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 asinh.
asinhOp :: Floating a => Op N1 a a
-- | Optimized version of op1 acosh.
acoshOp :: Floating a => Op N1 a a
-- | Optimized version of op1 atanh.
atanhOp :: Floating a => Op N1 a a