module MachineLearning.Optimization
(
MinimizeMethod(..)
, minimize
, checkGradient
)
where
import MachineLearning.Types (R, Vector, Matrix)
import MachineLearning.Model as Model
import MachineLearning.Regularization (Regularization)
import qualified MachineLearning.Optimization.GradientDescent as GD
import qualified MachineLearning.Optimization.MinibatchGradientDescent as MGD
import qualified Data.Vector.Storable as V
import qualified Numeric.LinearAlgebra as LA
import qualified Numeric.GSL.Minimization as Min
data MinimizeMethod = GradientDescent R
| MinibatchGradientDescent Int Int R
| ConjugateGradientFR R R
| ConjugateGradientPR R R
| BFGS2 R R
minimize :: Model.Model a =>
MinimizeMethod
-> a
-> R
-> Int
-> Regularization
-> Matrix
-> Vector
-> Vector
-> (Vector, Matrix)
minimize (BFGS2 firstStepSize tol) = minimizeVD Min.VectorBFGS2 firstStepSize tol
minimize (ConjugateGradientFR firstStepSize tol) = minimizeVD Min.ConjugateFR firstStepSize tol
minimize (ConjugateGradientPR firstStepSize tol) = minimizeVD Min.ConjugatePR firstStepSize tol
minimize (GradientDescent alpha) = GD.gradientDescent alpha
minimize (MinibatchGradientDescent seed batchSize alpha) = MGD.minibatchGradientDescent seed batchSize alpha
minimizeVD method firstStepSize tol model epsilon niters reg x y initialTheta
= Min.minimizeVD method epsilon niters firstStepSize tol costf gradientf initialTheta
where costf = Model.cost model reg x y
gradientf = Model.gradient model reg x y
checkGradient :: Model a => a -> Regularization -> Matrix -> Vector -> Vector -> R -> R
checkGradient model reg x y theta eps = LA.norm_2 $ grad1 - grad2
where theta_m = LA.asColumn theta
eps_v = V.replicate (V.length theta) eps
eps_m = LA.diag eps_v
theta_m1 = theta_m + eps_m
theta_m2 = theta_m - eps_m
cost1 = LA.vector $ map (cost model reg x y) $ LA.toColumns theta_m1
cost2 = LA.vector $ map (cost model reg x y) $ LA.toColumns theta_m2
grad1 = (cost1 - cost2) / (LA.scalar $ 2*eps)
grad2 = gradient model reg x y theta