{-# OPTIONS_HADDOCK prune #-}
module Database.Postgres.Temp.Internal.Config where
import Database.Postgres.Temp.Internal.Core
import Control.Applicative.Lift
import Control.DeepSeq
import Control.Exception
import Control.Monad (join)
import Crypto.Hash.SHA1 (hash)
import Control.Monad.Trans.Class
import Control.Monad.Trans.Cont
import qualified Data.ByteString.Char8 as BSC
import qualified Data.ByteString.Base64.URL as Base64
import Data.Char
import qualified Data.Map.Strict as Map
import Data.Map.Strict (Map)
import Data.Maybe
import Data.Monoid
import Data.Monoid.Generic
import Data.List
import Data.Traversable
import qualified Database.PostgreSQL.Simple.Options as Client
import GHC.Generics (Generic)
import Network.Socket.Free (getFreePort)
import System.Directory
import System.Environment
import System.Exit (ExitCode(..))
import System.IO
import System.IO.Error
import System.IO.Temp (createTempDirectory)
import System.IO.Unsafe (unsafePerformIO)
import System.Process
import Text.PrettyPrint.ANSI.Leijen hiding ((<$>))
data Accum a = DontCare | Zlich | Merge a
deriving stock (Show, Eq, Ord, Functor)
instance Applicative Accum where
pure = Merge
af <*> ax = case (af, ax) of
(Merge f, Merge x) -> Merge $ f x
(DontCare, _) -> DontCare
(_, DontCare) -> DontCare
(Zlich, _) -> Zlich
(_, Zlich) -> Zlich
instance Semigroup a => Semigroup (Accum a) where
x <> y = case (x, y) of
(DontCare, b) -> b
(a , DontCare ) -> a
(Zlich , _ ) -> Zlich
(_ , Zlich) -> Zlich
(Merge a, Merge b) -> Merge $ a <> b
getAccum :: Accum a -> Maybe a
getAccum = \case
Merge a -> Just a
_ -> Nothing
instance Monoid a => Monoid (Accum a) where
mempty = DontCare
prettyMap :: (Pretty a, Pretty b) => Map a b -> Doc
prettyMap theMap =
let xs = Map.toList theMap
in vsep $ map (uncurry prettyKeyPair) xs
data EnvironmentVariables = EnvironmentVariables
{ inherit :: Last Bool
, specific :: Map String String
}
deriving stock (Generic, Show, Eq)
instance Semigroup EnvironmentVariables where
x <> y = EnvironmentVariables
{ inherit =
inherit x <> inherit y
, specific =
specific y <> specific x
}
instance Monoid EnvironmentVariables where
mempty = EnvironmentVariables mempty mempty
instance Pretty EnvironmentVariables where
pretty EnvironmentVariables {..}
= text "inherit:"
<+> pretty (getLast inherit)
<> hardline
<> text "specific:"
<> softline
<> indent 2 (prettyMap specific)
completeEnvironmentVariables
:: [(String, String)]
-> EnvironmentVariables
-> Either [String] [(String, String)]
completeEnvironmentVariables envs EnvironmentVariables {..} = case getLast inherit of
Nothing -> Left ["Inherit not specified"]
Just x -> Right $ (if x then envs else [])
<> Map.toList specific
data CommandLineArgs = CommandLineArgs
{ keyBased :: Map String (Maybe String)
, indexBased :: Map Int String
}
deriving stock (Generic, Show, Eq)
deriving Monoid via GenericMonoid CommandLineArgs
instance Semigroup CommandLineArgs where
x <> y = CommandLineArgs
{ keyBased =
keyBased y <> keyBased x
, indexBased =
indexBased y <> indexBased x
}
instance Pretty CommandLineArgs where
pretty p@CommandLineArgs {..}
= text "keyBased:"
<> softline
<> indent 2 (prettyMap keyBased)
<> hardline
<> text "indexBased:"
<> softline
<> indent 2 (prettyMap indexBased)
<> hardline
<> text "completed:" <+> text (unwords (completeCommandLineArgs p))
takeWhileInSequence :: [(Int, a)] -> [a]
takeWhileInSequence ((0, x):xs) = x : go 0 xs where
go _ [] = []
go prev ((next, a):rest)
| prev + 1 == next = a : go next rest
| otherwise = []
takeWhileInSequence _ = []
completeCommandLineArgs :: CommandLineArgs -> [String]
completeCommandLineArgs CommandLineArgs {..}
= map (\(name, mvalue) -> maybe name (name <>) mvalue)
(Map.toList keyBased)
<> takeWhileInSequence (Map.toList indexBased)
data ProcessConfig = ProcessConfig
{ environmentVariables :: EnvironmentVariables
, commandLine :: CommandLineArgs
, stdIn :: Last Handle
, stdOut :: Last Handle
, stdErr :: Last Handle
}
deriving stock (Generic, Eq, Show)
deriving Semigroup via GenericSemigroup ProcessConfig
deriving Monoid via GenericMonoid ProcessConfig
instance Pretty ProcessConfig where
pretty ProcessConfig {..}
= text "environmentVariables:"
<> softline
<> indent 2 (pretty environmentVariables)
<> hardline
<> text "commandLine:"
<> softline
<> indent 2 (pretty environmentVariables)
<> hardline
<> text "stdIn:" <+>
pretty (prettyHandle <$> getLast stdIn)
<> hardline
<> text "stdOut:" <+>
pretty (prettyHandle <$> getLast stdOut)
<> hardline
<> text "stdErr:" <+>
pretty (prettyHandle <$> getLast stdErr)
standardProcessConfig :: ProcessConfig
standardProcessConfig = mempty
{ environmentVariables = mempty
{ inherit = pure True
}
, stdIn = pure stdin
, stdOut = pure stdout
, stdErr = pure stderr
}
devNull :: Handle
devNull = unsafePerformIO (openFile "/dev/null" WriteMode)
{-# NOINLINE devNull #-}
silentProcessConfig :: ProcessConfig
silentProcessConfig = mempty
{ environmentVariables = mempty
{ inherit = pure True
}
, stdIn = pure devNull
, stdOut = pure devNull
, stdErr = pure devNull
}
addErrorContext :: String -> Either [String] a -> Either [String] a
addErrorContext cxt = either (Left . map (cxt <>)) Right
getOption :: String -> Last a -> Errors [String] a
getOption optionName = \case
Last (Just x) -> pure x
Last Nothing -> failure ["Missing " ++ optionName ++ " option"]
completeProcessConfig
:: [(String, String)] -> ProcessConfig -> Either [String] CompleteProcessConfig
completeProcessConfig envs ProcessConfig {..} = runErrors $ do
let completeProcessConfigCmdLine = completeCommandLineArgs commandLine
completeProcessConfigEnvVars <- eitherToErrors $
completeEnvironmentVariables envs environmentVariables
completeProcessConfigStdIn <-
getOption "stdIn" stdIn
completeProcessConfigStdOut <-
getOption "stdOut" stdOut
completeProcessConfigStdErr <-
getOption "stdErr" stdErr
pure CompleteProcessConfig {..}
data CompleteDirectoryType = CPermanent FilePath | CTemporary FilePath
deriving stock (Show, Eq, Ord, Generic)
deriving anyclass (NFData)
toFilePath :: CompleteDirectoryType -> FilePath
toFilePath = \case
CPermanent x -> x
CTemporary x -> x
instance Pretty CompleteDirectoryType where
pretty = \case
CPermanent x -> text "CPermanent" <+> pretty x
CTemporary x -> text "CTemporary" <+> pretty x
makePermanent :: CompleteDirectoryType -> CompleteDirectoryType
makePermanent = \case
CTemporary x -> CPermanent x
x -> x
data DirectoryType
= Permanent FilePath
| Temporary
deriving(Show, Eq, Ord)
instance Pretty DirectoryType where
pretty = \case
Permanent x -> text "Permanent" <+> pretty x
Temporary -> text "Temporary"
instance Semigroup DirectoryType where
x <> y = case (x, y) of
(a, Temporary ) -> a
(_, a@Permanent {}) -> a
instance Monoid DirectoryType where
mempty = Temporary
fixPath :: FilePath -> IO FilePath
fixPath x = case x of
'~':rest -> do
homeDir <- getHomeDirectory
pure $ homeDir <> "/" <> rest
xs -> pure xs
setupDirectoryType
:: String
-> String
-> DirectoryType
-> IO CompleteDirectoryType
setupDirectoryType tempDir pat dirType = case dirType of
Temporary -> CTemporary <$> createTempDirectory tempDir pat
Permanent x -> CPermanent <$> fixPath x
rmDirIgnoreErrors :: FilePath -> IO ()
rmDirIgnoreErrors mainDir = do
let ignoreDirIsMissing e
| isDoesNotExistError e = return ()
| otherwise = throwIO e
handle ignoreDirIsMissing $
try (removeDirectoryRecursive mainDir) >>= \case
Left (_ :: IOError) -> try (removeDirectoryRecursive mainDir) >>= \case
Left (_ :: IOError) -> removeDirectoryRecursive mainDir
Right _ -> pure ()
Right _ -> pure ()
cleanupDirectoryType :: CompleteDirectoryType -> IO ()
cleanupDirectoryType = \case
CPermanent _ -> pure ()
CTemporary filePath -> rmDirIgnoreErrors filePath
completePostgresPlan :: [(String, String)] -> Config -> Either [String] CompletePostgresPlan
completePostgresPlan envs Config {..} = runErrors $ do
let completePostgresPlanClientOptions = connectionOptions
completePostgresPlanProcessConfig <-
eitherToErrors $ addErrorContext "postgresConfig: " $
completeProcessConfig envs postgresConfig
pure CompletePostgresPlan {..}
flattenConfig :: [(String, String)] -> String
flattenConfig = unlines . map (\(x, y) -> x <> "=" <> y) .
Map.toList . Map.fromList
completePlan :: [(String, String)] -> String -> Config -> Either [String] Plan
completePlan envs dataDirectoryString config@Config {..} = do
( completePlanLogger
, completePlanCreateDb
, completePlanPostgres
, completePlanDataDirectory
, completePlanConnectionTimeout
) <- runErrors
$ (,,,,)
<$> getOption "logger" logger
<*> eitherToErrors (addErrorContext "createDbConfig: " $
traverse (completeProcessConfig envs) $ getAccum createDbConfig)
<*> eitherToErrors (addErrorContext "postgresPlan: "
(completePostgresPlan envs config))
<*> pure dataDirectoryString
<*> getOption "connectionTimeout" connectionTimeout
let completePlanConfig = flattenConfig postgresConfigFile
completePlanCopy = completeCopyDirectory completePlanDataDirectory <$>
join (getLast copyConfig)
completePlanInitDb <- addErrorContext "initDbConfig: " $ completeInitDb envs dataDirectoryString config
pure Plan {..}
removeDataDirectory :: ProcessConfig -> ProcessConfig
removeDataDirectory processConfig@ProcessConfig{..} =
let CommandLineArgs {..} = commandLine
newCommandLine = commandLine
{ keyBased = Map.delete "--pgdata=" $ Map.delete "-D" keyBased
}
newEnvironmentVariables = environmentVariables
{ specific = Map.delete "PGDATA" $ specific environmentVariables
}
in processConfig
{ commandLine = newCommandLine
, environmentVariables = newEnvironmentVariables
}
completeInitDb :: [(String, String)] -> FilePath -> Config -> Either [String] (Maybe (Either CompleteProcessConfig InitDbCachePlan))
completeInitDb envs theDataDirectory Config {..} = for (getAccum initDbConfig) $ \theInitDbConfig -> case join $ getLast initDbCache of
Nothing -> Left <$> completeProcessConfig envs theInitDbConfig
Just (cow, cacheDirectory) -> do
let
clearedConfig = removeDataDirectory theInitDbConfig
completeClearedPlan <- completeProcessConfig envs clearedConfig
let
cachePath = makeCachePath cacheDirectory theCommandLine
cachePlanDataDirectory = cachePath <> "/data"
cachePlanCopy = CompleteCopyDirectoryCommand
{ copyDirectoryCommandSrc = cachePlanDataDirectory
, copyDirectoryCommandDst = theDataDirectory
, copyDirectoryCommandCow = cow
}
theCommandLine = makeInitDbCommandLine completeClearedPlan
modifiedConfig = clearedConfig <> mempty
{ commandLine = mempty
{ keyBased = Map.fromList [("--pgdata=", Just cachePlanDataDirectory)]
}
}
cachePlanInitDb <- completeProcessConfig envs modifiedConfig
pure $ Right InitDbCachePlan {..}
hasInitDb :: Config -> Bool
hasInitDb Config {..} = isJust $ getAccum initDbConfig
hasCreateDb :: Config -> Bool
hasCreateDb Config {..} = isJust $ getAccum createDbConfig
data Config = Config
{ logger :: Last Logger
, initDbConfig :: Accum ProcessConfig
, copyConfig :: Last (Maybe CopyDirectoryCommand)
, createDbConfig :: Accum ProcessConfig
, postgresConfig :: ProcessConfig
, connectionOptions :: Client.Options
, postgresConfigFile :: [(String, String)]
, connectionTimeout :: Last Int
, socketDirectory :: DirectoryType
, dataDirectory :: DirectoryType
, port :: Last (Maybe Int)
, temporaryDirectory :: Last FilePath
, initDbCache :: Last (Maybe (Bool, FilePath))
}
deriving stock (Generic)
deriving Semigroup via GenericSemigroup Config
deriving Monoid via GenericMonoid Config
instance Pretty Config where
pretty Config {..}
= text "socketDirectory:"
<> softline
<> pretty socketDirectory
<> hardline
<> text "dataDirectory:"
<> softline
<> pretty dataDirectory
<> hardline
<> text "port:" <+> pretty (getLast port)
<> hardline
<> text "temporaryDirectory:"
<> softline
<> pretty (getLast temporaryDirectory)
<> hardline
<> text "initDbCache:" <+> pretty (getLast initDbCache)
<> hardline
<> text "initDbConfig:"
<> softline
<> indent 2 (pretty $ getAccum initDbConfig)
<> hardline
<> text "initDbConfig:"
<> softline
<> indent 2 (pretty $ getAccum createDbConfig)
<> text "copyConfig:"
<> softline
<> indent 2 (pretty (getLast copyConfig))
<> hardline
<> text "postgresConfig:"
<> softline
<> indent 2 (pretty postgresConfig)
<> hardline
<> text "connectionOptions:"
<> softline
<> indent 2 (prettyOptions connectionOptions)
<> hardline
<> text "postgresConfigFile:"
<> softline
<> indent 2 (vsep $ map (\(x, y) -> text x <> "=" <> text y) postgresConfigFile)
<> hardline
<> text "connectionTimeout:" <+> pretty (getLast connectionTimeout)
socketDirectoryToConfig :: FilePath -> [(String, String)]
socketDirectoryToConfig dir =
[ ("listen_addresses", "'127.0.0.1,::1'")
, ("unix_socket_directories", "'" <> dir <> "'")
]
data CopyDirectoryCommand = CopyDirectoryCommand
{ sourceDirectory :: FilePath
, destinationDirectory :: Maybe FilePath
, useCopyOnWrite :: Bool
} deriving (Show, Eq, Ord)
instance Pretty CopyDirectoryCommand where
pretty CopyDirectoryCommand {..}
= text "sourceDirectory:"
<> softline
<> indent 2 (text sourceDirectory)
<> hardline
<> text "destinationDirectory:"
<> softline
<> indent 2 (pretty destinationDirectory)
<> hardline
<> text "useCopyOnWrite:"
<+> pretty useCopyOnWrite
completeCopyDirectory
:: FilePath
-> CopyDirectoryCommand
-> CompleteCopyDirectoryCommand
completeCopyDirectory theDataDirectory CopyDirectoryCommand {..} =
CompleteCopyDirectoryCommand
{ copyDirectoryCommandSrc = sourceDirectory
, copyDirectoryCommandDst = fromMaybe theDataDirectory destinationDirectory
, copyDirectoryCommandCow = useCopyOnWrite
}
getInitDbVersion :: String
getInitDbVersion = unsafePerformIO $ readProcessWithExitCode "initdb" ["--version"] "" >>= \case
(ExitSuccess, outputString, _) -> do
let
theLastPart = last $ words outputString
versionPart = takeWhile (\x -> isDigit x || x == '.' || x == '-') theLastPart
humanReadable = if last versionPart == '.'
then init versionPart
else versionPart
pure $ humanReadable <> take 8 (makeArgumentHash outputString)
(startErrorExitCode, startErrorStdOut, startErrorStdErr) ->
throwIO InitDbFailed {..}
{-# NOINLINE getInitDbVersion #-}
makeCommandLine :: String -> CompleteProcessConfig -> String
makeCommandLine command CompleteProcessConfig {..} =
let envs = unwords $ map (\(x, y) -> x <> "=" <> y)
$ filter ((`elem` envsToKeep) . fst) completeProcessConfigEnvVars
args = unwords completeProcessConfigCmdLine
in envs <> " " <> command <> args
makeInitDbCommandLine :: CompleteProcessConfig -> String
makeInitDbCommandLine = makeCommandLine "initdb"
makeArgumentHash :: String -> String
makeArgumentHash = BSC.unpack . Base64.encode . hash . BSC.pack
makeCachePath :: FilePath -> String -> String
makeCachePath cacheFolder cmdLine =
let
version = getInitDbVersion
theHash = makeArgumentHash cmdLine
in cacheFolder <> "/" <> version <> "/" <> theHash
envsToKeep :: [String]
envsToKeep =
[ "PGHOST"
, "PGHOSTADDR"
, "PGPORT"
, "PGDATABASE"
, "PGUSER"
, "PGPASSWORD"
, "PGPASSFILE"
, "PGSERVICE"
, "PGSERVICEFILE"
, "PGOPTIONS"
, "PGAPPNAME"
, "PGSSLMODE"
, "PGREQUIRESSL"
, "PGSSLCOMPRESSION"
, "PGSSLCERT"
, "PGSSLKEY"
, "PGSSLROOTCERT"
, "PGSSLCRL"
, "PGREQUIREPEER"
, "PGKRBSRVNAME"
, "PGGSSLIB"
, "PGCONNECT_TIMEOUT"
, "PGCLIENTENCODING"
, "PGTARGETSESSIONATTRS"
, "PGDATESTYLE"
, "PGTZ"
, "PGGEQO"
, "PGSYSCONFDIR"
, "PGLOCALEDIR"
]
splitDataDirectory :: CompleteProcessConfig -> (Maybe String, CompleteProcessConfig)
splitDataDirectory old =
let isDataDirectoryFlag xs = "-D" `isPrefixOf` xs || "--pgdata=" `isPrefixOf` xs
(dataDirectoryArgs, otherArgs) =
partition isDataDirectoryFlag $ completeProcessConfigCmdLine old
firstDataDirectoryArg = flip fmap (listToMaybe dataDirectoryArgs) $ \case
'-':'D':' ':theDir -> theDir
'-':'D':theDir -> theDir
'-':'-':'p':'g':'d':'a':'t':'a':'=':theDir -> theDir
_ -> error "splitDataDirectory not possible"
filteredEnvs = filter (("PGDATA" /=) . fst) $
completeProcessConfigEnvVars old
clearedConfig = old
{ completeProcessConfigCmdLine = otherArgs
, completeProcessConfigEnvVars = filteredEnvs
}
in (firstDataDirectoryArg, clearedConfig)
addDataDirectory :: String -> CompleteProcessConfig -> CompleteProcessConfig
addDataDirectory theDataDirectory x = x
{ completeProcessConfigCmdLine =
("--pgdata=" <> theDataDirectory) : completeProcessConfigCmdLine x
}
toPlan
:: Bool
-> Bool
-> Int
-> FilePath
-> FilePath
-> Config
toPlan _makeInitDb makeCreateDb port socketDirectory dataDirectoryString = mempty
{ postgresConfigFile = socketDirectoryToConfig socketDirectory
, connectionTimeout = pure (60 * 1000000)
, logger = pure $ const $ pure ()
, postgresConfig = silentProcessConfig
{ commandLine = mempty
{ keyBased = Map.fromList
[ ("-p", Just $ show port)
, ("-D", Just dataDirectoryString)
]
}
}
, connectionOptions = mempty
{ Client.host = pure socketDirectory
, Client.port = pure port
, Client.dbname = pure "postgres"
}
, createDbConfig = if makeCreateDb
then pure silentProcessConfig
{ commandLine = mempty
{ keyBased = Map.fromList
[ ("-h", Just socketDirectory)
, ("-p ", Just $ show port)
]
}
}
else mempty
, initDbConfig = pure silentProcessConfig
{ commandLine = mempty
{ keyBased = Map.fromList
[("--pgdata=", Just dataDirectoryString)]
}
}
, copyConfig = pure Nothing
}
setupConfig
:: Config
-> IO Resources
setupConfig config@Config {..} = evalContT $ do
envs <- lift getEnvironment
thePort <- lift $ maybe getFreePort pure $ join $ getLast port
let resourcesTemporaryDir = fromMaybe "/tmp" $ getLast temporaryDirectory
resourcesInitDbCache = join $ getLast initDbCache
resourcesSocketDirectory <- ContT $ bracketOnError
(setupDirectoryType resourcesTemporaryDir "tmp-postgres-socket" socketDirectory) cleanupDirectoryType
resourcesDataDir <- ContT $ bracketOnError
(setupDirectoryType resourcesTemporaryDir "tmp-postgres-data" dataDirectory) cleanupDirectoryType
let hostAndDir = toPlan
(hasInitDb config)
(hasCreateDb config)
thePort
(toFilePath resourcesSocketDirectory)
(toFilePath resourcesDataDir)
finalPlan = hostAndDir <> config
resourcesPlan <- lift $
either (throwIO . PlanFailed (show $ pretty finalPlan)) pure $
completePlan envs (toFilePath resourcesDataDir) finalPlan
pure Resources {..}
cleanupConfig :: Resources -> IO ()
cleanupConfig Resources {..} = do
cleanupDirectoryType resourcesSocketDirectory
cleanupDirectoryType resourcesDataDir
prettyPrintConfig :: Config -> String
prettyPrintConfig = show . pretty
data Resources = Resources
{ resourcesPlan :: Plan
, resourcesSocketDirectory :: CompleteDirectoryType
, resourcesDataDir :: CompleteDirectoryType
, resourcesTemporaryDir :: FilePath
, resourcesInitDbCache :: Maybe (Bool, FilePath)
}
instance Pretty Resources where
pretty Resources {..}
= text "resourcePlan:"
<> softline
<> indent 2 (pretty resourcesPlan)
<> hardline
<> text "resourcesSocket:"
<+> pretty resourcesSocketDirectory
<> hardline
<> text "resourcesDataDir:"
<+> pretty resourcesDataDir
makeResourcesDataDirPermanent :: Resources -> Resources
makeResourcesDataDirPermanent r = r
{ resourcesDataDir = makePermanent $ resourcesDataDir r
}
optionsToConfig :: Client.Options -> Config
optionsToConfig opts@Client.Options {..}
= ( mempty
{ port = maybe (Last Nothing) (pure . pure) $ getLast port
, socketDirectory = maybe mempty hostToSocketClass $ getLast host
}
) <> optionsToPlan opts
optionsToPlan :: Client.Options -> Config
optionsToPlan opts@Client.Options {..}
= maybe mempty (dbnameToPlan (getLast user) (getLast password)) (getLast dbname)
<> maybe mempty userToPlan (getLast user)
<> maybe mempty passwordToPlan (getLast password)
<> clientOptionsToPlan opts
clientOptionsToPlan :: Client.Options -> Config
clientOptionsToPlan opts = mempty
{ connectionOptions = opts
}
userToPlan :: String -> Config
userToPlan user = mempty
{ initDbConfig = pure mempty
{ commandLine = mempty
{ keyBased = Map.singleton "--username=" $ Just user
}
}
}
dbnameToPlan :: Maybe String -> Maybe String -> String -> Config
dbnameToPlan muser mpassword dbName
| dbName == "template1" || dbName == "postgres" = mempty
| otherwise = mempty
{ createDbConfig = pure mempty
{ commandLine = mempty
{ indexBased = Map.singleton 0 dbName
, keyBased = maybe mempty (Map.singleton "--username=" . Just) muser
}
, environmentVariables = mempty
{ specific = maybe mempty (Map.singleton "PGPASSWORD") mpassword
}
}
}
passwordToPlan :: String -> Config
passwordToPlan password = mempty
{ initDbConfig = pure mempty
{ environmentVariables = mempty
{ specific = Map.singleton "PGPASSWORD" password
}
}
}
hostToSocketClass :: String -> DirectoryType
hostToSocketClass hostOrSocketPath = case hostOrSocketPath of
'/' : _ -> Permanent hostOrSocketPath
_ -> Temporary