{-# LANGUAGE CPP #-}
{-# LANGUAGE ViewPatterns #-}
{-# LANGUAGE NamedFieldPuns #-}
{-# LANGUAGE MultiWayIf #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE OverloadedLabels #-}

-- | New-style @.travis.yml@ script generator using cabal 1.24's nix-style
-- tech-preview facilities.
--
-- See also <https://github.com/haskell-CI/haskell-ci>
--
-- NB: This code deliberately avoids relying on non-standard packages and
--     is expected to compile/work with at least GHC 7.0 through GHC 8.0
module HaskellCI (
    main,
    -- * for tests
    Result (..),
    Diagnostic (..),
    parseTravis,
    formatDiagnostic, formatDiagnostics,
    travisFromConfigFile, MakeTravisOutput, Options (..), defaultOptions,
    ) where

import Prelude ()
import Prelude.Compat

import Control.DeepSeq (force)
import Control.Exception (evaluate)
import Control.Monad (when, unless, liftM, forM_, mzero)
import qualified Data.ByteString as BS
import qualified Data.Foldable as F
import qualified Data.Traversable as T
import Data.Function
import Data.List
import Data.Maybe
import Data.Set (Set)
import qualified Data.Set as S
import qualified Data.Map as M
import System.Directory (doesDirectoryExist, doesFileExist)
import System.Exit
import System.FilePath.Posix ((</>), takeDirectory, takeFileName, takeExtension)
import System.IO
import Control.Monad.IO.Class
import Control.Monad.Trans.Maybe
import System.Environment (getArgs)
import Control.Monad.Trans.Writer
import Text.Read (readMaybe)
import Distribution.Compat.ReadP (readP_to_S)


import qualified Options.Applicative         as O

import Distribution.Compiler (CompilerFlavor(..))
import Distribution.Package hiding (Package, pkgName)
import qualified Distribution.Package as Pkg
import Distribution.PackageDescription (GenericPackageDescription,packageDescription, testedWith, package, condLibrary, condTestSuites)
import Distribution.PackageDescription.Configuration (flattenPackageDescription)
import qualified Distribution.PackageDescription as PD
import qualified Distribution.ParseUtils as PU
import Distribution.Text
import Distribution.Version
#if MIN_VERSION_Cabal(2,2,0)
import Distribution.PackageDescription.Parsec (readGenericPackageDescription)
#elif MIN_VERSION_Cabal(2,0,0)
import Distribution.PackageDescription.Parse (readGenericPackageDescription)
#else
import Distribution.PackageDescription.Parse (readPackageDescription)
import Distribution.Verbosity (Verbosity)
#endif

import qualified Distribution.FieldGrammar                    as C
import qualified Distribution.Fields.Pretty                   as C
import qualified Distribution.PackageDescription.FieldGrammar as C
import qualified Distribution.Types.SourceRepo                as C
import qualified Distribution.Types.VersionRange              as C
import qualified Text.PrettyPrint                             as PP



#if MIN_VERSION_base(4,9,0)
import Data.Semigroup (Semigroup (..))
#else
import Data.Monoid ((<>))
#endif

-- lens
import Lens.Micro
import Data.Generics.Labels () -- IsLabel (->) ...

import qualified Distribution.Types.BuildInfo.Lens          as L
import qualified Distribution.Types.PackageDescription.Lens as L

import HaskellCI.Cli
import HaskellCI.Config
import HaskellCI.Config.CopyFields
import HaskellCI.Config.ConstraintSet
import HaskellCI.Config.Doctest
import HaskellCI.Config.Dump
import HaskellCI.Config.Folds
import HaskellCI.Config.HLint
import HaskellCI.Config.Installed
import HaskellCI.Config.Jobs
import HaskellCI.Extras
import HaskellCI.GHC
import HaskellCI.Glob
import HaskellCI.MakeTravisOutput
import HaskellCI.Optimization
import HaskellCI.Package
import HaskellCI.Project
import HaskellCI.TestedWith
import HaskellCI.Version

#ifndef CURRENT_PACKAGE_VERSION
#define CURRENT_PACKAGE_VERSION "???"
#endif

-------------------------------------------------------------------------------
-- Main
-------------------------------------------------------------------------------

main :: IO ()
main = do
    argv0 <- getArgs
    (cmd, opts) <- O.execParser cliParserInfo
    case cmd of
        CommandListGHC -> do
            putStrLn $ "Supported GHC versions:"
            forM_ groupedVersions $ \(v, vs) -> do
                putStr $ prettyMajVersion v ++ ": "
                putStrLn $ intercalate ", " (map display vs)
        CommandDumpConfig -> do
            putStr $ unlines $ runDG configGrammar

        CommandRegenerate -> do
            let fp = ".travis.yml" -- make configurable?
            contents <- readFile fp
            case findArgv (lines contents) of
                Nothing     -> do
                    hPutStrLn stderr $ "Error: expected REGENDATA line in " ++ fp
                    exitFailure
                Just argv   -> do
                    (f, opts') <- parseTravis argv
                    doTravis argv f (opts' <> opts)
        CommandTravis f -> doTravis argv0 f opts
  where
    findArgv :: [String] -> Maybe [String]
    findArgv ls = do
        l <- findMaybe (afterInfix "REGENDATA") ls
        readMaybe l

    groupedVersions :: [(Version, [Version])]
    groupedVersions = map ((\vs -> (head vs, vs)) . sortBy (flip compare))
                    . groupBy ((==) `on` ghcMajVer)
                    $ sort knownGhcVersions

    prettyMajVersion :: Version -> String
    prettyMajVersion v
        | Just v == ghcAlpha = "alpha"
        | otherwise = case ghcMajVer v of (x,y) -> show x ++ "." ++ show y

doTravis :: [String] -> FilePath -> Options -> IO ()
doTravis args path opts = do
    runYamlWriter (optOutput opts) $ travisFromConfigFile args opts path

runYamlWriter :: Maybe FilePath -> YamlWriter IO () -> IO ()
runYamlWriter mfp m = do
    result <- execWriterT (runMaybeT m)
    case result of
        Failure (formatDiagnostics -> errors) -> hPutStr stderr errors >> exitFailure
        Success (formatDiagnostics -> warnings) (unlines -> contents) -> do
            contents' <- evaluate (force contents)
            hPutStr stderr warnings
            case mfp of
                Nothing -> putStr contents'
                Just fp -> writeFile fp contents'

travisFromConfigFile
    :: MonadIO m
    => [String]
    -> Options
    -> FilePath
    -> YamlWriter m ()
travisFromConfigFile args opts path = do
    cabalFiles <- getCabalFiles
    config' <- maybe (return emptyConfig) readConfigFile (optConfig opts)
    let config = optConfigMorphism opts config'
    pkgs <- T.mapM (configFromCabalFile config opts) cabalFiles
    (ghcs, prj) <- case checkVersions (cfgTestedWith config) pkgs of
        Right x     -> return x
        Left errors -> putStrLnErrs errors >> mzero
    genTravisFromConfigs args opts isCabalProject config prj ghcs
  where
    isCabalProject :: Maybe FilePath
    isCabalProject
        | "cabal.project" `isPrefixOf` takeFileName path = Just path
        | otherwise = Nothing

    getCabalFiles :: MonadIO m => YamlWriter m (Project FilePath)
    getCabalFiles
        | isNothing isCabalProject = return $ emptyProject & #prjPackages .~ [path]
        | otherwise = do
            contents <- liftIO $ BS.readFile path
            pkgs <- either putStrLnErr return $ parseProjectFile path contents
            over #prjPackages concat `liftM` T.mapM findProjectPackage pkgs

    rootdir = takeDirectory path

    -- See findProjectPackages in cabal-install codebase
    -- this is simple variant.
    findProjectPackage :: MonadIO m => String -> YamlWriter m [FilePath]
    findProjectPackage pkglocstr = do
        mfp <- checkisFileGlobPackage pkglocstr `mplusMaybeT`
               checkIsSingleFilePackage pkglocstr
        maybe (putStrLnErr $ "bad package location: " ++ pkglocstr) return mfp

    checkIsSingleFilePackage pkglocstr = do
        let abspath = rootdir </> pkglocstr
        isFile <- liftIO $ doesFileExist abspath
        isDir  <- liftIO $ doesDirectoryExist abspath
        if | isFile && takeExtension pkglocstr == ".cabal" -> return (Just [abspath])
           | isDir -> checkisFileGlobPackage (pkglocstr </> "*.cabal")
           | otherwise -> return Nothing

    -- if it looks like glob, glob
    checkisFileGlobPackage pkglocstr =
        case filter (null . snd) $ readP_to_S parseFilePathGlobRel pkglocstr of
            [(g, "")] -> do
                files <- liftIO $ expandRelGlob rootdir g
                let files' = filter ((== ".cabal") . takeExtension) files
                -- if nothing is matched, skip.
                if null files' then return Nothing else return (Just files')
            _         -> return Nothing

    mplusMaybeT :: Monad m => m (Maybe a) -> m (Maybe a) -> m (Maybe a)
    mplusMaybeT ma mb = do
        mx <- ma
        case mx of
            Nothing -> mb
            Just x  -> return (Just x)

configFromCabalFile
    :: MonadIO m => Config ->  Options -> FilePath -> YamlWriter m (Package, Set Version)
configFromCabalFile cfg opts cabalFile = do
    gpd <- liftIO $ readGenericPackageDescription maxBound cabalFile

    let compilers = testedWith $ packageDescription gpd
        pkgNameStr = display $ Pkg.pkgName $ package $ packageDescription gpd

    let unknownComps = nub [ c | (c,_) <- compilers, c /= GHC ]
        ghcVerConstrs = [ vc | (GHC,vc) <- compilers ]
        ghcVerConstrs' = simplifyVersionRange $ foldr unionVersionRanges noVersion ghcVerConstrs
        twoDigitGhcVerConstrs = mapMaybe isTwoDigitGhcVersion ghcVerConstrs :: [Version]
        specificGhcVers = nub $ mapMaybe isSpecificVersion ghcVerConstrs

    unless (null twoDigitGhcVerConstrs) $ do
        putStrLnWarn $ "'tested-with:' uses two digit GHC versions (which don't match any existing GHC version): " ++ intercalate ", " (map display twoDigitGhcVerConstrs)
        putStrLnInfo $ "Either use wild-card format, for example 'tested-with: GHC ==7.10.*' or a specific existing version 'tested-with: GHC ==7.10.3'"

    when (null compilers) $ do
        putStrLnErr (unlines $
                     [ "empty or missing top-level 'tested-with:' definition in " ++ cabalFile ++ " file; example definition:"
                     , ""
                     , "tested-with: " ++ intercalate ", " [ "GHC==" ++ display v | v <- lastStableGhcVers ]
                     ])

    unless (null unknownComps) $ do
        putStrLnWarn $ "ignoring unsupported compilers mentioned in tested-with: " ++ show unknownComps

    when (null ghcVerConstrs) $ do
        putStrLnErr "'tested-with:' doesn't mention any 'GHC' version"

    when (isNoVersion ghcVerConstrs') $ do
        putStrLnErr "'tested-with:' describes an empty version range for 'GHC'"

    when (isAnyVersion ghcVerConstrs') $ do
        putStrLnErr "'tested-with:' allows /any/ 'GHC' version"

    let unknownGhcVers = sort $ specificGhcVers \\ knownGhcVersions

    unless (null unknownGhcVers) $ do
        putStrLnErr ("'tested-with:' specifically refers to unknown 'GHC' versions: "
                     ++ intercalate ", " (map display unknownGhcVers) ++ "\n"
                     ++ "Known GHC versions: " ++ intercalate ", " (map display knownGhcVersions))

    let knownGhcVersions'
            | cfgLastInSeries cfg = filterLastMajor knownGhcVersions
            | otherwise           = knownGhcVersions

    let testedGhcVersions = filter (`withinRange` ghcVerConstrs') knownGhcVersions'

    when (null testedGhcVersions) $ do
        putStrLnErr "no known GHC version is allowed by the 'tested-with' specification"

    forM_ (optCollections opts) $ \c -> do
        let v = collToGhcVer c
        unless (v `elem` testedGhcVersions) $
            putStrLnErr $ unlines
               [ "collection " ++ c ++ " requires GHC " ++ display v
               , "add 'tested-width: GHC == " ++ display v ++ "' to your .cabal file"
               ]

    let pkg = Pkg pkgNameStr anyVersion (takeDirectory cabalFile) gpd

    return (pkg, S.fromList testedGhcVersions)
  where
    lastStableGhcVers
        = nubBy ((==) `on` ghcMajVer)
        $ sortBy (flip compare)
        $ filter (not . previewGHC . Just)
        $ knownGhcVersions

    isTwoDigitGhcVersion :: VersionRange -> Maybe Version
    isTwoDigitGhcVersion vr = isSpecificVersion vr >>= t
      where
        t v | [_,_] <- versionNumbers v = Just v
        t _                             = Nothing

    filterLastMajor = map maximum . groupBy ((==) `on` ghcMajVer)

genTravisFromConfigs
    :: Monad m
    => [String]
    -> Options
    -> Maybe FilePath
    -> Config
    -> Project Package
    -> Set Version
    -> YamlWriter m ()
genTravisFromConfigs argv opts isCabalProject config prj@Project { prjPackages = pkgs } versions' = do
    let folds = cfgFolds config

    putStrLnInfo $
        "Generating Travis-CI config for testing for GHC versions: " ++ ghcVersions

    unless (null $ cfgOsx config) $  do
        putStrLnInfo $ "Also OSX jobs for: " ++ ghcOsxVersions
        unless (S.null omittedOsxVersions) $
            putStrLnWarn $ "Not all GHC versions specified with --osx are generated: " ++ ghcOmittedOsxVersions

    ---------------------------------------------------------------------------
    -- travis.yml generation starts here

    tellStrLns
        [ "# This Travis job script has been generated by a script via"
        , "#"
        , rawRow $ "#   haskell-ci " ++ unwords [ "'" ++ a ++ "'" | a <- argv ]
        , "#"
        , "# For more information, see https://github.com/haskell-CI/haskell-ci"
        , "#"
        , rawRow $ "# version: " ++ CURRENT_PACKAGE_VERSION
        , "#"
        , "language: c"
        , "dist: xenial"
        , ""
        , "git:"
        , "  submodules: false  # whether to recursively clone submodules"
        , ""
        ]

    let projectName = fromMaybe (pkgName $ head pkgs) (cfgProjectName config)
    unless (null $ cfgIrcChannels config) $ tellStrLnsRaw $
        [ "notifications:"
        , "  irc:"
        , "    channels:"
        ] ++
        [ "      - \"" ++ chan ++ "\"" | chan <- cfgIrcChannels config ] ++
        [ "    skip_join: true"
        , "    template:"
        , "      - \"\\x0313" ++ projectName ++ "\\x03/\\x0306%{branch}\\x03 \\x0314%{commit}\\x03 %{build_url} %{message}\""
        , ""
        ]

    unless (null $ cfgOnlyBranches config) $ tellStrLnsRaw $
        [ "branches:"
        , "  only:"
        ] ++
        [ "    - " ++ branch
        | branch <- cfgOnlyBranches config
        ] ++
        [ ""
        ]

    -- cache directories
    when (cfgCache config) $ tellStrLns
        [ "cache:"
        , "  directories:"
        , "    - $HOME/.cabal/packages"
        , "    - $HOME/.cabal/store"
        ]


    -- on OSX ghc is installed in $HOME so we can cache it
    -- independently of linux
    when (cfgCache config && not (null (cfgOsx config))) $ tellStrLns
        [ "    - $HOME/.ghc-install"
        ]

    tellStrLn ""

    -- postgresql
    when (cfgPostgres config) $ tellStrLns
        [ "services:"
        , "- postgresql"
        , "addons:"
        , "  postgresql: '10'"
        , ""
        ]

    -- before caching: clear some redundant stuff
    when (cfgCache config) $ tellStrLns
        [ "before_cache:"
        , "  - rm -fv $CABALHOME/packages/hackage.haskell.org/build-reports.log"
        , "  # remove files that are regenerated by 'cabal update'"
        , "  - rm -fv $CABALHOME/packages/hackage.haskell.org/00-index.*" -- legacy
        , "  - rm -fv $CABALHOME/packages/hackage.haskell.org/*.json" -- TUF meta-data
        , "  - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.cache"
        , "  - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar"
        , "  - rm -fv $CABALHOME/packages/hackage.haskell.org/01-index.tar.idx"
        , ""
        , "  - rm -rfv $CABALHOME/packages/head.hackage" -- if we cache, it will break builds.
        , ""
        ]

    tellStrLn "matrix:"
    tellStrLn "  include:"

    let colls = [ (collToGhcVer cid,cid) | cid <- reverse $ optCollections opts ]

    let tellJob :: Monad m => Bool -> Maybe Version -> YamlWriter m ()
        tellJob osx gv = do
            let cvs = dispGhcVersion $ gv >> cfgCabalInstallVersion config
                gvs = dispGhcVersion gv

                xpkgs' = concatMap (',':) (S.toList $ cfgApt config)

                colls' = [ cid | (v,cid) <- colls, Just v == gv ]

            tellStrLnsRaw $ catMaybes
                [ Just $ "    - compiler: \"ghc-" <> gvs <> "\""
                , if | Just e <- gv >>= \v -> M.lookup v (cfgEnv config)
                                     -> Just $ "      env: " ++ e
                     | previewGHC gv -> Just $ "      env: GHCHEAD=true"
                     | null colls'   -> Nothing
                     | otherwise     -> Just $ "      env: 'COLLECTIONS=" ++ intercalate "," colls' ++ "'"
                , Just $ "      addons: {apt: {packages: [ghc-ppa-tools,cabal-install-" <> cvs <> ",ghc-" <> gvs <> xpkgs' <> "], sources: [hvr-ghc]}}"
                ]

            when osx $ tellStrLnsRaw
                [ "      os: osx"
                ]

    -- newer GHC first, -head last (which is great).
    -- Alpha release would go first though.
    F.forM_ (reverse $ S.toList versions) $ tellJob False
    F.forM_ (reverse $ S.toList osxVersions) $ tellJob True . Just

    let allowFailures = headGhcVers `S.union` S.map Just (S.filter (`C.withinRange` cfgAllowFailures config) versions')
    unless (S.null allowFailures) $ do
        tellStrLn ""
        tellStrLn "  allow_failures:"

        F.forM_ allowFailures $ \gv -> do
            let gvs = dispGhcVersion gv
            tellStrLn $ concat [ "    - compiler: \"ghc-", gvs, "\"" ]

    tellStrLns
        [ ""
        , "before_install:"
        , sh "HC=/opt/ghc/bin/${CC}"
        , sh' [2034,2039] "HCPKG=${HC/ghc/ghc-pkg}" -- SC2039. In POSIX sh, string replacement is undefined.
        , sh "unset CC"
        -- cabal
        , sh "CABAL=/opt/ghc/bin/cabal"
        , sh "CABALHOME=$HOME/.cabal"
        -- PATH
        , sh "export PATH=\"$CABALHOME/bin:$PATH\""
        -- rootdir is useful for manual script additions
        , sh "ROOTDIR=$(pwd)"
        ]

    -- macOS installing
    let haskellOnMacos = "https://haskell.futurice.com/haskell-on-macos.py"
    unless (null (cfgOsx config)) $ tellStrLns
        [ sh $ "if [ \"$TRAVIS_OS_NAME\" = \"osx\" ]; then brew update; brew upgrade python@3; curl " ++ haskellOnMacos ++ " | python3 - --make-dirs --install-dir=$HOME/.ghc-install --cabal-alias=head install cabal-install-head ${TRAVIS_COMPILER}; fi"
        , sh' [2034,2039] "if [ \"$TRAVIS_OS_NAME\" = \"osx\" ]; then HC=$HOME/.ghc-install/ghc/bin/$TRAVIS_COMPILER; HCPKG=${HC/ghc/ghc-pkg}; CABAL=$HOME/.ghc-install/ghc/bin/cabal; fi"
        ]

    -- HCNUMVER, numeric HC version, e.g. ghc 7.8.4 is 70804 and 7.10.3 is 71003
    tellStrLns
        [ sh $ "HCNUMVER=$(( $(${HC} --numeric-version|sed -E 's/([0-9]+)\\.([0-9]+)\\.([0-9]+).*/\\1 * 10000 + \\2 * 100 + \\3/') ))"
        , sh "echo $HCNUMVER"
        ]

    unless (null colls) $
       tellStrLn " - IFS=', ' read -a COLLS <<< \"$COLLECTIONS\""

    tellStrLns
        [ ""
        , "install:"
        , sh "${CABAL} --version"
        , sh "echo \"$(${HC} --version) [$(${HC} --print-project-git-commit-id 2> /dev/null || echo '?')]\""
        , sh "TEST=--enable-tests"
        , shForJob versions' (invertVersionRange $ cfgTests config) "TEST=--disable-tests"
        , sh "BENCH=--enable-benchmarks"
        , shForJob versions' (invertVersionRange $ cfgBenchmarks config) "BENCH=--disable-benchmarks"
        , sh "GHCHEAD=${GHCHEAD-false}"
        ]

    -- Update hackage index. Side-effect: ~/.cabal.config is created.
    tellStrLns
        [ sh "travis_retry ${CABAL} update -v"
        , sh "sed -i.bak 's/^jobs:/-- jobs:/' $CABALHOME/config"
        , sh "rm -fv cabal.project cabal.project.local"
        ]

    -- Cabal jobs
    case cfgJobs config >>= cabalJobs of
        Just n -> tellStrLns
            [ sh $ "sed -i.bak 's/^-- jobs:.*/jobs: " ++ show n ++ "/' $CABALHOME/config"
            ]
        _ -> return ()

    -- GHC jobs
    case cfgJobs config >>= ghcJobs of
        Just m -> tellStrLns
            [ shForJob versions' (orLaterVersion (mkVersion [7,8])) $
              "sed -i.bak 's/-- ghc-options:.*/ghc-options: -j" ++ show m ++ "/' $CABALHOME/config"
            ]
        _ -> return ()

    -- Add head.hackage repository to ~/.cabal/config
    -- (locally you want to add it to cabal.project)
    unless (S.null headGhcVers) $ tellStrLns
        [ "  # Overlay Hackage Package Index for GHC HEAD: https://github.com/hvr/head.hackage"
        , "  - |"
        , "    if $GHCHEAD; then"
        , "      sed -i 's/-- allow-newer: .*/allow-newer: *:base/' $CABALHOME/config"
        , "      for pkg in $($HCPKG list --simple-output); do pkg=$(echo $pkg | sed 's/-[^-]*$//'); sed -i \"s/allow-newer: /allow-newer: *:$pkg, /\" $CABALHOME/config; done"
        , ""
        , "      echo 'repository head.hackage'                                                        >> $CABALHOME/config"
        , "      echo '   url: http://head.hackage.haskell.org/'                                       >> $CABALHOME/config"
        , "      echo '   secure: True'                                                                >> $CABALHOME/config"
        , "      echo '   root-keys: 07c59cb65787dedfaef5bd5f987ceb5f7e5ebf88b904bbd4c5cbdeb2ff71b740' >> $CABALHOME/config"
        , "      echo '              2e8555dde16ebd8df076f1a8ef13b8f14c66bad8eafefd7d9e37d0ed711821fb' >> $CABALHOME/config"
        , "      echo '              8f79fd2389ab2967354407ec852cbe73f2e8635793ac446d09461ffb99527f6e' >> $CABALHOME/config"
        , "      echo '   key-threshold: 3'                                                            >> $CABALHOME.config"
        , ""
        , "      grep -Ev -- '^\\s*--' $CABALHOME/config | grep -Ev '^\\s*$'"
        , ""
        , "      ${CABAL} new-update head.hackage -v"
        , "    fi"
        ]

    -- Output cabal.config
    tellStrLns
        [ sh "grep -Ev -- '^\\s*--' $CABALHOME/config | grep -Ev '^\\s*$'"
        ]

    -- Install doctest
    let doctestVersionConstraint
            | isAnyVersion (cfgDoctestVersion doctestConfig) = ""
            | otherwise = " --constraint='doctest " ++ display (cfgDoctestVersion doctestConfig) ++ "'"
    when (cfgDoctestEnabled doctestConfig) $ tellStrLns
        [ shForJob versions' doctestJobVersionRange $ "${CABAL} new-install -w ${HC} -j2 doctest" ++ doctestVersionConstraint
        ]

    -- Install hlint
    let hlintVersionConstraint
            | isAnyVersion (cfgHLintVersion hlintConfig) = ""
            | otherwise = " --constraint='hlint " ++ display (cfgHLintVersion hlintConfig) ++ "'"
    when (cfgHLintEnabled hlintConfig) $ tellStrLns
        [ shForJob versions' (hlintJobVersionRange versions (cfgHLintJob hlintConfig)) $
          "${CABAL} new-install -w ${HC} -j2 hlint" ++ hlintVersionConstraint
        ]

    -- create cabal.project file
    generateCabalProject False

    let pkgFilter = intercalate " | " $ map (wrap.pkgName) pkgs
        wrap s = "grep -Fv \"" ++ s ++ " ==\""
    unless (null colls) $ tellStrLnsRaw
        [ "  - for COLL in \"${COLLS[@]}\"; do"
        , "      echo \"== collection $COLL ==\";"
        , "      ghc-travis collection ${COLL} > /dev/null || break;"
        , "      ghc-travis collection ${COLL} | " ++ pkgFilter ++ " > cabal.project.freeze;"
        , "      grep ' collection-id' cabal.project.freeze;"
        , "      rm -rf dist-newstyle/;"
        , "      $(CABAL} new-build -w ${HC} ${TEST} ${BENCH} --project-file=\"" ++ projectFile ++ "\" --dep -j2 all;"
        , "    done"
        , ""
        ]

    forM_ pkgs $ \Pkg{pkgDir} -> tellStrLns
        [ sh $ "if [ -f \"" ++ pkgDir ++ "/configure.ac\" ]; then (cd \"" ++ pkgDir ++ "\" && autoreconf -i); fi"
        ]

    let quotedRmPaths =
          ".ghc.environment.*"
          ++ " " ++
          unwords (quotedPaths (\Pkg{pkgDir} -> pkgDir ++ "/dist"))

    tellStrLns
        [ sh $ "rm -f cabal.project.freeze"
        ]

    -- Install dependencies
    when (cfgInstallDeps config) $ do
        tellStrLns
            -- dump install plan
            [ sh $ "${CABAL} new-freeze -w ${HC} ${TEST} ${BENCH} --project-file=\"" ++ projectFile ++"\" --dry"
            , sh $ "cat \"" ++ projectFile ++ ".freeze\" | sed -E 's/^(constraints: *| *)//' | sed 's/any.//'"
            , sh $ "rm  \"" ++ projectFile ++ ".freeze\""

            -- install dependencies
            , sh $ "${CABAL} new-build -w ${HC} ${TEST} ${BENCH} --project-file=\"" ++ projectFile ++"\" --dep -j2 all"
            ]
        tellStrLns
            [ shForJob versions' (cfgNoTestsNoBench config) $ "${CABAL} new-build -w ${HC} --disable-tests --disable-benchmarks --project-file=\"" ++ projectFile ++ "\" --dep -j2 all"
            ]

    tellStrLns
        [ sh $ "rm -rf " ++ quotedRmPaths
        , sh $  "DISTDIR=$(mktemp -d /tmp/dist-test.XXXX)"
        ]

    tellStrLns
        [ ""
        , "# Here starts the actual work to be performed for the package under test;"
        , "# any command which exits with a non-zero exit code causes the build to fail."
        , "script:"
        , "  # test that source-distributions can be generated"
        ]

    foldedTellStrLns FoldSDist "Packaging..." folds $ do
        tellStrLns
            [ sh $ "${CABAL} new-sdist all"
            ]

    let tarFiles = "dist-newstyle" </> "sdist" </> "*.tar.gz"


    foldedTellStrLns FoldUnpack "Unpacking..." folds $ do
        tellStrLns
            [ sh $ "mv " ++ tarFiles ++ " ${DISTDIR}/"
            , sh $ "cd ${DISTDIR} || false" -- fail explicitly, makes SC happier
            , sh $ "find . -maxdepth 1 -name '*.tar.gz' -exec tar -xvf '{}' \\;"
            ]
        generateCabalProject True

    unless (equivVersionRanges noVersion $ cfgNoTestsNoBench config) $ foldedTellStrLns FoldBuild "Building..." folds $ tellStrLns
        [ comment "this builds all libraries and executables (without tests/benchmarks)"
        , shForJob versions' (cfgNoTestsNoBench config) $ "${CABAL} new-build -w ${HC} --disable-tests --disable-benchmarks all"
        ]

    tellStrLns [""]


    foldedTellStrLns FoldBuildEverything
        "Building with tests and benchmarks..." folds $ tellStrLns
        [ comment "build & run tests, build benchmarks"
        , sh "${CABAL} new-build -w ${HC} ${TEST} ${BENCH} all"
        ]

    -- cabal new-test fails if there are no test-suites.
    when hasTests $
        foldedTellStrLns FoldTest "Testing..." folds $ tellStrLns
            [ sh $ mconcat
                [ "if [ \"x$TEST\" = \"x--enable-tests\" ]; then "
                , if cfgNoise config
                     then "${CABAL} "
                     else "(set -o pipefail; ${CABAL} -vnormal+nowrap+markoutput "
                , "new-test -w ${HC} ${TEST} ${BENCH} all"
                , if cfgNoise config
                     then ""
                     else " 2>&1 | sed '/^-----BEGIN CABAL OUTPUT-----$/,/^-----END CABAL OUTPUT-----$/d' )"
                , "; fi"
                ]
            ]

    tellStrLns [""]

    when (cfgDoctestEnabled doctestConfig) $ do
        let doctestOptions = unwords $ cfgDoctestOptions doctestConfig
        tellStrLns [ comment "doctest" ]
        foldedTellStrLns FoldDoctest "Doctest..." folds $ do
            forM_ pkgs $ \Pkg{pkgName,pkgGpd,pkgJobs} -> do
                forM_ (doctestArgs pkgGpd) $ \args -> do
                    let args' = unwords args
                    unless (null args) $ tellStrLns
                        [ shForJob versions' (doctestJobVersionRange `intersectVersionRanges` pkgJobs) $
                          "(cd " ++ pkgName ++ "-* && doctest " ++ doctestOptions ++ " " ++ args' ++ ")"
                        ]
        tellStrLns [ "" ]

    when (cfgHLintEnabled hlintConfig) $ do
        let "" <+> ys = ys
            xs <+> "" = xs
            xs <+> ys = xs ++ " " ++ ys

            prependSpace "" = ""
            prependSpace xs = " " ++ xs

        let hlintOptions = prependSpace $ maybe "" ("-h ${ROOTDIR}/" ++) (cfgHLintYaml hlintConfig) <+> unwords (cfgHLintOptions hlintConfig)

        tellStrLns [ comment "hlint" ]
        foldedTellStrLns FoldHLint "HLint.." folds $ do
            forM_ pkgs $ \Pkg{pkgName,pkgGpd,pkgJobs} -> do
                -- note: similar arguments work so far for doctest and hlint
                forM_ (hlintArgs pkgGpd) $ \args -> do
                    let args' = unwords args
                    unless (null args) $ tellStrLns
                        [ shForJob versions' (hlintJobVersionRange versions (cfgHLintJob hlintConfig) `intersectVersionRanges` pkgJobs) $
                          "(cd " ++ pkgName ++ "-* && hlint" ++ hlintOptions ++ " " ++ args' ++ ")"
                        ]
        tellStrLns [ "" ]

    when (cfgCheck config) $
        foldedTellStrLns FoldCheck "cabal check..." folds $ do
            tellStrLns [ comment "cabal check" ]
            forM_ pkgs $ \Pkg{pkgName,pkgJobs} -> tellStrLns
                [ shForJob versions' pkgJobs $
                  "(cd " ++ pkgName ++ "-* && ${CABAL} check)"
                ]
            tellStrLns [ "" ]

    when (hasLibrary && not (equivVersionRanges noVersion $ cfgHaddock config)) $
        foldedTellStrLns FoldHaddock "Haddock..." folds $ tellStrLns
            [ comment "haddock"
            , shForJob versions' (cfgHaddock config) "${CABAL} new-haddock -w ${HC} ${TEST} ${BENCH} all"
            , ""
            ]

    unless (null colls) $
        foldedTellStrLns FoldStackage "Stackage builds..." folds $ tellStrLnsRaw
            [ "  # try building & testing for package collections"
            , "  - for COLL in \"${COLLS[@]}\"; do"
            , "      echo \"== collection $COLL ==\";"
            , "      ghc-travis collection ${COLL} > /dev/null || break;"
            , "      ghc-travis collection ${COLL} | " ++ pkgFilter ++ " > cabal.project.freeze;"
            , "      grep ' collection-id' cabal.project.freeze;"
            , "      rm -rf dist-newstyle/;"
            , "      ${CABAL} new-build -w ${HC} ${TEST} ${BENCH} all || break;"
            , "      if [ \"x$TEST\" = \"x--enable-tests\" ]; then ${CABAL} new-test -w ${HC} ${TEST} ${BENCH} all || break; fi;"
            , "    done"
            , ""
            ]

    -- Have to build last, as we remove cabal.project.local
    unless (equivVersionRanges noVersion $ cfgUnconstrainted config) $ foldedTellStrLns FoldBuildInstalled
        "Building without installed constraints for packages in global-db..." folds $ tellStrLns
        [ comment "Build without installed constraints for packages in global-db"
        , shForJob versions' (cfgUnconstrainted config) "rm -f cabal.project.local; ${CABAL} new-build -w ${HC} --disable-tests --disable-benchmarks all;"
        , ""
        ]

    -- and now, as we don't have cabal.project.local;
    -- we can test with other constraint sets
    let constraintSets = cfgConstraintSets config
    unless (null constraintSets) $ do
        tellStrLns
            [ comment "Constraint sets"
            , sh "rm -rf cabal.project.local"
            , ""
            ]
        forM_ constraintSets $ \cs -> do
            let name = csName cs
            let constraintFlags = concatMap (\x ->  " --constraint='" ++ x ++ "'") (csConstraints cs)
            let cmd | csRunTests cs = "${CABAL} new-test  -w ${HC} --enable-tests  --enable-benchmarks"
                    | otherwise     = "${CABAL} new-build -w ${HC} --disable-tests --disable-benchmarks"
            tellStrLns [ comment $ "Constraint set " ++ name ]
            foldedTellStrLns' FoldConstraintSets name ("Constraint set " ++ name) folds $ tellStrLns
                [ shForJob versions' (csGhcVersions cs) $
                  cmd ++ " " ++ constraintFlags ++ " all"
                , ""
                ]
        tellStrLns [""]

    tellStrLnsRaw
        [ "# REGENDATA " ++ show argv
        , "# EOF"
        ]

    return ()
  where
    doctestConfig = cfgDoctest config
    hlintConfig   = cfgHLint config

    hasTests   = F.any (\Pkg{pkgGpd} -> not . null $ condTestSuites pkgGpd) pkgs
    hasLibrary = F.any (\Pkg{pkgGpd} -> isJust $ condLibrary pkgGpd) pkgs

    -- GHC versions which need head.hackage
    headGhcVers = S.filter previewGHC versions

    generateCabalProject dist = do
        tellStrLns
            [ sh "rm -f cabal.project"
            , sh "touch cabal.project"
            ]
        F.forM_  pkgs $ \pkg -> do
            let p | dist      = pkgName pkg ++ "-*/*.cabal"
                  | otherwise = pkgDir pkg
            tellStrLns $
                [ shForJob versions' (pkgJobs pkg) $ "printf 'packages: \"" ++ p ++ "\"\\n' >> cabal.project"
                ]

        case cfgCopyFields config of
            CopyFieldsNone -> return ()
            CopyFieldsAll  -> unless (null (prjOrigFields prj)) $ tellStrLns
                [ sh $ "echo '" ++ l ++ "' >> cabal.project"
                | l <- lines $ C.showFields' 2 $ prjOrigFields prj
                , not (null l)
                ]
            CopyFieldsSome -> do
                F.forM_ (prjConstraints prj) $ \xs -> do
                    let s = concat (lines xs)
                    tellStrLns
                        [ sh $ "echo 'constraints: " ++ s ++ "' >> cabal.project"
                        ]
                F.forM_ (prjAllowNewer prj) $ \xs -> do
                    let s = concat (lines xs)
                    tellStrLns
                        [ sh $ "echo 'allow-newer: " ++ s ++ "' >> cabal.project"
                        ]
                unless (null (cfgLocalGhcOptions config)) $ forM_ pkgs $ \Pkg{pkgName} -> do
                    let s = unwords $ map (show . PU.showToken) $ cfgLocalGhcOptions config
                    tellStrLns
                        [ sh $ "echo 'package " ++ pkgName ++ "' >> cabal.project"
                        , sh $ "echo '  ghc-options: " ++ s ++ "' >> cabal.project"
                        ]

                when (prjReorderGoals prj) $
                    tellStrLns
                        [ sh $ "echo 'reorder-goals: True' >> cabal.project"
                        ]

                F.forM_ (prjMaxBackjumps prj) $ \bj ->
                    tellStrLns
                        [ sh $ "echo 'max-backjumps: " ++ show bj ++ "' >> cabal.project"
                        ]

                case prjOptimization prj of
                    OptimizationOn      -> return ()
                    OptimizationOff     -> tellStrLns [ sh $ "echo 'optimization: False' >> cabal.project " ]
                    OptimizationLevel l -> tellStrLns [ sh $ "echo 'optimization: " ++ show l ++ "' >> cabal.project " ]

                F.forM_ (prjSourceRepos prj) $ \repo -> do
                    let repo' = PP.render $ C.prettyFieldGrammar (C.sourceRepoFieldGrammar $ C.RepoKindUnknown "unused") repo
                    tellStrLns [ sh $ "echo 'source-repository-package' >> cabal.project" ]
                    tellStrLns [ sh $ "echo '  " ++ l ++ "' >> cabal.project" | l <- lines repo' ]

        -- mandatory cabal.project setup
        tellStrLns
            [ sh $ "printf 'write-ghc-environment-files: always\\n' >> cabal.project"
            ]

        unless (null (cfgRawProject config)) $ tellStrLns
            [ sh $ "echo '" ++ l ++ "' >> cabal.project"
            | l <- lines $ C.showFields' 2 $ cfgRawProject config
            , not (null l)
            ]

        -- also write cabal.project.local file with
        -- @
        -- constraints: base installed
        -- constraints: array installed
        -- ...
        --
        -- omitting any local package names
        case normaliseInstalled (cfgInstalled config) of
            InstalledDiff pns -> tellStrLns
                [ sh $ "touch cabal.project.local"
                , sh $ unwords
                    [ "for pkg in $($HCPKG list --simple-output); do"
                    , "echo $pkg"
                    , "| sed 's/-[^-]*$//'"
                    , "| grep -vE -- " ++ re
                    , "| sed 's/^/constraints: /'"
                    , "| sed 's/$/ installed/'"
                    , ">> cabal.project.local; done"
                    ]
                ]
              where
                pns' = S.map unPackageName pns `S.union` foldMap (S.singleton . pkgName) pkgs
                re = "'^(" ++ intercalate "|" (S.toList pns') ++ ")$'"

            InstalledOnly pns | not (null pns') -> tellStrLns
                [ sh $ "touch cabal.project.local"
                , sh' [2043] $ unwords
                    [ "for pkg in " ++ unwords (S.toList pns') ++ "; do"
                    , "echo \"constraints: $pkg installed\" >> cabal.project"
                    , ">> cabal.project.local; done"
                    ]
                ]
              where
                pns' = S.map unPackageName pns `S.difference` foldMap (S.singleton . pkgName) pkgs

            -- otherwise: nothing
            _ -> pure ()

        tellStrLns
            [ sh $ "cat cabal.project || true"
            , sh $ "cat cabal.project.local || true"
            ]

    projectFile :: FilePath
    projectFile = fromMaybe "cabal.project" isCabalProject

    quotedPaths :: (Package -> FilePath) -> [String]
    quotedPaths f = map (f . quote) pkgs
      where
        quote pkg = pkg{ pkgDir = "\"" ++ pkgDir pkg ++ "\"" }

    showVersions :: Set (Maybe Version) -> String
    showVersions = unwords . map dispGhcVersion . S.toList

    -- specified ersions
    osxVersions' :: Set Version
    osxVersions' = cfgOsx config

    versions :: Set (Maybe Version)
    versions
        | cfgGhcHead config = S.insert Nothing $ S.map Just versions'
        | otherwise         = S.map Just versions'

    ghcVersions :: String
    ghcVersions = showVersions versions

    osxVersions, omittedOsxVersions :: Set Version
    (osxVersions, omittedOsxVersions) = S.partition (`S.member` versions') osxVersions'

    ghcOsxVersions :: String
    ghcOsxVersions = showVersions $ S.map Just osxVersions

    ghcOmittedOsxVersions :: String
    ghcOmittedOsxVersions = showVersions $ S.map Just omittedOsxVersions


collToGhcVer :: String -> Version
collToGhcVer cid = case simpleParse cid of
  Nothing -> error ("invalid collection-id syntax " ++ show cid)
  Just (PackageIdentifier n (versionNumbers -> v))
    | display n /= "lts" -> error ("unknown collection " ++ show cid)
    | isPrefixOf [0] v -> mkVersion [7,8,3]
    | isPrefixOf [1] v -> mkVersion [7,8,4]
    | isPrefixOf [2] v -> mkVersion [7,8,4]
    | isPrefixOf [3] v -> mkVersion [7,10,2]
    | isPrefixOf [4] v -> mkVersion [7,10,3]
    | isPrefixOf [5] v -> mkVersion [7,10,3]
    | isPrefixOf [6] v -> mkVersion [7,10,3]
    | isPrefixOf [7] v -> mkVersion [8,0,1]
    | otherwise -> error ("unknown collection " ++ show cid)

-------------------------------------------------------------------------------
-- Doctest
-------------------------------------------------------------------------------

doctestJobVersionRange :: VersionRange
doctestJobVersionRange = orLaterVersion $ mkVersion [8,0]

-- | Modules arguments to the library
--
-- * We check the library component
--
-- * If there are hs-source-dirs, use them
--
-- * otherwise use exposed + other modules
--
-- * Also add default-extensions
--
-- /Note:/ same argument work for hlint too, but not exactly
--
doctestArgs :: GenericPackageDescription -> [[String]]
doctestArgs gpd =
    [ libraryModuleArgs c
    | c <- flattenPackageDescription gpd ^.. L.library . traverse
    ] ++
    [ libraryModuleArgs c
    | c <- flattenPackageDescription gpd ^.. L.subLibraries . traverse
    ]

libraryModuleArgs :: PD.Library -> [String]
libraryModuleArgs l
    | null dirsOrMods = []
    | otherwise       = exts ++ dirsOrMods
  where
    bi = l ^. L.buildInfo

    dirsOrMods
        | null (PD.hsSourceDirs bi) = map display (PD.exposedModules l)
        | otherwise                 = PD.hsSourceDirs bi

    exts = map (("-X" ++) . display) (PD.defaultExtensions bi)

executableModuleArgs :: PD.Executable -> [String]
executableModuleArgs e
    | null dirsOrMods = []
    | otherwise       = exts ++ dirsOrMods
  where
    bi = e ^. L.buildInfo

    dirsOrMods
        -- note: we don't try to find main_is location, if hsSourceDirs is empty.
        | null (PD.hsSourceDirs bi) = map display (PD.otherModules bi)
        | otherwise                 = PD.hsSourceDirs bi

    exts = map (("-X" ++) . display) (PD.defaultExtensions bi)

-------------------------------------------------------------------------------
-- HLint
-------------------------------------------------------------------------------

hlintJobVersionRange :: Set (Maybe Version) -> HLintJob -> VersionRange
hlintJobVersionRange vs HLintJobLatest = case S.maxView vs of
    Just (Just v, _) -> thisVersion v
    _                -> thisVersion $ mkVersion [8,6,3]
hlintJobVersionRange _ (HLintJob v)   = thisVersion v

hlintArgs :: GenericPackageDescription -> [[String]]
hlintArgs gpd =
    [ libraryModuleArgs c
    | c <- flattenPackageDescription gpd ^.. L.library . traverse
    ] ++
    [ libraryModuleArgs c
    | c <- flattenPackageDescription gpd ^.. L.subLibraries . traverse
    ] ++
    [ executableModuleArgs c
    | c <- flattenPackageDescription gpd ^.. L.executables . traverse
    ]