{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE DeriveDataTypeable #-}
{-# LANGUAGE BangPatterns #-}

module Commonmark.Tokens
  ( Tok(..)
  , TokType(..)
  , SourcePos
  , tokenize
  , untokenize
  ) where

import           Unicode.Char    (isAlphaNum)
import           Unicode.Char.General.Compat  (isSpace)
import           Data.Text       (Text)
import qualified Data.Text       as T
import           Data.Data       (Data, Typeable)
import           Text.Parsec.Pos
import           Data.Text.Normalize (normalize, NormalizationMode(NFC))

data Tok = Tok { Tok -> TokType
tokType     :: !TokType
               , Tok -> SourcePos
tokPos      :: !SourcePos
               , Tok -> Text
tokContents :: {-# UNPACK #-} !Text
               }
               deriving (Int -> Tok -> ShowS
[Tok] -> ShowS
Tok -> String
(Int -> Tok -> ShowS)
-> (Tok -> String) -> ([Tok] -> ShowS) -> Show Tok
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [Tok] -> ShowS
$cshowList :: [Tok] -> ShowS
show :: Tok -> String
$cshow :: Tok -> String
showsPrec :: Int -> Tok -> ShowS
$cshowsPrec :: Int -> Tok -> ShowS
Show, Tok -> Tok -> Bool
(Tok -> Tok -> Bool) -> (Tok -> Tok -> Bool) -> Eq Tok
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: Tok -> Tok -> Bool
$c/= :: Tok -> Tok -> Bool
== :: Tok -> Tok -> Bool
$c== :: Tok -> Tok -> Bool
Eq, Typeable Tok
DataType
Constr
Typeable Tok
-> (forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> Tok -> c Tok)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c Tok)
-> (Tok -> Constr)
-> (Tok -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c Tok))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok))
-> ((forall b. Data b => b -> b) -> Tok -> Tok)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r)
-> (forall u. (forall d. Data d => d -> u) -> Tok -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Tok -> m Tok)
-> Data Tok
Tok -> DataType
Tok -> Constr
(forall b. Data b => b -> b) -> Tok -> Tok
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Tok -> c Tok
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Tok
forall a.
Typeable a
-> (forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> a -> c a)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c a)
-> (a -> Constr)
-> (a -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c a))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
-> ((forall b. Data b => b -> b) -> a -> a)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall u. (forall d. Data d => d -> u) -> a -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> Data a
forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u
forall u. (forall d. Data d => d -> u) -> Tok -> [u]
forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Tok -> m Tok
forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Tok -> m Tok
forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Tok
forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Tok -> c Tok
forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Tok)
forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok)
$cTok :: Constr
$tTok :: DataType
gmapMo :: (forall d. Data d => d -> m d) -> Tok -> m Tok
$cgmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Tok -> m Tok
gmapMp :: (forall d. Data d => d -> m d) -> Tok -> m Tok
$cgmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Tok -> m Tok
gmapM :: (forall d. Data d => d -> m d) -> Tok -> m Tok
$cgmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Tok -> m Tok
gmapQi :: Int -> (forall d. Data d => d -> u) -> Tok -> u
$cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Tok -> u
gmapQ :: (forall d. Data d => d -> u) -> Tok -> [u]
$cgmapQ :: forall u. (forall d. Data d => d -> u) -> Tok -> [u]
gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
$cgmapQr :: forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
$cgmapQl :: forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Tok -> r
gmapT :: (forall b. Data b => b -> b) -> Tok -> Tok
$cgmapT :: (forall b. Data b => b -> b) -> Tok -> Tok
dataCast2 :: (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok)
$cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Tok)
dataCast1 :: (forall d. Data d => c (t d)) -> Maybe (c Tok)
$cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Tok)
dataTypeOf :: Tok -> DataType
$cdataTypeOf :: Tok -> DataType
toConstr :: Tok -> Constr
$ctoConstr :: Tok -> Constr
gunfold :: (forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Tok
$cgunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Tok
gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Tok -> c Tok
$cgfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Tok -> c Tok
$cp1Data :: Typeable Tok
Data, Typeable)

data TokType =
       Spaces
     | UnicodeSpace
     | LineEnd
     | WordChars
     | Symbol {-# UNPACK #-} !Char
     deriving (Int -> TokType -> ShowS
[TokType] -> ShowS
TokType -> String
(Int -> TokType -> ShowS)
-> (TokType -> String) -> ([TokType] -> ShowS) -> Show TokType
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
showList :: [TokType] -> ShowS
$cshowList :: [TokType] -> ShowS
show :: TokType -> String
$cshow :: TokType -> String
showsPrec :: Int -> TokType -> ShowS
$cshowsPrec :: Int -> TokType -> ShowS
Show, TokType -> TokType -> Bool
(TokType -> TokType -> Bool)
-> (TokType -> TokType -> Bool) -> Eq TokType
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
/= :: TokType -> TokType -> Bool
$c/= :: TokType -> TokType -> Bool
== :: TokType -> TokType -> Bool
$c== :: TokType -> TokType -> Bool
Eq, Eq TokType
Eq TokType
-> (TokType -> TokType -> Ordering)
-> (TokType -> TokType -> Bool)
-> (TokType -> TokType -> Bool)
-> (TokType -> TokType -> Bool)
-> (TokType -> TokType -> Bool)
-> (TokType -> TokType -> TokType)
-> (TokType -> TokType -> TokType)
-> Ord TokType
TokType -> TokType -> Bool
TokType -> TokType -> Ordering
TokType -> TokType -> TokType
forall a.
Eq a
-> (a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
min :: TokType -> TokType -> TokType
$cmin :: TokType -> TokType -> TokType
max :: TokType -> TokType -> TokType
$cmax :: TokType -> TokType -> TokType
>= :: TokType -> TokType -> Bool
$c>= :: TokType -> TokType -> Bool
> :: TokType -> TokType -> Bool
$c> :: TokType -> TokType -> Bool
<= :: TokType -> TokType -> Bool
$c<= :: TokType -> TokType -> Bool
< :: TokType -> TokType -> Bool
$c< :: TokType -> TokType -> Bool
compare :: TokType -> TokType -> Ordering
$ccompare :: TokType -> TokType -> Ordering
$cp1Ord :: Eq TokType
Ord, Typeable TokType
DataType
Constr
Typeable TokType
-> (forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> TokType -> c TokType)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c TokType)
-> (TokType -> Constr)
-> (TokType -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c TokType))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType))
-> ((forall b. Data b => b -> b) -> TokType -> TokType)
-> (forall r r'.
    (r -> r' -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r)
-> (forall r r'.
    (r' -> r -> r)
    -> r -> (forall d. Data d => d -> r') -> TokType -> r)
-> (forall u. (forall d. Data d => d -> u) -> TokType -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> TokType -> m TokType)
-> Data TokType
TokType -> DataType
TokType -> Constr
(forall b. Data b => b -> b) -> TokType -> TokType
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> TokType -> c TokType
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c TokType
forall a.
Typeable a
-> (forall (c :: * -> *).
    (forall d b. Data d => c (d -> b) -> d -> c b)
    -> (forall g. g -> c g) -> a -> c a)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c a)
-> (a -> Constr)
-> (a -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c a))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
-> ((forall b. Data b => b -> b) -> a -> a)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall u. (forall d. Data d => d -> u) -> a -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> Data a
forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u
forall u. (forall d. Data d => d -> u) -> TokType -> [u]
forall r r'.
(r -> r' -> r)
-> r -> (forall d. Data d => d -> r') -> TokType -> r
forall r r'.
(r' -> r -> r)
-> r -> (forall d. Data d => d -> r') -> TokType -> r
forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> TokType -> m TokType
forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> TokType -> m TokType
forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c TokType
forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> TokType -> c TokType
forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c TokType)
forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType)
$cSymbol :: Constr
$cWordChars :: Constr
$cLineEnd :: Constr
$cUnicodeSpace :: Constr
$cSpaces :: Constr
$tTokType :: DataType
gmapMo :: (forall d. Data d => d -> m d) -> TokType -> m TokType
$cgmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> TokType -> m TokType
gmapMp :: (forall d. Data d => d -> m d) -> TokType -> m TokType
$cgmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> TokType -> m TokType
gmapM :: (forall d. Data d => d -> m d) -> TokType -> m TokType
$cgmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> TokType -> m TokType
gmapQi :: Int -> (forall d. Data d => d -> u) -> TokType -> u
$cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> TokType -> u
gmapQ :: (forall d. Data d => d -> u) -> TokType -> [u]
$cgmapQ :: forall u. (forall d. Data d => d -> u) -> TokType -> [u]
gmapQr :: (r' -> r -> r)
-> r -> (forall d. Data d => d -> r') -> TokType -> r
$cgmapQr :: forall r r'.
(r' -> r -> r)
-> r -> (forall d. Data d => d -> r') -> TokType -> r
gmapQl :: (r -> r' -> r)
-> r -> (forall d. Data d => d -> r') -> TokType -> r
$cgmapQl :: forall r r'.
(r -> r' -> r)
-> r -> (forall d. Data d => d -> r') -> TokType -> r
gmapT :: (forall b. Data b => b -> b) -> TokType -> TokType
$cgmapT :: (forall b. Data b => b -> b) -> TokType -> TokType
dataCast2 :: (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType)
$cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c TokType)
dataCast1 :: (forall d. Data d => c (t d)) -> Maybe (c TokType)
$cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c TokType)
dataTypeOf :: TokType -> DataType
$cdataTypeOf :: TokType -> DataType
toConstr :: TokType -> Constr
$ctoConstr :: TokType -> Constr
gunfold :: (forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c TokType
$cgunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c TokType
gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> TokType -> c TokType
$cgfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> TokType -> c TokType
$cp1Data :: Typeable TokType
Data, Typeable)

-- | Convert a 'Text' into a list of 'Tok'. The first parameter
-- species the source name.
tokenize :: String -> Text -> [Tok]
tokenize :: String -> Text -> [Tok]
tokenize String
name =
  {-# SCC tokenize #-} SourcePos -> [Text] -> [Tok]
go (String -> SourcePos
initialPos String
name) ([Text] -> [Tok]) -> (Text -> [Text]) -> Text -> [Tok]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Char -> Char -> Bool) -> Text -> [Text]
T.groupBy Char -> Char -> Bool
f (Text -> [Text]) -> (Text -> Text) -> Text -> [Text]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. NormalizationMode -> Text -> Text
normalize NormalizationMode
NFC
  where
    -- We group \r\n, consecutive spaces, and consecutive alphanums;
    -- everything else gets in a token by itself.
    f :: Char -> Char -> Bool
f Char
'\r' Char
'\n' = Bool
True
    f Char
' ' Char
' '   = Bool
True
    f Char
x   Char
y     = Char -> Bool
isAlphaNum Char
x Bool -> Bool -> Bool
&& Char -> Bool
isAlphaNum Char
y

    go :: SourcePos -> [Text] -> [Tok]
go !SourcePos
_pos [] = []
    go !SourcePos
pos (!Text
t:[Text]
ts) = -- note that t:ts are guaranteed to be nonempty
      case Text -> Char
T.head Text
t of
         Char
' ' ->  TokType -> SourcePos -> Text -> Tok
Tok TokType
Spaces SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceColumn SourcePos
pos (Text -> Int
T.length Text
t)) [Text]
ts
         Char
'\t' -> TokType -> SourcePos -> Text -> Tok
Tok TokType
Spaces SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceColumn SourcePos
pos
                       (Int
4 Int -> Int -> Int
forall a. Num a => a -> a -> a
- (SourcePos -> Int
sourceColumn SourcePos
pos Int -> Int -> Int
forall a. Num a => a -> a -> a
- Int
1) Int -> Int -> Int
forall a. Integral a => a -> a -> a
`mod` Int
4)) [Text]
ts
         Char
'\r' -> TokType -> SourcePos -> Text -> Tok
Tok TokType
LineEnd SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceLine (SourcePos -> Int -> SourcePos
setSourceColumn SourcePos
pos Int
1) Int
1) [Text]
ts
         Char
'\n' -> TokType -> SourcePos -> Text -> Tok
Tok TokType
LineEnd SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceLine (SourcePos -> Int -> SourcePos
setSourceColumn SourcePos
pos Int
1) Int
1) [Text]
ts
         Char
thead
           | Char -> Bool
isAlphaNum Char
thead ->
                 TokType -> SourcePos -> Text -> Tok
Tok TokType
WordChars SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceColumn SourcePos
pos (Text -> Int
T.length Text
t)) [Text]
ts
           | Char -> Bool
isSpace Char
thead ->
                 TokType -> SourcePos -> Text -> Tok
Tok TokType
UnicodeSpace SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceColumn SourcePos
pos Int
1) [Text]
ts
           | Bool
otherwise ->
                 TokType -> SourcePos -> Text -> Tok
Tok (Char -> TokType
Symbol Char
thead) SourcePos
pos Text
t Tok -> [Tok] -> [Tok]
forall a. a -> [a] -> [a]
:
                 SourcePos -> [Text] -> [Tok]
go (SourcePos -> Int -> SourcePos
incSourceColumn SourcePos
pos Int
1) [Text]
ts

-- | Reverses 'tokenize'.  @untokenize . tokenize@ should be
-- the identity.
untokenize :: [Tok] -> Text
untokenize :: [Tok] -> Text
untokenize = {-# SCC untokenize #-} [Text] -> Text
forall a. Monoid a => [a] -> a
mconcat ([Text] -> Text) -> ([Tok] -> [Text]) -> [Tok] -> Text
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Tok -> Text) -> [Tok] -> [Text]
forall a b. (a -> b) -> [a] -> [b]
map Tok -> Text
tokContents