haskell-token-utils- Utilities to tie up tokens to an AST

Safe HaskellSafe-Inferred




data Entry a Source

An entry in the data structure for a particular srcspan.


Entry !ForestSpan !Layout ![a]

Entry has * the source span contained in this Node * how the sub-elements nest * the tokens for the SrcSpan if subtree is empty

Deleted !ForestSpan !RowOffset !SimpPos

Deleted has * the source span has been deleted * prior gap in lines * the gap between this span end and the start of the next in the fringe of the tree.


IsToken a => Eq (Entry a) 
IsToken t => Ord (LayoutTree t) 
Show a => Show (Entry a) 
HasLoc (Entry a) 
Outputable (LayoutTree GhcPosToken) 
Outputable (Entry GhcPosToken) 

data TokenCache a Source




tkCache :: !(Map TreeId (Tree (Entry a)))
tkLastTreeId :: !TreeId

data TreeId Source


TId !Int 


Eq TreeId 
Ord TreeId 
Show TreeId 

mainTid :: TreeIdSource

Identifies the tree carrying the main tokens, not any work in progress or deleted ones

type ForestSpan = (ForestPos, ForestPos)Source

Match a SrcSpan, using a ForestLine as the marker

data ForestLine Source




flSpanLengthChanged :: !Bool

The length of the span may have changed due to updated tokens.

flTreeSelector :: !Int
flInsertVersion :: !Int
flLine :: !Int

type RowOffset = IntSource

type ColOffset = IntSource

type Row = IntSource

type Col = IntSource

type SimpPos = (Int, Int)Source

data Layout Source


Above EndOffset (Row, Col) (Row, Col) EndOffset

Initial offset from token before the stacked list of items, the (r,c) of the first non-comment token, the (r,c) of the end of the last non-comment token in the stacked list to be able to calculate the (RowOffset,ColOffset) between the last token and the start of the next item.



Eq Layout 
Show Layout 
Monoid Layout 
Monoid Layout 
Outputable Layout 

type LayoutTree a = Tree (Entry a)Source

ghcLineToForestLine :: Int -> ForestLineSource

Extract an encoded ForestLine from a GHC line

class (Show a, HasLoc a) => IsToken a whereSource

The IsToken class captures the different token type in use. For GHC it represents the type returned by getRichTokenStream, namely [(GHC.Located GHC.Token, String)] For haskell-src-exts this is the reult of lexTokenStream, namely `[HSE.Loc HSE.Token]`


tokenLen :: a -> IntSource

tokenLen returns the length of the string representation of the token, not just the difference in the location, as the string may have changed without the position being updated, e.g. in a renaming

isComment :: a -> BoolSource

isEmpty :: a -> BoolSource

Zero-length tokens, as appear in GHC as markers

mkZeroToken :: aSource

isDo :: a -> BoolSource

isElse :: a -> BoolSource

isIn :: a -> BoolSource

isLet :: a -> BoolSource

isOf :: a -> BoolSource

isThen :: a -> BoolSource

isWhere :: a -> BoolSource

tokenToString :: a -> StringSource

showTokenStream :: [a] -> StringSource

lexStringToTokens :: SimpSpan -> String -> [a]Source

Create a stream of tokens from source, with first token start at given location

markToken :: a -> aSource

Mark a token so that it can be use to trigger layout checking later when the toks are retrieved

isMarked :: a -> BoolSource


IsToken (Loc TuToken) 
IsToken (Located Token, String) 

notWhiteSpace :: IsToken a => a -> BoolSource

isIgnored :: IsToken a => a -> BoolSource

isIgnoredNonComment :: IsToken a => a -> BoolSource

Tokens that are ignored when determining the first non-comment token in a span

isWhereOrLet :: IsToken a => a -> BoolSource

showFriendlyToks :: IsToken a => [a] -> StringSource

class HasLoc a whereSource


HasLoc SrcSpan 
HasLoc ForestSpan 
HasLoc SrcSpan 
HasLoc (Loc a) 
HasLoc (Entry a) 
HasLoc (Located a) 
HasLoc (Located Token, String) 

class Allocatable b a whereSource


allocTokens :: IsToken a => b -> [a] -> LayoutTree aSource

Construct a LayoutTree from a Haskell AST and a stream of tokens.


Allocatable ParsedSource GhcPosToken 
Allocatable (Module SrcSpanInfo) (Loc TuToken)

This instance is the purpose of this module