License	GPL-2
Maintainer	yi-devel@googlegroups.com
Stability	experimental
Portability	portable
Safe Haskell	None
Language	Haskell2010
Extensions	TemplateHaskell TemplateHaskellQuotes DisambiguateRecordFields RecordWildCards DeriveFunctor

Yi.Lexer.Alex

Description

Utilities to turn a lexer generated by Alex into a Scanner that can be used by Yi. Most lexers will use the types defined here. Some things are exported for use by lexers themselves through the use of YiLexerscommon.hsinc.

Synopsis

type StyleLexerASI s t = StyleLexer AlexState s t AlexInput
data StyleLexer l s t i = StyleLexer {
- _tokenToStyle :: t -> StyleName
- _styleLexer :: Lexer l s (Tok t) i
}
data Lexer l s t i = Lexer {
- _step :: TokenLexer l s t i
- _starting :: s -> Point -> Posn -> l s
- _withChars :: Char -> [(Point, Char)] -> i
- _looked :: l s -> Point
- _statePosn :: l s -> Posn
- _lexEmpty :: t
- _startingState :: s
}
type CharScanner = Scanner Point Char
type TokenLexer l s t i = (l s, i) -> Maybe (t, (l s, i))
type ASI s = (AlexState s, AlexInput)
data Posn = Posn {
- posnOfs :: !Point
- posnLine :: !Int
- posnCol :: !Int
}
data Tok t = Tok {
- tokT :: t
- tokLen :: Size
- tokPosn :: Posn
}
data AlexState lexerState = AlexState {
- stLexer :: lexerState
- lookedOffset :: !Point
- stPosn :: !Posn
}
type Action hlState token = IndexedStr -> hlState -> (hlState, token)
type AlexInput = (Char, [Byte], IndexedStr)
type IndexedStr = [(Point, Char)]
type Byte = Word8
utf8Encode :: Char -> [Word8]
tokToSpan :: Tok t -> Span t
tokFromT :: t -> Tok t
tokBegin :: Tok t -> Point
tokEnd :: Tok t -> Point
startPosn :: Posn
moveStr :: Posn -> IndexedStr -> Posn
moveCh :: Posn -> Char -> Posn
alexGetChar :: AlexInput -> Maybe (Char, AlexInput)
alexGetByte :: AlexInput -> Maybe (Word8, AlexInput)
alexCollectChar :: AlexInput -> [Char]
alexInputPrevChar :: AlexInput -> Char
actionConst :: token -> Action lexState token
actionAndModify :: (lexState -> lexState) -> token -> Action lexState token
actionStringAndModify :: (s -> s) -> (String -> token) -> Action s token
actionStringConst :: (String -> token) -> Action lexState token
commonLexer :: (ASI s -> Maybe (Tok t, ASI s)) -> s -> Lexer AlexState s (Tok t) AlexInput
lexScanner :: Lexer l s t i -> CharScanner -> Scanner (l s) t
unfoldLexer :: ((state, input) -> Maybe (token, (state, input))) -> (state, input) -> [(state, token)]
posnColA :: Lens' Posn Int
posnLineA :: Lens' Posn Int
posnOfsA :: Lens' Posn Point
tokLenA :: forall t. Lens' (Tok t) Size
tokPosnA :: forall t. Lens' (Tok t) Posn
tokTA :: forall t t. Lens (Tok t) (Tok t) t t
lexEmpty :: forall l s t i. Lens' (Lexer l s t i) t
looked :: forall l s t i. Lens' (Lexer l s t i) (l s -> Point)
starting :: forall l s t i. Lens' (Lexer l s t i) (s -> Point -> Posn -> l s)
startingState :: forall l s t i. Lens' (Lexer l s t i) s
statePosn :: forall l s t i. Lens' (Lexer l s t i) (l s -> Posn)
step :: forall l s t i. Lens' (Lexer l s t i) (TokenLexer l s t i)
withChars :: forall l s t i. Lens' (Lexer l s t i) (Char -> [(Point, Char)] -> i)
styleLexer :: forall l s t i l s i. Lens (StyleLexer l s t i) (StyleLexer l s t i) (Lexer l s (Tok t) i) (Lexer l s (Tok t) i)
tokenToStyle :: forall l s t i. Lens' (StyleLexer l s t i) (t -> StyleName)
(+~) :: SemiNum absolute relative => absolute -> relative -> absolute
(~-) :: SemiNum absolute relative => absolute -> absolute -> relative
newtype Size = Size {
- fromSize :: Int
}
type Stroke = Span StyleName

Documentation

type StyleLexerASI s t = StyleLexer AlexState s t AlexInput #

StyleLexer over ASI.

data StyleLexer l s t i #

Just like Lexer but also knows how to turn its tokens into StyleNames.

Constructors

StyleLexer
Fields _tokenToStyle :: t -> StyleName _styleLexer :: Lexer l s (Tok t) i

data Lexer l s t i #

Generalises lexers. This allows us to easily use lexers which don't want to be cornered into the types we have predefined here and use in common.hsinc.

Constructors

Lexer
Fields _step :: TokenLexer l s t i _starting :: s -> Point -> Posn -> l s _withChars :: Char -> [(Point, Char)] -> i _looked :: l s -> Point _statePosn :: l s -> Posn _lexEmpty :: t _startingState :: s

type CharScanner = Scanner Point Char #

Handy alias

type TokenLexer l s t i = (l s, i) -> Maybe (t, (l s, i)) #

Function to (possibly) lex a single token and give us the remaining input.

type ASI s = (AlexState s, AlexInput) #

data Posn #

Constructors

Posn
Fields posnOfs :: !Point posnLine :: !Int posnCol :: !Int

Instances

Eq Posn #
Instance details Defined in Yi.Lexer.Alex Methods (==) :: Posn -> Posn -> Bool # (/=) :: Posn -> Posn -> Bool #
Ord Posn #
Instance details Defined in Yi.Lexer.Alex Methods compare :: Posn -> Posn -> Ordering # (<) :: Posn -> Posn -> Bool # (<=) :: Posn -> Posn -> Bool # (>) :: Posn -> Posn -> Bool # (>=) :: Posn -> Posn -> Bool # max :: Posn -> Posn -> Posn # min :: Posn -> Posn -> Posn #
Show Posn #
Instance details Defined in Yi.Lexer.Alex Methods showsPrec :: Int -> Posn -> ShowS # show :: Posn -> String # showList :: [Posn] -> ShowS #
Ix Posn #
Instance details Defined in Yi.Lexer.Alex Methods range :: (Posn, Posn) -> [Posn] # index :: (Posn, Posn) -> Posn -> Int # unsafeIndex :: (Posn, Posn) -> Posn -> Int inRange :: (Posn, Posn) -> Posn -> Bool # rangeSize :: (Posn, Posn) -> Int # unsafeRangeSize :: (Posn, Posn) -> Int

data Tok t #

Constructors

Tok
Fields tokT :: t tokLen :: Size tokPosn :: Posn

Instances

Functor Tok #
Instance details Defined in Yi.Lexer.Alex Methods fmap :: (a -> b) -> Tok a -> Tok b # (<$) :: a -> Tok b -> Tok a #
Eq (Tok a) #
Instance details Defined in Yi.Lexer.Alex Methods (==) :: Tok a -> Tok a -> Bool # (/=) :: Tok a -> Tok a -> Bool #
Show t => Show (Tok t) #
Instance details Defined in Yi.Lexer.Alex Methods showsPrec :: Int -> Tok t -> ShowS # show :: Tok t -> String # showList :: [Tok t] -> ShowS #

data AlexState lexerState #

Lexer state

Constructors

AlexState
Fields stLexer :: lexerState lookedOffset :: !Point stPosn :: !Posn

Instances

Show lexerState => Show (AlexState lexerState) #
Instance details Defined in Yi.Lexer.Alex Methods showsPrec :: Int -> AlexState lexerState -> ShowS # show :: AlexState lexerState -> String # showList :: [AlexState lexerState] -> ShowS #

type Action hlState token = IndexedStr -> hlState -> (hlState, token) #

type AlexInput = (Char, [Byte], IndexedStr) #

type IndexedStr = [(Point, Char)] #

type Byte = Word8 #

utf8Encode :: Char -> [Word8] #

Encode a Haskell String to a list of Word8 values, in UTF8 format.

tokToSpan :: Tok t -> Span t #

tokFromT :: t -> Tok t #

tokBegin :: Tok t -> Point #

tokEnd :: Tok t -> Point #

startPosn :: Posn #

moveStr :: Posn -> IndexedStr -> Posn #

moveCh :: Posn -> Char -> Posn #

alexGetChar :: AlexInput -> Maybe (Char, AlexInput) #

alexGetByte :: AlexInput -> Maybe (Word8, AlexInput) #

alexCollectChar :: AlexInput -> [Char] #

alexInputPrevChar :: AlexInput -> Char #

actionConst :: token -> Action lexState token #

Return a constant token

actionAndModify :: (lexState -> lexState) -> token -> Action lexState token #

Return a constant token, and modify the lexer state

actionStringAndModify :: (s -> s) -> (String -> token) -> Action s token #

Convert the parsed string into a token, and also modify the lexer state

actionStringConst :: (String -> token) -> Action lexState token #

Convert the parsed string into a token

commonLexer :: (ASI s -> Maybe (Tok t, ASI s)) -> s -> Lexer AlexState s (Tok t) AlexInput #

Defines a Lexer for ASI. This exists to make using the new lexScanner easier if you're using ASI as all our lexers do today, 23-08-2014.

lexScanner :: Lexer l s t i -> CharScanner -> Scanner (l s) t #

Combine a character scanner with a lexer to produce a token scanner. May be used together with mkHighlighter to produce a Highlighter, or with linearSyntaxMode to produce a Mode.

unfoldLexer :: ((state, input) -> Maybe (token, (state, input))) -> (state, input) -> [(state, token)] #

unfold lexer into a function that returns a stream of (state, token)

posnColA :: Lens' Posn Int #

posnLineA :: Lens' Posn Int #

posnOfsA :: Lens' Posn Point #

tokLenA :: forall t. Lens' (Tok t) Size #

tokPosnA :: forall t. Lens' (Tok t) Posn #

tokTA :: forall t t. Lens (Tok t) (Tok t) t t #

lexEmpty :: forall l s t i. Lens' (Lexer l s t i) t #

looked :: forall l s t i. Lens' (Lexer l s t i) (l s -> Point) #

starting :: forall l s t i. Lens' (Lexer l s t i) (s -> Point -> Posn -> l s) #

startingState :: forall l s t i. Lens' (Lexer l s t i) s #

statePosn :: forall l s t i. Lens' (Lexer l s t i) (l s -> Posn) #

step :: forall l s t i. Lens' (Lexer l s t i) (TokenLexer l s t i) #

withChars :: forall l s t i. Lens' (Lexer l s t i) (Char -> [(Point, Char)] -> i) #

styleLexer :: forall l s t i l s i. Lens (StyleLexer l s t i) (StyleLexer l s t i) (Lexer l s (Tok t) i) (Lexer l s (Tok t) i) #

tokenToStyle :: forall l s t i. Lens' (StyleLexer l s t i) (t -> StyleName) #

(+~) :: SemiNum absolute relative => absolute -> relative -> absolute #

(~-) :: SemiNum absolute relative => absolute -> absolute -> relative #

newtype Size #

Size of a buffer region

Constructors

Size
Fields fromSize :: Int

Instances

Enum Size #
Instance details Defined in Yi.Buffer.Basic Methods succ :: Size -> Size # pred :: Size -> Size # toEnum :: Int -> Size # fromEnum :: Size -> Int # enumFrom :: Size -> [Size] # enumFromThen :: Size -> Size -> [Size] # enumFromTo :: Size -> Size -> [Size] # enumFromThenTo :: Size -> Size -> Size -> [Size] #
Eq Size #
Instance details Defined in Yi.Buffer.Basic Methods (==) :: Size -> Size -> Bool # (/=) :: Size -> Size -> Bool #
Integral Size #
Instance details Defined in Yi.Buffer.Basic Methods quot :: Size -> Size -> Size # rem :: Size -> Size -> Size # div :: Size -> Size -> Size # mod :: Size -> Size -> Size # quotRem :: Size -> Size -> (Size, Size) # divMod :: Size -> Size -> (Size, Size) # toInteger :: Size -> Integer #
Num Size #
Instance details Defined in Yi.Buffer.Basic Methods (+) :: Size -> Size -> Size # (-) :: Size -> Size -> Size # (*) :: Size -> Size -> Size # negate :: Size -> Size # abs :: Size -> Size # signum :: Size -> Size # fromInteger :: Integer -> Size #
Ord Size #
Instance details Defined in Yi.Buffer.Basic Methods compare :: Size -> Size -> Ordering # (<) :: Size -> Size -> Bool # (<=) :: Size -> Size -> Bool # (>) :: Size -> Size -> Bool # (>=) :: Size -> Size -> Bool # max :: Size -> Size -> Size # min :: Size -> Size -> Size #
Real Size #
Instance details Defined in Yi.Buffer.Basic Methods toRational :: Size -> Rational #
Show Size #
Instance details Defined in Yi.Buffer.Basic Methods showsPrec :: Int -> Size -> ShowS # show :: Size -> String # showList :: [Size] -> ShowS #
Binary Size #
Instance details Defined in Yi.Buffer.Basic Methods put :: Size -> Put # get :: Get Size # putList :: [Size] -> Put #
SemiNum Point Size #
Instance details Defined in Yi.Buffer.Basic Methods (+~) :: Point -> Size -> Point # (-~) :: Point -> Size -> Point # (~-) :: Point -> Point -> Size #

type Stroke = Span StyleName #