Copyright	(c) roki 2019
License	MIT
Maintainer	falgon53@yahoo.co.jp
Stability	experimental
Portability	POSIX
Safe Haskell	None
Language	Haskell2010

Htcc.Tokenizer.Token

Contents

Token data types
Utilities for accessing to token data

Description

Types used in lexical analysis and their utility functions

Synopsis

data TokenLCNums i = TokenLCNums {
- tkLn :: !i
- tkCn :: !i
}
type TokenLC i = (TokenLCNums i, Token i)
data Token i
- = TKReserved Text
- | TKNum !i
- | TKIdent Text
- | TKReturn
- | TKIf
- | TKSwitch
- | TKCase
- | TKDefault
- | TKElse
- | TKWhile
- | TKFor
- | TKBreak
- | TKContinue
- | TKEnum
- | TKSizeof
- | TKAlignof
- | TKStruct
- | TKGoto
- | TKType (StorageClass i)
- | TKTypedef
- | TKString ByteString
- | TKMacro Macros Text
- | TKEmpty
length :: Show i => Token i -> Int
emptyToken :: Num i => TokenLC i
isTKNum :: Token i -> Bool
isTKType :: Token i -> Bool
isTKStruct :: Token i -> Bool
isTKEnum :: Token i -> Bool
isTKIdent :: Token i -> Bool
isTKReserved :: Token i -> Bool
isTKMacro :: Token i -> Bool
isTKString :: Token i -> Bool
spanStrLiteral :: Text -> Maybe (Text, Text)
spanCharLiteral :: Text -> Maybe (Text, Text)
spanIntLit :: (Eq i, Num i, Read i) => Text -> Maybe (Natural, Token i, Text)
lookupKeyword :: forall i. Show i => Text -> Maybe (Token i)
altEmptyToken :: Num i => [TokenLC i] -> TokenLC i

Token data types

data TokenLCNums i Source #

TokenLCNums is data structure for storing the line number and character number of each token

Constructors

TokenLCNums	The constructor of `TokenLCNums`
Fields tkLn :: !i line number tkCn :: !i character number

Instances

Instances details

NFData1 TokenLCNums Source #
Instance details Defined in Htcc.Tokenizer.Token Methods liftRnf :: (a -> ()) -> TokenLCNums a -> () #
Eq i => Eq (TokenLCNums i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods (==) :: TokenLCNums i -> TokenLCNums i -> Bool # (/=) :: TokenLCNums i -> TokenLCNums i -> Bool #
Show i => Show (TokenLCNums i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods showsPrec :: Int -> TokenLCNums i -> ShowS # show :: TokenLCNums i -> String # showList :: [TokenLCNums i] -> ShowS #
Generic (TokenLCNums i) Source #
Instance details Defined in Htcc.Tokenizer.Token Associated Types type Rep (TokenLCNums i) :: Type -> Type # Methods from :: TokenLCNums i -> Rep (TokenLCNums i) x # to :: Rep (TokenLCNums i) x -> TokenLCNums i #
NFData i => NFData (TokenLCNums i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods rnf :: TokenLCNums i -> () #
Generic1 TokenLCNums Source #
Instance details Defined in Htcc.Tokenizer.Token Associated Types type Rep1 TokenLCNums :: k -> Type # Methods from1 :: forall (a :: k). TokenLCNums a -> Rep1 TokenLCNums a # to1 :: forall (a :: k). Rep1 TokenLCNums a -> TokenLCNums a #
type Rep (TokenLCNums i) Source #
Instance details Defined in Htcc.Tokenizer.Token type Rep (TokenLCNums i) = D1 ('MetaData "TokenLCNums" "Htcc.Tokenizer.Token" "htcc-0.0.0.1-I8tHuOdNzco4DzUthNftqr" 'False) (C1 ('MetaCons "TokenLCNums" 'PrefixI 'True) (S1 ('MetaSel ('Just "tkLn") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 i) :*: S1 ('MetaSel ('Just "tkCn") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 i)))
type Rep1 TokenLCNums Source #
Instance details Defined in Htcc.Tokenizer.Token type Rep1 TokenLCNums = D1 ('MetaData "TokenLCNums" "Htcc.Tokenizer.Token" "htcc-0.0.0.1-I8tHuOdNzco4DzUthNftqr" 'False) (C1 ('MetaCons "TokenLCNums" 'PrefixI 'True) (S1 ('MetaSel ('Just "tkLn") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) Par1 :*: S1 ('MetaSel ('Just "tkCn") 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) Par1))

type TokenLC i = (TokenLCNums i, Token i) Source #

Token and its TokenLCNums.

data Token i Source #

Token type

Constructors

TKReserved Text	The reserved token
TKNum !i	The number data
TKIdent Text	The identifier
TKReturn	The `return` keyword
TKIf	The `if` keyword
TKSwitch	The `switch` keyword
TKCase	The `case` keyword
TKDefault	The `default` keyword
TKElse	The `else` keyword
TKWhile	The `while` keyword
TKFor	The `for` keyword
TKBreak	The `break` keyword
TKContinue	The `continue` keyword
TKEnum	The `enum` keyword
TKSizeof	The `sizeof` keyword
TKAlignof	The `_Alignof` keyword
TKStruct	The `struct` keyword
TKGoto	THe `goto` keyword
TKType (StorageClass i)	Types
TKTypedef	The `typedef` keyword
TKString ByteString	The string literal
TKMacro Macros Text	The C macro
TKEmpty	The empty token (This is not used by `tokenize`, but when errors are detected during parsing, the token at error locations cannot be specified)

Instances

Instances details

Eq i => Eq (Token i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods (==) :: Token i -> Token i -> Bool # (/=) :: Token i -> Token i -> Bool #
Read i => Read (Token i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods readsPrec :: Int -> ReadS (Token i) # readList :: ReadS [Token i] # readPrec :: ReadPrec (Token i) # readListPrec :: ReadPrec [Token i] #
Show i => Show (Token i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods showsPrec :: Int -> Token i -> ShowS # show :: Token i -> String # showList :: [Token i] -> ShowS #
Generic (Token i) Source #
Instance details Defined in Htcc.Tokenizer.Token Associated Types type Rep (Token i) :: Type -> Type # Methods from :: Token i -> Rep (Token i) x # to :: Rep (Token i) x -> Token i #
NFData i => NFData (Token i) Source #
Instance details Defined in Htcc.Tokenizer.Token Methods rnf :: Token i -> () #
type Rep (Token i) Source #
Instance details Defined in Htcc.Tokenizer.Token type Rep (Token i) = D1 ('MetaData "Token" "Htcc.Tokenizer.Token" "htcc-0.0.0.1-I8tHuOdNzco4DzUthNftqr" 'False) ((((C1 ('MetaCons "TKReserved" 'PrefixI 'False) (S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text)) :+: C1 ('MetaCons "TKNum" 'PrefixI 'False) (S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'SourceStrict 'DecidedStrict) (Rec0 i))) :+: (C1 ('MetaCons "TKIdent" 'PrefixI 'False) (S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text)) :+: (C1 ('MetaCons "TKReturn" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "TKIf" 'PrefixI 'False) (U1 :: Type -> Type)))) :+: ((C1 ('MetaCons "TKSwitch" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "TKCase" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "TKDefault" 'PrefixI 'False) (U1 :: Type -> Type))) :+: (C1 ('MetaCons "TKElse" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "TKWhile" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "TKFor" 'PrefixI 'False) (U1 :: Type -> Type))))) :+: (((C1 ('MetaCons "TKBreak" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "TKContinue" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "TKEnum" 'PrefixI 'False) (U1 :: Type -> Type))) :+: (C1 ('MetaCons "TKSizeof" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "TKAlignof" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "TKStruct" 'PrefixI 'False) (U1 :: Type -> Type)))) :+: ((C1 ('MetaCons "TKGoto" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "TKType" 'PrefixI 'False) (S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 (StorageClass i))) :+: C1 ('MetaCons "TKTypedef" 'PrefixI 'False) (U1 :: Type -> Type))) :+: (C1 ('MetaCons "TKString" 'PrefixI 'False) (S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 ByteString)) :+: (C1 ('MetaCons "TKMacro" 'PrefixI 'False) (S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Macros) :*: S1 ('MetaSel ('Nothing :: Maybe Symbol) 'NoSourceUnpackedness 'NoSourceStrictness 'DecidedLazy) (Rec0 Text)) :+: C1 ('MetaCons "TKEmpty" 'PrefixI 'False) (U1 :: Type -> Type))))))

Utilities for accessing to token data

length :: Show i => Token i -> Int Source #

length returns the token length

emptyToken :: Num i => TokenLC i Source #

emptyToken is used when it cannot be referenced

isTKNum :: Token i -> Bool Source #

Utility for TKNum. When the argument is TKNum, it returns True, otherwise False.

isTKType :: Token i -> Bool Source #

Utility for TKType. When the argument is TKType, it returns True, otherwise False.

isTKStruct :: Token i -> Bool Source #

Utility for TKStruct. When the argument is TKStruct, it returns True, otherwise False.

isTKEnum :: Token i -> Bool Source #

Utility for TKEnum. When the argument is TKEnum, it returns True, otherwise False.

isTKIdent :: Token i -> Bool Source #

Utility for TKIdent. When the argument is TKIdent, it returns True, otherwise False.

isTKReserved :: Token i -> Bool Source #

Utility for TKReserved. When the argument is TKReserved, it returns True, otherwise False.

isTKMacro :: Token i -> Bool Source #

Utility for TKMacro. When the argument is TKMacro, it returns True, otherwise False.

isTKString :: Token i -> Bool Source #

Utility for TKString. When the argument is TKString, it returns True, otherwise False.

spanStrLiteral :: Text -> Maybe (Text, Text) Source #

spanStrLiteral separate the string literal part and the non-string literal part from the input text

spanCharLiteral :: Text -> Maybe (Text, Text) Source #

spanCharLiteral separate the string literal part and the non-string literal part from the input text

spanIntLit :: (Eq i, Num i, Read i) => Text -> Maybe (Natural, Token i, Text) Source #

Take the integer literal from given text.

lookupKeyword :: forall i. Show i => Text -> Maybe (Token i) Source #

Lookup keyword from Text. If the specified Text is not keyword as C language, lookupKeyword returns Nothing.

altEmptyToken :: Num i => [TokenLC i] -> TokenLC i Source #

altEmptyToken returns emptyToken if the first token is empty. Otherwise, returns the first token in the token sequence.