{-|
Module      : Htcc.Tokenizer
Description : Tokenizer
Copyright   : (c) roki, 2019
License     : MIT
Maintainer  : falgon53@yahoo.co.jp
Stability   : experimental
Portability : POSIX

The tokenizer
-}
module Htcc.Tokenizer (
    module Htcc.Tokenizer.Token,
    tokenize
) where

import           Control.Monad                                   ((>=>))
import qualified Data.Text                                       as T

import           Htcc.CRules.Preprocessor                        as CP
import           Htcc.Parser.ConstructionData.Scope.ManagedScope (ASTError)
import           Htcc.Tokenizer.Core                             (tokenize')
import           Htcc.Tokenizer.Token

-- | Tokenize the `T.Text`. If an invalid chraracter matches as C language, the part and the character are returned.
-- Otherwise, @[TokenIdx i]@ is returned.
tokenize :: (Integral i, Read i, Show i) => T.Text -> Either (ASTError i) [TokenLC i]
tokenize :: Text -> Either (ASTError i) [TokenLC i]
tokenize = Text -> Either (ASTError i) [TokenLC i]
forall i.
(Enum i, Num i, Eq i, Read i, Show i) =>
Text -> Either (ASTError i) [TokenLC i]
tokenize' (Text -> Either (ASTError i) [TokenLC i])
-> ([TokenLC i] -> Either (ASTError i) [TokenLC i])
-> Text
-> Either (ASTError i) [TokenLC i]
forall (m :: * -> *) a b c.
Monad m =>
(a -> m b) -> (b -> m c) -> a -> m c
>=> [TokenLC i] -> Either (ASTError i) [TokenLC i]
forall i. [TokenLC i] -> Either (ASTError i) [TokenLC i]
CP.preprocess