{-|
Module      : Htcc.Tokenizer.Core
Description : Tokenizer
Copyright   : (c) roki, 2019
License     : MIT
Maintainer  : falgon53@yahoo.co.jp
Stability   : experimental
Portability : POSIX

The tokenizer
-}
{-# LANGUAGE LambdaCase, MultiWayIf, OverloadedStrings, TupleSections #-}
module Htcc.Tokenizer.Core (
    -- * Tokenizer
    tokenize'
) where

import           Control.Applicative                             (Alternative (..))
import           Control.Conditional                             (ifM)
import           Control.Monad.Extra                             (firstJustM)
import           Control.Monad.State
import           Data.Char                                       (digitToInt,
                                                                  isDigit, ord)
import           Data.List                                       (find)
import           Data.Maybe                                      (fromJust,
                                                                  isJust,
                                                                  isNothing)
import qualified Data.Text                                       as T
import qualified Data.Text.Encoding                              as T
import           Numeric                                         (readHex,
                                                                  showHex)

import qualified Htcc.CRules                                     as CR
import qualified Htcc.CRules.Preprocessor.Punctuators            as CP
import           Htcc.Parser.ConstructionData.Scope.ManagedScope (ASTError)
import           Htcc.Tokenizer.Token
import           Htcc.Utils                                      (isStrictSpace,
                                                                  lor, maybe',
                                                                  spanLenT,
                                                                  subTextIndex,
                                                                  tshow)
import qualified Htcc.Utils.CompilationState                     as C


{-# INLINE isNewLine #-}
isNewLine :: Char -> Bool
isNewLine :: Char -> Bool
isNewLine = [Char -> Bool] -> Char -> Bool
forall a. [a -> Bool] -> a -> Bool
lor [(Char -> Char -> Bool
forall a. Eq a => a -> a -> Bool
=='\n'), (Char -> Char -> Bool
forall a. Eq a => a -> a -> Bool
=='\r')]

{-# INLINE headToken #-}
headToken :: (Char -> Token i) -> T.Text -> Token i
headToken :: (Char -> Token i) -> Text -> Token i
headToken f :: Char -> Token i
f txt :: Text
txt
    | Text -> Bool
T.null Text
txt = Token i
forall i. Token i
TKEmpty
    | Bool
otherwise = Char -> Token i
f (Char -> Token i) -> Char -> Token i
forall a b. (a -> b) -> a -> b
$ Text -> Char
T.head Text
txt

type Tokenizer i a = C.CompilationState (TokenLCNums i) T.Text i a

{-# INLINE advanceLC #-}
advanceLC :: (Enum i, Num i) => TokenLCNums i -> Char -> TokenLCNums i
advanceLC :: TokenLCNums i -> Char -> TokenLCNums i
advanceLC lc :: TokenLCNums i
lc e :: Char
e
    | Char -> Bool
isNewLine Char
e = TokenLCNums i
lc { tkCn :: i
tkCn = 1, tkLn :: i
tkLn = i -> i
forall a. Enum a => a -> a
succ (i -> i) -> i -> i
forall a b. (a -> b) -> a -> b
$ TokenLCNums i -> i
forall i. TokenLCNums i -> i
tkLn TokenLCNums i
lc }
    | Bool
otherwise = TokenLCNums i
lc { tkCn :: i
tkCn = i -> i
forall a. Enum a => a -> a
succ (i -> i) -> i -> i
forall a b. (a -> b) -> a -> b
$ TokenLCNums i -> i
forall i. TokenLCNums i -> i
tkCn TokenLCNums i
lc }

{-# INLINE itemP #-}
itemP :: Tokenizer i (Maybe Char)
itemP :: Tokenizer i (Maybe Char)
itemP = Tokenizer i (Maybe Char)
forall mono cd i.
MonoFoldable mono =>
CompilationState cd mono i (Maybe (Element mono))
C.itemP

{-# INLINE itemC #-}
itemC :: (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC :: Tokenizer i (Maybe Char)
itemC = (TokenLCNums i -> Element Text -> TokenLCNums i)
-> CompilationState (TokenLCNums i) Text i (Maybe (Element Text))
forall mono cd i.
IsSequence mono =>
(cd -> Element mono -> cd)
-> CompilationState cd mono i (Maybe (Element mono))
C.itemC TokenLCNums i -> Element Text -> TokenLCNums i
forall i. (Enum i, Num i) => TokenLCNums i -> Char -> TokenLCNums i
advanceLC

{-# INLINE itemsP #-}
itemsP :: Int -> Tokenizer i (Maybe T.Text)
itemsP :: Int -> Tokenizer i (Maybe Text)
itemsP = Int -> Tokenizer i (Maybe Text)
forall mono cd i.
(MonoFoldable mono, IsSequence mono) =>
Index mono -> CompilationState cd mono i (Maybe mono)
C.itemsP

{-# INLINE curLC #-}
curLC :: Tokenizer i (TokenLCNums i)
curLC :: Tokenizer i (TokenLCNums i)
curLC = Tokenizer i (TokenLCNums i)
forall cd mono i. CompilationState cd mono i cd
C.curCD

char :: (Enum i, Num i) => (Char -> Bool) -> Tokenizer i (Maybe Char)
char :: (Char -> Bool) -> Tokenizer i (Maybe Char)
char = (TokenLCNums i -> Element Text -> TokenLCNums i)
-> (Element Text -> Bool)
-> CompilationState (TokenLCNums i) Text i (Maybe (Element Text))
forall mono cd i.
(MonoFoldable mono, IsSequence mono) =>
(cd -> Element mono -> cd)
-> (Element mono -> Bool)
-> CompilationState cd mono i (Maybe (Element mono))
C.itemCWhen TokenLCNums i -> Element Text -> TokenLCNums i
forall i. (Enum i, Num i) => TokenLCNums i -> Char -> TokenLCNums i
advanceLC

string :: (Enum i, Num i) => (Char -> Bool) -> Tokenizer i T.Text
string :: (Char -> Bool) -> Tokenizer i Text
string = (TokenLCNums i -> Element Text -> TokenLCNums i)
-> (Element Text -> Bool) -> Tokenizer i Text
forall mono cd i.
(MonoFoldable mono, IsSequence mono) =>
(cd -> Element mono -> cd)
-> (Element mono -> Bool) -> CompilationState cd mono i mono
C.itemsCWhen TokenLCNums i -> Element Text -> TokenLCNums i
forall i. (Enum i, Num i) => TokenLCNums i -> Char -> TokenLCNums i
advanceLC

isPrefixOf :: Enum i => T.Text -> Tokenizer i Bool
isPrefixOf :: Text -> Tokenizer i Bool
isPrefixOf = (Text -> Bool) -> Tokenizer i Bool
forall mono cd i. (mono -> Bool) -> CompilationState cd mono i Bool
C.isSatisfied ((Text -> Bool) -> Tokenizer i Bool)
-> (Text -> Text -> Bool) -> Text -> Tokenizer i Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Text -> Text -> Bool
T.isPrefixOf

consumeSpace :: (Enum i, Num i) => Tokenizer i Bool
consumeSpace :: Tokenizer i Bool
consumeSpace = Bool -> Bool
not (Bool -> Bool) -> (Text -> Bool) -> Text -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Text -> Bool
T.null (Text -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
-> Tokenizer i Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
forall i. (Enum i, Num i) => (Char -> Bool) -> Tokenizer i Text
string Char -> Bool
isStrictSpace

consumeNewLine :: (Enum i, Num i) => Tokenizer i Bool
consumeNewLine :: Tokenizer i Bool
consumeNewLine = Bool -> Bool
not (Bool -> Bool) -> (Text -> Bool) -> Text -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Text -> Bool
T.null (Text -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
-> Tokenizer i Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
forall i. (Enum i, Num i) => (Char -> Bool) -> Tokenizer i Text
string Char -> Bool
isNewLine

consumeComment :: (Enum i, Num i) => Tokenizer i Bool
consumeComment :: Tokenizer i Bool
consumeComment = Tokenizer i Bool
-> Tokenizer i Bool -> Tokenizer i Bool -> Tokenizer i Bool
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM (Text -> Tokenizer i Bool
forall i. Enum i => Text -> Tokenizer i Bool
isPrefixOf "//") (Bool
True Bool
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
-> Tokenizer i Bool
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
forall i. (Enum i, Num i) => Tokenizer i Text
line) (Tokenizer i Bool -> Tokenizer i Bool)
-> Tokenizer i Bool -> Tokenizer i Bool
forall a b. (a -> b) -> a -> b
$ (Tokenizer i Bool -> Tokenizer i Bool -> Tokenizer i Bool)
-> Tokenizer i Bool -> Tokenizer i Bool -> Tokenizer i Bool
forall a b c. (a -> b -> c) -> b -> a -> c
flip (Tokenizer i Bool
-> Tokenizer i Bool -> Tokenizer i Bool -> Tokenizer i Bool
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM (Text -> Tokenizer i Bool
forall i. Enum i => Text -> Tokenizer i Bool
isPrefixOf "/*")) (Bool -> Tokenizer i Bool
forall (m :: * -> *) a. Monad m => a -> m a
return Bool
False) (Tokenizer i Bool -> Tokenizer i Bool)
-> Tokenizer i Bool -> Tokenizer i Bool
forall a b. (a -> b) -> a -> b
$ do
    TokenLCNums i
cur <- Tokenizer i (TokenLCNums i)
forall i. Tokenizer i (TokenLCNums i)
curLC
    Int
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall (m :: * -> *) a. Applicative m => Int -> m a -> m ()
replicateM_ 2 StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC
    Maybe Int
ind <- ((TokenLCNums i, Text) -> Maybe Int)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Int)
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (Text -> Text -> Maybe Int
subTextIndex "*/" (Text -> Maybe Int)
-> ((TokenLCNums i, Text) -> Text)
-> (TokenLCNums i, Text)
-> Maybe Int
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (TokenLCNums i, Text) -> Text
forall a b. (a, b) -> b
snd)
    Bool
True Bool
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i Bool
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Maybe Int
-> (Int -> StateT (TokenLCNums i, Text) (Either (ASTError i)) ())
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall b a. b -> Maybe a -> (a -> b) -> b
maybe' (Either (ASTError i) ()
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) ()
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) ())
-> Either (ASTError i) ()
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall a b. (a -> b) -> a -> b
$ ASTError i -> Either (ASTError i) ()
forall a b. a -> Either a b
Left ("unterminated comment, expected to '*/'", (TokenLCNums i
cur, Text -> Token i
forall i. Text -> Token i
TKReserved "/"))) Maybe Int
ind ((Int
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) ())
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> Int
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall a b c. (a -> b -> c) -> b -> a -> c
flip Int
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall (m :: * -> *) a. Applicative m => Int -> m a -> m ()
replicateM_ StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC (Int -> StateT (TokenLCNums i, Text) (Either (ASTError i)) ())
-> (Int -> Int)
-> Int
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Int -> Int -> Int
forall a. Num a => a -> a -> a
+2))

line :: (Enum i, Num i) => Tokenizer i T.Text
line :: Tokenizer i Text
line = (Char -> Bool) -> Tokenizer i Text
forall i. (Enum i, Num i) => (Char -> Bool) -> Tokenizer i Text
string (Bool -> Bool
not (Bool -> Bool) -> (Char -> Bool) -> Char -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Char -> Bool
isNewLine)

macro :: (Enum i, Num i) => Tokenizer i (Maybe (Token i))
macro :: Tokenizer i (Maybe (Token i))
macro = StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM ((Maybe Char -> Maybe Char -> Bool
forall a. Eq a => a -> a -> Bool
/= Char -> Maybe Char
forall a. a -> Maybe a
Just '#') (Maybe Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. Tokenizer i (Maybe Char)
itemP) (Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing) Tokenizer i (Maybe (Token i))
forall i.
StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
f
    where
        f :: StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
f = StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> (Maybe Char
    -> StateT
         (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
(>>=) StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC ((Maybe Char
  -> StateT
       (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
 -> StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> (Maybe Char
    -> StateT
         (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
-> (Char
    -> StateT
         (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> Maybe Char
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (Maybe (Token i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing) ((Char
  -> StateT
       (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
 -> Maybe Char
 -> StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> (Char
    -> StateT
         (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> Maybe Char
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ \case
            '#' -> do
                TokenLCNums i
cur <- Tokenizer i (TokenLCNums i)
forall i. Tokenizer i (TokenLCNums i)
curLC
                Text
ln <- Tokenizer i Text
forall i. (Enum i, Num i) => Tokenizer i Text
line
                if Text -> Bool
T.null Text
ln then Maybe (Token i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing else let (_, kmc :: Text
kmc, ds :: Text
ds) = (Char -> Bool) -> Text -> (Int, Text, Text)
spanLenT Char -> Bool
CR.isValidChar Text
ln in
                    StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
-> Maybe Macros
-> (Macros
    -> StateT
         (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall b a. b -> Maybe a -> (a -> b) -> b
maybe' (Either (ASTError i) (Maybe (Token i))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) (Maybe (Token i))
 -> StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> Either (ASTError i) (Maybe (Token i))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ ASTError i -> Either (ASTError i) (Maybe (Token i))
forall a b. a -> Either a b
Left ("invalid macro in program", (TokenLCNums i
cur, Text -> Token i
forall i. Text -> Token i
TKReserved Text
ln))) ((Macros -> Bool) -> [Macros] -> Maybe Macros
forall (t :: * -> *) a. Foldable t => (a -> Bool) -> t a -> Maybe a
find ((Text -> Text -> Bool
forall a. Eq a => a -> a -> Bool
==Text
kmc) (Text -> Bool) -> (Macros -> Text) -> Macros -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Macros -> Text
forall a. Show a => a -> Text
tshow) [Macros]
CP.macros) ((Macros
  -> StateT
       (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
 -> StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> (Macros
    -> StateT
         (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ \m :: Macros
m -> Maybe (Token i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return (Maybe (Token i)
 -> StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> Maybe (Token i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (Token i -> Maybe (Token i)) -> Token i -> Maybe (Token i)
forall a b. (a -> b) -> a -> b
$ Macros -> Text -> Token i
forall i. Macros -> Text -> Token i
TKMacro Macros
m Text
ds
            _ -> Maybe (Token i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing

natLit :: (Enum i, Num i, Eq i, Read i) => Tokenizer i (Maybe (Token i))
natLit :: Tokenizer i (Maybe (Token i))
natLit = do
    TokenLCNums i
lc <- ((TokenLCNums i, Text) -> TokenLCNums i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (TokenLCNums i)
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (TokenLCNums i, Text) -> TokenLCNums i
forall a b. (a, b) -> a
fst
    StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM (Bool -> (Char -> Bool) -> Maybe Char -> Bool
forall b a. b -> (a -> b) -> Maybe a -> b
maybe Bool
True (Bool -> Bool
not (Bool -> Bool) -> (Char -> Bool) -> Char -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Char -> Bool
isDigit) (Maybe Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. Tokenizer i (Maybe Char)
itemP) (Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing) (Tokenizer i (Maybe (Token i)) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i)) -> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ do
        Text
txt <- ((TokenLCNums i, Text) -> Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (TokenLCNums i, Text) -> Text
forall a b. (a, b) -> b
snd
        Tokenizer i (Maybe (Token i))
-> Maybe (Natural, Token i, Text)
-> ((Natural, Token i, Text) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall b a. b -> Maybe a -> (a -> b) -> b
maybe' (Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) (Maybe (Token i))
 -> Tokenizer i (Maybe (Token i)))
-> Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ ASTError i -> Either (ASTError i) (Maybe (Token i))
forall a b. a -> Either a b
Left ("invalid number in program", (TokenLCNums i
lc, (Char -> Token i) -> Text -> Token i
forall i. (Char -> Token i) -> Text -> Token i
headToken (i -> Token i
forall i. i -> Token i
TKNum (i -> Token i) -> (Char -> i) -> Char -> Token i
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Int -> i
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Int -> i) -> (Char -> Int) -> Char -> i
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Char -> Int
digitToInt) Text
txt))) (Text -> Maybe (Natural, Token i, Text)
forall i.
(Eq i, Num i, Read i) =>
Text -> Maybe (Natural, Token i, Text)
spanIntLit Text
txt) (((Natural, Token i, Text) -> Tokenizer i (Maybe (Token i)))
 -> Tokenizer i (Maybe (Token i)))
-> ((Natural, Token i, Text) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ \(n :: Natural
n, tk :: Token i
tk, ds :: Text
ds) ->
            Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just Token i
tk Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ (TokenLCNums i, Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall s (m :: * -> *). MonadState s m => s -> m ()
put (TokenLCNums i
lc { tkCn :: i
tkCn = TokenLCNums i -> i
forall i. TokenLCNums i -> i
tkCn TokenLCNums i
lc i -> i -> i
forall a. Num a => a -> a -> a
+ Natural -> i
forall a b. (Integral a, Num b) => a -> b
fromIntegral Natural
n }, Text
ds)

strLit :: (Enum i, Num i) => Tokenizer i (Maybe (Token i))
strLit :: Tokenizer i (Maybe (Token i))
strLit = do
    TokenLCNums i
lc <- ((TokenLCNums i, Text) -> TokenLCNums i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (TokenLCNums i)
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (TokenLCNums i, Text) -> TokenLCNums i
forall a b. (a, b) -> a
fst
    StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM (Maybe Char -> Bool
forall a. Maybe a -> Bool
isNothing (Maybe Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i.
(Enum i, Num i) =>
(Char -> Bool) -> Tokenizer i (Maybe Char)
char (Char -> Char -> Bool
forall a. Eq a => a -> a -> Bool
=='\"')) (Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing) (Tokenizer i (Maybe (Token i)) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i)) -> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ do
        Text
txt <- ((TokenLCNums i, Text) -> Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (TokenLCNums i, Text) -> Text
forall a b. (a, b) -> b
snd
        Tokenizer i (Maybe (Token i))
-> Maybe (Text, Text)
-> ((Text, Text) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall b a. b -> Maybe a -> (a -> b) -> b
maybe' (Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) (Maybe (Token i))
 -> Tokenizer i (Maybe (Token i)))
-> Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ ASTError i -> Either (ASTError i) (Maybe (Token i))
forall a b. a -> Either a b
Left ("invalid string literal in program", (TokenLCNums i
lc, Text -> Token i
forall i. Text -> Token i
TKReserved "\""))) (Text -> Maybe (Text, Text)
spanStrLiteral Text
txt) (((Text, Text) -> Tokenizer i (Maybe (Token i)))
 -> Tokenizer i (Maybe (Token i)))
-> ((Text, Text) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ \(lit :: Text
lit, ds :: Text
ds) ->
            -- The meaning of adding 2 is to add the two "characters surrounding the string literal.
            Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (ByteString -> Token i
forall i. ByteString -> Token i
TKString (Text -> ByteString
T.encodeUtf8 (Text -> ByteString) -> Text -> ByteString
forall a b. (a -> b) -> a -> b
$ Text -> Text -> Text
T.append Text
lit "\0")) Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ (TokenLCNums i, Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall s (m :: * -> *). MonadState s m => s -> m ()
put (TokenLCNums i
lc { tkCn :: i
tkCn = 2 i -> i -> i
forall a. Num a => a -> a -> a
+ TokenLCNums i -> i
forall i. TokenLCNums i -> i
tkCn TokenLCNums i
lc i -> i -> i
forall a. Num a => a -> a -> a
+ Int -> i
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Text -> Int
T.length Text
lit) }, Text
ds)

charLit :: (Enum i, Num i, Eq i) => Tokenizer i (Maybe (Token i))
charLit :: Tokenizer i (Maybe (Token i))
charLit = do
    TokenLCNums i
lc <- ((TokenLCNums i, Text) -> TokenLCNums i)
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (TokenLCNums i)
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (TokenLCNums i, Text) -> TokenLCNums i
forall a b. (a, b) -> a
fst
    StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM (Maybe Char -> Bool
forall a. Maybe a -> Bool
isNothing (Maybe Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> (Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i.
(Enum i, Num i) =>
(Char -> Bool) -> Tokenizer i (Maybe Char)
char (Char -> Char -> Bool
forall a. Eq a => a -> a -> Bool
=='\'')) (Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing) (Tokenizer i (Maybe (Token i)) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i)) -> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ do
        Text
txt <- ((TokenLCNums i, Text) -> Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Text
forall s (m :: * -> *) a. MonadState s m => (s -> a) -> m a
gets (TokenLCNums i, Text) -> Text
forall a b. (a, b) -> b
snd
        Tokenizer i (Maybe (Token i))
-> Maybe (Text, Text)
-> ((Text, Text) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall b a. b -> Maybe a -> (a -> b) -> b
maybe' (Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) (Maybe (Token i))
 -> Tokenizer i (Maybe (Token i)))
-> Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ ASTError i -> Either (ASTError i) (Maybe (Token i))
forall a b. a -> Either a b
Left ("invalid char literal in program", (TokenLCNums i
lc, Text -> Token i
forall i. Text -> Token i
TKReserved "\'"))) (Text -> Maybe (Text, Text)
spanCharLiteral Text
txt) (((Text, Text) -> Tokenizer i (Maybe (Token i)))
 -> Tokenizer i (Maybe (Token i)))
-> ((Text, Text) -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ \(lit :: Text
lit, ds :: Text
ds) ->
            -- Adding 3 means to add a single character literal and two @"@
            if  | Text -> Int
T.length Text
lit Int -> Int -> Bool
forall a. Eq a => a -> a -> Bool
== 1 -> Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (i -> Token i
forall i. i -> Token i
TKNum (Int -> i
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Int -> i) -> Int -> i
forall a b. (a -> b) -> a -> b
$ Char -> Int
ord (Char -> Int) -> Char -> Int
forall a b. (a -> b) -> a -> b
$ Text -> Char
T.head Text
lit)) Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ (TokenLCNums i, Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall s (m :: * -> *). MonadState s m => s -> m ()
put (TokenLCNums i
lc { tkCn :: i
tkCn = 3 i -> i -> i
forall a. Num a => a -> a -> a
+ TokenLCNums i -> i
forall i. TokenLCNums i -> i
tkCn TokenLCNums i
lc }, Text
ds)
                -- For multi-character constants.
                -- The standard states that this is an implementation definition.
                -- Here it follows the implementation definitions of GCC and Clang.
                | Bool
otherwise -> Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (i -> Token i
forall i. i -> Token i
TKNum (i -> Token i) -> i -> Token i
forall a b. (a -> b) -> a -> b
$ (i, String) -> i
forall a b. (a, b) -> a
fst ((i, String) -> i) -> (i, String) -> i
forall a b. (a -> b) -> a -> b
$ [(i, String)] -> (i, String)
forall a. [a] -> a
head ([(i, String)] -> (i, String)) -> [(i, String)] -> (i, String)
forall a b. (a -> b) -> a -> b
$ ReadS i
forall a. (Eq a, Num a) => ReadS a
readHex ReadS i -> ReadS i
forall a b. (a -> b) -> a -> b
$ (Char -> String -> String) -> String -> String -> String
forall (t :: * -> *) a b.
Foldable t =>
(a -> b -> b) -> b -> t a -> b
foldr (\x :: Char
x acc :: String
acc -> Int -> String -> String
forall a. (Integral a, Show a) => a -> String -> String
showHex (Char -> Int
ord Char
x) "" String -> String -> String
forall a. Semigroup a => a -> a -> a
<> String
acc) [] (String -> String) -> String -> String
forall a b. (a -> b) -> a -> b
$ Text -> String
T.unpack Text
lit) Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$
                    (TokenLCNums i, Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall s (m :: * -> *). MonadState s m => s -> m ()
put (TokenLCNums i
lc { tkCn :: i
tkCn = 2 i -> i -> i
forall a. Num a => a -> a -> a
+ Int -> i
forall a b. (Integral a, Num b) => a -> b
fromIntegral (Text -> Int
T.length Text
lit) i -> i -> i
forall a. Num a => a -> a -> a
+ TokenLCNums i -> i
forall i. TokenLCNums i -> i
tkCn TokenLCNums i
lc }, Text
ds)

operators :: (Enum i, Num i) => Tokenizer i (Maybe (Token i))
operators :: Tokenizer i (Maybe (Token i))
operators = do
    Maybe Text
s3 <- Int -> Tokenizer i (Maybe Text)
forall i. Int -> Tokenizer i (Maybe Text)
itemsP 3
    if Maybe Text -> Bool
forall a. Maybe a -> Bool
isJust Maybe Text
s3 Bool -> Bool -> Bool
&& Maybe Text -> Text
forall a. HasCallStack => Maybe a -> a
fromJust Maybe Text
s3 Text -> [Text] -> Bool
forall (t :: * -> *) a. (Foldable t, Eq a) => a -> t a -> Bool
`elem` [Text]
CR.strOps3 then Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (Text -> Token i
forall i. Text -> Token i
TKReserved (Text -> Token i) -> Text -> Token i
forall a b. (a -> b) -> a -> b
$ Maybe Text -> Text
forall a. HasCallStack => Maybe a -> a
fromJust Maybe Text
s3) Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Int
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall (m :: * -> *) a. Applicative m => Int -> m a -> m ()
replicateM_ 3 StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC else do
        Maybe Text
s2 <- Int -> Tokenizer i (Maybe Text)
forall i. Int -> Tokenizer i (Maybe Text)
itemsP 2
        if Maybe Text -> Bool
forall a. Maybe a -> Bool
isJust Maybe Text
s2 Bool -> Bool -> Bool
&& Maybe Text -> Text
forall a. HasCallStack => Maybe a -> a
fromJust Maybe Text
s2 Text -> [Text] -> Bool
forall (t :: * -> *) a. (Foldable t, Eq a) => a -> t a -> Bool
`elem` [Text]
CR.strOps2 then Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (Text -> Token i
forall i. Text -> Token i
TKReserved (Text -> Token i) -> Text -> Token i
forall a b. (a -> b) -> a -> b
$ Maybe Text -> Text
forall a. HasCallStack => Maybe a -> a
fromJust Maybe Text
s2) Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ Int
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) ()
forall (m :: * -> *) a. Applicative m => Int -> m a -> m ()
replicateM_ 2 StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC else do
            Maybe Char
s1 <- StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. Tokenizer i (Maybe Char)
itemP
            if Maybe Char -> Bool
forall a. Maybe a -> Bool
isJust Maybe Char
s1 Bool -> Bool -> Bool
&& Maybe Char -> Char
forall a. HasCallStack => Maybe a -> a
fromJust Maybe Char
s1 Char -> String -> Bool
forall (t :: * -> *) a. (Foldable t, Eq a) => a -> t a -> Bool
`elem` String
CR.charOps then Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (Text -> Token i
forall i. Text -> Token i
TKReserved (Text -> Token i) -> Text -> Token i
forall a b. (a -> b) -> a -> b
$ Char -> Text
T.singleton (Char -> Text) -> Char -> Text
forall a b. (a -> b) -> a -> b
$ Maybe Char -> Char
forall a. HasCallStack => Maybe a -> a
fromJust Maybe Char
s1) Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> Tokenizer i (Maybe (Token i))
forall (f :: * -> *) a b. Functor f => a -> f b -> f a
<$ StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. (Enum i, Num i) => Tokenizer i (Maybe Char)
itemC else
                Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return Maybe (Token i)
forall a. Maybe a
Nothing

keyWordOrIdent :: (Enum i, Num i, Show i) => Tokenizer i (Maybe (Token i))
keyWordOrIdent :: Tokenizer i (Maybe (Token i))
keyWordOrIdent = do
    (lc :: TokenLCNums i
lc, txt :: Text
txt) <- StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (TokenLCNums i, Text)
forall s (m :: * -> *). MonadState s m => m s
get
    (Char -> Bool) -> Tokenizer i Text
forall i. (Enum i, Num i) => (Char -> Bool) -> Tokenizer i Text
string Char -> Bool
CR.isValidChar Tokenizer i Text
-> (Text -> Tokenizer i (Maybe (Token i)))
-> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= \s :: Text
s -> if Text -> Bool
T.null Text
s then Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) (Maybe (Token i))
 -> Tokenizer i (Maybe (Token i)))
-> Either (ASTError i) (Maybe (Token i))
-> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ ASTError i -> Either (ASTError i) (Maybe (Token i))
forall a b. a -> Either a b
Left ("stray token in program", (TokenLCNums i
lc, Text -> Token i
forall i. Text -> Token i
TKReserved (Text -> Token i) -> Text -> Token i
forall a b. (a -> b) -> a -> b
$ (Char -> Bool) -> Text -> Text
T.takeWhile (Bool -> Bool
not (Bool -> Bool) -> (Char -> Bool) -> Char -> Bool
forall b c a. (b -> c) -> (a -> b) -> a -> c
. Char -> Bool
CR.isValidChar) Text
txt)) else
        Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall (m :: * -> *) a. Monad m => a -> m a
return (Maybe (Token i) -> Tokenizer i (Maybe (Token i)))
-> Maybe (Token i) -> Tokenizer i (Maybe (Token i))
forall a b. (a -> b) -> a -> b
$ Text -> Maybe (Token i)
forall i. Show i => Text -> Maybe (Token i)
lookupKeyword Text
s Maybe (Token i) -> Maybe (Token i) -> Maybe (Token i)
forall (f :: * -> *) a. Alternative f => f a -> f a -> f a
<|> Token i -> Maybe (Token i)
forall a. a -> Maybe a
Just (Text -> Token i
forall i. Text -> Token i
TKIdent Text
s)

-- | The core function of `Htcc.Tokenizer.tokenize`
tokenize' :: (Enum i, Num i, Eq i, Read i, Show i) => T.Text -> Either (ASTError i) [TokenLC i]
tokenize' :: Text -> Either (ASTError i) [TokenLC i]
tokenize' = StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> (TokenLCNums i, Text) -> Either (ASTError i) [TokenLC i]
forall (m :: * -> *) s a. Monad m => StateT s m a -> s -> m a
evalStateT StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
runTokenizer' ((TokenLCNums i, Text) -> Either (ASTError i) [TokenLC i])
-> (Text -> (TokenLCNums i, Text))
-> Text
-> Either (ASTError i) [TokenLC i]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (i -> i -> TokenLCNums i
forall i. i -> i -> TokenLCNums i
TokenLCNums 1 1,)
    where
        next :: StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
next = StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (TokenLCNums i, Text)
forall s (m :: * -> *). MonadState s m => m s
get StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (TokenLCNums i, Text)
-> ((TokenLCNums i, Text)
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= Either (ASTError i) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> ((TokenLCNums i, Text) -> Either (ASTError i) [TokenLC i])
-> (TokenLCNums i, Text)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> (TokenLCNums i, Text) -> Either (ASTError i) [TokenLC i]
forall (m :: * -> *) s a. Monad m => StateT s m a -> s -> m a
evalStateT StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
runTokenizer'
        runTokenizer' :: StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
runTokenizer' = (StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
 -> (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
     -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> [StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (t :: * -> *) a b.
Foldable t =>
(a -> b -> b) -> b -> t a -> b
foldr ((StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall b c a. (b -> c) -> (a -> b) -> a -> c
(.) ((StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
  -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
 -> (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
     -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> (StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
`ifM` StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
next)) StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall a. a -> a
id [StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall i. (Enum i, Num i) => Tokenizer i Bool
consumeSpace, StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall i. (Enum i, Num i) => Tokenizer i Bool
consumeNewLine, StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall i. (Enum i, Num i) => Tokenizer i Bool
consumeComment] (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall a b. (a -> b) -> a -> b
$ do
            TokenLCNums i
cur <- Tokenizer i (TokenLCNums i)
forall i. Tokenizer i (TokenLCNums i)
curLC
            StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall bool (m :: * -> *) a.
(ToBool bool, Monad m) =>
m bool -> m a -> m a -> m a
ifM (Maybe Char -> Bool
forall a. Maybe a -> Bool
isNothing (Maybe Char -> Bool)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) Bool
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> StateT (TokenLCNums i, Text) (Either (ASTError i)) (Maybe Char)
forall i. Tokenizer i (Maybe Char)
itemP) (Either (ASTError i) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> Either (ASTError i) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall a b. (a -> b) -> a -> b
$ [TokenLC i] -> Either (ASTError i) [TokenLC i]
forall a b. b -> Either a b
Right []) (StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall a b. (a -> b) -> a -> b
$
                (StateT
   (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
 -> StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i)))
-> [StateT
      (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))]
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall (m :: * -> *) a b.
Monad m =>
(a -> m (Maybe b)) -> [a] -> m (Maybe b)
firstJustM StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
-> StateT
     (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall a. a -> a
id [StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall i. (Enum i, Num i) => Tokenizer i (Maybe (Token i))
macro, StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall i.
(Enum i, Num i, Eq i, Read i) =>
Tokenizer i (Maybe (Token i))
natLit, StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall i. (Enum i, Num i) => Tokenizer i (Maybe (Token i))
strLit, StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall i. (Enum i, Num i, Eq i) => Tokenizer i (Maybe (Token i))
charLit, StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall i. (Enum i, Num i) => Tokenizer i (Maybe (Token i))
operators, StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
forall i. (Enum i, Num i, Show i) => Tokenizer i (Maybe (Token i))
keyWordOrIdent] StateT
  (TokenLCNums i, Text) (Either (ASTError i)) (Maybe (Token i))
-> (Maybe (Token i)
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> (Token i
    -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> Maybe (Token i)
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall b a. b -> (a -> b) -> Maybe a -> b
maybe (Either (ASTError i) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (t :: (* -> *) -> * -> *) (m :: * -> *) a.
(MonadTrans t, Monad m) =>
m a -> t m a
lift (Either (ASTError i) [TokenLC i]
 -> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i])
-> Either (ASTError i) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall a b. (a -> b) -> a -> b
$ [TokenLC i] -> Either (ASTError i) [TokenLC i]
forall a b. b -> Either a b
Right []) (\tk :: Token i
tk -> ((TokenLCNums i
cur, Token i
tk)TokenLC i -> [TokenLC i] -> [TokenLC i]
forall a. a -> [a] -> [a]
:) ([TokenLC i] -> [TokenLC i])
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
-> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
<$> StateT (TokenLCNums i, Text) (Either (ASTError i)) [TokenLC i]
next)