package tokeniser
import (
"fmt"
"os"
"github.com/marzeq/quokka/shared"
)
type Tokeniser struct {
pos int
line int
col int
text []rune
fileOrigin string
tokens []Token
}
func NewTokeniserFromFile(path string) (*Tokeniser, error) {
text, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to open file '%s'", path)
}
return &Tokeniser{
pos: 0,
line: 1,
col: 1,
fileOrigin: path,
text: []rune(string(text)),
}, nil
}
func (t *Tokeniser) Peek() rune {
if t.pos >= len(t.text) || t.pos < 0 {
return 0
}
return t.text[t.pos]
}
func (t *Tokeniser) Next() rune {
pos := t.pos + 1
if pos >= len(t.text) || pos < 0 {
return 0
}
return t.text[pos]
}
func (t *Tokeniser) Inc() *Tokeniser {
if t.Next() == '\n' {
t.line++
t.col = 1
} else if t.Peek() != '\r' {
t.col++
}
t.pos++
return t
}
func (t *Tokeniser) Consume() rune {
c := t.Peek()
t.Inc()
return c
}
func IsAlpha(c rune) bool {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}
func IsNum(c rune) bool {
return (c >= '0' && c <= '9')
}
func IsLegalWordStart(c rune) bool {
return IsAlpha(c) || c == '_'
}
func IsLegalWordChar(c rune) bool {
return IsLegalWordStart(c) || IsNum(c)
}
func IsSpace(c rune) bool {
return c == ' ' || c == '\t' || c == '\r'
}
func (t *Tokeniser) ReadWord() string {
s := ""
for IsLegalWordChar(t.Peek()) {
s += string(t.Consume())
}
return s
}
func (t *Tokeniser) ReadNumber() (string, error) {
s := ""
if t.Peek() == '-' {
t.Inc()
s = "-"
}
for !IsSpace(t.Peek()) && t.Peek() != '\n' {
if IsAlpha(t.Peek()) {
ch := t.Consume()
return "", shared.NewError(t.GetLoc(), "invalid char in number literal: %c", ch)
}
if !IsNum(t.Peek()) {
break
}
s += string(t.Consume())
}
return s, nil
}
func (t *Tokeniser) HanldeEscape() (string, error) {
c := t.Consume()
if c != '\\' {
return string(c), nil
} else {
switch t.Consume() {
case '\\':
return "\\", nil
case '"':
return "\"", nil
case 'n':
return "\n", nil
case 'r':
return "\r", nil
case 't':
return "\t", nil
case 'b':
return "\b", nil
case 'f':
return "\f", nil
case 'v':
return "\v", nil
case 'a':
return "\a", nil
case '0':
return string(rune(0)), nil
default:
return "", shared.NewError(t.GetLoc(), "invalid escape sequence")
}
}
}
func (t *Tokeniser) ReadString() (string, error) {
s := ""
if t.Peek() != '"' {
return "", shared.NewError(t.GetLoc(), "expected '\"' to start a string")
}
t.Inc()
for t.Peek() != '"' && t.Peek() != '\n' {
ch, err := t.HanldeEscape()
if err != nil {
return "", err
}
s += ch
}
if t.Peek() != '"' {
return "", shared.NewError(t.GetLoc(), "expected '\"' to end a string")
}
t.Inc()
return s, nil
}
func (t *Tokeniser) IgnoreComment() {
t.Inc().Inc()
for t.Peek() != '\n' {
t.Inc()
}
}
func (t *Tokeniser) GetLoc() shared.Location {
return shared.Location{
FilePath: t.fileOrigin,
LC: shared.LineCol{
Line: t.line,
Col: t.col,
},
}
}
func (t *Tokeniser) AddToken(ttype TokenType, loc shared.Location, _value ...string) {
value := ""
if len(_value) > 0 {
value = _value[0]
}
t.tokens = append(t.tokens, Token{Type: ttype, Value: value, Loc: loc})
}
var keywords = map[string]struct{}{
string(KEYWORD_LET): {},
string(KEYWORD_VAR): {},
string(KEYWORD_EXTERN): {},
string(KEYWORD_STRUCT): {},
string(KEYWORD_IF): {},
string(KEYWORD_ELSE): {},
string(KEYWORD_GIVEN): {},
string(KEYWORD_FOR): {},
string(KEYWORD_BREAK): {},
string(KEYWORD_CONTINUE): {},
string(KEYWORD_RETURN): {},
string(KEYWORD_IMPORT): {},
string(KEYWORD_MODULE): {},
string(KEYWORD_PUB): {},
string(KEYWORD_AND): {},
string(KEYWORD_OR): {},
string(KEYWORD_NOT): {},
string(KEYWORD_TRUE): {},
string(KEYWORD_FALSE): {},
string(KEYWORD_NIL): {},
string(KEYWORD_AS): {},
string(KEYWORD_SIZEOF): {},
string(shared.PRIMITIVE_VOID): {},
string(shared.PRIMITIVE_BOOL): {},
string(shared.PRIMITIVE_CHAR): {},
string(shared.PRIMITIVE_U8): {},
string(shared.PRIMITIVE_U16): {},
string(shared.PRIMITIVE_U32): {},
string(shared.PRIMITIVE_U64): {},
string(shared.PRIMITIVE_I8): {},
string(shared.PRIMITIVE_I16): {},
string(shared.PRIMITIVE_I32): {},
string(shared.PRIMITIVE_I64): {},
}
func IsKeyword(w string) bool {
_, ok := keywords[w]
return ok
}
func (t *Tokeniser) Tokenise() ([]Token, error) {
for {
c := t.Peek()
if c == 0 {
t.AddToken(TOKEN_TYPE_EOF, t.GetLoc())
return t.tokens, nil
}
if IsSpace(c) {
t.Inc()
continue
}
if IsLegalWordStart(c) {
pos := t.GetLoc()
w := t.ReadWord()
if IsKeyword(w) {
t.AddToken(TOKEN_TYPE_KEYWORD, pos, w)
} else {
t.AddToken(TOKEN_TYPE_IDENT, pos, w)
}
continue
}
if IsNum(c) {
pos := t.GetLoc()
n, err := t.ReadNumber()
if err != nil {
return nil, err
}
t.AddToken(TOKEN_TYPE_NUMBER, pos, n)
continue
}
switch c {
case '"':
pos := t.GetLoc()
s, err := t.ReadString()
if err != nil {
return nil, err
}
t.AddToken(TOKEN_TYPE_STRING, pos, s)
continue
case '\\':
if t.Next() == '\n' {
t.Inc().Inc()
}
continue
case '\n':
t.AddToken(TOKEN_TYPE_NEWLINE, t.GetLoc())
t.Inc()
continue
case '(':
t.AddToken(TOKEN_TYPE_OPEN_PAREN, t.GetLoc())
t.Inc()
continue
case ')':
t.AddToken(TOKEN_TYPE_CLOSE_PAREN, t.GetLoc())
t.Inc()
continue
case '{':
t.AddToken(TOKEN_TYPE_OPEN_CURLY, t.GetLoc())
t.Inc()
continue
case '}':
t.AddToken(TOKEN_TYPE_CLOSE_CURLY, t.GetLoc())
t.Inc()
continue
case '=':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_EQUALS_EQUALS, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_EQUALS, t.GetLoc())
t.Inc()
}
continue
case '!':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_NOT_EQUALS, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_EXCLAM, t.GetLoc())
t.Inc()
}
continue
case '<':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_LESS_EQUALS, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_LESS, t.GetLoc())
t.Inc()
}
continue
case '>':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_GREATER_EQUALS, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_GREATER, t.GetLoc())
t.Inc()
}
continue
case '+':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_INC_BY, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_PLUS, t.GetLoc())
t.Inc()
}
continue
case '-':
if t.Next() == '-' {
t.IgnoreComment()
} else if IsNum(t.Next()) {
pos := t.GetLoc()
n, err := t.ReadNumber()
if err != nil {
return nil, err
}
t.AddToken(TOKEN_TYPE_NUMBER, pos, n)
} else if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_DEC_BY, t.GetLoc())
t.Inc().Inc()
} else if t.Next() == '>' {
t.AddToken(TOKEN_TYPE_ARROW, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_MINUS, t.GetLoc())
t.Inc()
}
continue
case '*':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_MUL_BY, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_ASTERISK, t.GetLoc())
t.Inc()
}
continue
case '/':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_DIV_BY, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_SLASH, t.GetLoc())
t.Inc()
}
continue
case '%':
if t.Next() == '=' {
t.AddToken(TOKEN_TYPE_MOD_BY, t.GetLoc())
t.Inc().Inc()
} else {
t.AddToken(TOKEN_TYPE_PERCENT, t.GetLoc())
t.Inc()
}
continue
case '&':
t.AddToken(TOKEN_TYPE_AMPERSAND, t.GetLoc())
t.Inc()
continue
case ';':
t.AddToken(TOKEN_TYPE_SEMICOLON, t.GetLoc())
t.Inc()
continue
case ',':
t.AddToken(TOKEN_TYPE_COMMA, t.GetLoc())
t.Inc()
continue
case ':':
t.AddToken(TOKEN_TYPE_COLON, t.GetLoc())
t.Inc()
continue
case '.':
loc := t.GetLoc()
if t.Next() == '.' {
t.Inc()
nextloc := t.GetLoc()
if t.Next() == '.' {
t.Inc().Inc()
t.AddToken(TOKEN_TYPE_3DOTS, loc)
continue
}
t.AddToken(TOKEN_TYPE_DOT, loc)
t.AddToken(TOKEN_TYPE_DOT, nextloc)
continue
}
t.AddToken(TOKEN_TYPE_DOT, t.GetLoc())
t.Inc()
continue
case '\'':
loc := t.GetLoc()
t.Inc()
ch, err := t.HanldeEscape()
if err != nil {
return nil, err
}
if ch == "\n" {
return nil, shared.NewError(loc, "Unexpected newline in char literal")
}
if t.Consume() != '\'' {
return nil, shared.NewError(loc, "Expected ' to end char literal")
}
t.AddToken(TOKEN_TYPE_CHAR, loc, ch)
continue
}
return nil, shared.NewError(t.GetLoc(), "unexpected char: %c", c)
}
}