marzeq/qk revision: d678a2a

package tokeniser

import "github.com/marzeq/qk/shared"

type Token struct {
	Type  TokenKind
	Value string
	Loc   shared.Location
}
type TokenKind uint

const (
	TokenEof TokenKind = iota
	TokenNewline

	TokenKeyword
	TokenIdentifier
	TokenNumber
	TokenString
	TokenChar

	TokenOpenParen
	TokenCloseParen
	TokenOpenCurly
	TokenCloseCurly
	TokenOpenSquare
	TokenCloseSquare

	TokenEquals
	TokenEqualsEquals
	TokenNotEquals
	TokenLess
	TokenGreater
	TokenLessEquals
	TokenGreaterEquals

	TokenPlus
	TokenMinus
	TokenAsterisk
	TokenSlash
	TokenPercent
	TokenAmpersand

	TokenIncBy
	TokenDecBy
	TokenMulBy
	TokenDivBy
	TokenModBy

	TokenSemicolon
	TokenComma
	TokenExclam
	TokenColon
	TokenDot
	Token3Dots
	TokenArrow
)

type KeywordKind string

const (
	KeywordLet      KeywordKind = "let"
	KeywordMut      KeywordKind = "mut"
	KeywordExtern   KeywordKind = "extern"
	KeywordForeign  KeywordKind = "foreign"
	KeywordStruct   KeywordKind = "struct"
	KeywordType     KeywordKind = "type"
	KeywordIf       KeywordKind = "if"
	KeywordElse     KeywordKind = "else"
	KeywordGiven    KeywordKind = "given"
	KeywordFor      KeywordKind = "for"
	KeywordBreak    KeywordKind = "break"
	KeywordContinue KeywordKind = "continue"
	KeywordReturn   KeywordKind = "return"
	KeywordImport   KeywordKind = "import"
	KeywordModule   KeywordKind = "module"
	KeywordPub      KeywordKind = "pub"
	KeywordAnd      KeywordKind = "and"
	KeywordOr       KeywordKind = "or"
	KeywordNot      KeywordKind = "not"
	KeywordTrue     KeywordKind = "true"
	KeywordFalse    KeywordKind = "false"
	KeywordNil      KeywordKind = "nil"
	KeywordAs       KeywordKind = "as"
	KeywordSizeof   KeywordKind = "sizeof"
	KeywordLen      KeywordKind = "len"
)

func (t Token) String() string {
	switch t.Type {
	case TokenEof:
		return "<eof>"
	case TokenNewline:
		return "\\n"
	case TokenKeyword:
		return "kw(" + t.Value + ")"
	case TokenIdentifier:
		return "ident(" + t.Value + ")"
	case TokenNumber:
		return "num(" + t.Value + ")"
	case TokenString:
		return "str(" + t.Value + ")"
	case TokenChar:
		return "ch(" + t.Value + ")"
	case TokenOpenParen:
		return "("
	case TokenCloseParen:
		return ")"
	case TokenOpenCurly:
		return "{"
	case TokenCloseCurly:
		return "}"
	case TokenOpenSquare:
		return "["
	case TokenCloseSquare:
		return "]"
	case TokenEquals:
		return "="
	case TokenEqualsEquals:
		return "=="
	case TokenNotEquals:
		return "!="
	case TokenLess:
		return "<"
	case TokenGreater:
		return ">"
	case TokenLessEquals:
		return "<="
	case TokenGreaterEquals:
		return ">="
	case TokenPlus:
		return "+"
	case TokenMinus:
		return "-"
	case TokenAsterisk:
		return "*"
	case TokenSlash:
		return "/"
	case TokenPercent:
		return "%"
	case TokenAmpersand:
		return "&"
	case TokenIncBy:
		return "+="
	case TokenDecBy:
		return "-="
	case TokenMulBy:
		return "*="
	case TokenDivBy:
		return "/="
	case TokenModBy:
		return "%="
	case TokenSemicolon:
		return ";"
	case TokenComma:
		return ","
	case TokenExclam:
		return "!"
	case TokenColon:
		return ":"
	case TokenDot:
		return "."
	case Token3Dots:
		return "..."
	case TokenArrow:
		return "->"
	default:
		return "{UNKNOWN}"
	}
}