restructure project

internal/token/token.go

@@ -0,0 +1,108 @@
+package token
+
+type TokenType string
+
+type Token struct {
+	Type    TokenType
+	Literal string
+}
+
+const (
+	ILLEGAL = "ILLEGAL"
+	EOF     = "EOF"
+
+	// COMMENT a line comment, e.g: # this is a comment
+	COMMENT = "COMMENT"
+
+	// Identifiers + literals
+	IDENT  = "IDENT" // add, foobar, x, y
+	INT    = "INT"   // 123456
+	STRING = "STRING"
+
+	// Operators
+	BIND     = ":="
+	ASSIGN   = "="
+	PLUS     = "+"
+	MINUS    = "-"
+	MULTIPLY = "*"
+	DIVIDE   = "/"
+	MODULO   = "%"
+
+	// Bitwise / Logical operators
+
+	// BITWISE_AND AND
+	BITWISE_AND = "&"
+	// BITWISE_OR OR
+	BITWISE_OR = "|"
+	// BITWISE_XOR XOR
+	BITWISE_XOR = "^"
+	// BITWISE_NOT NOT
+	BITWISE_NOT = "~"
+	// LeftShift
+	LEFT_SHIFT = "<<"
+	// RightShift
+	RIGHT_SHIFT = ">>"
+
+	//
+	// Comparision operators
+	//
+
+	// NOT the not operator
+	NOT = "!"
+	// AND the and operator
+	AND = "&&"
+	// OR the or operator
+	OR = "||"
+
+	LT  = "<"
+	LTE = "<="
+	GT  = ">"
+	GTE = ">="
+
+	EQ     = "=="
+	NOT_EQ = "!="
+
+	// Delimiters
+	COMMA     = ","
+	SEMICOLON = ";"
+	COLON     = ":"
+	DOT       = "."
+
+	LPAREN   = "("
+	RPAREN   = ")"
+	LBRACE   = "{"
+	RBRACE   = "}"
+	LBRACKET = "["
+	RBRACKET = "]"
+
+	// Keywords
+	FUNCTION = "FUNCTION"
+	TRUE     = "TRUE"
+	FALSE    = "FALSE"
+	NULL     = "NULL"
+	IF       = "IF"
+	ELSE     = "ELSE"
+	RETURN   = "RETURN"
+	WHILE    = "WHILE"
+	// IMPORT the `import` keyword (import)
+	IMPORT = "IMPORT"
+)
+
+var keywords = map[string]TokenType{
+	"fn":     FUNCTION,
+	"true":   TRUE,
+	"false":  FALSE,
+	"if":     IF,
+	"null":   NULL,
+	"else":   ELSE,
+	"return": RETURN,
+	"while":  WHILE,
+	"import": IMPORT,
+}
+
+func LookupIdent(ident string) TokenType {
+	if tok, ok := keywords[ident]; ok {
+		return tok
+	}
+	return IDENT
+}