restructure project

internal/lexer/lexer.go

@@ -0,0 +1,287 @@
+package lexer
+
+import (
+	"encoding/hex"
+	"monkey/internal/token"
+	"strings"
+)
+
+type Lexer struct {
+	input        string
+	position     int  // current position in input (point to current char)
+	readPosition int  // current reading position in input (after current char)
+	ch           byte // current char under exclamation
+	prevCh       byte // previous char read
+}
+
+func New(input string) *Lexer {
+	l := &Lexer{input: input}
+	l.readChar()
+	return l
+}
+
+func (l *Lexer) readChar() {
+	l.prevCh = l.ch
+	if l.readPosition >= len(l.input) {
+		l.ch = 0
+	} else {
+		l.ch = l.input[l.readPosition]
+	}
+	l.position = l.readPosition
+	l.readPosition += 1
+}
+
+func (l *Lexer) peekChar() byte {
+	if l.readPosition >= len(l.input) {
+		return 0
+	} else {
+		return l.input[l.readPosition]
+	}
+}
+
+func (l *Lexer) NextToken() token.Token {
+	var tok token.Token
+
+	l.skipWhitespace()
+
+	switch l.ch {
+	case '#':
+		tok.Type = token.COMMENT
+		tok.Literal = l.readLine()
+	case '=':
+		if l.peekChar() == '=' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{
+				Type:    token.EQ,
+				Literal: literal,
+			}
+		} else {
+			tok = newToken(token.ASSIGN, l.ch)
+		}
+	case '+':
+		tok = newToken(token.PLUS, l.ch)
+	case '-':
+		tok = newToken(token.MINUS, l.ch)
+	case '!':
+		if l.peekChar() == '=' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{
+				Type:    token.NOT_EQ,
+				Literal: literal,
+			}
+		} else {
+			tok = newToken(token.NOT, l.ch)
+		}
+
+	case '/':
+		if l.peekChar() == '/' {
+			l.readChar()
+			tok.Type = token.COMMENT
+			tok.Literal = l.readLine()
+		} else {
+			tok = newToken(token.DIVIDE, l.ch)
+		}
+	case '*':
+		tok = newToken(token.MULTIPLY, l.ch)
+	case '%':
+		tok = newToken(token.MODULO, l.ch)
+	case '&':
+		if l.peekChar() == '&' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.AND, Literal: literal}
+		} else {
+			tok = newToken(token.BITWISE_AND, l.ch)
+		}
+	case '|':
+		if l.peekChar() == '|' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.OR, Literal: literal}
+		} else {
+			tok = newToken(token.BITWISE_OR, l.ch)
+		}
+	case '^':
+		tok = newToken(token.BITWISE_XOR, l.ch)
+	case '~':
+		tok = newToken(token.BITWISE_NOT, l.ch)
+	case '<':
+		if l.peekChar() == '=' {
+			l.readChar()
+			tok = newToken(token.LTE, l.ch)
+			tok.Literal = "<="
+		} else if l.peekChar() == '<' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.LEFT_SHIFT, Literal: literal}
+		} else {
+			tok = newToken(token.LT, l.ch)
+		}
+	case '>':
+		if l.peekChar() == '=' {
+			l.readChar()
+			tok = newToken(token.GTE, l.ch)
+			tok.Literal = ">="
+		} else if l.peekChar() == '>' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.RIGHT_SHIFT, Literal: literal}
+		} else {
+			tok = newToken(token.GT, l.ch)
+		}
+	case ';':
+		tok = newToken(token.SEMICOLON, l.ch)
+	case ',':
+		tok = newToken(token.COMMA, l.ch)
+	case '.':
+		tok = newToken(token.DOT, l.ch)
+	case '(':
+		tok = newToken(token.LPAREN, l.ch)
+	case ')':
+		tok = newToken(token.RPAREN, l.ch)
+	case '{':
+		tok = newToken(token.LBRACE, l.ch)
+	case '}':
+		tok = newToken(token.RBRACE, l.ch)
+	case 0:
+		tok.Literal = ""
+		tok.Type = token.EOF
+	case '"':
+		tok.Type = token.STRING
+		str, err := l.readString()
+		if err != nil {
+			tok = newToken(token.ILLEGAL, l.prevCh)
+		} else {
+			tok.Type = token.STRING
+			tok.Literal = str
+		}
+	case '[':
+		tok = newToken(token.LBRACKET, l.ch)
+	case ']':
+		tok = newToken(token.RBRACKET, l.ch)
+	case ':':
+		if l.peekChar() == '=' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.BIND, Literal: literal}
+		} else {
+			tok = newToken(token.COLON, l.ch)
+		}
+	default:
+		if isLetter(l.ch) {
+			tok.Literal = l.readIdentifier()
+			tok.Type = token.LookupIdent(tok.Literal)
+			return tok
+		} else if isDigit(l.ch) {
+			tok.Type = token.INT
+			tok.Literal = l.readNumber()
+			return tok
+		} else {
+			tok = newToken(token.ILLEGAL, l.ch)
+		}
+	}
+
+	l.readChar()
+	return tok
+}
+
+func newToken(tokenType token.TokenType, ch byte) token.Token {
+	return token.Token{
+		Type:    tokenType,
+		Literal: string(ch),
+	}
+}
+
+func (l *Lexer) readIdentifier() string {
+	position := l.position
+	for isLetter(l.ch) {
+		l.readChar()
+	}
+	return l.input[position:l.position]
+}
+
+func (l *Lexer) readNumber() string {
+	position := l.position
+	for isDigit(l.ch) {
+		l.readChar()
+	}
+	return l.input[position:l.position]
+}
+
+func isLetter(ch byte) bool {
+	return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
+}
+
+func isDigit(ch byte) bool {
+	return '0' <= ch && ch <= '9'
+}
+
+func (l *Lexer) skipWhitespace() {
+	for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
+		l.readChar()
+	}
+}
+
+func (l *Lexer) readString() (string, error) {
+	b := &strings.Builder{}
+	for {
+		l.readChar()
+		if l.ch == '\\' {
+			switch l.peekChar() {
+			case '"':
+				b.WriteByte('"')
+			case 'n':
+				b.WriteString("\n")
+			case 'r':
+				b.WriteString("\r")
+			case 't':
+				b.WriteString("\t")
+			case '\\':
+				b.WriteString("\\")
+			case 'x':
+				// Skip over the '\\', 'x' and the next two bytes (hex)
+				l.readChar()
+				l.readChar()
+				l.readChar()
+				src := string([]byte{l.prevCh, l.ch})
+				dst, err := hex.DecodeString(src)
+				if err != nil {
+					return "", err
+				}
+				b.Write(dst)
+				continue
+			}
+
+			// Skip over the '\\' and the matched single escape char
+			l.readChar()
+			continue
+		} else {
+			if l.ch == '"' || l.ch == 0 {
+				break
+			}
+		}
+
+		b.WriteByte(l.ch)
+	}
+	return b.String(), nil
+}
+
+func (l *Lexer) readLine() string {
+	position := l.position + 1
+	for {
+		l.readChar()
+		if l.ch == '\r' || l.ch == '\n' || l.ch == 0 {
+			break
+		}
+	}
+	return l.input[position:l.position]
+}

internal/lexer/lexer_test.go

@@ -0,0 +1,212 @@
+package lexer
+
+import (
+	"monkey/internal/token"
+	"testing"
+)
+
+func TestNextToken(t *testing.T) {
+	input := `#!./monkey-lang
+				five := 5;
+				ten := 10;
+				
+				add := fn(x, y) {
+				  x + y;
+				};
+				
+				# this is a comment
+				result := add(five, ten);
+				!-/*5;
+				5 < 10 > 5;
+				
+				if (5 < 10) {
+					return true;
+				} else {
+					return false;
+				}
+				
+				// this is another comment
+				10 == 10;
+				10 != 9;
+				"foobar"
+				"foo bar"
+				[1, 2];
+				{"foo": "bar"}
+				d.foo
+				&|^~
+				!&&||
+				<<>>
+			`
+
+	tests := []struct {
+		expectedType    token.TokenType
+		expectedLiteral string
+	}{
+		{token.COMMENT, "!./monkey-lang"},
+		{token.IDENT, "five"},
+		{token.BIND, ":="},
+		{token.INT, "5"},
+		{token.SEMICOLON, ";"},
+		{token.IDENT, "ten"},
+		{token.BIND, ":="},
+		{token.INT, "10"},
+		{token.SEMICOLON, ";"},
+		{token.IDENT, "add"},
+		{token.BIND, ":="},
+		{token.FUNCTION, "fn"},
+		{token.LPAREN, "("},
+		{token.IDENT, "x"},
+		{token.COMMA, ","},
+		{token.IDENT, "y"},
+		{token.RPAREN, ")"},
+		{token.LBRACE, "{"},
+		{token.IDENT, "x"},
+		{token.PLUS, "+"},
+		{token.IDENT, "y"},
+		{token.SEMICOLON, ";"},
+		{token.RBRACE, "}"},
+		{token.SEMICOLON, ";"},
+		{token.COMMENT, " this is a comment"},
+		{token.IDENT, "result"},
+		{token.BIND, ":="},
+		{token.IDENT, "add"},
+		{token.LPAREN, "("},
+		{token.IDENT, "five"},
+		{token.COMMA, ","},
+		{token.IDENT, "ten"},
+		{token.RPAREN, ")"},
+		{token.SEMICOLON, ";"},
+
+		{token.NOT, "!"},
+		{token.MINUS, "-"},
+		{token.DIVIDE, "/"},
+		{token.MULTIPLY, "*"},
+		{token.INT, "5"},
+		{token.SEMICOLON, ";"},
+		{token.INT, "5"},
+		{token.LT, "<"},
+		{token.INT, "10"},
+		{token.GT, ">"},
+		{token.INT, "5"},
+		{token.SEMICOLON, ";"},
+
+		{token.IF, "if"},
+		{token.LPAREN, "("},
+		{token.INT, "5"},
+		{token.LT, "<"},
+		{token.INT, "10"},
+		{token.RPAREN, ")"},
+		{token.LBRACE, "{"},
+
+		{token.RETURN, "return"},
+		{token.TRUE, "true"},
+		{token.SEMICOLON, ";"},
+
+		{token.RBRACE, "}"},
+		{token.ELSE, "else"},
+		{token.LBRACE, "{"},
+
+		{token.RETURN, "return"},
+		{token.FALSE, "false"},
+		{token.SEMICOLON, ";"},
+
+		{token.RBRACE, "}"},
+		{token.COMMENT, " this is another comment"},
+		{token.INT, "10"},
+		{token.EQ, "=="},
+		{token.INT, "10"},
+		{token.SEMICOLON, ";"},
+
+		{token.INT, "10"},
+		{token.NOT_EQ, "!="},
+		{token.INT, "9"},
+		{token.SEMICOLON, ";"},
+
+		{token.STRING, "foobar"},
+		{token.STRING, "foo bar"},
+
+		{token.LBRACKET, "["},
+		{token.INT, "1"},
+		{token.COMMA, ","},
+		{token.INT, "2"},
+		{token.RBRACKET, "]"},
+		{token.SEMICOLON, ";"},
+
+		{token.LBRACE, "{"},
+		{token.STRING, "foo"},
+		{token.COLON, ":"},
+		{token.STRING, "bar"},
+		{token.RBRACE, "}"},
+		{token.IDENT, "d"},
+		{token.DOT, "."},
+		{token.IDENT, "foo"},
+		{token.BITWISE_AND, "&"},
+		{token.BITWISE_OR, "|"},
+		{token.BITWISE_XOR, "^"},
+		{token.BITWISE_NOT, "~"},
+		{token.NOT, "!"},
+		{token.AND, "&&"},
+		{token.OR, "||"},
+		{token.LEFT_SHIFT, "<<"},
+		{token.RIGHT_SHIFT, ">>"},
+		{token.EOF, ""},
+	}
+
+	l := New(input)
+
+	for i, tt := range tests {
+		tok := l.NextToken()
+
+		if tok.Type != tt.expectedType {
+			t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q",
+				i, tt.expectedType, tok.Type)
+		}
+
+		if tok.Literal != tt.expectedLiteral {
+			t.Fatalf("tests[%d] - literal wrong. expected=%q, got=%q",
+				i, tt.expectedLiteral, tok.Literal)
+		}
+	}
+}
+
+func TestStringEscapes(t *testing.T) {
+	input := `#!./monkey-lang
+a := "\"foo\""
+b := "\x00\x0a\x7f"
+c := "\r\n\t"
+`
+
+	tests := []struct {
+		expectedType    token.TokenType
+		expectedLiteral string
+	}{
+		{token.COMMENT, "!./monkey-lang"},
+		{token.IDENT, "a"},
+		{token.BIND, ":="},
+		{token.STRING, "\"foo\""},
+		{token.IDENT, "b"},
+		{token.BIND, ":="},
+		{token.STRING, "\x00\n\u007f"},
+		{token.IDENT, "c"},
+		{token.BIND, ":="},
+		{token.STRING, "\r\n\t"},
+		{token.EOF, ""},
+	}
+
+	lexer := New(input)
+
+	for i, test := range tests {
+		token := lexer.NextToken()
+
+		if token.Type != test.expectedType {
+			t.Fatalf("tests[%d] - token type wrong. expected=%q, got=%q",
+				i, test.expectedType, token.Type)
+		}
+
+		if token.Literal != test.expectedLiteral {
+			t.Fatalf("tests[%d] - literal wrong. expected=%q, got=%q",
+				i, test.expectedLiteral, token.Literal)
+		}
+	}
+
+}