package lexer import ( "encoding/hex" "monkey/token" "strings" ) type Lexer struct { input string position int // current position in input (point to current char) readPosition int // current reading position in input (after current char) ch byte // current char under exclamation prevCh byte // previous char read } func New(input string) *Lexer { l := &Lexer{input: input} l.readChar() return l } func (l *Lexer) readChar() { l.prevCh = l.ch if l.readPosition >= len(l.input) { l.ch = 0 } else { l.ch = l.input[l.readPosition] } l.position = l.readPosition l.readPosition += 1 } func (l *Lexer) peekChar() byte { if l.readPosition >= len(l.input) { return 0 } else { return l.input[l.readPosition] } } func (l *Lexer) NextToken() token.Token { var tok token.Token l.skipWhitespace() switch l.ch { case '#': tok.Type = token.COMMENT tok.Literal = l.readLine() case '=': if l.peekChar() == '=' { ch := l.ch l.readChar() literal := string(ch) + string(l.ch) tok = token.Token{ Type: token.EQ, Literal: literal, } } else { tok = newToken(token.ASSIGN, l.ch) } case '+': tok = newToken(token.PLUS, l.ch) case '-': tok = newToken(token.MINUS, l.ch) case '!': if l.peekChar() == '=' { ch := l.ch l.readChar() literal := string(ch) + string(l.ch) tok = token.Token{ Type: token.NOT_EQ, Literal: literal, } } else { tok = newToken(token.BANG, l.ch) } case '/': if l.peekChar() == '/' { l.readChar() tok.Type = token.COMMENT tok.Literal = l.readLine() } else { tok = newToken(token.SLASH, l.ch) } case '*': tok = newToken(token.ASTERISK, l.ch) case '<': if l.peekChar() == '=' { l.readChar() tok = newToken(token.LTE, l.ch) tok.Literal = "<=" } else { tok = newToken(token.LT, l.ch) } case '>': if l.peekChar() == '=' { l.readChar() tok = newToken(token.GTE, l.ch) tok.Literal = ">=" } else { tok = newToken(token.GT, l.ch) } case ';': tok = newToken(token.SEMICOLON, l.ch) case ',': tok = newToken(token.COMMA, l.ch) case '(': tok = newToken(token.LPAREN, l.ch) case ')': tok = newToken(token.RPAREN, l.ch) case '{': tok = newToken(token.LBRACE, l.ch) case '}': tok = newToken(token.RBRACE, l.ch) case 0: tok.Literal = "" tok.Type = token.EOF case '"': tok.Type = token.STRING str, err := l.readString() if err != nil { tok = newToken(token.ILLEGAL, l.prevCh) } else { tok.Type = token.STRING tok.Literal = str } case '[': tok = newToken(token.LBRACKET, l.ch) case ']': tok = newToken(token.RBRACKET, l.ch) case ':': tok = newToken(token.COLON, l.ch) default: if isLetter(l.ch) { tok.Literal = l.readIdentifier() tok.Type = token.LookupIdent(tok.Literal) return tok } else if isDigit(l.ch) { tok.Type = token.INT tok.Literal = l.readNumber() return tok } else { tok = newToken(token.ILLEGAL, l.ch) } } l.readChar() return tok } func newToken(tokenType token.TokenType, ch byte) token.Token { return token.Token{ Type: tokenType, Literal: string(ch), } } func (l *Lexer) readIdentifier() string { position := l.position for isLetter(l.ch) { l.readChar() } return l.input[position:l.position] } func (l *Lexer) readNumber() string { position := l.position for isDigit(l.ch) { l.readChar() } return l.input[position:l.position] } func isLetter(ch byte) bool { return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' } func isDigit(ch byte) bool { return '0' <= ch && ch <= '9' } func (l *Lexer) skipWhitespace() { for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' { l.readChar() } } func (l *Lexer) readString() (string, error) { b := &strings.Builder{} for { l.readChar() if l.ch == '\\' { switch l.peekChar() { case '"': b.WriteByte('"') case 'n': b.WriteString("\n") case 'r': b.WriteString("\r") case 't': b.WriteString("\t") case '\\': b.WriteString("\\") case 'x': // Skip over the '\\', 'x' and the next two bytes (hex) l.readChar() l.readChar() l.readChar() src := string([]byte{l.prevCh, l.ch}) dst, err := hex.DecodeString(src) if err != nil { return "", err } b.Write(dst) continue } // Skip over the '\\' and the matched single escape char l.readChar() continue } else { if l.ch == '"' || l.ch == 0 { break } } b.WriteByte(l.ch) } return b.String(), nil } func (l *Lexer) readLine() string { position := l.position + 1 for { l.readChar() if l.ch == '\r' || l.ch == '\n' || l.ch == 0 { break } } return l.input[position:l.position] }