Writing An Interpreter in Go - Thorsten Ball - Chapter 1 Lexer

// token/token.go

package token

// This to distinguish between integer and symbol and other things
// Using string might not be the most efficient but for this book it is GE

// Type token type
type Type string

// Token main representation of the token
type Token struct {
 Type    Type
 Literal string
}

// We have limited number of tokens in Monkeylang

const (
 ILLEGAL = "ILLEGAL" // This means the token and character we don't know
 EOF     = "EOF"     // EOF tells our parser it can stop

 // Identifiers + literals

 IDENT = "IDENT" // add, foobar, x, y, ...
 INT   = "INT"   // 123456789

 // Operators

 ASSIGN   = "="
 PLUS     = "+"
 MINUS    = "-"
 BANG     = "!"
 ASTERISK = "*"
 SLASH    = "/"
 LT       = "<"
 GT       = ">"

 // Delimiters

 COMMA     = ","
 SEMICOLON = ";"

 LPAREN = "("
 RPAREN = ")"
 LBRACE = "{"
 RBRACE = "}"

 // Keywords

 FUNCTION = "FUNCTION"
 LET      = "LET"
 TRUE     = "TRUE"
 FALSE    = "FALSE"
 IF       = "IF"
 ELSE     = "ELSE"
 RETURN   = "RETURN"

 EQ    = "=="
 NOTEQ = "!="
)

// We have a map of all the keywords in our language
var keywords = map[string]Type{
 "fn":     FUNCTION,
 "let":    LET,
 "true":   TRUE,
 "false":  FALSE,
 "if":     IF,
 "else":   ELSE,
 "return": RETURN,
}

// LookUpIdent We look up an identifier in the keywords if it does not exist
// it is an identifier (variable name)
func LookUpIdent(ident string) Type {
 if tok, ok := keywords[ident]; ok {
  return tok
 }
 return IDENT
}

package lexer

import (
 "monkey/token"
)

// Lexer Holds the source code and pointers to operate
// The sliding window of operation
// current pointer next pointer and current value
type Lexer struct {
 input        string
 position     int  // current position in input (points to current char)
 readPosition int  // current reading position in input (after current char)
 ch           byte // current char under examination
}

// New Constructor for Lexer
func New(input string) *Lexer {
 l := &Lexer{input: input}
 l.readChar() // calling readchar() in lexer now l.ch is first character in input
 return l
}

// FEL: Future Enhancement to the Language
// Monkey does not support UTF-8 only ASCII to keep it simple
// if so we would have to use runes instead bytes in ch

// readChar gives us the next char and advance our position in the input string
func (l *Lexer) readChar() {
 if l.readPosition >= len(l.input) {
  l.ch = 0
 } else {
  l.ch = l.input[l.readPosition]
 }
 l.position = l.readPosition
 l.readPosition += 1
}

// NextToken when we have identifiers and keywords. Our algorithm is to
// read letters until we hit a non letter
// take that window and decide whether it is
// a keyword or identifier to decide token.TokenType
func (l *Lexer) NextToken() token.Token {
 var tok token.Token

 l.skipWhitespace()

 switch l.ch {

 case '=':
  if l.peekChar() == '=' {
   ch := l.ch                                          // we keep a copy of current '='
   l.readChar()                                        // go move forward
   literal := string(ch) + string(l.ch)                // we concat previous and new l.ch to get ==
   tok = token.Token{Type: token.EQ, Literal: literal} // Finally create the token with correct type and literal
  } else {
   tok = newToken(token.ASSIGN, l.ch) // Lesson: assumptions kill
  }
 case '!':
  if l.peekChar() == '=' {
   ch := l.ch
   l.readChar()
   literal := string(ch) + string(l.ch)
   tok = token.Token{Type: token.NOTEQ, Literal: literal}
  } else {
   tok = newToken(token.BANG, l.ch)
  }
 case ';':
  tok = newToken(token.SEMICOLON, l.ch)
 case '(':
  tok = newToken(token.LPAREN, l.ch)
 case ')':
  tok = newToken(token.RPAREN, l.ch)
 case ',':
  tok = newToken(token.COMMA, l.ch)
 case '+':
  tok = newToken(token.PLUS, l.ch)
 case '{':
  tok = newToken(token.LBRACE, l.ch)
 case '}':
  tok = newToken(token.RBRACE, l.ch)
 case '-':
  tok = newToken(token.MINUS, l.ch)
 case '/':
  tok = newToken(token.SLASH, l.ch)
 case '*':
  tok = newToken(token.ASTERISK, l.ch)
 case '<':
  tok = newToken(token.LT, l.ch)
 case '>':
  tok = newToken(token.GT, l.ch)
 case 0:
  tok.Literal = ""
  tok.Type = token.EOF
 default:
  if isLetter(l.ch) { // identifier loop: we hit the first letter
   tok.Literal = l.readIdentifier()          // we hand off to readIdentifier
   tok.Type = token.LookUpIdent(tok.Literal) // given the literal we look up if it is a keyword
   return tok
  } else if isDigit(l.ch) {
   tok.Type = token.INT         // what about float,hex.octal
   tok.Literal = l.readNumber() // Monkey does not support those for ease of development
   return tok
  }

  tok = newToken(token.ILLEGAL, l.ch)

  // ISU: IDE Suggestions - Redundant else branch
  // OBO: Obsolete as book goes on moved to else branch
  //tok = newToken(token.ILLEGAL, l.ch)
 }

 l.readChar()
 return tok
}

// newToken Helper function
func newToken(tokenType token.Type, ch byte) token.Token {
 return token.Token{Type: tokenType, Literal: string(ch)}
}

// readIdentifier Helper function to pick identifier slices from input
func (l *Lexer) readIdentifier() string {
 start := l.position  // renamed var to start for clarity
 for isLetter(l.ch) { // after handoff, we keep the start of the letter location
  l.readChar() // we continue to iterate until next character is not a letter
 }
 return l.input[start:l.position] // finally we return the slice of start and where letter ended
}

// To check whether the input is a letter by range check with azAz  or '_'.
// This function impact the language a lot since it dictate whether we can
// have something like foo_bar as an identifier.
// since this allows the underscore as a letter.
func isLetter(ch byte) bool {
 return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_'
}

// A lot of lexers have this kind of function which might be named eatWhitespace
func (l *Lexer) skipWhitespace() {
 for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' {
  l.readChar()
 }
}

// similar to isLetter start loop and return the window of digits
func (l *Lexer) readNumber() string {
 position := l.position
 for isDigit(l.ch) {
  l.readChar()
 }
 return l.input[position:l.position]
}

// is a digit range check between 0 and 9
func isDigit(ch byte) bool {
 return '0' <= ch && ch <= '9'
}

// peekChar is only used to peek ahead of the current position just to read it
func (l *Lexer) peekChar() byte {
 if l.readPosition >= len(l.input) {
  return 0
 }
 return l.input[l.readPosition]
}

package lexer

import (
 "monkey/token"
 "testing"
)

func TestNextToken(t *testing.T) {

 input := `let five = 5;
let ten = 10;

let add = fn(x, y) {
  x + y;
};

let result = add(five, ten);
!-/*5;
5 < 10 > 5;

if (5 < 10) {
 return true;
} else {
 return false;
}

10 == 10;
10 != 9;
`

 // some of the above look gibberish because this is a test of lexer not
 // parse. We don't check syntax rules here we test tokenization
 // we try to provoke off-by-one errors, edge cases, at end-of-file
 // newline handling multi digit number parsing and so on.

 tests := []struct {
  expectedType    token.Type
  expectedLiteral string
 }{
  {token.LET, "let"},
  {token.IDENT, "five"},
  {token.ASSIGN, "="},
  {token.INT, "5"},
  {token.SEMICOLON, ";"},
  {token.LET, "let"},
  {token.IDENT, "ten"},
  {token.ASSIGN, "="},
  {token.INT, "10"},
  {token.SEMICOLON, ";"},
  {token.LET, "let"},
  {token.IDENT, "add"},
  {token.ASSIGN, "="},
  {token.FUNCTION, "fn"},
  {token.LPAREN, "("},
  {token.IDENT, "x"},
  {token.COMMA, ","},
  {token.IDENT, "y"},
  {token.RPAREN, ")"},
  {token.LBRACE, "{"},
  {token.IDENT, "x"},
  {token.PLUS, "+"},
  {token.IDENT, "y"},
  {token.SEMICOLON, ";"},
  {token.RBRACE, "}"},
  {token.SEMICOLON, ";"},
  {token.LET, "let"},
  {token.IDENT, "result"},
  {token.ASSIGN, "="},
  {token.IDENT, "add"},
  {token.LPAREN, "("},
  {token.IDENT, "five"},
  {token.COMMA, ","},
  {token.IDENT, "ten"},
  {token.RPAREN, ")"},
  {token.SEMICOLON, ";"},
  {token.BANG, "!"},
  {token.MINUS, "-"},
  {token.SLASH, "/"},
  {token.ASTERISK, "*"},
  {token.INT, "5"},
  {token.SEMICOLON, ";"},
  {token.INT, "5"},
  {token.LT, "<"},
  {token.INT, "10"},
  {token.GT, ">"},
  {token.INT, "5"},
  {token.SEMICOLON, ";"},
  {token.IF, "if"},
  {token.LPAREN, "("},
  {token.INT, "5"},
  {token.LT, "<"},
  {token.INT, "10"},
  {token.RPAREN, ")"},
  {token.LBRACE, "{"},
  {token.RETURN, "return"},
  {token.TRUE, "true"},
  {token.SEMICOLON, ";"},
  {token.RBRACE, "}"},
  {token.ELSE, "else"},
  {token.LBRACE, "{"},
  {token.RETURN, "return"},
  {token.FALSE, "false"},
  {token.SEMICOLON, ";"},
  {token.RBRACE, "}"},
  {token.INT, "10"},
  {token.EQ, "=="},
  {token.INT, "10"},
  {token.SEMICOLON, ";"},
  {token.INT, "10"},
  {token.NOTEQ, "!="},
  {token.INT, "9"},
  {token.SEMICOLON, ";"},
  {token.EOF, ""},
 }

 l := New(input)

 for i, tt := range tests {
  tok := l.NextToken()

  if tok.Type != tt.expectedType {
   t.Fatalf("tests[%d] - token type wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type)
  }

  if tok.Literal != tt.expectedLiteral {
   t.Fatalf("test[%d] - literal wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal)
  }
 }
}

package repl

import (
 "bufio"
 "fmt"
 "io"
 "monkey/lexer"
 "monkey/token"
)

const PROMPT = ">> "

func Start(in io.Reader, out io.Writer) {
 scanner := bufio.NewScanner(in)

 for {
  _, err := fmt.Fprintf(out, PROMPT) // We print the prompt sign
  if err != nil {
   return
  }
  scanned := scanner.Scan() // We advance the scanner to read
  if !scanned {
   return // return if nothing
  }

  line := scanner.Text() // We can access the things there using either Text() or Bytes()

  switch line {
  case ":exit":
   return // this actually breaks the loop not break
  case ":help":
   replHelp()
   continue // continue skips the lexer below and restart the loop
  }

  l := lexer.New(line) // We initiate the lexer with the input

  // Until we meet EOF print the tokens that were processed
  for tok := l.NextToken(); tok.Type != token.EOF; tok = l.NextToken() {
   _, err := fmt.Fprintf(out, "%+v\n", tok)
   if err != nil {
    return
   }
  }
 }
}

func replHelp() {
 fmt.Println("Help \n :help - to get help \n :exit - to exit the program")
}

package main

import (
 "fmt"
 "monkey/repl"
 "os"
 "os/user"
)

func main() {
 // IDE: Variable collides with imported package apparently
 current, err := user.Current()
 if err != nil {
  panic(err)
 }

 fmt.Printf("Hello %s! This is the Monkey Programming Language!\n",
  current.Username)
 fmt.Printf("Feel free to type in commands\n")
 repl.Start(os.Stdin, os.Stdout)
}