From 4fb91ad4622e099f798e01873bac914b64ed48f4 Mon Sep 17 00:00:00 2001
From: franck cuny <franck@fcuny.net>
Date: Sat, 11 Jan 2020 14:40:32 +0100
Subject: lexer: support tokens for equal and not equal.

The tokens for equal (`==`) and not equal (`!=`) are composed of two
characters. We introduce a new helper (`peekChar`) that we use when we
encounter the token `=` or `!` to see if this is a token composed of two
characters.

Add some tests to ensure they are parsed correctly.
---
 users/fcuny/exp/monkey/pkg/lexer/lexer.go      | 28 ++++++++++++++++++++++++--
 users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 13 ++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

(limited to 'users')

diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go
index d538cf5..06d526e 100644
--- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go
+++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go
@@ -56,6 +56,16 @@ func (l *Lexer) skipWhitespace() {
 	}
 }
 
+// peekChar returns the character at position (which is the next charatecter),
+// but does not increment `readPosition` and `position`.
+// This is needed to read tokens that are composed of two characters (e.g. `==`).
+func (l *Lexer) peekChar() byte {
+	if l.readPosition >= len(l.input) {
+		return 0
+	}
+	return l.input[l.readPosition]
+}
+
 // NextToken reads the next token from the lexer and returns the current token.
 func (l *Lexer) NextToken() token.Token {
 	var tok token.Token
@@ -64,13 +74,27 @@ func (l *Lexer) NextToken() token.Token {
 
 	switch l.ch {
 	case '=':
-		tok = newToken(token.ASSIGN, l.ch)
+		if l.peekChar() == '=' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.EQ, Literal: literal}
+		} else {
+			tok = newToken(token.ASSIGN, l.ch)
+		}
 	case '+':
 		tok = newToken(token.PLUS, l.ch)
 	case '-':
 		tok = newToken(token.MINUS, l.ch)
 	case '!':
-		tok = newToken(token.BANG, l.ch)
+		if l.peekChar() == '=' {
+			ch := l.ch
+			l.readChar()
+			literal := string(ch) + string(l.ch)
+			tok = token.Token{Type: token.NOT_EQ, Literal: literal}
+		} else {
+			tok = newToken(token.BANG, l.ch)
+		}
 	case '*':
 		tok = newToken(token.ASTERISK, l.ch)
 	case '/':
diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go
index df1b392..fdea1d3 100644
--- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go
+++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go
@@ -22,6 +22,9 @@ if (5 < 10) {
 } else {
   return false;
 }
+
+10 == 10;
+10 != 9;
 `
 
 	tests := []struct {
@@ -96,6 +99,16 @@ if (5 < 10) {
 		{token.FALSE, "false"},
 		{token.SEMICOLON, ";"},
 		{token.RBRACE, "}"},
+
+		{token.INT, "10"},
+		{token.EQ, "=="},
+		{token.INT, "10"},
+		{token.SEMICOLON, ";"},
+
+		{token.INT, "10"},
+		{token.NOT_EQ, "!="},
+		{token.INT, "9"},
+		{token.SEMICOLON, ";"},
 	}
 
 	l := New(input)
-- 
cgit 1.4.1