From 2522d0dd102f930898a35e3019f50117e4de9c74 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sun, 29 Dec 2019 11:18:50 -0800 Subject: Add README.md, LICENSE.txt --- users/fcuny/exp/monkey/LICENSE.txt | 20 ++++++++++++++++++++ users/fcuny/exp/monkey/README.md | 1 + 2 files changed, 21 insertions(+) create mode 100644 users/fcuny/exp/monkey/LICENSE.txt create mode 100644 users/fcuny/exp/monkey/README.md (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/LICENSE.txt b/users/fcuny/exp/monkey/LICENSE.txt new file mode 100644 index 0000000..b928a6d --- /dev/null +++ b/users/fcuny/exp/monkey/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2019 franck cuny + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/users/fcuny/exp/monkey/README.md b/users/fcuny/exp/monkey/README.md new file mode 100644 index 0000000..1ceb74e --- /dev/null +++ b/users/fcuny/exp/monkey/README.md @@ -0,0 +1 @@ +# monkey \ No newline at end of file -- cgit 1.4.1 From 2b27536980956f07d2ac94b763eca4b2262fecd4 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:31:43 +0100 Subject: go.mod: create the module 'monkey' The project is named monkey, we add a mod file to ensure that the tooling / dependencies are set up correctly when we import various modules in this project. --- users/fcuny/exp/monkey/go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 users/fcuny/exp/monkey/go.mod (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/go.mod b/users/fcuny/exp/monkey/go.mod new file mode 100644 index 0000000..34c713d --- /dev/null +++ b/users/fcuny/exp/monkey/go.mod @@ -0,0 +1,3 @@ +module monkey + +go 1.12 -- cgit 1.4.1 From a4260f907f740bb57246832ee7aca75899bf4aff Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:32:55 +0100 Subject: token: initial tokenizer. This is the initial tokenizer for the monkey language. For now we recognize a limited number of tokens. We only have two keywords at this stage: `fn` and `let`. `fn` is used to create function, while `let` is used for assigning variables. The other tokens are mostly to parse the source code, and recognize things like brackets, parentheses, etc. --- users/fcuny/exp/monkey/pkg/token/token.go | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 users/fcuny/exp/monkey/pkg/token/token.go (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go new file mode 100644 index 0000000..fda4449 --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -0,0 +1,48 @@ +package token + +// TokenType represents the type of the token +type TokenType string + +// Token represents a token, with the type and the literal value of the token +type Token struct { + Type TokenType + Literal string +} + +const ( + ILLEGAL = "ILLEGAL" + EOF = "EOF" + + IDENT = "IDENT" + INT = "INT" + + ASSIGN = "=" + PLUS = "+" + + COMMA = "," + SEMICOLON = ";" + + LPAREN = "(" + RPAREN = ")" + LBRACE = "{" + RBRACE = "}" + + FUNCTION = "FUNCTION" + LET = "LET" +) + +// List of our keywords for the language +var keywords = map[string]TokenType{ + "fn": FUNCTION, + "let": LET, +} + +// LookupIdent returns the token type for a given identifier. If the identifier +// is one of our keyword, we return the corresponding value, otherwise we return +// the given identifier. +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[ident]; ok { + return tok + } + return IDENT +} -- cgit 1.4.1 From d7b15313dc554871cc0973924323f7407722121a Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:34:54 +0100 Subject: lexer: initial lexer The initial lexer for the monkey language. We only support a small subset at this stage. We have some simple tests to ensure that we can parse some small snippet, and that the minimum number of tokens we need are also all supported correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 114 +++++++++++++++++++++++++ users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 104 ++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 users/fcuny/exp/monkey/pkg/lexer/lexer.go create mode 100644 users/fcuny/exp/monkey/pkg/lexer/lexer_test.go (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go new file mode 100644 index 0000000..fc29371 --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -0,0 +1,114 @@ +package lexer + +import "monkey/pkg/token" + +// Lexer represents the lexer +type Lexer struct { + input string + // current position in input + position int + // current reading position in input (after a char) + readPosition int + // current character under examination + ch byte +} + +// New returns a new lexer +func New(input string) *Lexer { + l := &Lexer{input: input} + l.readChar() + return l +} + +// Read the current character and advances our position in the input string. +func (l *Lexer) readChar() { + // if we've reached the end of the input, we set the current character to 0, + // which is the ASCII code for NUL. + if l.readPosition >= len(l.input) { + l.ch = 0 + } else { + l.ch = l.input[l.readPosition] + } + l.position = l.readPosition + l.readPosition++ +} + +func (l *Lexer) readIdentifier() string { + position := l.position + for isLetter(l.ch) { + l.readChar() + } + return l.input[position:l.position] +} + +func (l *Lexer) readNumber() string { + position := l.position + for isDigit(l.ch) { + l.readChar() + } + return l.input[position:l.position] +} + +// we don't care about white space characters, we skip them when we find them. +func (l *Lexer) skipWhitespace() { + for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' { + l.readChar() + } +} + +// NextToken reads the next token from the lexer and returns the current token. +func (l *Lexer) NextToken() token.Token { + var tok token.Token + + l.skipWhitespace() + + switch l.ch { + case '=': + tok = newToken(token.ASSIGN, l.ch) + case '+': + tok = newToken(token.PLUS, l.ch) + case ';': + tok = newToken(token.SEMICOLON, l.ch) + case ',': + tok = newToken(token.COMMA, l.ch) + case '(': + tok = newToken(token.LPAREN, l.ch) + case ')': + tok = newToken(token.RPAREN, l.ch) + case '{': + tok = newToken(token.LBRACE, l.ch) + case '}': + tok = newToken(token.RBRACE, l.ch) + case 0: + tok.Literal = "" + tok.Type = token.EOF + default: + if isLetter(l.ch) { + tok.Literal = l.readIdentifier() + tok.Type = token.LookupIdent(tok.Literal) + return tok + } else if isDigit(l.ch) { + tok.Type = token.INT + tok.Literal = l.readNumber() + return tok + } else { + tok = newToken(token.ILLEGAL, l.ch) + } + + } + + l.readChar() + return tok +} + +func newToken(tokenType token.TokenType, ch byte) token.Token { + return token.Token{Type: tokenType, Literal: string(ch)} +} + +func isLetter(ch byte) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' +} + +func isDigit(ch byte) bool { + return '0' <= ch && ch <= '9' +} diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go new file mode 100644 index 0000000..73b27fb --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -0,0 +1,104 @@ +package lexer + +import ( + "monkey/pkg/token" + "testing" +) + +func TestNextTokenBasic(t *testing.T) { + input := `=+(){},;` + + tests := []struct { + expectedType token.TokenType + expectedLiteral string + }{ + {token.ASSIGN, "="}, + {token.PLUS, "+"}, + {token.LPAREN, "("}, + {token.RPAREN, ")"}, + {token.LBRACE, "{"}, + {token.RBRACE, "}"}, + {token.COMMA, ","}, + {token.SEMICOLON, ";"}, + } + + l := New(input) + + for i, tt := range tests { + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - tokenliteral wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal) + } + } +} + +func TestNextTokenMonkey(t *testing.T) { + input := `let five = 5; +let ten = 10; + +let add = fn(x, y) { + x + y +}; + +let result = add(five, ten);` + + tests := []struct { + expectedType token.TokenType + expectedLiteral string + }{ + {token.LET, "let"}, + {token.IDENT, "five"}, + {token.ASSIGN, "="}, + {token.INT, "5"}, + {token.SEMICOLON, ";"}, + + {token.LET, "let"}, + {token.IDENT, "ten"}, + {token.ASSIGN, "="}, + {token.INT, "10"}, + {token.SEMICOLON, ";"}, + + {token.LET, "let"}, + {token.IDENT, "add"}, + {token.ASSIGN, "="}, + {token.FUNCTION, "fn"}, + {token.LPAREN, "("}, + {token.IDENT, "x"}, + {token.COMMA, ","}, + {token.IDENT, "y"}, + {token.RPAREN, ")"}, + {token.LBRACE, "{"}, + {token.IDENT, "x"}, + {token.PLUS, "+"}, + {token.IDENT, "y"}, + {token.RBRACE, "}"}, + {token.SEMICOLON, ";"}, + + {token.LET, "let"}, + {token.IDENT, "result"}, + {token.ASSIGN, "="}, + {token.IDENT, "add"}, + {token.LPAREN, "("}, + {token.IDENT, "five"}, + {token.COMMA, ","}, + {token.IDENT, "ten"}, + {token.RPAREN, ")"}, + {token.SEMICOLON, ";"}, + } + + l := New(input) + for i, tt := range tests { + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - tokenliteral wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal) + } + } +} -- cgit 1.4.1 From 88621a87ddc263b0560aa927118464d50c15c697 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:47:40 +0100 Subject: token: support more operator tokens Support additional tokens for operators (`-`, `*`, etc). This change only adds the tokens to the list of constants, and group all the tokens related to operators together. --- users/fcuny/exp/monkey/pkg/token/token.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index fda4449..bc9e563 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -16,9 +16,6 @@ const ( IDENT = "IDENT" INT = "INT" - ASSIGN = "=" - PLUS = "+" - COMMA = "," SEMICOLON = ";" @@ -29,6 +26,16 @@ const ( FUNCTION = "FUNCTION" LET = "LET" + + // The following tokens are for operators + ASSIGN = "=" + PLUS = "+" + MINUS = "-" + BANG = "!" + ASTERISK = "*" + SLASH = "/" + LT = "<" + GT = ">" ) // List of our keywords for the language -- cgit 1.4.1 From bcd7ed3ec9f59f350a29eb95ffbac71345d93e6d Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:53:44 +0100 Subject: lexer: support more operator tokens. Support the operator tokens that were added to our tokenizer. This also add a few more tests to ensure we handle them correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 13 +++++++++++++ users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index fc29371..d538cf5 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -67,6 +67,19 @@ func (l *Lexer) NextToken() token.Token { tok = newToken(token.ASSIGN, l.ch) case '+': tok = newToken(token.PLUS, l.ch) + case '-': + tok = newToken(token.MINUS, l.ch) + case '!': + tok = newToken(token.BANG, l.ch) + case '*': + tok = newToken(token.ASTERISK, l.ch) + case '/': + tok = newToken(token.SLASH, l.ch) + case '<': + tok = newToken(token.LT, l.ch) + case '>': + tok = newToken(token.GT, l.ch) + case ';': tok = newToken(token.SEMICOLON, l.ch) case ',': diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index 73b27fb..ba7fa07 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -44,7 +44,10 @@ let add = fn(x, y) { x + y }; -let result = add(five, ten);` +let result = add(five, ten); +!-/*5; +5 < 10 > 5; +` tests := []struct { expectedType token.TokenType @@ -88,6 +91,20 @@ let result = add(five, ten);` {token.IDENT, "ten"}, {token.RPAREN, ")"}, {token.SEMICOLON, ";"}, + + {token.BANG, "!"}, + {token.MINUS, "-"}, + {token.SLASH, "/"}, + {token.ASTERISK, "*"}, + {token.INT, "5"}, + {token.SEMICOLON, ";"}, + + {token.INT, "5"}, + {token.LT, "<"}, + {token.INT, "10"}, + {token.GT, ">"}, + {token.INT, "5"}, + {token.SEMICOLON, ";"}, } l := New(input) -- cgit 1.4.1 From 77c36fc5244cb46fdbf76b3f96d423271da11209 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:58:36 +0100 Subject: Makefile: add a Makefile For now, automate running the tests. --- users/fcuny/exp/monkey/Makefile | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 users/fcuny/exp/monkey/Makefile (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/Makefile b/users/fcuny/exp/monkey/Makefile new file mode 100644 index 0000000..61168f3 --- /dev/null +++ b/users/fcuny/exp/monkey/Makefile @@ -0,0 +1,4 @@ +test: + go test -v ./... + +.phony: test -- cgit 1.4.1 From be6ab89f58b0572d0999701d4f1b454e98dec581 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:01:49 +0100 Subject: lexer: delete redundant test. The test `TestNextTokenBasic` was not testing anything that `TestNextTokenMonkey` was not already testing. Rename `TestNextTokenMonkey` to `TestNextToken` for clarity. --- users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 33 +------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index ba7fa07..22dbfcb 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -5,38 +5,7 @@ import ( "testing" ) -func TestNextTokenBasic(t *testing.T) { - input := `=+(){},;` - - tests := []struct { - expectedType token.TokenType - expectedLiteral string - }{ - {token.ASSIGN, "="}, - {token.PLUS, "+"}, - {token.LPAREN, "("}, - {token.RPAREN, ")"}, - {token.LBRACE, "{"}, - {token.RBRACE, "}"}, - {token.COMMA, ","}, - {token.SEMICOLON, ";"}, - } - - l := New(input) - - for i, tt := range tests { - tok := l.NextToken() - if tok.Type != tt.expectedType { - t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type) - } - - if tok.Literal != tt.expectedLiteral { - t.Fatalf("tests[%d] - tokenliteral wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal) - } - } -} - -func TestNextTokenMonkey(t *testing.T) { +func TestNextToken(t *testing.T) { input := `let five = 5; let ten = 10; -- cgit 1.4.1 From 18594e5caf3dda013e6f6fb9cfe558d7153e383d Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:17:06 +0100 Subject: token: rewrite documentation for `LookupIdent`. --- users/fcuny/exp/monkey/pkg/token/token.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index bc9e563..6937595 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -44,9 +44,10 @@ var keywords = map[string]TokenType{ "let": LET, } -// LookupIdent returns the token type for a given identifier. If the identifier -// is one of our keyword, we return the corresponding value, otherwise we return -// the given identifier. +// LookupIdent returns the token type for a given identifier. +// First we check if the identifier is a keyword. If it is, we return they +// keyword TokenType constant. If it isn't, we return the token.IDENT which is +// the TokenType for all user-defined identifiers. func LookupIdent(ident string) TokenType { if tok, ok := keywords[ident]; ok { return tok -- cgit 1.4.1 From 2c4068a5be7f60e8d0376720f70791d23a8c3ef1 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:26:19 +0100 Subject: token: support more keywords Add support for a few more keywords (`true`, `false`, `if`, `else`, `return`). All keywords are grouped together in the constant declaration. --- users/fcuny/exp/monkey/pkg/token/token.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index 6937595..a211c55 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -24,8 +24,14 @@ const ( LBRACE = "{" RBRACE = "}" + // The following tokens are keywords FUNCTION = "FUNCTION" LET = "LET" + TRUE = "TRUE" + FALSE = "FALSE" + IF = "IF" + ELSE = "ELSE" + RETURN = "RETURN" // The following tokens are for operators ASSIGN = "=" @@ -40,8 +46,13 @@ const ( // List of our keywords for the language var keywords = map[string]TokenType{ - "fn": FUNCTION, - "let": LET, + "fn": FUNCTION, + "let": LET, + "true": TRUE, + "false": FALSE, + "if": IF, + "else": ELSE, + "return": RETURN, } // LookupIdent returns the token type for a given identifier. -- cgit 1.4.1 From a2991b978a309d4c3a9c480aad4e4e657ae82597 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:27:18 +0100 Subject: lexer: test the new keywords are parsed correctly. Ensure that the new keywords added (`if`, `else`, `true`, `false`, `return`) are parsed correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 28 +++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index 22dbfcb..df1b392 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -15,7 +15,13 @@ let add = fn(x, y) { let result = add(five, ten); !-/*5; -5 < 10 > 5; +10 > 5; + +if (5 < 10) { + return true; +} else { + return false; +} ` tests := []struct { @@ -68,12 +74,28 @@ let result = add(five, ten); {token.INT, "5"}, {token.SEMICOLON, ";"}, - {token.INT, "5"}, - {token.LT, "<"}, {token.INT, "10"}, {token.GT, ">"}, {token.INT, "5"}, {token.SEMICOLON, ";"}, + + {token.IF, "if"}, + {token.LPAREN, "("}, + {token.INT, "5"}, + {token.LT, "<"}, + {token.INT, "10"}, + {token.RPAREN, ")"}, + {token.LBRACE, "{"}, + {token.RETURN, "return"}, + {token.TRUE, "true"}, + {token.SEMICOLON, ";"}, + {token.RBRACE, "}"}, + {token.ELSE, "else"}, + {token.LBRACE, "{"}, + {token.RETURN, "return"}, + {token.FALSE, "false"}, + {token.SEMICOLON, ";"}, + {token.RBRACE, "}"}, } l := New(input) -- cgit 1.4.1 From 72ce9e990e8df7ad4ddb591dd0a0b96cf89503f6 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:37:52 +0100 Subject: token: add tokens for equal and not equal. --- users/fcuny/exp/monkey/pkg/token/token.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index a211c55..b2342e7 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -42,6 +42,9 @@ const ( SLASH = "/" LT = "<" GT = ">" + + EQ = "==" + NOT_EQ = "!=" ) // List of our keywords for the language -- cgit 1.4.1 From 4fb91ad4622e099f798e01873bac914b64ed48f4 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:40:32 +0100 Subject: lexer: support tokens for equal and not equal. The tokens for equal (`==`) and not equal (`!=`) are composed of two characters. We introduce a new helper (`peekChar`) that we use when we encounter the token `=` or `!` to see if this is a token composed of two characters. Add some tests to ensure they are parsed correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 28 ++++++++++++++++++++++++-- users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 13 ++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index d538cf5..06d526e 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -56,6 +56,16 @@ func (l *Lexer) skipWhitespace() { } } +// peekChar returns the character at position (which is the next charatecter), +// but does not increment `readPosition` and `position`. +// This is needed to read tokens that are composed of two characters (e.g. `==`). +func (l *Lexer) peekChar() byte { + if l.readPosition >= len(l.input) { + return 0 + } + return l.input[l.readPosition] +} + // NextToken reads the next token from the lexer and returns the current token. func (l *Lexer) NextToken() token.Token { var tok token.Token @@ -64,13 +74,27 @@ func (l *Lexer) NextToken() token.Token { switch l.ch { case '=': - tok = newToken(token.ASSIGN, l.ch) + if l.peekChar() == '=' { + ch := l.ch + l.readChar() + literal := string(ch) + string(l.ch) + tok = token.Token{Type: token.EQ, Literal: literal} + } else { + tok = newToken(token.ASSIGN, l.ch) + } case '+': tok = newToken(token.PLUS, l.ch) case '-': tok = newToken(token.MINUS, l.ch) case '!': - tok = newToken(token.BANG, l.ch) + if l.peekChar() == '=' { + ch := l.ch + l.readChar() + literal := string(ch) + string(l.ch) + tok = token.Token{Type: token.NOT_EQ, Literal: literal} + } else { + tok = newToken(token.BANG, l.ch) + } case '*': tok = newToken(token.ASTERISK, l.ch) case '/': diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index df1b392..fdea1d3 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -22,6 +22,9 @@ if (5 < 10) { } else { return false; } + +10 == 10; +10 != 9; ` tests := []struct { @@ -96,6 +99,16 @@ if (5 < 10) { {token.FALSE, "false"}, {token.SEMICOLON, ";"}, {token.RBRACE, "}"}, + + {token.INT, "10"}, + {token.EQ, "=="}, + {token.INT, "10"}, + {token.SEMICOLON, ";"}, + + {token.INT, "10"}, + {token.NOT_EQ, "!="}, + {token.INT, "9"}, + {token.SEMICOLON, ";"}, } l := New(input) -- cgit 1.4.1 From 7e0195b0a753bd990307587926bfc0fe11927f77 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:50:44 +0100 Subject: repl: support a simple REPL for some early testing The REPL reads the input, send it to the lexer, and prints the token to STDOUT. For now nothing else is done since we still don't parse the tokens. --- users/fcuny/exp/monkey/cmd/repl/main.go | 12 ++++++++++++ users/fcuny/exp/monkey/pkg/repl/repl.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 users/fcuny/exp/monkey/cmd/repl/main.go create mode 100644 users/fcuny/exp/monkey/pkg/repl/repl.go (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/cmd/repl/main.go b/users/fcuny/exp/monkey/cmd/repl/main.go new file mode 100644 index 0000000..46b865c --- /dev/null +++ b/users/fcuny/exp/monkey/cmd/repl/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "fmt" + "monkey/pkg/repl" + "os" +) + +func main() { + fmt.Printf("Welcome to monkey's REPL.") + repl.Start(os.Stdin, os.Stdout) +} diff --git a/users/fcuny/exp/monkey/pkg/repl/repl.go b/users/fcuny/exp/monkey/pkg/repl/repl.go new file mode 100644 index 0000000..e8b3b1f --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/repl/repl.go @@ -0,0 +1,29 @@ +package repl + +import ( + "bufio" + "fmt" + "io" + lexer "monkey/pkg/lexer" + token "monkey/pkg/token" +) + +const PROMPT = ">> " + +func Start(in io.Reader, out io.Writer) { + scanner := bufio.NewScanner(in) + for { + fmt.Printf(PROMPT) + scanned := scanner.Scan() + + if !scanned { + return + } + + line := scanner.Text() + l := lexer.New(line) + for tok := l.NextToken(); tok.Type != token.EOF; tok = l.NextToken() { + fmt.Printf("%+v\n", tok) + } + } +} -- cgit 1.4.1 From 64da030dc88edb71e264ae1abfa95b79ebf2a11b Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Mon, 10 May 2021 19:21:26 -0700 Subject: git: ignore binary for the REPL --- users/fcuny/exp/monkey/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 users/fcuny/exp/monkey/.gitignore (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/.gitignore b/users/fcuny/exp/monkey/.gitignore new file mode 100644 index 0000000..2f226a3 --- /dev/null +++ b/users/fcuny/exp/monkey/.gitignore @@ -0,0 +1 @@ +/cmd/repl/repl -- cgit 1.4.1 From 8437218bdaed90cab7374a752f8aaf128225aa1a Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Mon, 10 May 2021 19:21:39 -0700 Subject: lint: fix a few issues --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 1 + users/fcuny/exp/monkey/pkg/repl/repl.go | 3 ++- users/fcuny/exp/monkey/pkg/token/token.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index 06d526e..3e98cf0 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -1,3 +1,4 @@ +// Package lexer provides a lexer to the monkey language. package lexer import "monkey/pkg/token" diff --git a/users/fcuny/exp/monkey/pkg/repl/repl.go b/users/fcuny/exp/monkey/pkg/repl/repl.go index e8b3b1f..5e7b1d1 100644 --- a/users/fcuny/exp/monkey/pkg/repl/repl.go +++ b/users/fcuny/exp/monkey/pkg/repl/repl.go @@ -1,3 +1,4 @@ +// Package repl provides a REPL to the monkey language. package repl import ( @@ -13,7 +14,7 @@ const PROMPT = ">> " func Start(in io.Reader, out io.Writer) { scanner := bufio.NewScanner(in) for { - fmt.Printf(PROMPT) + fmt.Print(PROMPT) scanned := scanner.Scan() if !scanned { diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index b2342e7..5eadc5e 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -1,3 +1,4 @@ +// Package token provides a tokenizer for the monkey language. package token // TokenType represents the type of the token -- cgit 1.4.1 From 89f0d0d1ad82ff6858d001b758317dd224b76032 Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Mon, 10 May 2021 20:06:06 -0700 Subject: readme: convert to org-mode --- users/fcuny/exp/monkey/README.md | 1 - users/fcuny/exp/monkey/README.org | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 users/fcuny/exp/monkey/README.md create mode 100644 users/fcuny/exp/monkey/README.org (limited to 'users/fcuny/exp') diff --git a/users/fcuny/exp/monkey/README.md b/users/fcuny/exp/monkey/README.md deleted file mode 100644 index 1ceb74e..0000000 --- a/users/fcuny/exp/monkey/README.md +++ /dev/null @@ -1 +0,0 @@ -# monkey \ No newline at end of file diff --git a/users/fcuny/exp/monkey/README.org b/users/fcuny/exp/monkey/README.org new file mode 100644 index 0000000..d968f4c --- /dev/null +++ b/users/fcuny/exp/monkey/README.org @@ -0,0 +1,3 @@ +#+TITLE: monkey + +Implementation of https://interpreterbook.com/ -- cgit 1.4.1