From a4260f907f740bb57246832ee7aca75899bf4aff Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:32:55 +0100 Subject: token: initial tokenizer. This is the initial tokenizer for the monkey language. For now we recognize a limited number of tokens. We only have two keywords at this stage: `fn` and `let`. `fn` is used to create function, while `let` is used for assigning variables. The other tokens are mostly to parse the source code, and recognize things like brackets, parentheses, etc. --- users/fcuny/exp/monkey/pkg/token/token.go | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 users/fcuny/exp/monkey/pkg/token/token.go (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go new file mode 100644 index 0000000..fda4449 --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -0,0 +1,48 @@ +package token + +// TokenType represents the type of the token +type TokenType string + +// Token represents a token, with the type and the literal value of the token +type Token struct { + Type TokenType + Literal string +} + +const ( + ILLEGAL = "ILLEGAL" + EOF = "EOF" + + IDENT = "IDENT" + INT = "INT" + + ASSIGN = "=" + PLUS = "+" + + COMMA = "," + SEMICOLON = ";" + + LPAREN = "(" + RPAREN = ")" + LBRACE = "{" + RBRACE = "}" + + FUNCTION = "FUNCTION" + LET = "LET" +) + +// List of our keywords for the language +var keywords = map[string]TokenType{ + "fn": FUNCTION, + "let": LET, +} + +// LookupIdent returns the token type for a given identifier. If the identifier +// is one of our keyword, we return the corresponding value, otherwise we return +// the given identifier. +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[ident]; ok { + return tok + } + return IDENT +} -- cgit 1.4.1 From d7b15313dc554871cc0973924323f7407722121a Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:34:54 +0100 Subject: lexer: initial lexer The initial lexer for the monkey language. We only support a small subset at this stage. We have some simple tests to ensure that we can parse some small snippet, and that the minimum number of tokens we need are also all supported correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 114 +++++++++++++++++++++++++ users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 104 ++++++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 users/fcuny/exp/monkey/pkg/lexer/lexer.go create mode 100644 users/fcuny/exp/monkey/pkg/lexer/lexer_test.go (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go new file mode 100644 index 0000000..fc29371 --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -0,0 +1,114 @@ +package lexer + +import "monkey/pkg/token" + +// Lexer represents the lexer +type Lexer struct { + input string + // current position in input + position int + // current reading position in input (after a char) + readPosition int + // current character under examination + ch byte +} + +// New returns a new lexer +func New(input string) *Lexer { + l := &Lexer{input: input} + l.readChar() + return l +} + +// Read the current character and advances our position in the input string. +func (l *Lexer) readChar() { + // if we've reached the end of the input, we set the current character to 0, + // which is the ASCII code for NUL. + if l.readPosition >= len(l.input) { + l.ch = 0 + } else { + l.ch = l.input[l.readPosition] + } + l.position = l.readPosition + l.readPosition++ +} + +func (l *Lexer) readIdentifier() string { + position := l.position + for isLetter(l.ch) { + l.readChar() + } + return l.input[position:l.position] +} + +func (l *Lexer) readNumber() string { + position := l.position + for isDigit(l.ch) { + l.readChar() + } + return l.input[position:l.position] +} + +// we don't care about white space characters, we skip them when we find them. +func (l *Lexer) skipWhitespace() { + for l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r' { + l.readChar() + } +} + +// NextToken reads the next token from the lexer and returns the current token. +func (l *Lexer) NextToken() token.Token { + var tok token.Token + + l.skipWhitespace() + + switch l.ch { + case '=': + tok = newToken(token.ASSIGN, l.ch) + case '+': + tok = newToken(token.PLUS, l.ch) + case ';': + tok = newToken(token.SEMICOLON, l.ch) + case ',': + tok = newToken(token.COMMA, l.ch) + case '(': + tok = newToken(token.LPAREN, l.ch) + case ')': + tok = newToken(token.RPAREN, l.ch) + case '{': + tok = newToken(token.LBRACE, l.ch) + case '}': + tok = newToken(token.RBRACE, l.ch) + case 0: + tok.Literal = "" + tok.Type = token.EOF + default: + if isLetter(l.ch) { + tok.Literal = l.readIdentifier() + tok.Type = token.LookupIdent(tok.Literal) + return tok + } else if isDigit(l.ch) { + tok.Type = token.INT + tok.Literal = l.readNumber() + return tok + } else { + tok = newToken(token.ILLEGAL, l.ch) + } + + } + + l.readChar() + return tok +} + +func newToken(tokenType token.TokenType, ch byte) token.Token { + return token.Token{Type: tokenType, Literal: string(ch)} +} + +func isLetter(ch byte) bool { + return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' +} + +func isDigit(ch byte) bool { + return '0' <= ch && ch <= '9' +} diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go new file mode 100644 index 0000000..73b27fb --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -0,0 +1,104 @@ +package lexer + +import ( + "monkey/pkg/token" + "testing" +) + +func TestNextTokenBasic(t *testing.T) { + input := `=+(){},;` + + tests := []struct { + expectedType token.TokenType + expectedLiteral string + }{ + {token.ASSIGN, "="}, + {token.PLUS, "+"}, + {token.LPAREN, "("}, + {token.RPAREN, ")"}, + {token.LBRACE, "{"}, + {token.RBRACE, "}"}, + {token.COMMA, ","}, + {token.SEMICOLON, ";"}, + } + + l := New(input) + + for i, tt := range tests { + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - tokenliteral wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal) + } + } +} + +func TestNextTokenMonkey(t *testing.T) { + input := `let five = 5; +let ten = 10; + +let add = fn(x, y) { + x + y +}; + +let result = add(five, ten);` + + tests := []struct { + expectedType token.TokenType + expectedLiteral string + }{ + {token.LET, "let"}, + {token.IDENT, "five"}, + {token.ASSIGN, "="}, + {token.INT, "5"}, + {token.SEMICOLON, ";"}, + + {token.LET, "let"}, + {token.IDENT, "ten"}, + {token.ASSIGN, "="}, + {token.INT, "10"}, + {token.SEMICOLON, ";"}, + + {token.LET, "let"}, + {token.IDENT, "add"}, + {token.ASSIGN, "="}, + {token.FUNCTION, "fn"}, + {token.LPAREN, "("}, + {token.IDENT, "x"}, + {token.COMMA, ","}, + {token.IDENT, "y"}, + {token.RPAREN, ")"}, + {token.LBRACE, "{"}, + {token.IDENT, "x"}, + {token.PLUS, "+"}, + {token.IDENT, "y"}, + {token.RBRACE, "}"}, + {token.SEMICOLON, ";"}, + + {token.LET, "let"}, + {token.IDENT, "result"}, + {token.ASSIGN, "="}, + {token.IDENT, "add"}, + {token.LPAREN, "("}, + {token.IDENT, "five"}, + {token.COMMA, ","}, + {token.IDENT, "ten"}, + {token.RPAREN, ")"}, + {token.SEMICOLON, ";"}, + } + + l := New(input) + for i, tt := range tests { + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - tokenliteral wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal) + } + } +} -- cgit 1.4.1 From 88621a87ddc263b0560aa927118464d50c15c697 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:47:40 +0100 Subject: token: support more operator tokens Support additional tokens for operators (`-`, `*`, etc). This change only adds the tokens to the list of constants, and group all the tokens related to operators together. --- users/fcuny/exp/monkey/pkg/token/token.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index fda4449..bc9e563 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -16,9 +16,6 @@ const ( IDENT = "IDENT" INT = "INT" - ASSIGN = "=" - PLUS = "+" - COMMA = "," SEMICOLON = ";" @@ -29,6 +26,16 @@ const ( FUNCTION = "FUNCTION" LET = "LET" + + // The following tokens are for operators + ASSIGN = "=" + PLUS = "+" + MINUS = "-" + BANG = "!" + ASTERISK = "*" + SLASH = "/" + LT = "<" + GT = ">" ) // List of our keywords for the language -- cgit 1.4.1 From bcd7ed3ec9f59f350a29eb95ffbac71345d93e6d Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:53:44 +0100 Subject: lexer: support more operator tokens. Support the operator tokens that were added to our tokenizer. This also add a few more tests to ensure we handle them correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 13 +++++++++++++ users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 19 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index fc29371..d538cf5 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -67,6 +67,19 @@ func (l *Lexer) NextToken() token.Token { tok = newToken(token.ASSIGN, l.ch) case '+': tok = newToken(token.PLUS, l.ch) + case '-': + tok = newToken(token.MINUS, l.ch) + case '!': + tok = newToken(token.BANG, l.ch) + case '*': + tok = newToken(token.ASTERISK, l.ch) + case '/': + tok = newToken(token.SLASH, l.ch) + case '<': + tok = newToken(token.LT, l.ch) + case '>': + tok = newToken(token.GT, l.ch) + case ';': tok = newToken(token.SEMICOLON, l.ch) case ',': diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index 73b27fb..ba7fa07 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -44,7 +44,10 @@ let add = fn(x, y) { x + y }; -let result = add(five, ten);` +let result = add(five, ten); +!-/*5; +5 < 10 > 5; +` tests := []struct { expectedType token.TokenType @@ -88,6 +91,20 @@ let result = add(five, ten);` {token.IDENT, "ten"}, {token.RPAREN, ")"}, {token.SEMICOLON, ";"}, + + {token.BANG, "!"}, + {token.MINUS, "-"}, + {token.SLASH, "/"}, + {token.ASTERISK, "*"}, + {token.INT, "5"}, + {token.SEMICOLON, ";"}, + + {token.INT, "5"}, + {token.LT, "<"}, + {token.INT, "10"}, + {token.GT, ">"}, + {token.INT, "5"}, + {token.SEMICOLON, ";"}, } l := New(input) -- cgit 1.4.1 From be6ab89f58b0572d0999701d4f1b454e98dec581 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:01:49 +0100 Subject: lexer: delete redundant test. The test `TestNextTokenBasic` was not testing anything that `TestNextTokenMonkey` was not already testing. Rename `TestNextTokenMonkey` to `TestNextToken` for clarity. --- users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 33 +------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index ba7fa07..22dbfcb 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -5,38 +5,7 @@ import ( "testing" ) -func TestNextTokenBasic(t *testing.T) { - input := `=+(){},;` - - tests := []struct { - expectedType token.TokenType - expectedLiteral string - }{ - {token.ASSIGN, "="}, - {token.PLUS, "+"}, - {token.LPAREN, "("}, - {token.RPAREN, ")"}, - {token.LBRACE, "{"}, - {token.RBRACE, "}"}, - {token.COMMA, ","}, - {token.SEMICOLON, ";"}, - } - - l := New(input) - - for i, tt := range tests { - tok := l.NextToken() - if tok.Type != tt.expectedType { - t.Fatalf("tests[%d] - tokentype wrong. expected=%q, got=%q", i, tt.expectedType, tok.Type) - } - - if tok.Literal != tt.expectedLiteral { - t.Fatalf("tests[%d] - tokenliteral wrong. expected=%q, got=%q", i, tt.expectedLiteral, tok.Literal) - } - } -} - -func TestNextTokenMonkey(t *testing.T) { +func TestNextToken(t *testing.T) { input := `let five = 5; let ten = 10; -- cgit 1.4.1 From 18594e5caf3dda013e6f6fb9cfe558d7153e383d Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:17:06 +0100 Subject: token: rewrite documentation for `LookupIdent`. --- users/fcuny/exp/monkey/pkg/token/token.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index bc9e563..6937595 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -44,9 +44,10 @@ var keywords = map[string]TokenType{ "let": LET, } -// LookupIdent returns the token type for a given identifier. If the identifier -// is one of our keyword, we return the corresponding value, otherwise we return -// the given identifier. +// LookupIdent returns the token type for a given identifier. +// First we check if the identifier is a keyword. If it is, we return they +// keyword TokenType constant. If it isn't, we return the token.IDENT which is +// the TokenType for all user-defined identifiers. func LookupIdent(ident string) TokenType { if tok, ok := keywords[ident]; ok { return tok -- cgit 1.4.1 From 2c4068a5be7f60e8d0376720f70791d23a8c3ef1 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:26:19 +0100 Subject: token: support more keywords Add support for a few more keywords (`true`, `false`, `if`, `else`, `return`). All keywords are grouped together in the constant declaration. --- users/fcuny/exp/monkey/pkg/token/token.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index 6937595..a211c55 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -24,8 +24,14 @@ const ( LBRACE = "{" RBRACE = "}" + // The following tokens are keywords FUNCTION = "FUNCTION" LET = "LET" + TRUE = "TRUE" + FALSE = "FALSE" + IF = "IF" + ELSE = "ELSE" + RETURN = "RETURN" // The following tokens are for operators ASSIGN = "=" @@ -40,8 +46,13 @@ const ( // List of our keywords for the language var keywords = map[string]TokenType{ - "fn": FUNCTION, - "let": LET, + "fn": FUNCTION, + "let": LET, + "true": TRUE, + "false": FALSE, + "if": IF, + "else": ELSE, + "return": RETURN, } // LookupIdent returns the token type for a given identifier. -- cgit 1.4.1 From a2991b978a309d4c3a9c480aad4e4e657ae82597 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:27:18 +0100 Subject: lexer: test the new keywords are parsed correctly. Ensure that the new keywords added (`if`, `else`, `true`, `false`, `return`) are parsed correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 28 +++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index 22dbfcb..df1b392 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -15,7 +15,13 @@ let add = fn(x, y) { let result = add(five, ten); !-/*5; -5 < 10 > 5; +10 > 5; + +if (5 < 10) { + return true; +} else { + return false; +} ` tests := []struct { @@ -68,12 +74,28 @@ let result = add(five, ten); {token.INT, "5"}, {token.SEMICOLON, ";"}, - {token.INT, "5"}, - {token.LT, "<"}, {token.INT, "10"}, {token.GT, ">"}, {token.INT, "5"}, {token.SEMICOLON, ";"}, + + {token.IF, "if"}, + {token.LPAREN, "("}, + {token.INT, "5"}, + {token.LT, "<"}, + {token.INT, "10"}, + {token.RPAREN, ")"}, + {token.LBRACE, "{"}, + {token.RETURN, "return"}, + {token.TRUE, "true"}, + {token.SEMICOLON, ";"}, + {token.RBRACE, "}"}, + {token.ELSE, "else"}, + {token.LBRACE, "{"}, + {token.RETURN, "return"}, + {token.FALSE, "false"}, + {token.SEMICOLON, ";"}, + {token.RBRACE, "}"}, } l := New(input) -- cgit 1.4.1 From 72ce9e990e8df7ad4ddb591dd0a0b96cf89503f6 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:37:52 +0100 Subject: token: add tokens for equal and not equal. --- users/fcuny/exp/monkey/pkg/token/token.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index a211c55..b2342e7 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -42,6 +42,9 @@ const ( SLASH = "/" LT = "<" GT = ">" + + EQ = "==" + NOT_EQ = "!=" ) // List of our keywords for the language -- cgit 1.4.1 From 4fb91ad4622e099f798e01873bac914b64ed48f4 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:40:32 +0100 Subject: lexer: support tokens for equal and not equal. The tokens for equal (`==`) and not equal (`!=`) are composed of two characters. We introduce a new helper (`peekChar`) that we use when we encounter the token `=` or `!` to see if this is a token composed of two characters. Add some tests to ensure they are parsed correctly. --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 28 ++++++++++++++++++++++++-- users/fcuny/exp/monkey/pkg/lexer/lexer_test.go | 13 ++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index d538cf5..06d526e 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -56,6 +56,16 @@ func (l *Lexer) skipWhitespace() { } } +// peekChar returns the character at position (which is the next charatecter), +// but does not increment `readPosition` and `position`. +// This is needed to read tokens that are composed of two characters (e.g. `==`). +func (l *Lexer) peekChar() byte { + if l.readPosition >= len(l.input) { + return 0 + } + return l.input[l.readPosition] +} + // NextToken reads the next token from the lexer and returns the current token. func (l *Lexer) NextToken() token.Token { var tok token.Token @@ -64,13 +74,27 @@ func (l *Lexer) NextToken() token.Token { switch l.ch { case '=': - tok = newToken(token.ASSIGN, l.ch) + if l.peekChar() == '=' { + ch := l.ch + l.readChar() + literal := string(ch) + string(l.ch) + tok = token.Token{Type: token.EQ, Literal: literal} + } else { + tok = newToken(token.ASSIGN, l.ch) + } case '+': tok = newToken(token.PLUS, l.ch) case '-': tok = newToken(token.MINUS, l.ch) case '!': - tok = newToken(token.BANG, l.ch) + if l.peekChar() == '=' { + ch := l.ch + l.readChar() + literal := string(ch) + string(l.ch) + tok = token.Token{Type: token.NOT_EQ, Literal: literal} + } else { + tok = newToken(token.BANG, l.ch) + } case '*': tok = newToken(token.ASTERISK, l.ch) case '/': diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go index df1b392..fdea1d3 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer_test.go @@ -22,6 +22,9 @@ if (5 < 10) { } else { return false; } + +10 == 10; +10 != 9; ` tests := []struct { @@ -96,6 +99,16 @@ if (5 < 10) { {token.FALSE, "false"}, {token.SEMICOLON, ";"}, {token.RBRACE, "}"}, + + {token.INT, "10"}, + {token.EQ, "=="}, + {token.INT, "10"}, + {token.SEMICOLON, ";"}, + + {token.INT, "10"}, + {token.NOT_EQ, "!="}, + {token.INT, "9"}, + {token.SEMICOLON, ";"}, } l := New(input) -- cgit 1.4.1 From 7e0195b0a753bd990307587926bfc0fe11927f77 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:50:44 +0100 Subject: repl: support a simple REPL for some early testing The REPL reads the input, send it to the lexer, and prints the token to STDOUT. For now nothing else is done since we still don't parse the tokens. --- users/fcuny/exp/monkey/cmd/repl/main.go | 12 ++++++++++++ users/fcuny/exp/monkey/pkg/repl/repl.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 users/fcuny/exp/monkey/cmd/repl/main.go create mode 100644 users/fcuny/exp/monkey/pkg/repl/repl.go (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/cmd/repl/main.go b/users/fcuny/exp/monkey/cmd/repl/main.go new file mode 100644 index 0000000..46b865c --- /dev/null +++ b/users/fcuny/exp/monkey/cmd/repl/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "fmt" + "monkey/pkg/repl" + "os" +) + +func main() { + fmt.Printf("Welcome to monkey's REPL.") + repl.Start(os.Stdin, os.Stdout) +} diff --git a/users/fcuny/exp/monkey/pkg/repl/repl.go b/users/fcuny/exp/monkey/pkg/repl/repl.go new file mode 100644 index 0000000..e8b3b1f --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/repl/repl.go @@ -0,0 +1,29 @@ +package repl + +import ( + "bufio" + "fmt" + "io" + lexer "monkey/pkg/lexer" + token "monkey/pkg/token" +) + +const PROMPT = ">> " + +func Start(in io.Reader, out io.Writer) { + scanner := bufio.NewScanner(in) + for { + fmt.Printf(PROMPT) + scanned := scanner.Scan() + + if !scanned { + return + } + + line := scanner.Text() + l := lexer.New(line) + for tok := l.NextToken(); tok.Type != token.EOF; tok = l.NextToken() { + fmt.Printf("%+v\n", tok) + } + } +} -- cgit 1.4.1 From 8437218bdaed90cab7374a752f8aaf128225aa1a Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Mon, 10 May 2021 19:21:39 -0700 Subject: lint: fix a few issues --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 1 + users/fcuny/exp/monkey/pkg/repl/repl.go | 3 ++- users/fcuny/exp/monkey/pkg/token/token.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'users/fcuny/exp/monkey/pkg') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index 06d526e..3e98cf0 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -1,3 +1,4 @@ +// Package lexer provides a lexer to the monkey language. package lexer import "monkey/pkg/token" diff --git a/users/fcuny/exp/monkey/pkg/repl/repl.go b/users/fcuny/exp/monkey/pkg/repl/repl.go index e8b3b1f..5e7b1d1 100644 --- a/users/fcuny/exp/monkey/pkg/repl/repl.go +++ b/users/fcuny/exp/monkey/pkg/repl/repl.go @@ -1,3 +1,4 @@ +// Package repl provides a REPL to the monkey language. package repl import ( @@ -13,7 +14,7 @@ const PROMPT = ">> " func Start(in io.Reader, out io.Writer) { scanner := bufio.NewScanner(in) for { - fmt.Printf(PROMPT) + fmt.Print(PROMPT) scanned := scanner.Scan() if !scanned { diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index b2342e7..5eadc5e 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -1,3 +1,4 @@ +// Package token provides a tokenizer for the monkey language. package token // TokenType represents the type of the token -- cgit 1.4.1