From a4260f907f740bb57246832ee7aca75899bf4aff Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:32:55 +0100 Subject: token: initial tokenizer. This is the initial tokenizer for the monkey language. For now we recognize a limited number of tokens. We only have two keywords at this stage: `fn` and `let`. `fn` is used to create function, while `let` is used for assigning variables. The other tokens are mostly to parse the source code, and recognize things like brackets, parentheses, etc. --- users/fcuny/exp/monkey/pkg/token/token.go | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 users/fcuny/exp/monkey/pkg/token/token.go (limited to 'users/fcuny/exp/monkey/pkg/token/token.go') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go new file mode 100644 index 0000000..fda4449 --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -0,0 +1,48 @@ +package token + +// TokenType represents the type of the token +type TokenType string + +// Token represents a token, with the type and the literal value of the token +type Token struct { + Type TokenType + Literal string +} + +const ( + ILLEGAL = "ILLEGAL" + EOF = "EOF" + + IDENT = "IDENT" + INT = "INT" + + ASSIGN = "=" + PLUS = "+" + + COMMA = "," + SEMICOLON = ";" + + LPAREN = "(" + RPAREN = ")" + LBRACE = "{" + RBRACE = "}" + + FUNCTION = "FUNCTION" + LET = "LET" +) + +// List of our keywords for the language +var keywords = map[string]TokenType{ + "fn": FUNCTION, + "let": LET, +} + +// LookupIdent returns the token type for a given identifier. If the identifier +// is one of our keyword, we return the corresponding value, otherwise we return +// the given identifier. +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[ident]; ok { + return tok + } + return IDENT +} -- cgit 1.4.1 From 88621a87ddc263b0560aa927118464d50c15c697 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 13:47:40 +0100 Subject: token: support more operator tokens Support additional tokens for operators (`-`, `*`, etc). This change only adds the tokens to the list of constants, and group all the tokens related to operators together. --- users/fcuny/exp/monkey/pkg/token/token.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg/token/token.go') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index fda4449..bc9e563 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -16,9 +16,6 @@ const ( IDENT = "IDENT" INT = "INT" - ASSIGN = "=" - PLUS = "+" - COMMA = "," SEMICOLON = ";" @@ -29,6 +26,16 @@ const ( FUNCTION = "FUNCTION" LET = "LET" + + // The following tokens are for operators + ASSIGN = "=" + PLUS = "+" + MINUS = "-" + BANG = "!" + ASTERISK = "*" + SLASH = "/" + LT = "<" + GT = ">" ) // List of our keywords for the language -- cgit 1.4.1 From 18594e5caf3dda013e6f6fb9cfe558d7153e383d Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:17:06 +0100 Subject: token: rewrite documentation for `LookupIdent`. --- users/fcuny/exp/monkey/pkg/token/token.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg/token/token.go') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index bc9e563..6937595 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -44,9 +44,10 @@ var keywords = map[string]TokenType{ "let": LET, } -// LookupIdent returns the token type for a given identifier. If the identifier -// is one of our keyword, we return the corresponding value, otherwise we return -// the given identifier. +// LookupIdent returns the token type for a given identifier. +// First we check if the identifier is a keyword. If it is, we return they +// keyword TokenType constant. If it isn't, we return the token.IDENT which is +// the TokenType for all user-defined identifiers. func LookupIdent(ident string) TokenType { if tok, ok := keywords[ident]; ok { return tok -- cgit 1.4.1 From 2c4068a5be7f60e8d0376720f70791d23a8c3ef1 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:26:19 +0100 Subject: token: support more keywords Add support for a few more keywords (`true`, `false`, `if`, `else`, `return`). All keywords are grouped together in the constant declaration. --- users/fcuny/exp/monkey/pkg/token/token.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'users/fcuny/exp/monkey/pkg/token/token.go') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index 6937595..a211c55 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -24,8 +24,14 @@ const ( LBRACE = "{" RBRACE = "}" + // The following tokens are keywords FUNCTION = "FUNCTION" LET = "LET" + TRUE = "TRUE" + FALSE = "FALSE" + IF = "IF" + ELSE = "ELSE" + RETURN = "RETURN" // The following tokens are for operators ASSIGN = "=" @@ -40,8 +46,13 @@ const ( // List of our keywords for the language var keywords = map[string]TokenType{ - "fn": FUNCTION, - "let": LET, + "fn": FUNCTION, + "let": LET, + "true": TRUE, + "false": FALSE, + "if": IF, + "else": ELSE, + "return": RETURN, } // LookupIdent returns the token type for a given identifier. -- cgit 1.4.1 From 72ce9e990e8df7ad4ddb591dd0a0b96cf89503f6 Mon Sep 17 00:00:00 2001 From: franck cuny Date: Sat, 11 Jan 2020 14:37:52 +0100 Subject: token: add tokens for equal and not equal. --- users/fcuny/exp/monkey/pkg/token/token.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'users/fcuny/exp/monkey/pkg/token/token.go') diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index a211c55..b2342e7 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -42,6 +42,9 @@ const ( SLASH = "/" LT = "<" GT = ">" + + EQ = "==" + NOT_EQ = "!=" ) // List of our keywords for the language -- cgit 1.4.1 From 8437218bdaed90cab7374a752f8aaf128225aa1a Mon Sep 17 00:00:00 2001 From: Franck Cuny Date: Mon, 10 May 2021 19:21:39 -0700 Subject: lint: fix a few issues --- users/fcuny/exp/monkey/pkg/lexer/lexer.go | 1 + users/fcuny/exp/monkey/pkg/repl/repl.go | 3 ++- users/fcuny/exp/monkey/pkg/token/token.go | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'users/fcuny/exp/monkey/pkg/token/token.go') diff --git a/users/fcuny/exp/monkey/pkg/lexer/lexer.go b/users/fcuny/exp/monkey/pkg/lexer/lexer.go index 06d526e..3e98cf0 100644 --- a/users/fcuny/exp/monkey/pkg/lexer/lexer.go +++ b/users/fcuny/exp/monkey/pkg/lexer/lexer.go @@ -1,3 +1,4 @@ +// Package lexer provides a lexer to the monkey language. package lexer import "monkey/pkg/token" diff --git a/users/fcuny/exp/monkey/pkg/repl/repl.go b/users/fcuny/exp/monkey/pkg/repl/repl.go index e8b3b1f..5e7b1d1 100644 --- a/users/fcuny/exp/monkey/pkg/repl/repl.go +++ b/users/fcuny/exp/monkey/pkg/repl/repl.go @@ -1,3 +1,4 @@ +// Package repl provides a REPL to the monkey language. package repl import ( @@ -13,7 +14,7 @@ const PROMPT = ">> " func Start(in io.Reader, out io.Writer) { scanner := bufio.NewScanner(in) for { - fmt.Printf(PROMPT) + fmt.Print(PROMPT) scanned := scanner.Scan() if !scanned { diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go index b2342e7..5eadc5e 100644 --- a/users/fcuny/exp/monkey/pkg/token/token.go +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -1,3 +1,4 @@ +// Package token provides a tokenizer for the monkey language. package token // TokenType represents the type of the token -- cgit 1.4.1