diff options
author | franck cuny <franck@fcuny.net> | 2020-01-11 13:32:55 +0100 |
---|---|---|
committer | franck cuny <franck@fcuny.net> | 2020-01-11 13:32:55 +0100 |
commit | a4260f907f740bb57246832ee7aca75899bf4aff (patch) | |
tree | d33d11238031b991f4d11a2466f3edfc2d73d04c /users/fcuny/exp/monkey/pkg | |
parent | go.mod: create the module 'monkey' (diff) | |
download | world-a4260f907f740bb57246832ee7aca75899bf4aff.tar.gz |
token: initial tokenizer.
This is the initial tokenizer for the monkey language. For now we recognize a limited number of tokens. We only have two keywords at this stage: `fn` and `let`. `fn` is used to create function, while `let` is used for assigning variables. The other tokens are mostly to parse the source code, and recognize things like brackets, parentheses, etc.
Diffstat (limited to 'users/fcuny/exp/monkey/pkg')
-rw-r--r-- | users/fcuny/exp/monkey/pkg/token/token.go | 48 |
1 files changed, 48 insertions, 0 deletions
diff --git a/users/fcuny/exp/monkey/pkg/token/token.go b/users/fcuny/exp/monkey/pkg/token/token.go new file mode 100644 index 0000000..fda4449 --- /dev/null +++ b/users/fcuny/exp/monkey/pkg/token/token.go @@ -0,0 +1,48 @@ +package token + +// TokenType represents the type of the token +type TokenType string + +// Token represents a token, with the type and the literal value of the token +type Token struct { + Type TokenType + Literal string +} + +const ( + ILLEGAL = "ILLEGAL" + EOF = "EOF" + + IDENT = "IDENT" + INT = "INT" + + ASSIGN = "=" + PLUS = "+" + + COMMA = "," + SEMICOLON = ";" + + LPAREN = "(" + RPAREN = ")" + LBRACE = "{" + RBRACE = "}" + + FUNCTION = "FUNCTION" + LET = "LET" +) + +// List of our keywords for the language +var keywords = map[string]TokenType{ + "fn": FUNCTION, + "let": LET, +} + +// LookupIdent returns the token type for a given identifier. If the identifier +// is one of our keyword, we return the corresponding value, otherwise we return +// the given identifier. +func LookupIdent(ident string) TokenType { + if tok, ok := keywords[ident]; ok { + return tok + } + return IDENT +} |