cybercyst/go
1
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-05-17 05:14:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

- introduce explicit Token type

Browse Source 
- convert some functions into methods
- corresponding changes in pretty

R=r
DELTA=57  (3 added, 0 deleted, 54 changed)
OCL=26764
CL=26777
This commit is contained in:
Robert Griesemer 2009-03-26 10:53:14 -07:00
parent 99cc2fee81
commit cc8e4fb485
6 changed files with 53 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/lib/go/scanner.go
View File

@ -167,7 +167,7 @@ func isDigit(ch int) bool {
} }
func (S *Scanner) scanIdentifier() int { func (S *Scanner) scanIdentifier() token.Token {
pos := S.loc.Pos; pos := S.loc.Pos;
for isLetter(S.ch) || isDigit(S.ch) { for isLetter(S.ch) || isDigit(S.ch) {
S.next(); S.next();
@ -193,7 +193,7 @@ func (S *Scanner) scanMantissa(base int) {
} }
func (S *Scanner) scanNumber(seen_decimal_point bool) int { func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
tok := token.INT; tok := token.INT;
if seen_decimal_point { if seen_decimal_point {
@ -335,7 +335,7 @@ func (S *Scanner) scanRawString(loc Location) {
// respectively. Otherwise, the result is tok0 if there was no other // respectively. Otherwise, the result is tok0 if there was no other
// matching character, or tok2 if the matching character was ch2. // matching character, or tok2 if the matching character was ch2.
func (S *Scanner) switch2(tok0, tok1 int) int { func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
if S.ch == '=' { if S.ch == '=' {
S.next(); S.next();
return tok1; return tok1;
@ -344,7 +344,7 @@ func (S *Scanner) switch2(tok0, tok1 int) int {
} }
func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int { func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
if S.ch == '=' { if S.ch == '=' {
S.next(); S.next();
return tok1; return tok1;
@ -357,7 +357,7 @@ func (S *Scanner) switch3(tok0, tok1, ch2, tok2 int) int {
} }
func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int { func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
if S.ch == '=' { if S.ch == '=' {
S.next(); S.next();
return tok1; return tok1;
@ -378,7 +378,7 @@ func (S *Scanner) switch4(tok0, tok1, ch2, tok2, tok3 int) int {
// the token tok, and the literal text lit corresponding to the // the token tok, and the literal text lit corresponding to the
// token. The source end is indicated by token.EOF. // token. The source end is indicated by token.EOF.
// //
func (S *Scanner) Scan() (loc Location, tok int, lit []byte) { func (S *Scanner) Scan() (loc Location, tok token.Token, lit []byte) {
scan_again: scan_again:
// skip white space // skip white space
for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' || S.ch == '\r' { for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' || S.ch == '\r' {
@ -468,7 +468,7 @@ scan_again:
// meaning as for the Init function. Tokenize keeps scanning until f returns // meaning as for the Init function. Tokenize keeps scanning until f returns
// false (usually when the token value is token.EOF). // false (usually when the token value is token.EOF).
// //
func Tokenize(src []byte, err ErrorHandler, scan_comments bool, f func (loc Location, tok int, lit []byte) bool) { func Tokenize(src []byte, err ErrorHandler, scan_comments bool, f func (loc Location, tok token.Token, lit []byte) bool) {
var s Scanner; var s Scanner;
s.Init(src, err, scan_comments); s.Init(src, err, scan_comments);
for f(s.Scan()) { for f(s.Scan()) {

16
src/lib/go/scanner_test.go
View File

@ -20,18 +20,18 @@ const /* class */ (
) )
func tokenclass(tok int) int { func tokenclass(tok token.Token) int {
switch { switch {
case token.IsLiteral(tok): return literal; case tok.IsLiteral(): return literal;
case token.IsOperator(tok): return operator; case tok.IsOperator(): return operator;
case token.IsKeyword(tok): return keyword; case tok.IsKeyword(): return keyword;
} }
return special; return special;
} }
type elt struct { type elt struct {
tok int; tok token.Token;
lit string; lit string;
class int; class int;
} }
@ -188,7 +188,7 @@ func Test(t *testing.T) {
index := 0; index := 0;
eloc := scanner.Location{0, 1, 1}; eloc := scanner.Location{0, 1, 1};
scanner.Tokenize(io.StringBytes(src), &TestErrorHandler{t}, true, scanner.Tokenize(io.StringBytes(src), &TestErrorHandler{t}, true,
func (loc Location, tok int, litb []byte) bool { func (loc Location, tok token.Token, litb []byte) bool {
e := elt{token.EOF, "", special}; e := elt{token.EOF, "", special};
if index < len(tokens) { if index < len(tokens) {
e = tokens[index]; e = tokens[index];
@ -208,9 +208,9 @@ func Test(t *testing.T) {
t.Errorf("bad column for %s: got %d, expected %d", lit, loc.Col, eloc.Col); t.Errorf("bad column for %s: got %d, expected %d", lit, loc.Col, eloc.Col);
} }
if tok != e.tok { if tok != e.tok {
t.Errorf("bad token for %s: got %s, expected %s", lit, token.TokenString(tok), token.TokenString(e.tok)); t.Errorf("bad token for %s: got %s, expected %s", lit, tok.String(), e.tok.String());
} }
if token.IsLiteral(e.tok) && lit != e.lit { if e.tok.IsLiteral() && lit != e.lit {
t.Errorf("bad literal for %s: got %s, expected %s", lit, lit, e.lit); t.Errorf("bad literal for %s: got %s, expected %s", lit, lit, e.lit);
} }
if tokenclass(tok) != e.class { if tokenclass(tok) != e.class {

25
src/lib/go/token.go
View File

@ -10,10 +10,13 @@ package token
import "strconv" import "strconv"
// Token is the set of lexical tokens of the Go programming language.
type Token int
// The list of tokens. // The list of tokens.
const ( const (
// Special tokens // Special tokens
ILLEGAL = iota; ILLEGAL Token = iota;
EOF; EOF;
COMMENT; COMMENT;
@ -124,7 +127,7 @@ const (
// At the moment we have no array literal syntax that lets us describe // At the moment we have no array literal syntax that lets us describe
// the index for each element - use a map for now to make sure they are // the index for each element - use a map for now to make sure they are
// in sync. // in sync.
var tokens = map [int] string { var tokens = map [Token] string {
ILLEGAL : "ILLEGAL", ILLEGAL : "ILLEGAL",
EOF : "EOF", EOF : "EOF",
@ -224,13 +227,13 @@ var tokens = map [int] string {
} }
// TokenString returns the string corresponding to the token tok. // String returns the string corresponding to the token tok.
// For operators, delimiters, and keywords the string is the actual // For operators, delimiters, and keywords the string is the actual
// token character sequence (e.g., for the token ADD, the string is // token character sequence (e.g., for the token ADD, the string is
// "+"). For all other tokens the string corresponds to the token // "+"). For all other tokens the string corresponds to the token
// constant name (e.g. for the token IDENT, the string is "IDENT"). // constant name (e.g. for the token IDENT, the string is "IDENT").
// //
func TokenString(tok int) string { func (tok Token) String() string {
if str, exists := tokens[tok]; exists { if str, exists := tokens[tok]; exists {
return str; return str;
} }
@ -254,7 +257,7 @@ const (
// Precedence returns the syntax precedence of the operator // Precedence returns the syntax precedence of the operator
// token tok or LowestPrecedence if tok is not an operator. // token tok or LowestPrecedence if tok is not an operator.
// //
func Precedence(tok int) int { func (tok Token) Precedence() int {
switch tok { switch tok {
case COLON: case COLON:
return 0; return 0;
@ -275,10 +278,10 @@ func Precedence(tok int) int {
} }
var keywords map [string] int; var keywords map [string] Token;
func init() { func init() {
keywords = make(map [string] int); keywords = make(map [string] Token);
for i := keyword_beg + 1; i < keyword_end; i++ { for i := keyword_beg + 1; i < keyword_end; i++ {
keywords[tokens[i]] = i; keywords[tokens[i]] = i;
} }
@ -287,7 +290,7 @@ func init() {
// Lookup maps an identifier to its keyword token or IDENT (if not a keyword). // Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
// //
func Lookup(ident []byte) int { func Lookup(ident []byte) Token {
// TODO Maps with []byte key are illegal because []byte does not // TODO Maps with []byte key are illegal because []byte does not
// support == . Should find a more efficient solution eventually. // support == . Should find a more efficient solution eventually.
if tok, is_keyword := keywords[string(ident)]; is_keyword { if tok, is_keyword := keywords[string(ident)]; is_keyword {
@ -302,20 +305,20 @@ func Lookup(ident []byte) int {
// IsLiteral returns true for tokens corresponding to identifiers // IsLiteral returns true for tokens corresponding to identifiers
// and basic type literals; returns false otherwise. // and basic type literals; returns false otherwise.
// //
func IsLiteral(tok int) bool { func (tok Token) IsLiteral() bool {
return literal_beg < tok && tok < literal_end; return literal_beg < tok && tok < literal_end;
} }
// IsOperator returns true for tokens corresponding to operators and // IsOperator returns true for tokens corresponding to operators and
// delimiters; returns false otherwise. // delimiters; returns false otherwise.
// //
func IsOperator(tok int) bool { func (tok Token) IsOperator() bool {
return operator_beg < tok && tok < operator_end; return operator_beg < tok && tok < operator_end;
} }
// IsKeyword returns true for tokens corresponding to keywords; // IsKeyword returns true for tokens corresponding to keywords;
// returns false otherwise. // returns false otherwise.
// //
func IsKeyword(tok int) bool { func (tok Token) IsKeyword() bool {
return keyword_beg < tok && tok < keyword_end; return keyword_beg < tok && tok < keyword_end;
} }

16
usr/gri/pretty/ast.go
View File

@ -247,7 +247,7 @@ type (
// //
UnaryExpr struct { UnaryExpr struct {
Pos_ Position; // token position Pos_ Position; // token position
Tok int; // operator Tok token.Token; // operator
X Expr; // operand X Expr; // operand
}; };
@ -255,7 +255,7 @@ type (
BinaryExpr struct { BinaryExpr struct {
X Expr; // left operand X Expr; // left operand
Pos_ Position; // token position Pos_ Position; // token position
Tok int; // operator Tok token.Token; // operator
Y Expr; // right operand Y Expr; // right operand
}; };
) )
@ -472,7 +472,7 @@ type (
// An IncDecStmt node represents an increment or decrement statement. // An IncDecStmt node represents an increment or decrement statement.
IncDecStmt struct { IncDecStmt struct {
X Expr; X Expr;
Tok int; // INC or DEC Tok token.Token; // INC or DEC
}; };
// An AssignStmt node represents an assignment or // An AssignStmt node represents an assignment or
@ -480,7 +480,7 @@ type (
AssignStmt struct { AssignStmt struct {
Lhs []Expr; Lhs []Expr;
Pos_ Position; // token position Pos_ Position; // token position
Tok int; // assignment token, DEFINE Tok token.Token; // assignment token, DEFINE
Rhs []Expr; Rhs []Expr;
}; };
@ -507,7 +507,7 @@ type (
// //
BranchStmt struct { BranchStmt struct {
Pos_ Position; // position of keyword Pos_ Position; // position of keyword
Tok int; // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH) Tok token.Token; // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH)
Label *Ident; Label *Ident;
}; };
@ -562,7 +562,7 @@ type (
// A CommClause node represents a case of a select statement. // A CommClause node represents a case of a select statement.
CommClause struct { CommClause struct {
Case Position; // position of "case" or "default" keyword Case Position; // position of "case" or "default" keyword
Tok int; // ASSIGN or DEFINE (valid only if Lhs != nil) Tok token.Token; // ASSIGN or DEFINE (valid only if Lhs != nil)
Lhs, Rhs Expr; // Rhs == nil means default case Lhs, Rhs Expr; // Rhs == nil means default case
Colon Position; // position of ":" Colon Position; // position of ":"
Body []Stmt; // statement list; or nil Body []Stmt; // statement list; or nil
@ -588,7 +588,7 @@ type (
For Position; // position of "for" keyword For Position; // position of "for" keyword
Key, Value Expr; // Value may be nil Key, Value Expr; // Value may be nil
Pos_ Position; // token position Pos_ Position; // token position
Tok int; // ASSIGN or DEFINE Tok token.Token; // ASSIGN or DEFINE
X Expr; // value to range over X Expr; // value to range over
Body *BlockStmt; Body *BlockStmt;
}; };
@ -730,7 +730,7 @@ type (
DeclList struct { DeclList struct {
Doc Comments; // associated documentation; or nil Doc Comments; // associated documentation; or nil
Pos_ Position; // position of token Pos_ Position; // position of token
Tok int; // IMPORT, CONST, VAR, TYPE Tok token.Token; // IMPORT, CONST, VAR, TYPE
Lparen Position; // position of '(' Lparen Position; // position of '('
List []Decl; // the list of parenthesized declarations List []Decl; // the list of parenthesized declarations
Rparen Position; // position of ')' Rparen Position; // position of ')'

20
usr/gri/pretty/parser.go
View File

@ -47,7 +47,7 @@ type Parser struct {
// The next token // The next token
pos Position; // token location pos Position; // token location
tok int; // one token look-ahead tok token.Token; // one token look-ahead
val []byte; // token value val []byte; // token value
// Non-syntactic parser control // Non-syntactic parser control
@ -107,7 +107,7 @@ func (P *Parser) next0() {
P.printIndent(); P.printIndent();
switch P.tok { switch P.tok {
case token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING: case token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING:
fmt.Printf("%d:%d: %s = %s\n", P.pos.Line, P.pos.Col, token.TokenString(P.tok), P.val); fmt.Printf("%d:%d: %s = %s\n", P.pos.Line, P.pos.Col, P.tok.String(), P.val);
case token.LPAREN: case token.LPAREN:
// don't print '(' - screws up selection in terminal window // don't print '(' - screws up selection in terminal window
fmt.Printf("%d:%d: LPAREN\n", P.pos.Line, P.pos.Col); fmt.Printf("%d:%d: LPAREN\n", P.pos.Line, P.pos.Col);
@ -115,7 +115,7 @@ func (P *Parser) next0() {
// don't print ')' - screws up selection in terminal window // don't print ')' - screws up selection in terminal window
fmt.Printf("%d:%d: RPAREN\n", P.pos.Line, P.pos.Col); fmt.Printf("%d:%d: RPAREN\n", P.pos.Line, P.pos.Col);
default: default:
fmt.Printf("%d:%d: %s\n", P.pos.Line, P.pos.Col, token.TokenString(P.tok)); fmt.Printf("%d:%d: %s\n", P.pos.Line, P.pos.Col, P.tok.String());
} }
} }
} }
@ -178,10 +178,10 @@ func (P *Parser) error(pos Position, msg string) {
} }
func (P *Parser) expect(tok int) Position { func (P *Parser) expect(tok token.Token) Position {
if P.tok != tok { if P.tok != tok {
msg := "expected '" + token.TokenString(tok) + "', found '" + token.TokenString(P.tok) + "'"; msg := "expected '" + tok.String() + "', found '" + P.tok.String() + "'";
if token.IsLiteral(P.tok) { if P.tok.IsLiteral() {
msg += " " + string(P.val); msg += " " + string(P.val);
} }
P.error(P.pos, msg); P.error(P.pos, msg);
@ -1082,8 +1082,8 @@ func (P *Parser) parseBinaryExpr(prec1 int) ast.Expr {
} }
x := P.parseUnaryExpr(); x := P.parseUnaryExpr();
for prec := token.Precedence(P.tok); prec >= prec1; prec-- { for prec := P.tok.Precedence(); prec >= prec1; prec-- {
for token.Precedence(P.tok) == prec { for P.tok.Precedence() == prec {
pos, tok := P.pos, P.tok; pos, tok := P.pos, P.tok;
P.next(); P.next();
y := P.parseBinaryExpr(prec + 1); y := P.parseBinaryExpr(prec + 1);
@ -1217,7 +1217,7 @@ func (P *Parser) parseReturnStmt() *ast.ReturnStmt {
} }
func (P *Parser) parseBranchStmt(tok int) *ast.BranchStmt { func (P *Parser) parseBranchStmt(tok token.Token) *ast.BranchStmt {
if P.trace { if P.trace {
defer un(trace(P, "BranchStmt")); defer un(trace(P, "BranchStmt"));
} }
@ -1396,7 +1396,7 @@ func (P *Parser) parseCommClause() *ast.CommClause {
// CommCase // CommCase
loc := P.pos; loc := P.pos;
var tok int; var tok token.Token;
var lhs, rhs ast.Expr; var lhs, rhs ast.Expr;
if P.tok == token.CASE { if P.tok == token.CASE {
P.next(); P.next();

12
usr/gri/pretty/printer.go
View File

@ -393,14 +393,14 @@ func (P *Printer) String(loc scanner.Location, s string) {
} }
func (P *Printer) Token(loc scanner.Location, tok int) { func (P *Printer) Token(loc scanner.Location, tok token.Token) {
P.String(loc, token.TokenString(tok)); P.String(loc, tok.String());
//P.TaggedString(pos, "<b>", token.TokenString(tok), "</b>"); //P.TaggedString(pos, "<b>", tok.String(), "</b>");
} }
func (P *Printer) Error(loc scanner.Location, tok int, msg string) { func (P *Printer) Error(loc scanner.Location, tok token.Token, msg string) {
fmt.Printf("\ninternal printing error: pos = %d, tok = %s, %s\n", loc.Pos, token.TokenString(tok), msg); fmt.Printf("\ninternal printing error: pos = %d, tok = %s, %s\n", loc.Pos, tok.String(), msg);
panic(); panic();
} }
@ -576,7 +576,7 @@ func (P *Printer) DoIdent(x *ast.Ident) {
func (P *Printer) DoBinaryExpr(x *ast.BinaryExpr) { func (P *Printer) DoBinaryExpr(x *ast.BinaryExpr) {
prec := token.Precedence(x.Tok); prec := x.Tok.Precedence();
if prec < P.prec { if prec < P.prec {
P.Token(nopos, token.LPAREN); P.Token(nopos, token.LPAREN);
} }