Improved performance

src/build/token/Tokenize_test.go

@@ -25,11 +25,15 @@ func TestFunction(t *testing.T) {
 }
 
 func TestKeyword(t *testing.T) {
-	tokens := token.Tokenize([]byte("return x"))
+	tokens := token.Tokenize([]byte("assert if import else loop return"))
 
 	expected := []token.Kind{
+		token.Assert,
+		token.If,
+		token.Import,
+		token.Else,
+		token.Loop,
 		token.Return,
-		token.Identifier,
 		token.EOF,
 	}
 
@@ -103,6 +107,37 @@ func TestOperator(t *testing.T) {
 	}
 }
 
+func TestOperatorAssign(t *testing.T) {
+	tokens := token.Tokenize([]byte(`a += b -= c *= d /= e &= f |= g ^= h <<= i >>= j`))
+
+	expected := []token.Kind{
+		token.Identifier,
+		token.AddAssign,
+		token.Identifier,
+		token.SubAssign,
+		token.Identifier,
+		token.MulAssign,
+		token.Identifier,
+		token.DivAssign,
+		token.Identifier,
+		token.AndAssign,
+		token.Identifier,
+		token.OrAssign,
+		token.Identifier,
+		token.XorAssign,
+		token.Identifier,
+		token.ShlAssign,
+		token.Identifier,
+		token.ShrAssign,
+		token.Identifier,
+		token.EOF,
+	}
+
+	for i, kind := range expected {
+		assert.Equal(t, tokens[i].Kind, kind)
+	}
+}
+
 func TestNegateFirstToken(t *testing.T) {
 	tokens := token.Tokenize([]byte(`-a`))
 
@@ -244,37 +279,6 @@ func TestLeadingZero(t *testing.T) {
 	}
 }
 
-func TestOperatorAssign(t *testing.T) {
-	tokens := token.Tokenize([]byte(`a += b -= c *= d /= e &= f |= g ^= h <<= i >>= j`))
-
-	expected := []token.Kind{
-		token.Identifier,
-		token.AddAssign,
-		token.Identifier,
-		token.SubAssign,
-		token.Identifier,
-		token.MulAssign,
-		token.Identifier,
-		token.DivAssign,
-		token.Identifier,
-		token.AndAssign,
-		token.Identifier,
-		token.OrAssign,
-		token.Identifier,
-		token.XorAssign,
-		token.Identifier,
-		token.ShlAssign,
-		token.Identifier,
-		token.ShrAssign,
-		token.Identifier,
-		token.EOF,
-	}
-
-	for i, kind := range expected {
-		assert.Equal(t, tokens[i].Kind, kind)
-	}
-}
-
 func TestSeparator(t *testing.T) {
 	tokens := token.Tokenize([]byte("a,b,c"))