Added tokenizer benchmark

src/build/token/bench_test.go

@@ -0,0 +1,15 @@
+package token_test
+
+import (
+	"testing"
+
+	"git.akyoto.dev/cli/q/src/build/token"
+)
+
+func BenchmarkTokenize(b *testing.B) {
+	input := []byte("hello := 123\nworld := 456")
+
+	for i := 0; i < b.N; i++ {
+		token.Tokenize(input)
+	}
+}