Added a tokenizer

src/token/Token.go

@@ -0,0 +1,15 @@
+package token
+
+// Token represents a single element in a source file.
+// The characters that make up an identifier are grouped into a single token.
+// This makes parsing easier and allows us to do better syntax checks.
+type Token struct {
+	Kind     Kind
+	Position int
+	Bytes    []byte
+}
+
+// String returns the token text.
+func (t Token) String() string {
+	return string(t.Bytes)
+}