Simplified file structure

src/token/Token.go

@@ -0,0 +1,71 @@
+package token
+
+import (
+	"unsafe"
+)
+
+// Token represents a single element in a source file.
+// The characters that make up an identifier are grouped into a single token.
+// This makes parsing easier and allows us to do better syntax checks.
+type Token struct {
+	Position Position
+	Length   Length
+	Kind     Kind
+}
+
+// Bytes returns the byte slice.
+func (t Token) Bytes(buffer []byte) []byte {
+	return buffer[t.Position : t.Position+Position(t.Length)]
+}
+
+// End returns the position after the token.
+func (t Token) End() Position {
+	return t.Position + Position(t.Length)
+}
+
+// IsAssignment returns true if the token is an assignment operator.
+func (t Token) IsAssignment() bool {
+	return t.Kind > _assignments && t.Kind < _assignmentsEnd
+}
+
+// IsComparison returns true if the token is a comparison operator.
+func (t Token) IsComparison() bool {
+	return t.Kind > _comparisons && t.Kind < _comparisonsEnd
+}
+
+// IsExpressionStart returns true if the token starts an expression.
+func (t Token) IsExpressionStart() bool {
+	return t.Kind == GroupStart || t.Kind == ArrayStart || t.Kind == BlockStart
+}
+
+// IsKeyword returns true if the token is a keyword.
+func (t Token) IsKeyword() bool {
+	return t.Kind > _keywords && t.Kind < _keywordsEnd
+}
+
+// IsNumeric returns true if the token is a number or rune.
+func (t Token) IsNumeric() bool {
+	return t.Kind == Number || t.Kind == Rune
+}
+
+// IsOperator returns true if the token is an operator.
+func (t Token) IsOperator() bool {
+	return t.Kind > _operators && t.Kind < _operatorsEnd
+}
+
+// IsUnaryOperator returns true if the token is a unary operator.
+func (t Token) IsUnaryOperator() bool {
+	return t.Kind > _unary && t.Kind < _unaryEnd
+}
+
+// Reset resets the token to default values.
+func (t *Token) Reset() {
+	t.Position = 0
+	t.Length = 0
+	t.Kind = Invalid
+}
+
+// Text returns the token text.
+func (t Token) Text(buffer []byte) string {
+	return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)
+}