34 lines
783 B
Go
34 lines
783 B
Go
package token
|
|
|
|
import "fmt"
|
|
|
|
// Token represents a single element in a source file.
|
|
// The characters that make up an identifier are grouped into a single token.
|
|
// This makes parsing easier and allows us to do better syntax checks.
|
|
type Token struct {
|
|
Bytes []byte
|
|
Position int
|
|
Kind Kind
|
|
}
|
|
|
|
// End returns the position after the token.
|
|
func (t *Token) End() int {
|
|
return t.Position + len(t.Bytes)
|
|
}
|
|
|
|
// String creates a human readable representation for debugging purposes.
|
|
func (t *Token) String() string {
|
|
return fmt.Sprintf("%s %s", t.Kind, t.Text())
|
|
}
|
|
|
|
// Reset resets the token to default values.
|
|
func (t *Token) Reset() {
|
|
t.Kind = Invalid
|
|
t.Position = 0
|
|
t.Bytes = nil
|
|
}
|
|
|
|
// Text returns the token text.
|
|
func (t *Token) Text() string {
|
|
return string(t.Bytes)
|
|
}
|