27 lines
661 B
Go
27 lines
661 B
Go
package token
|
|
|
|
import "fmt"
|
|
|
|
// Token represents a single element in a source file.
|
|
// The characters that make up an identifier are grouped into a single token.
|
|
// This makes parsing easier and allows us to do better syntax checks.
|
|
type Token struct {
|
|
Kind Kind
|
|
Position int
|
|
Bytes []byte
|
|
}
|
|
|
|
// After returns the position after the token.
|
|
func (t Token) After() int {
|
|
return t.Position + len(t.Bytes)
|
|
}
|
|
|
|
// String creates a human readable representation for debugging purposes.
|
|
func (t Token) String() string {
|
|
return fmt.Sprintf("%s %s", t.Kind, t.Text())
|
|
}
|
|
|
|
// Text returns the token text.
|
|
func (t Token) Text() string {
|
|
return string(t.Bytes)
|
|
}
|