cli/q
1
0
Fork 0
Code Issues Pull requests Releases Activity

Reduced token size

Browse source
This commit is contained in:
Eduard Urbach 2024-07-21 14:35:06 +02:00
parent 7bfd0e731d
commit 1e3705df55
Signed by: eduard
GPG key ID: 49226B848C78F6C8
47 changed files with 543 additions and 764 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/build/scanner/scanFile.go
View file

@ -11,7 +11,6 @@ import (
"git.akyoto.dev/cli/q/src/build/errors"
"git.akyoto.dev/cli/q/src/build/expression"
"git.akyoto.dev/cli/q/src/build/fs"
"git.akyoto.dev/cli/q/src/build/keyword"
"git.akyoto.dev/cli/q/src/build/scope"
"git.akyoto.dev/cli/q/src/build/token"
)
@ -27,8 +26,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
tokens := token.Tokenize(contents)
file := &fs.File{
Tokens: tokens,
Path: path,
Bytes: contents,
Tokens: tokens,
}
var (
@ -42,14 +42,14 @@ func (s *Scanner) scanFile(path string, pkg string) error {
)
for {
for i < len(tokens) && tokens[i].Kind == token.Keyword && tokens[i].Text() == keyword.Import {
for i < len(tokens) && tokens[i].Kind == token.Import {
i++
if tokens[i].Kind != token.Identifier {
panic("expected package name")
}
packageName := tokens[i].Text()
packageName := tokens[i].Text(contents)
s.queueDirectory(filepath.Join(config.Library, packageName), packageName)
i++
@ -75,7 +75,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
}
if tokens[i].Kind == token.Invalid {
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
@ -116,7 +116,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
}
if tokens[i].Kind == token.Invalid {
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
@ -168,7 +168,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
}
if tokens[i].Kind == token.Invalid {
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text()}, file, tokens[i].Position)
return errors.New(&errors.InvalidCharacter{Character: tokens[i].Text(contents)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
@ -191,7 +191,7 @@ func (s *Scanner) scanFile(path string, pkg string) error {
return errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
}
name := tokens[nameStart].Text()
name := tokens[nameStart].Text(contents)
body := tokens[bodyStart:i]
if pkg != "" {
@ -207,9 +207,9 @@ func (s *Scanner) scanFile(path string, pkg string) error {
return errors.New(errors.NotImplemented, file, tokens[0].Position)
}
name := tokens[0].Text()
name := tokens[0].Text(contents)
register := x64.CallRegisters[count]
uses := token.Count(function.Body, token.Identifier, name)
uses := token.Count(function.Body, contents, token.Identifier, name)
if uses == 0 {
return errors.New(&errors.UnusedVariable{Name: name}, file, tokens[0].Position)