Implemented function scanning
All checks were successful
/ test (push) Successful in 16s

This commit is contained in:
Eduard Urbach 2025-06-19 23:31:52 +02:00
parent 9b51680af5
commit c2b8db238e
Signed by: akyoto
GPG key ID: 49226B848C78F6C8
24 changed files with 624 additions and 232 deletions

View file

@ -10,7 +10,7 @@ import (
func TestExec(t *testing.T) { func TestExec(t *testing.T) {
assert.Equal(t, cli.Exec(nil), 2) assert.Equal(t, cli.Exec(nil), 2)
assert.Equal(t, cli.Exec([]string{"_"}), 2) assert.Equal(t, cli.Exec([]string{"_"}), 2)
assert.Equal(t, cli.Exec([]string{"build"}), 0) assert.Equal(t, cli.Exec([]string{"build"}), 1)
assert.Equal(t, cli.Exec([]string{"build", "--invalid-parameter"}), 2) assert.Equal(t, cli.Exec([]string{"build", "--invalid-parameter"}), 2)
assert.Equal(t, cli.Exec([]string{"build", "../../examples/hello", "--invalid-parameter"}), 2) assert.Equal(t, cli.Exec([]string{"build", "../../examples/hello", "--invalid-parameter"}), 2)
assert.Equal(t, cli.Exec([]string{"build", "../../examples/hello", "--dry"}), 0) assert.Equal(t, cli.Exec([]string{"build", "../../examples/hello", "--dry"}), 0)

View file

@ -1,36 +1,34 @@
package compiler package compiler
import ( import (
"fmt"
"git.urbach.dev/cli/q/src/build" "git.urbach.dev/cli/q/src/build"
"git.urbach.dev/cli/q/src/scanner" "git.urbach.dev/cli/q/src/errors"
) )
// Compile waits for the scan to finish and compiles all functions. // Compile waits for the scan to finish and compiles all functions.
func Compile(b *build.Build) (Result, error) { func Compile(b *build.Build) (Result, error) {
result := Result{} result := Result{}
files, errors := scanner.Scan(b) all, err := scan(b)
for files != nil || errors != nil { if err != nil {
select { return result, err
case file, ok := <-files: }
if !ok {
files = nil
continue
}
fmt.Println(file.Path) if len(all.Files) == 0 {
return result, errors.NoInputFiles
}
case err, ok := <-errors: // Resolve the types
if !ok { for _, function := range all.Functions {
errors = nil err := function.ResolveTypes()
continue
}
if err != nil {
return result, err return result, err
} }
} }
// Parallel compilation
compileFunctions(all.Functions)
return result, nil return result, nil
} }

View file

@ -0,0 +1,27 @@
package compiler
import (
"sync"
"git.urbach.dev/cli/q/src/core"
)
// compileFunctions starts a goroutine for each function compilation and waits for completion.
func compileFunctions(functions map[string]*core.Function) {
wg := sync.WaitGroup{}
for _, function := range functions {
if function.IsExtern() {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
function.Compile()
}()
}
wg.Wait()
}

47
src/compiler/scan.go Normal file
View file

@ -0,0 +1,47 @@
package compiler
import (
"git.urbach.dev/cli/q/src/build"
"git.urbach.dev/cli/q/src/core"
"git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/scanner"
)
func scan(b *build.Build) (*core.Environment, error) {
functions, files, errs := scanner.Scan(b)
all := &core.Environment{
Files: make([]*fs.File, 0, 8),
Functions: make(map[string]*core.Function, 32),
}
for functions != nil || files != nil || errs != nil {
select {
case f, ok := <-functions:
if !ok {
functions = nil
continue
}
all.Functions[f.String()] = f
case file, ok := <-files:
if !ok {
files = nil
continue
}
all.Files = append(all.Files, file)
case err, ok := <-errs:
if !ok {
errs = nil
continue
}
return all, err
}
}
return all, nil
}

4
src/core/Compile.go Normal file
View file

@ -0,0 +1,4 @@
package core
// Compile turns a function into machine code.
func (f *Function) Compile() {}

11
src/core/Environment.go Normal file
View file

@ -0,0 +1,11 @@
package core
import (
"git.urbach.dev/cli/q/src/fs"
)
// Environment holds information about the entire build.
type Environment struct {
Functions map[string]*Function
Files []*fs.File
}

44
src/core/Function.go Normal file
View file

@ -0,0 +1,44 @@
package core
import (
"fmt"
"git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/token"
)
// Function is the smallest unit of code.
type Function struct {
name string
file *fs.File
Input []*Parameter
Output []*Parameter
Body token.List
}
// NewFunction creates a new function.
func NewFunction(name string, file *fs.File) *Function {
return &Function{
name: name,
file: file,
}
}
// IsExtern returns true if the function has no body.
func (f *Function) IsExtern() bool {
return f.Body == nil
}
// ResolveTypes parses the input and output types.
func (f *Function) ResolveTypes() error {
for _, param := range f.Input {
param.name = param.tokens[0].String(f.file.Bytes)
}
return nil
}
// String returns the package and function name.
func (f *Function) String() string {
return fmt.Sprintf("%s.%s", f.file.Package, f.name)
}

28
src/core/Parameter.go Normal file
View file

@ -0,0 +1,28 @@
package core
import (
"git.urbach.dev/cli/q/src/token"
"git.urbach.dev/cli/q/src/types"
)
// Parameter is an input or output parameter in a function.
type Parameter struct {
name string
typ types.Type
tokens token.List
}
// NewParameter creates a new parameter with the given list of tokens.
func NewParameter(tokens token.List) *Parameter {
return &Parameter{tokens: tokens}
}
// Name returns the name of the parameter.
func (p *Parameter) Name() string {
return p.name
}
// Type returns the type of the parameter.
func (p *Parameter) Type() types.Type {
return p.typ
}

14
src/errors/Common.go Normal file
View file

@ -0,0 +1,14 @@
package errors
var (
ExpectedFunctionDefinition = &String{"Expected function definition"}
ExpectedPackageName = &String{"Expected package name"}
InvalidDefinition = &String{"Invalid definition"}
MissingBlockStart = &String{"Missing '{'"}
MissingBlockEnd = &String{"Missing '}'"}
MissingGroupStart = &String{"Missing '('"}
MissingGroupEnd = &String{"Missing ')'"}
MissingParameter = &String{"Missing parameter"}
MissingType = &String{"Missing type"}
NoInputFiles = &String{"No input files"}
)

View file

@ -0,0 +1,13 @@
package errors
import "fmt"
// InvalidCharacter is created when an invalid character appears.
type InvalidCharacter struct {
Character string
}
// Error implements the error interface.
func (err *InvalidCharacter) Error() string {
return fmt.Sprintf("Invalid character '%s'", err.Character)
}

View file

@ -0,0 +1,13 @@
package errors
import "fmt"
// InvalidTopLevel error is created when a top-level instruction is not valid.
type InvalidTopLevel struct {
Instruction string
}
// Error implements the error interface.
func (err *InvalidTopLevel) Error() string {
return fmt.Sprintf("Invalid top level instruction '%s'", err.Instruction)
}

View file

@ -0,0 +1,13 @@
package errors
import "fmt"
// IsNotDirectory error is created when a path is not a directory.
type IsNotDirectory struct {
Path string
}
// Error implements the error interface.
func (err *IsNotDirectory) Error() string {
return fmt.Sprintf("'%s' is not a directory", err.Path)
}

View file

@ -2,23 +2,26 @@ package scanner
import ( import (
"git.urbach.dev/cli/q/src/build" "git.urbach.dev/cli/q/src/build"
"git.urbach.dev/cli/q/src/core"
"git.urbach.dev/cli/q/src/fs" "git.urbach.dev/cli/q/src/fs"
) )
// Scan scans all the files included in the build. // Scan scans all the files included in the build.
func Scan(b *build.Build) (<-chan *fs.File, <-chan error) { func Scan(b *build.Build) (<-chan *core.Function, <-chan *fs.File, <-chan error) {
s := scanner{ s := scanner{
files: make(chan *fs.File), functions: make(chan *core.Function),
errors: make(chan error), files: make(chan *fs.File),
build: b, errors: make(chan error),
build: b,
} }
go func() { go func() {
s.queue(b.Files...) s.queue(b.Files...)
s.group.Wait() s.group.Wait()
close(s.functions)
close(s.files) close(s.files)
close(s.errors) close(s.errors)
}() }()
return s.files, s.errors return s.functions, s.files, s.errors
} }

View file

@ -4,6 +4,7 @@ import (
"testing" "testing"
"git.urbach.dev/cli/q/src/build" "git.urbach.dev/cli/q/src/build"
"git.urbach.dev/cli/q/src/core"
"git.urbach.dev/cli/q/src/fs" "git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/scanner" "git.urbach.dev/cli/q/src/scanner"
"git.urbach.dev/go/assert" "git.urbach.dev/go/assert"
@ -11,30 +12,45 @@ import (
func TestScanDirectory(t *testing.T) { func TestScanDirectory(t *testing.T) {
b := build.New("testdata") b := build.New("testdata")
files, errors := scanner.Scan(b) functions, files, errors := scanner.Scan(b)
err := consume(t, files, errors) err := consume(t, functions, files, errors)
assert.Nil(t, err) assert.Nil(t, err)
} }
func TestScanFile(t *testing.T) { func TestScanFile(t *testing.T) {
b := build.New("testdata/file.q") b := build.New("testdata/file.q")
files, errors := scanner.Scan(b) functions, files, errors := scanner.Scan(b)
err := consume(t, files, errors) err := consume(t, functions, files, errors)
assert.Nil(t, err) assert.Nil(t, err)
} }
func TestScanNotExisting(t *testing.T) { func TestScanNotExisting(t *testing.T) {
b := build.New("_") b := build.New("_")
files, errors := scanner.Scan(b) functions, files, errors := scanner.Scan(b)
err := consume(t, files, errors) err := consume(t, functions, files, errors)
assert.NotNil(t, err) assert.NotNil(t, err)
} }
func consume(t *testing.T, files <-chan *fs.File, errors <-chan error) error { func TestScanHelloExample(t *testing.T) {
b := build.New("../../examples/hello")
functions, files, errors := scanner.Scan(b)
err := consume(t, functions, files, errors)
assert.Nil(t, err)
}
func consume(t *testing.T, functions <-chan *core.Function, files <-chan *fs.File, errors <-chan error) error {
var lastError error var lastError error
for files != nil || errors != nil { for functions != nil || files != nil || errors != nil {
select { select {
case function, ok := <-functions:
if !ok {
functions = nil
continue
}
t.Log(function)
case file, ok := <-files: case file, ok := <-files:
if !ok { if !ok {
files = nil files = nil

View file

@ -1,41 +0,0 @@
package scanner
import (
"fmt"
"git.urbach.dev/cli/q/src/errors"
)
var (
expectedPackageName = &errors.String{Message: "Expected package name"}
)
// invalidCharacter is created when an invalid character appears.
type invalidCharacter struct {
Character string
}
// Error implements the error interface.
func (err *invalidCharacter) Error() string {
return fmt.Sprintf("Invalid character '%s'", err.Character)
}
// invalidTopLevel error is created when a top-level instruction is not valid.
type invalidTopLevel struct {
Instruction string
}
// Error generates the string representation.
func (err *invalidTopLevel) Error() string {
return fmt.Sprintf("Invalid top level instruction '%s'", err.Instruction)
}
// isNotDirectory error is created when a path is not a directory.
type isNotDirectory struct {
Path string
}
// Error generates the string representation.
func (err *isNotDirectory) Error() string {
return fmt.Sprintf("Import path '%s' is not a directory", err.Path)
}

21
src/scanner/queue.go Normal file
View file

@ -0,0 +1,21 @@
package scanner
import "os"
// queue scans the list of files.
func (s *scanner) queue(files ...string) {
for _, file := range files {
stat, err := os.Stat(file)
if err != nil {
s.errors <- err
return
}
if stat.IsDir() {
s.queueDirectory(file, "main")
} else {
s.queueFile(file, "main")
}
}
}

View file

@ -0,0 +1,80 @@
package scanner
import (
"path/filepath"
"strings"
"git.urbach.dev/cli/q/src/build"
"git.urbach.dev/cli/q/src/fs"
)
// queueDirectory queues an entire directory to be scanned.
func (s *scanner) queueDirectory(directory string, pkg string) {
_, loaded := s.queued.LoadOrStore(directory, nil)
if loaded {
return
}
err := fs.Walk(directory, func(name string) {
if !strings.HasSuffix(name, ".q") {
return
}
tmp := name[:len(name)-2]
for {
underscore := strings.LastIndexByte(tmp, '_')
if underscore == -1 {
break
}
condition := tmp[underscore+1:]
switch condition {
case "linux":
if s.build.OS != build.Linux {
return
}
case "mac":
if s.build.OS != build.Mac {
return
}
case "unix":
if s.build.OS != build.Linux && s.build.OS != build.Mac {
return
}
case "windows":
if s.build.OS != build.Windows {
return
}
case "x86":
if s.build.Arch != build.X86 {
return
}
case "arm":
if s.build.Arch != build.ARM {
return
}
default:
return
}
tmp = tmp[:underscore]
}
fullPath := filepath.Join(directory, name)
s.queueFile(fullPath, pkg)
})
if err != nil {
s.errors <- err
}
}

15
src/scanner/queueFile.go Normal file
View file

@ -0,0 +1,15 @@
package scanner
// queueFile queues a single file to be scanned.
func (s *scanner) queueFile(file string, pkg string) {
s.group.Add(1)
go func() {
defer s.group.Done()
err := s.scanFile(file, pkg)
if err != nil {
s.errors <- err
}
}()
}

52
src/scanner/scanFile.go Normal file
View file

@ -0,0 +1,52 @@
package scanner
import (
"os"
"git.urbach.dev/cli/q/src/errors"
"git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/token"
)
// scanFile scans a single file.
func (s *scanner) scanFile(path string, pkg string) error {
contents, err := os.ReadFile(path)
if err != nil {
return err
}
tokens := token.Tokenize(contents)
file := &fs.File{
Path: path,
Package: pkg,
Bytes: contents,
Tokens: tokens,
}
s.files <- file
for i := 0; i < len(tokens); i++ {
switch tokens[i].Kind {
case token.NewLine:
case token.Comment:
case token.Identifier:
i, err = s.scanFunction(file, tokens, i)
case token.Import:
i, err = s.scanImport(file, tokens, i)
case token.EOF:
return nil
case token.Invalid:
return errors.New(&errors.InvalidCharacter{Character: tokens[i].String(file.Bytes)}, file, tokens[i].Position)
default:
return errors.New(&errors.InvalidTopLevel{Instruction: tokens[i].String(file.Bytes)}, file, tokens[i].Position)
}
if err != nil {
return err
}
}
return nil
}

View file

@ -1,15 +1,77 @@
package scanner package scanner
import ( import (
"git.urbach.dev/cli/q/src/errors"
"git.urbach.dev/cli/q/src/fs" "git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/token" "git.urbach.dev/cli/q/src/token"
) )
// scanFunction scans a function. // scanFunction scans a function.
func (s *scanner) scanFunction(file *fs.File, tokens token.List, i int) (int, error) { func (s *scanner) scanFunction(file *fs.File, tokens token.List, i int) (int, error) {
for i < len(tokens) && tokens[i].Kind != token.BlockEnd { function, i, err := scanSignature(file, tokens, i, token.BlockStart)
i++
if err != nil {
return i, err
} }
var (
blockLevel = 0
bodyStart = -1
)
// Function definition
for i < len(tokens) {
if tokens[i].Kind == token.BlockStart {
blockLevel++
i++
if blockLevel == 1 {
bodyStart = i
}
continue
}
if tokens[i].Kind == token.BlockEnd {
blockLevel--
if blockLevel < 0 {
return i, errors.New(errors.MissingBlockStart, file, tokens[i].Position)
}
if blockLevel == 0 {
break
}
i++
continue
}
if tokens[i].Kind == token.Invalid {
return i, errors.New(&errors.InvalidCharacter{Character: tokens[i].String(file.Bytes)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
if blockLevel > 0 {
return i, errors.New(errors.MissingBlockEnd, file, tokens[i].Position)
}
if bodyStart == -1 {
return i, errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
}
return i, nil
}
if blockLevel > 0 {
i++
continue
}
return i, errors.New(errors.ExpectedFunctionDefinition, file, tokens[i].Position)
}
function.Body = tokens[bodyStart:i]
s.functions <- function
return i, nil return i, nil
} }

View file

@ -15,7 +15,7 @@ func (s *scanner) scanImport(file *fs.File, tokens token.List, i int) (int, erro
i++ i++
if tokens[i].Kind != token.Identifier { if tokens[i].Kind != token.Identifier {
return i, errors.New(expectedPackageName, file, tokens[i].Position) return i, errors.New(errors.ExpectedPackageName, file, tokens[i].Position)
} }
packageName := tokens[i].String(file.Bytes) packageName := tokens[i].String(file.Bytes)
@ -27,7 +27,7 @@ func (s *scanner) scanImport(file *fs.File, tokens token.List, i int) (int, erro
} }
if !stat.IsDir() { if !stat.IsDir() {
return i, errors.New(&isNotDirectory{Path: fullPath}, file, tokens[i].Position) return i, errors.New(&errors.IsNotDirectory{Path: fullPath}, file, tokens[i].Position)
} }
s.queueDirectory(fullPath, packageName) s.queueDirectory(fullPath, packageName)

View file

@ -0,0 +1,118 @@
package scanner
import (
"git.urbach.dev/cli/q/src/core"
"git.urbach.dev/cli/q/src/errors"
"git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/token"
)
// scanSignature scans only the function signature without the body.
func scanSignature(file *fs.File, tokens token.List, i int, delimiter token.Kind) (*core.Function, int, error) {
var (
groupLevel = 0
nameStart = i
paramsStart = -1
paramsEnd = -1
typeStart = -1
typeEnd = -1
)
i++
// Function parameters
for i < len(tokens) {
if tokens[i].Kind == token.GroupStart {
groupLevel++
i++
if groupLevel == 1 {
paramsStart = i
}
continue
}
if tokens[i].Kind == token.GroupEnd {
groupLevel--
if groupLevel < 0 {
return nil, i, errors.New(errors.MissingGroupStart, file, tokens[i].Position)
}
if groupLevel == 0 {
paramsEnd = i
i++
break
}
i++
continue
}
if tokens[i].Kind == token.Invalid {
return nil, i, errors.New(&errors.InvalidCharacter{Character: tokens[i].String(file.Bytes)}, file, tokens[i].Position)
}
if tokens[i].Kind == token.EOF {
if groupLevel > 0 {
return nil, i, errors.New(errors.MissingGroupEnd, file, tokens[i].Position)
}
if paramsStart == -1 {
return nil, i, errors.New(errors.InvalidDefinition, file, tokens[i].Position)
}
return nil, i, nil
}
if groupLevel > 0 {
i++
continue
}
return nil, i, errors.New(errors.InvalidDefinition, file, tokens[i].Position)
}
// Return type
if i < len(tokens) && tokens[i].Kind == token.ReturnType {
typeStart = i + 1
for i < len(tokens) && tokens[i].Kind != delimiter {
i++
}
typeEnd = i
}
name := tokens[nameStart].String(file.Bytes)
function := core.NewFunction(name, file)
parameters := tokens[paramsStart:paramsEnd]
for param := range parameters.Split {
if len(param) == 0 {
return nil, i, errors.New(errors.MissingParameter, file, parameters[0].Position)
}
if len(param) == 1 {
return nil, i, errors.New(errors.MissingType, file, param[0].End())
}
function.Input = append(function.Input, core.NewParameter(param))
}
if typeStart != -1 {
if tokens[typeStart].Kind == token.GroupStart && tokens[typeEnd-1].Kind == token.GroupEnd {
typeStart++
typeEnd--
}
outputTokens := tokens[typeStart:typeEnd]
for param := range outputTokens.Split {
function.Output = append(function.Output, core.NewParameter(param))
}
}
return function, i, nil
}

View file

@ -1,168 +1,19 @@
package scanner package scanner
import ( import (
"os"
"path/filepath"
"strings"
"sync" "sync"
"git.urbach.dev/cli/q/src/build" "git.urbach.dev/cli/q/src/build"
"git.urbach.dev/cli/q/src/errors" "git.urbach.dev/cli/q/src/core"
"git.urbach.dev/cli/q/src/fs" "git.urbach.dev/cli/q/src/fs"
"git.urbach.dev/cli/q/src/token"
) )
// scanner is used to scan files before the actual compilation step. // scanner is used to scan files before the actual compilation step.
type scanner struct { type scanner struct {
files chan *fs.File functions chan *core.Function
errors chan error files chan *fs.File
build *build.Build errors chan error
queued sync.Map build *build.Build
group sync.WaitGroup queued sync.Map
} group sync.WaitGroup
// queue scans the list of files.
func (s *scanner) queue(files ...string) {
for _, file := range files {
stat, err := os.Stat(file)
if err != nil {
s.errors <- err
return
}
if stat.IsDir() {
s.queueDirectory(file, "main")
} else {
s.queueFile(file, "main")
}
}
}
// queueDirectory queues an entire directory to be scanned.
func (s *scanner) queueDirectory(directory string, pkg string) {
_, loaded := s.queued.LoadOrStore(directory, nil)
if loaded {
return
}
err := fs.Walk(directory, func(name string) {
if !strings.HasSuffix(name, ".q") {
return
}
tmp := name[:len(name)-2]
for {
underscore := strings.LastIndexByte(tmp, '_')
if underscore == -1 {
break
}
condition := tmp[underscore+1:]
switch condition {
case "linux":
if s.build.OS != build.Linux {
return
}
case "mac":
if s.build.OS != build.Mac {
return
}
case "unix":
if s.build.OS != build.Linux && s.build.OS != build.Mac {
return
}
case "windows":
if s.build.OS != build.Windows {
return
}
case "x86":
if s.build.Arch != build.X86 {
return
}
case "arm":
if s.build.Arch != build.ARM {
return
}
default:
return
}
tmp = tmp[:underscore]
}
fullPath := filepath.Join(directory, name)
s.queueFile(fullPath, pkg)
})
if err != nil {
s.errors <- err
}
}
// queueFile queues a single file to be scanned.
func (s *scanner) queueFile(file string, pkg string) {
s.group.Add(1)
go func() {
defer s.group.Done()
err := s.scanFile(file, pkg)
if err != nil {
s.errors <- err
}
}()
}
// scanFile scans a single file.
func (s *scanner) scanFile(path string, pkg string) error {
contents, err := os.ReadFile(path)
if err != nil {
return err
}
tokens := token.Tokenize(contents)
file := &fs.File{
Path: path,
Package: pkg,
Bytes: contents,
Tokens: tokens,
}
s.files <- file
for i := 0; i < len(tokens); i++ {
switch tokens[i].Kind {
case token.NewLine:
case token.Comment:
case token.Identifier:
i, err = s.scanFunction(file, tokens, i)
case token.Import:
i, err = s.scanImport(file, tokens, i)
case token.EOF:
return nil
case token.Invalid:
return errors.New(&invalidCharacter{Character: tokens[i].String(file.Bytes)}, file, tokens[i].Position)
default:
return errors.New(&invalidTopLevel{Instruction: tokens[i].String(file.Bytes)}, file, tokens[i].Position)
}
if err != nil {
return err
}
}
return nil
} }

3
src/types/Type.go Normal file
View file

@ -0,0 +1,3 @@
package types
type Type interface{}