From 154893d9f75f46018a7e1869c61556872a4da11e Mon Sep 17 00:00:00 2001 From: Eduard Urbach Date: Thu, 19 Jun 2025 15:17:50 +0200 Subject: [PATCH] Implemented a basic scanner --- src/compiler/Compile.go | 28 +++++- src/compiler/Compile_test.go | 6 ++ src/fs/File.go | 8 ++ src/fs/Walk_fast.go | 61 ++++++++++++ src/fs/Walk_slow.go | 28 ++++++ src/fs/Walk_test.go | 29 ++++++ src/fs/bench_test.go | 42 ++++++++ src/scanner/Result.go | 4 - src/scanner/Scan.go | 18 +++- src/scanner/Scan_test.go | 50 +++++++++- src/scanner/scanner.go | 147 ++++++++++++++++++++++++++++ src/scanner/testdata/file.q | 0 src/scanner/testdata/file_arm.q | 0 src/scanner/testdata/file_custom.q | 0 src/scanner/testdata/file_linux.q | 0 src/scanner/testdata/file_mac.q | 0 src/scanner/testdata/file_unix.q | 0 src/scanner/testdata/file_windows.q | 0 src/scanner/testdata/file_x86.q | 0 19 files changed, 410 insertions(+), 11 deletions(-) create mode 100644 src/fs/File.go create mode 100644 src/fs/Walk_fast.go create mode 100644 src/fs/Walk_slow.go create mode 100644 src/fs/Walk_test.go create mode 100644 src/fs/bench_test.go delete mode 100644 src/scanner/Result.go create mode 100644 src/scanner/scanner.go create mode 100644 src/scanner/testdata/file.q create mode 100644 src/scanner/testdata/file_arm.q create mode 100644 src/scanner/testdata/file_custom.q create mode 100644 src/scanner/testdata/file_linux.q create mode 100644 src/scanner/testdata/file_mac.q create mode 100644 src/scanner/testdata/file_unix.q create mode 100644 src/scanner/testdata/file_windows.q create mode 100644 src/scanner/testdata/file_x86.q diff --git a/src/compiler/Compile.go b/src/compiler/Compile.go index 4cf57e6..a126172 100644 --- a/src/compiler/Compile.go +++ b/src/compiler/Compile.go @@ -1,12 +1,36 @@ package compiler import ( + "fmt" + "git.urbach.dev/cli/q/src/build" "git.urbach.dev/cli/q/src/scanner" ) // Compile waits for the scan to finish and compiles all functions. func Compile(b *build.Build) (Result, error) { - scanner.Scan(b) - return Result{}, nil + result := Result{} + files, errors := scanner.Scan(b) + + for files != nil || errors != nil { + select { + case file, ok := <-files: + if !ok { + files = nil + continue + } + + fmt.Println(file.Path) + + case err, ok := <-errors: + if !ok { + errors = nil + continue + } + + return result, err + } + } + + return result, nil } \ No newline at end of file diff --git a/src/compiler/Compile_test.go b/src/compiler/Compile_test.go index 926ef78..cb4cafe 100644 --- a/src/compiler/Compile_test.go +++ b/src/compiler/Compile_test.go @@ -12,4 +12,10 @@ func TestCompile(t *testing.T) { b := build.New("../../examples/hello") _, err := compiler.Compile(b) assert.Nil(t, err) +} + +func TestCompileNotExisting(t *testing.T) { + b := build.New("_") + _, err := compiler.Compile(b) + assert.NotNil(t, err) } \ No newline at end of file diff --git a/src/fs/File.go b/src/fs/File.go new file mode 100644 index 0000000..ddbc618 --- /dev/null +++ b/src/fs/File.go @@ -0,0 +1,8 @@ +package fs + +// File represents a single source file. +type File struct { + Path string + Package string + Bytes []byte +} \ No newline at end of file diff --git a/src/fs/Walk_fast.go b/src/fs/Walk_fast.go new file mode 100644 index 0000000..e686f55 --- /dev/null +++ b/src/fs/Walk_fast.go @@ -0,0 +1,61 @@ +//go:build linux || darwin + +package fs + +import ( + "syscall" + "unsafe" +) + +// Walk calls your callback function for every file name inside the directory. +// It doesn't distinguish between files and directories. +func Walk(directory string, callBack func(string)) error { + fd, err := syscall.Open(directory, 0, 0) + + if err != nil { + return err + } + + defer syscall.Close(fd) + buffer := make([]byte, 1024) + + for { + n, err := syscall.ReadDirent(fd, buffer) + + if err != nil { + return err + } + + if n <= 0 { + break + } + + readBuffer := buffer[:n] + + for len(readBuffer) > 0 { + dirent := (*syscall.Dirent)(unsafe.Pointer(&readBuffer[0])) + readBuffer = readBuffer[dirent.Reclen:] + + if dirent.Ino == 0 { + continue + } + + if dirent.Name[0] == '.' { + continue + } + + for i, c := range dirent.Name { + if c != 0 { + continue + } + + bytePointer := (*byte)(unsafe.Pointer(&dirent.Name[0])) + name := unsafe.String(bytePointer, i) + callBack(name) + break + } + } + } + + return nil +} \ No newline at end of file diff --git a/src/fs/Walk_slow.go b/src/fs/Walk_slow.go new file mode 100644 index 0000000..2c11962 --- /dev/null +++ b/src/fs/Walk_slow.go @@ -0,0 +1,28 @@ +//go:build !linux && !darwin + +package fs + +import "os" + +// Walk calls your callback function for every file name inside the directory. +// It doesn't distinguish between files and directories. +func Walk(directory string, callBack func(string)) error { + f, err := os.Open(directory) + + if err != nil { + return err + } + + files, err := f.Readdirnames(0) + f.Close() + + if err != nil { + return err + } + + for _, file := range files { + callBack(file) + } + + return nil +} \ No newline at end of file diff --git a/src/fs/Walk_test.go b/src/fs/Walk_test.go new file mode 100644 index 0000000..1a59e7e --- /dev/null +++ b/src/fs/Walk_test.go @@ -0,0 +1,29 @@ +package fs_test + +import ( + "testing" + + "git.urbach.dev/cli/q/src/fs" + "git.urbach.dev/go/assert" +) + +func TestWalk(t *testing.T) { + var files []string + + err := fs.Walk(".", func(file string) { + files = append(files, file) + }) + + assert.Nil(t, err) + assert.Contains(t, files, "Walk_test.go") +} + +func TestWalkNotDirectory(t *testing.T) { + err := fs.Walk("Walk_test.go", func(file string) {}) + assert.NotNil(t, err) +} + +func TestWalkNotExisting(t *testing.T) { + err := fs.Walk("_", func(file string) {}) + assert.NotNil(t, err) +} \ No newline at end of file diff --git a/src/fs/bench_test.go b/src/fs/bench_test.go new file mode 100644 index 0000000..a8202db --- /dev/null +++ b/src/fs/bench_test.go @@ -0,0 +1,42 @@ +package fs_test + +import ( + "os" + "testing" + + "git.urbach.dev/cli/q/src/fs" + "git.urbach.dev/go/assert" +) + +func BenchmarkReadDir(b *testing.B) { + for b.Loop() { + files, err := os.ReadDir(".") + assert.Nil(b, err) + + for _, file := range files { + func(string) {}(file.Name()) + } + } +} + +func BenchmarkReaddirnames(b *testing.B) { + for b.Loop() { + f, err := os.Open(".") + assert.Nil(b, err) + files, err := f.Readdirnames(0) + assert.Nil(b, err) + + for _, file := range files { + func(string) {}(file) + } + + f.Close() + } +} + +func BenchmarkWalk(b *testing.B) { + for b.Loop() { + err := fs.Walk(".", func(string) {}) + assert.Nil(b, err) + } +} \ No newline at end of file diff --git a/src/scanner/Result.go b/src/scanner/Result.go deleted file mode 100644 index 69aa88f..0000000 --- a/src/scanner/Result.go +++ /dev/null @@ -1,4 +0,0 @@ -package scanner - -// Result contains everything the compiler needs to start a build. -type Result struct{} \ No newline at end of file diff --git a/src/scanner/Scan.go b/src/scanner/Scan.go index 4b6ed00..f677029 100644 --- a/src/scanner/Scan.go +++ b/src/scanner/Scan.go @@ -2,9 +2,23 @@ package scanner import ( "git.urbach.dev/cli/q/src/build" + "git.urbach.dev/cli/q/src/fs" ) // Scan scans all the files included in the build. -func Scan(b *build.Build) Result { - return Result{} +func Scan(b *build.Build) (<-chan *fs.File, <-chan error) { + s := scanner{ + files: make(chan *fs.File), + errors: make(chan error), + build: b, + } + + go func() { + s.queue(b.Files...) + s.group.Wait() + close(s.files) + close(s.errors) + }() + + return s.files, s.errors } \ No newline at end of file diff --git a/src/scanner/Scan_test.go b/src/scanner/Scan_test.go index 72a0502..3d90f40 100644 --- a/src/scanner/Scan_test.go +++ b/src/scanner/Scan_test.go @@ -4,10 +4,54 @@ import ( "testing" "git.urbach.dev/cli/q/src/build" + "git.urbach.dev/cli/q/src/fs" "git.urbach.dev/cli/q/src/scanner" + "git.urbach.dev/go/assert" ) -func TestScan(t *testing.T) { - b := build.New("../../examples/hello") - scanner.Scan(b) +func TestScanDirectory(t *testing.T) { + b := build.New("testdata") + files, errors := scanner.Scan(b) + err := consume(t, files, errors) + assert.Nil(t, err) +} + +func TestScanFile(t *testing.T) { + b := build.New("testdata/file.q") + files, errors := scanner.Scan(b) + err := consume(t, files, errors) + assert.Nil(t, err) +} + +func TestScanNotExisting(t *testing.T) { + b := build.New("_") + files, errors := scanner.Scan(b) + err := consume(t, files, errors) + assert.NotNil(t, err) +} + +func consume(t *testing.T, files <-chan *fs.File, errors <-chan error) error { + var lastError error + + for files != nil || errors != nil { + select { + case file, ok := <-files: + if !ok { + files = nil + continue + } + + t.Log(file) + + case err, ok := <-errors: + if !ok { + errors = nil + continue + } + + lastError = err + } + } + + return lastError } \ No newline at end of file diff --git a/src/scanner/scanner.go b/src/scanner/scanner.go new file mode 100644 index 0000000..f0f0ae5 --- /dev/null +++ b/src/scanner/scanner.go @@ -0,0 +1,147 @@ +package scanner + +import ( + "os" + "path/filepath" + "strings" + "sync" + + "git.urbach.dev/cli/q/src/build" + "git.urbach.dev/cli/q/src/fs" +) + +// scanner is used to scan files before the actual compilation step. +type scanner struct { + files chan *fs.File + errors chan error + build *build.Build + queued sync.Map + group sync.WaitGroup +} + +// queue scans the list of files. +func (s *scanner) queue(files ...string) { + for _, file := range files { + stat, err := os.Stat(file) + + if err != nil { + s.errors <- err + return + } + + if stat.IsDir() { + s.queueDirectory(file, "main") + } else { + s.queueFile(file, "main") + } + } +} + +// queueDirectory queues an entire directory to be scanned. +func (s *scanner) queueDirectory(directory string, pkg string) { + _, loaded := s.queued.LoadOrStore(directory, nil) + + if loaded { + return + } + + err := fs.Walk(directory, func(name string) { + if !strings.HasSuffix(name, ".q") { + return + } + + tmp := name[:len(name)-2] + + for { + underscore := strings.LastIndexByte(tmp, '_') + + if underscore == -1 { + break + } + + condition := tmp[underscore+1:] + + switch condition { + case "linux": + if s.build.OS != build.Linux { + return + } + + case "mac": + if s.build.OS != build.Mac { + return + } + + case "unix": + if s.build.OS != build.Linux && s.build.OS != build.Mac { + return + } + + case "windows": + if s.build.OS != build.Windows { + return + } + + case "x86": + if s.build.Arch != build.X86 { + return + } + + case "arm": + if s.build.Arch != build.ARM { + return + } + + default: + return + } + + tmp = tmp[:underscore] + } + + fullPath := filepath.Join(directory, name) + s.queueFile(fullPath, pkg) + }) + + if err != nil { + s.errors <- err + } +} + +// queueFile queues a single file to be scanned. +func (s *scanner) queueFile(file string, pkg string) { + _, loaded := s.queued.LoadOrStore(file, nil) + + if loaded { + return + } + + s.group.Add(1) + + go func() { + defer s.group.Done() + err := s.scanFile(file, pkg) + + if err != nil { + s.errors <- err + } + }() +} + +// scanFile scans a single file. +func (s *scanner) scanFile(path string, pkg string) error { + contents, err := os.ReadFile(path) + + if err != nil { + return err + } + + file := &fs.File{ + Path: path, + Package: pkg, + Bytes: contents, + } + + s.files <- file + return nil +} \ No newline at end of file diff --git a/src/scanner/testdata/file.q b/src/scanner/testdata/file.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_arm.q b/src/scanner/testdata/file_arm.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_custom.q b/src/scanner/testdata/file_custom.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_linux.q b/src/scanner/testdata/file_linux.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_mac.q b/src/scanner/testdata/file_mac.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_unix.q b/src/scanner/testdata/file_unix.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_windows.q b/src/scanner/testdata/file_windows.q new file mode 100644 index 0000000..e69de29 diff --git a/src/scanner/testdata/file_x86.q b/src/scanner/testdata/file_x86.q new file mode 100644 index 0000000..e69de29