Implemented a basic scanner

src/compiler/Compile.go

@@ -1,12 +1,36 @@
 package compiler
 
 import (
+	"fmt"
+
 	"git.urbach.dev/cli/q/src/build"
 	"git.urbach.dev/cli/q/src/scanner"
 )
 
 // Compile waits for the scan to finish and compiles all functions.
 func Compile(b *build.Build) (Result, error) {
-	scanner.Scan(b)
-	return Result{}, nil
+	result := Result{}
+	files, errors := scanner.Scan(b)
+
+	for files != nil || errors != nil {
+		select {
+		case file, ok := <-files:
+			if !ok {
+				files = nil
+				continue
+			}
+
+			fmt.Println(file.Path)
+
+		case err, ok := <-errors:
+			if !ok {
+				errors = nil
+				continue
+			}
+
+			return result, err
+		}
+	}
+
+	return result, nil
 }

src/compiler/Compile_test.go

@@ -12,4 +12,10 @@ func TestCompile(t *testing.T) {
 	b := build.New("../../examples/hello")
 	_, err := compiler.Compile(b)
 	assert.Nil(t, err)
+}
+
+func TestCompileNotExisting(t *testing.T) {
+	b := build.New("_")
+	_, err := compiler.Compile(b)
+	assert.NotNil(t, err)
 }

src/fs/File.go

@@ -0,0 +1,8 @@
+package fs
+
+// File represents a single source file.
+type File struct {
+	Path    string
+	Package string
+	Bytes   []byte
+}

src/fs/Walk_fast.go

@@ -0,0 +1,61 @@
+//go:build linux || darwin
+
+package fs
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Walk calls your callback function for every file name inside the directory.
+// It doesn't distinguish between files and directories.
+func Walk(directory string, callBack func(string)) error {
+	fd, err := syscall.Open(directory, 0, 0)
+
+	if err != nil {
+		return err
+	}
+
+	defer syscall.Close(fd)
+	buffer := make([]byte, 1024)
+
+	for {
+		n, err := syscall.ReadDirent(fd, buffer)
+
+		if err != nil {
+			return err
+		}
+
+		if n <= 0 {
+			break
+		}
+
+		readBuffer := buffer[:n]
+
+		for len(readBuffer) > 0 {
+			dirent := (*syscall.Dirent)(unsafe.Pointer(&readBuffer[0]))
+			readBuffer = readBuffer[dirent.Reclen:]
+
+			if dirent.Ino == 0 {
+				continue
+			}
+
+			if dirent.Name[0] == '.' {
+				continue
+			}
+
+			for i, c := range dirent.Name {
+				if c != 0 {
+					continue
+				}
+
+				bytePointer := (*byte)(unsafe.Pointer(&dirent.Name[0]))
+				name := unsafe.String(bytePointer, i)
+				callBack(name)
+				break
+			}
+		}
+	}
+
+	return nil
+}

src/fs/Walk_slow.go

@@ -0,0 +1,28 @@
+//go:build !linux && !darwin
+
+package fs
+
+import "os"
+
+// Walk calls your callback function for every file name inside the directory.
+// It doesn't distinguish between files and directories.
+func Walk(directory string, callBack func(string)) error {
+	f, err := os.Open(directory)
+
+	if err != nil {
+		return err
+	}
+
+	files, err := f.Readdirnames(0)
+	f.Close()
+
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		callBack(file)
+	}
+
+	return nil
+}

src/fs/Walk_test.go

@@ -0,0 +1,29 @@
+package fs_test
+
+import (
+	"testing"
+
+	"git.urbach.dev/cli/q/src/fs"
+	"git.urbach.dev/go/assert"
+)
+
+func TestWalk(t *testing.T) {
+	var files []string
+
+	err := fs.Walk(".", func(file string) {
+		files = append(files, file)
+	})
+
+	assert.Nil(t, err)
+	assert.Contains(t, files, "Walk_test.go")
+}
+
+func TestWalkNotDirectory(t *testing.T) {
+	err := fs.Walk("Walk_test.go", func(file string) {})
+	assert.NotNil(t, err)
+}
+
+func TestWalkNotExisting(t *testing.T) {
+	err := fs.Walk("_", func(file string) {})
+	assert.NotNil(t, err)
+}

src/fs/bench_test.go

@@ -0,0 +1,42 @@
+package fs_test
+
+import (
+	"os"
+	"testing"
+
+	"git.urbach.dev/cli/q/src/fs"
+	"git.urbach.dev/go/assert"
+)
+
+func BenchmarkReadDir(b *testing.B) {
+	for b.Loop() {
+		files, err := os.ReadDir(".")
+		assert.Nil(b, err)
+
+		for _, file := range files {
+			func(string) {}(file.Name())
+		}
+	}
+}
+
+func BenchmarkReaddirnames(b *testing.B) {
+	for b.Loop() {
+		f, err := os.Open(".")
+		assert.Nil(b, err)
+		files, err := f.Readdirnames(0)
+		assert.Nil(b, err)
+
+		for _, file := range files {
+			func(string) {}(file)
+		}
+
+		f.Close()
+	}
+}
+
+func BenchmarkWalk(b *testing.B) {
+	for b.Loop() {
+		err := fs.Walk(".", func(string) {})
+		assert.Nil(b, err)
+	}
+}

src/scanner/Result.go

@@ -1,4 +0,0 @@
-package scanner
-
-// Result contains everything the compiler needs to start a build.
-type Result struct{}

src/scanner/Scan.go

@@ -2,9 +2,23 @@ package scanner
 
 import (
 	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/fs"
 )
 
 // Scan scans all the files included in the build.
-func Scan(b *build.Build) Result {
-	return Result{}
+func Scan(b *build.Build) (<-chan *fs.File, <-chan error) {
+	s := scanner{
+		files:  make(chan *fs.File),
+		errors: make(chan error),
+		build:  b,
+	}
+
+	go func() {
+		s.queue(b.Files...)
+		s.group.Wait()
+		close(s.files)
+		close(s.errors)
+	}()
+
+	return s.files, s.errors
 }

src/scanner/Scan_test.go

@@ -4,10 +4,54 @@ import (
 	"testing"
 
 	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/fs"
 	"git.urbach.dev/cli/q/src/scanner"
+	"git.urbach.dev/go/assert"
 )
 
-func TestScan(t *testing.T) {
-	b := build.New("../../examples/hello")
-	scanner.Scan(b)
+func TestScanDirectory(t *testing.T) {
+	b := build.New("testdata")
+	files, errors := scanner.Scan(b)
+	err := consume(t, files, errors)
+	assert.Nil(t, err)
+}
+
+func TestScanFile(t *testing.T) {
+	b := build.New("testdata/file.q")
+	files, errors := scanner.Scan(b)
+	err := consume(t, files, errors)
+	assert.Nil(t, err)
+}
+
+func TestScanNotExisting(t *testing.T) {
+	b := build.New("_")
+	files, errors := scanner.Scan(b)
+	err := consume(t, files, errors)
+	assert.NotNil(t, err)
+}
+
+func consume(t *testing.T, files <-chan *fs.File, errors <-chan error) error {
+	var lastError error
+
+	for files != nil || errors != nil {
+		select {
+		case file, ok := <-files:
+			if !ok {
+				files = nil
+				continue
+			}
+
+			t.Log(file)
+
+		case err, ok := <-errors:
+			if !ok {
+				errors = nil
+				continue
+			}
+
+			lastError = err
+		}
+	}
+
+	return lastError
 }

src/scanner/scanner.go

@@ -0,0 +1,147 @@
+package scanner
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/fs"
+)
+
+// scanner is used to scan files before the actual compilation step.
+type scanner struct {
+	files  chan *fs.File
+	errors chan error
+	build  *build.Build
+	queued sync.Map
+	group  sync.WaitGroup
+}
+
+// queue scans the list of files.
+func (s *scanner) queue(files ...string) {
+	for _, file := range files {
+		stat, err := os.Stat(file)
+
+		if err != nil {
+			s.errors <- err
+			return
+		}
+
+		if stat.IsDir() {
+			s.queueDirectory(file, "main")
+		} else {
+			s.queueFile(file, "main")
+		}
+	}
+}
+
+// queueDirectory queues an entire directory to be scanned.
+func (s *scanner) queueDirectory(directory string, pkg string) {
+	_, loaded := s.queued.LoadOrStore(directory, nil)
+
+	if loaded {
+		return
+	}
+
+	err := fs.Walk(directory, func(name string) {
+		if !strings.HasSuffix(name, ".q") {
+			return
+		}
+
+		tmp := name[:len(name)-2]
+
+		for {
+			underscore := strings.LastIndexByte(tmp, '_')
+
+			if underscore == -1 {
+				break
+			}
+
+			condition := tmp[underscore+1:]
+
+			switch condition {
+			case "linux":
+				if s.build.OS != build.Linux {
+					return
+				}
+
+			case "mac":
+				if s.build.OS != build.Mac {
+					return
+				}
+
+			case "unix":
+				if s.build.OS != build.Linux && s.build.OS != build.Mac {
+					return
+				}
+
+			case "windows":
+				if s.build.OS != build.Windows {
+					return
+				}
+
+			case "x86":
+				if s.build.Arch != build.X86 {
+					return
+				}
+
+			case "arm":
+				if s.build.Arch != build.ARM {
+					return
+				}
+
+			default:
+				return
+			}
+
+			tmp = tmp[:underscore]
+		}
+
+		fullPath := filepath.Join(directory, name)
+		s.queueFile(fullPath, pkg)
+	})
+
+	if err != nil {
+		s.errors <- err
+	}
+}
+
+// queueFile queues a single file to be scanned.
+func (s *scanner) queueFile(file string, pkg string) {
+	_, loaded := s.queued.LoadOrStore(file, nil)
+
+	if loaded {
+		return
+	}
+
+	s.group.Add(1)
+
+	go func() {
+		defer s.group.Done()
+		err := s.scanFile(file, pkg)
+
+		if err != nil {
+			s.errors <- err
+		}
+	}()
+}
+
+// scanFile scans a single file.
+func (s *scanner) scanFile(path string, pkg string) error {
+	contents, err := os.ReadFile(path)
+
+	if err != nil {
+		return err
+	}
+
+	file := &fs.File{
+		Path:    path,
+		Package: pkg,
+		Bytes:   contents,
+	}
+
+	s.files <- file
+	return nil
+}