Improved tokenizer

src/compiler/Scan.go

@@ -0,0 +1,105 @@
+package compiler
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"git.akyoto.dev/cli/q/src/directory"
+	"git.akyoto.dev/cli/q/src/token"
+)
+
+// Scan scans the directory.
+func Scan(path string) (<-chan *Function, <-chan error) {
+	functions := make(chan *Function, 16)
+	errors := make(chan error)
+
+	go func() {
+		scan(path, functions, errors)
+		close(functions)
+		close(errors)
+	}()
+
+	return functions, errors
+}
+
+// scan scans the directory without channel allocations.
+func scan(path string, functions chan<- *Function, errors chan<- error) {
+	wg := sync.WaitGroup{}
+
+	directory.Walk(path, func(name string) {
+		if !strings.HasSuffix(name, ".q") {
+			return
+		}
+
+		fullPath := filepath.Join(path, name)
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+			err := scanFile(fullPath, functions)
+
+			if err != nil {
+				errors <- err
+			}
+		}()
+	})
+
+	wg.Wait()
+}
+
+// scanFile scans a single file.
+func scanFile(path string, functions chan<- *Function) error {
+	contents, err := os.ReadFile(path)
+
+	if err != nil {
+		return err
+	}
+
+	tokens := token.Tokenize(contents)
+
+	var (
+		groupLevel  = 0
+		blockLevel  = 0
+		headerStart = -1
+		bodyStart   = -1
+	)
+
+	for i, t := range tokens {
+		switch t.Kind {
+		case token.Identifier:
+			if blockLevel == 0 && groupLevel == 0 {
+				headerStart = i
+			}
+
+		case token.GroupStart:
+			groupLevel++
+
+		case token.GroupEnd:
+			groupLevel--
+
+		case token.BlockStart:
+			blockLevel++
+
+			if blockLevel == 1 {
+				bodyStart = i
+			}
+
+		case token.BlockEnd:
+			blockLevel--
+
+			if blockLevel == 0 {
+				function := &Function{
+					Name: tokens[headerStart].String(),
+					Head: tokens[headerStart:bodyStart],
+					Body: tokens[bodyStart : i+1],
+				}
+
+				functions <- function
+			}
+		}
+	}
+
+	return nil
+}