From bb460c918c77174600134b75618c5bd289e075ef Mon Sep 17 00:00:00 2001
From: Eduard Urbach <eduard@urbach.dev>
Date: Wed, 19 Mar 2025 22:56:17 +0100
Subject: [PATCH] Improved tokenizer

---
 src/scanner/scanFile.go |  2 +-
 src/token/Reader.go     | 12 +++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/scanner/scanFile.go b/src/scanner/scanFile.go
index faed4d6..19141b6 100644
--- a/src/scanner/scanFile.go
+++ b/src/scanner/scanFile.go
@@ -15,8 +15,8 @@ func (s *Scanner) scanFile(path string, pkg string) error {
 		return err
 	}
 
-	defer reader.File.Close()
 	tokens := token.Tokenize(&reader)
+	reader.File.Close()
 
 	file := &fs.File{
 		Path:    path,
diff --git a/src/token/Reader.go b/src/token/Reader.go
index 3a77cc0..318a7a1 100644
--- a/src/token/Reader.go
+++ b/src/token/Reader.go
@@ -37,7 +37,8 @@ func (t *Reader) read() error {
 		return nil
 	}
 
-	n, err := t.File.Read(t.Buffer[len(t.Buffer):cap(t.Buffer)])
+	end := min(len(t.Buffer)+4096, cap(t.Buffer))
+	n, err := t.File.Read(t.Buffer[len(t.Buffer):end])
 	t.Buffer = t.Buffer[:len(t.Buffer)+n]
 
 	if err != nil {
@@ -56,20 +57,21 @@ func (t *Reader) read() error {
 	return nil
 }
 
-func (t *Reader) Open(path string) (err error) {
-	t.File, err = os.Open(path)
+func (t *Reader) Open(path string) error {
+	f, err := os.Open(path)
 
 	if err != nil {
 		return err
 	}
 
-	info, err := t.File.Stat()
+	info, err := f.Stat()
 
 	if err != nil {
 		return err
 	}
 
+	t.File = f
 	t.Size = Position(info.Size())
-	t.Buffer = make([]byte, 0, 4096)
+	t.Buffer = make([]byte, 0, t.Size+1)
 	return t.read()
 }