Added a linker

docs/elf.md

@@ -0,0 +1,37 @@
+# ELF
+
+## Basic structure
+
+1. ELF header [0x00 : 0x40]
+2. Program headers [0x40 : 0xB0]
+3. Padding
+4. Executable code [0x1000 : ...]
+5. Padding
+6. Read-only data [0x2000 : ...]
+7. String table
+8. Section headers
+
+## Entry point
+
+The executables are compiled as position-independent executables (PIE). Therefore the entry point is defined as a file offset instead of a static virtual address.
+
+## Padding
+
+To ensure that execution permissions are properly applied,
+the code section and the data section are aligned on page boundaries. Permissions like read, write and execute can only be applied to an entire page in memory.
+
+## Initialization in Linux
+
+ELF loader:
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/fs/binfmt_elf.c
+
+ELF register definitions:
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/include/asm/elf.h
+
+## Links
+
+- https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/elf.h
+- https://lwn.net/Articles/631631/
+- https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
+- https://www.muppetlabs.com/~breadbox/software/tiny/teensy.html
+- https://nathanotterness.com/2021/10/tiny_elf_modernized.html

src/build/Build.go

@@ -2,9 +2,11 @@ package build
 
 // Build describes the parameters for the "build" command.
 type Build struct {
-	Files   []string
+	Files       []string
-	Arch    Arch
+	Arch        Arch
-	OS      OS
+	OS          OS
-	Dry     bool
+	FileAlign   int
-	ShowSSA bool
+	MemoryAlign int
+	Dry         bool
+	ShowSSA     bool
 }

src/build/New.go

@@ -12,9 +12,9 @@ func New(files ...string) *Build {
 
 	switch global.Arch {
 	case "amd64":
-		b.Arch = X86
+		b.SetArch(X86)
 	case "arm64":
-		b.Arch = ARM
+		b.SetArch(ARM)
 	}
 
 	switch global.OS {

src/build/SetArch.go

@@ -0,0 +1,15 @@
+package build
+
+// SetArch sets the architecture which also influences the default alignment.
+func (build *Build) SetArch(arch Arch) {
+	build.Arch = arch
+
+	switch arch {
+	case ARM:
+		build.MemoryAlign = 0x4000
+	default:
+		build.MemoryAlign = 0x1000
+	}
+
+	build.FileAlign = build.MemoryAlign
+}

src/cli/build.go

@@ -6,6 +6,7 @@ import (
 
 	"git.urbach.dev/cli/q/src/build"
 	"git.urbach.dev/cli/q/src/compiler"
+	"git.urbach.dev/cli/q/src/linker"
 )
 
 // _build parses the arguments and creates a build.
@@ -16,13 +17,18 @@ func _build(args []string) int {
 		return exit(err)
 	}
 
-	_, err = compiler.Compile(b)
+	result, err := compiler.Compile(b)
 
 	if err != nil {
 		return exit(err)
 	}
 
-	return 0
+	if b.Dry {
+		return 0
+	}
+
+	err = linker.WriteExecutable(b, result)
+	return exit(err)
 }
 
 // newBuildFromArgs creates a new build with the given arguments.
@@ -40,9 +46,9 @@ func newBuildFromArgs(args []string) (*build.Build, error) {
 
 			switch args[i] {
 			case "arm":
-				b.Arch = build.ARM
+				b.SetArch(build.ARM)
 			case "x86":
-				b.Arch = build.X86
+				b.SetArch(build.X86)
 			default:
 				return b, &invalidValueError{Value: args[i], Parameter: "arch"}
 			}

src/cli/exit.go

@@ -9,6 +9,10 @@ import (
 
 // exit returns the exit code depending on the error type.
 func exit(err error) int {
+	if err == nil {
+		return 0
+	}
+
 	fmt.Fprintln(os.Stderr, err)
 
 	var (

src/elf/AddSections.go

@@ -0,0 +1,37 @@
+package elf
+
+import "bytes"
+
+// AddSections adds section headers to the ELF file.
+func (elf *ELF) AddSections() {
+	elf.StringTable = []byte("\000.text\000.shstrtab\000")
+	stringTableStart := elf.DataHeader.Offset + elf.DataHeader.SizeInFile
+	sectionHeaderStart := stringTableStart + int64(len(elf.StringTable))
+
+	elf.SectionHeaders = []SectionHeader{
+		{
+			Type: SectionTypeNULL,
+		},
+		{
+			NameIndex:      int32(bytes.Index(elf.StringTable, []byte(".text\000"))),
+			Type:           SectionTypePROGBITS,
+			Flags:          SectionFlagsAllocate | SectionFlagsExecutable,
+			VirtualAddress: elf.CodeHeader.VirtualAddress,
+			Offset:         elf.CodeHeader.Offset,
+			SizeInFile:     elf.CodeHeader.SizeInFile,
+			Align:          elf.CodeHeader.Align,
+		},
+		{
+			NameIndex:  int32(bytes.Index(elf.StringTable, []byte(".shstrtab\000"))),
+			Type:       SectionTypeSTRTAB,
+			Offset:     int64(stringTableStart),
+			SizeInFile: int64(len(elf.StringTable)),
+			Align:      1,
+		},
+	}
+
+	elf.SectionHeaderEntrySize = SectionHeaderSize
+	elf.SectionHeaderEntryCount = int16(len(elf.SectionHeaders))
+	elf.SectionHeaderOffset = int64(sectionHeaderStart)
+	elf.SectionNameStringTableIndex = 2
+}

src/elf/Arch.go

@@ -0,0 +1,15 @@
+package elf
+
+import "git.urbach.dev/cli/q/src/build"
+
+// Arch converts the architecture variable to an ELF-specific constant.
+func Arch(arch build.Arch) int16 {
+	switch arch {
+	case build.ARM:
+		return ArchitectureARM64
+	case build.X86:
+		return ArchitectureAMD64
+	default:
+		return 0
+	}
+}

src/elf/Constants.go

@@ -0,0 +1,11 @@
+package elf
+
+const (
+	LittleEndian      = 1
+	TypeExecutable    = 2
+	TypeDynamic       = 3
+	ArchitectureAMD64 = 0x3E
+	ArchitectureARM64 = 0xB7
+	ArchitectureRISCV = 0xF3
+	HeaderEnd         = HeaderSize + ProgramHeaderSize*2
+)

src/elf/ELF.go

@@ -0,0 +1,79 @@
+package elf
+
+import (
+	"encoding/binary"
+	"io"
+
+	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/exe"
+)
+
+// ELF represents an ELF file.
+type ELF struct {
+	Header
+	CodeHeader     ProgramHeader
+	DataHeader     ProgramHeader
+	SectionHeaders []SectionHeader
+	StringTable    []byte
+}
+
+// Write writes the ELF64 format to the given writer.
+func Write(writer io.WriteSeeker, b *build.Build, codeBytes []byte, dataBytes []byte) {
+	x := exe.New(HeaderEnd, b.FileAlign, b.MemoryAlign)
+	x.InitSections(codeBytes, dataBytes)
+	code := x.Sections[0]
+	data := x.Sections[1]
+
+	elf := &ELF{
+		Header: Header{
+			Magic:                       [4]byte{0x7F, 'E', 'L', 'F'},
+			Class:                       2,
+			Endianness:                  LittleEndian,
+			Version:                     1,
+			OSABI:                       0,
+			ABIVersion:                  0,
+			Type:                        TypeDynamic,
+			Architecture:                Arch(b.Arch),
+			FileVersion:                 1,
+			EntryPointInMemory:          int64(code.MemoryOffset),
+			ProgramHeaderOffset:         HeaderSize,
+			SectionHeaderOffset:         0,
+			Flags:                       0,
+			Size:                        HeaderSize,
+			ProgramHeaderEntrySize:      ProgramHeaderSize,
+			ProgramHeaderEntryCount:     2,
+			SectionHeaderEntrySize:      0,
+			SectionHeaderEntryCount:     0,
+			SectionNameStringTableIndex: 0,
+		},
+		CodeHeader: ProgramHeader{
+			Type:           ProgramTypeLOAD,
+			Flags:          ProgramFlagsExecutable | ProgramFlagsReadable,
+			Offset:         int64(code.FileOffset),
+			VirtualAddress: int64(code.MemoryOffset),
+			SizeInFile:     int64(len(code.Bytes)),
+			SizeInMemory:   int64(len(code.Bytes)),
+			Align:          int64(b.MemoryAlign),
+		},
+		DataHeader: ProgramHeader{
+			Type:           ProgramTypeLOAD,
+			Flags:          ProgramFlagsReadable,
+			Offset:         int64(data.FileOffset),
+			VirtualAddress: int64(data.MemoryOffset),
+			SizeInFile:     int64(len(data.Bytes)),
+			SizeInMemory:   int64(len(data.Bytes)),
+			Align:          int64(b.MemoryAlign),
+		},
+	}
+
+	elf.AddSections()
+	binary.Write(writer, binary.LittleEndian, &elf.Header)
+	binary.Write(writer, binary.LittleEndian, &elf.CodeHeader)
+	binary.Write(writer, binary.LittleEndian, &elf.DataHeader)
+	writer.Seek(int64(code.Padding), io.SeekCurrent)
+	writer.Write(code.Bytes)
+	writer.Seek(int64(data.Padding), io.SeekCurrent)
+	writer.Write(data.Bytes)
+	writer.Write(elf.StringTable)
+	binary.Write(writer, binary.LittleEndian, &elf.SectionHeaders)
+}

src/elf/ELF_test.go

@@ -0,0 +1,15 @@
+package elf_test
+
+import (
+	"testing"
+
+	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/elf"
+	"git.urbach.dev/cli/q/src/exe"
+)
+
+func TestWrite(t *testing.T) {
+	elf.Write(&exe.Discard{}, &build.Build{Arch: build.ARM, FileAlign: 0x4000, MemoryAlign: 0x4000}, nil, nil)
+	elf.Write(&exe.Discard{}, &build.Build{Arch: build.X86, FileAlign: 0x1000, MemoryAlign: 0x1000}, nil, nil)
+	elf.Write(&exe.Discard{}, &build.Build{Arch: build.UnknownArch, FileAlign: 0x1000, MemoryAlign: 0x1000}, nil, nil)
+}

src/elf/Header.go

@@ -0,0 +1,28 @@
+package elf
+
+// HeaderSize is equal to the size of a header in bytes.
+const HeaderSize = 64
+
+// Header contains general information.
+type Header struct {
+	Magic                       [4]byte
+	Class                       byte
+	Endianness                  byte
+	Version                     byte
+	OSABI                       byte
+	ABIVersion                  byte
+	_                           [7]byte
+	Type                        int16
+	Architecture                int16
+	FileVersion                 int32
+	EntryPointInMemory          int64
+	ProgramHeaderOffset         int64
+	SectionHeaderOffset         int64
+	Flags                       int32
+	Size                        int16
+	ProgramHeaderEntrySize      int16
+	ProgramHeaderEntryCount     int16
+	SectionHeaderEntrySize      int16
+	SectionHeaderEntryCount     int16
+	SectionNameStringTableIndex int16
+}

src/elf/ProgramFlags.go

@@ -0,0 +1,10 @@
+package elf
+
+// ProgramFlags specifies the permissions for a segment.
+type ProgramFlags int32
+
+const (
+	ProgramFlagsExecutable ProgramFlags = 0x1
+	ProgramFlagsWritable   ProgramFlags = 0x2
+	ProgramFlagsReadable   ProgramFlags = 0x4
+)

src/elf/ProgramHeader.go

@@ -0,0 +1,16 @@
+package elf
+
+// ProgramHeaderSize is equal to the size of a program header in bytes.
+const ProgramHeaderSize = 56
+
+// ProgramHeader points to the executable part of our program.
+type ProgramHeader struct {
+	Type            ProgramType
+	Flags           ProgramFlags
+	Offset          int64
+	VirtualAddress  int64
+	PhysicalAddress int64
+	SizeInFile      int64
+	SizeInMemory    int64
+	Align           int64
+}

src/elf/ProgramType.go

@@ -0,0 +1,15 @@
+package elf
+
+// ProgramType indicates the program type.
+type ProgramType int32
+
+const (
+	ProgramTypeNULL    ProgramType = 0
+	ProgramTypeLOAD    ProgramType = 1
+	ProgramTypeDYNAMIC ProgramType = 2
+	ProgramTypeINTERP  ProgramType = 3
+	ProgramTypeNOTE    ProgramType = 4
+	ProgramTypeSHLIB   ProgramType = 5
+	ProgramTypePHDR    ProgramType = 6
+	ProgramTypeTLS     ProgramType = 7
+)

src/elf/SectionFlags.go

@@ -0,0 +1,12 @@
+package elf
+
+// SectionFlags defines flags for sections.
+type SectionFlags int64
+
+const (
+	SectionFlagsWritable   SectionFlags = 1 << 0
+	SectionFlagsAllocate   SectionFlags = 1 << 1
+	SectionFlagsExecutable SectionFlags = 1 << 2
+	SectionFlagsStrings    SectionFlags = 1 << 5
+	SectionFlagsTLS        SectionFlags = 1 << 10
+)

src/elf/SectionHeader.go

@@ -0,0 +1,18 @@
+package elf
+
+// SectionHeaderSize is equal to the size of a section header in bytes.
+const SectionHeaderSize = 64
+
+// SectionHeader points to the data sections of our program.
+type SectionHeader struct {
+	NameIndex      int32
+	Type           SectionType
+	Flags          SectionFlags
+	VirtualAddress int64
+	Offset         int64
+	SizeInFile     int64
+	Link           int32
+	Info           int32
+	Align          int64
+	EntrySize      int64
+}

src/elf/SectionType.go

@@ -0,0 +1,19 @@
+package elf
+
+// SectionType defines the type of the section.
+type SectionType int32
+
+const (
+	SectionTypeNULL     SectionType = 0
+	SectionTypePROGBITS SectionType = 1
+	SectionTypeSYMTAB   SectionType = 2
+	SectionTypeSTRTAB   SectionType = 3
+	SectionTypeRELA     SectionType = 4
+	SectionTypeHASH     SectionType = 5
+	SectionTypeDYNAMIC  SectionType = 6
+	SectionTypeNOTE     SectionType = 7
+	SectionTypeNOBITS   SectionType = 8
+	SectionTypeREL      SectionType = 9
+	SectionTypeSHLIB    SectionType = 10
+	SectionTypeDYNSYM   SectionType = 11
+)

src/exe/Align.go

@@ -0,0 +1,12 @@
+package exe
+
+// Align calculates the next aligned address.
+func Align[T int | uint | int64 | uint64 | int32 | uint32](n T, alignment T) T {
+	return (n + (alignment - 1)) & ^(alignment - 1)
+}
+
+// AlignPad calculates the next aligned address and the padding needed.
+func AlignPad[T int | uint | int64 | uint64 | int32 | uint32](n T, alignment T) (T, T) {
+	aligned := Align(n, alignment)
+	return aligned, aligned - n
+}

src/exe/Discard.go

@@ -0,0 +1,7 @@
+package exe
+
+// Discard implements a no-op WriteSeeker.
+type Discard struct{}
+
+func (w *Discard) Write(_ []byte) (int, error)        { return 0, nil }
+func (w *Discard) Seek(_ int64, _ int) (int64, error) { return 0, nil }

src/exe/Discard_test.go

@@ -0,0 +1,14 @@
+package exe_test
+
+import (
+	"io"
+	"testing"
+
+	"git.urbach.dev/cli/q/src/exe"
+)
+
+func TestDiscard(t *testing.T) {
+	discard := exe.Discard{}
+	discard.Write(nil)
+	discard.Seek(0, io.SeekCurrent)
+}

src/exe/Executable.go

@@ -0,0 +1,42 @@
+package exe
+
+// Executable is a generic definition of the binary that later gets translated to OS-specific formats.
+type Executable struct {
+	Sections    []*Section
+	headerEnd   int
+	fileAlign   int
+	memoryAlign int
+}
+
+// New creates a new executable.
+func New(headerEnd int, fileAlign int, memoryAlign int) *Executable {
+	return &Executable{
+		headerEnd:   headerEnd,
+		fileAlign:   fileAlign,
+		memoryAlign: memoryAlign,
+	}
+}
+
+// InitSections generates sections from raw byte slices.
+func (exe *Executable) InitSections(raw ...[]byte) {
+	exe.Sections = make([]*Section, len(raw))
+
+	for i, data := range raw {
+		exe.Sections[i] = &Section{Bytes: data}
+	}
+
+	exe.Update()
+}
+
+// Update recalculates all section offsets.
+func (exe *Executable) Update() {
+	first := exe.Sections[0]
+	first.FileOffset, first.Padding = AlignPad(exe.headerEnd, exe.fileAlign)
+	first.MemoryOffset = Align(exe.headerEnd, exe.memoryAlign)
+
+	for i, section := range exe.Sections[1:] {
+		previous := exe.Sections[i]
+		section.FileOffset, section.Padding = AlignPad(previous.FileOffset+len(previous.Bytes), exe.fileAlign)
+		section.MemoryOffset = Align(previous.MemoryOffset+len(previous.Bytes), exe.memoryAlign)
+	}
+}

src/exe/Executable_test.go

@@ -0,0 +1,19 @@
+package exe_test
+
+import (
+	"testing"
+
+	"git.urbach.dev/cli/q/src/exe"
+	"git.urbach.dev/go/assert"
+)
+
+func TestExecutable(t *testing.T) {
+	align := 0x1000
+	x := exe.New(1, align, align)
+	x.InitSections([]byte{1}, []byte{1})
+	assert.Equal(t, len(x.Sections), 2)
+	assert.Equal(t, x.Sections[0].Padding, align-1)
+	assert.Equal(t, x.Sections[0].FileOffset, align)
+	assert.Equal(t, x.Sections[1].Padding, align-1)
+	assert.Equal(t, x.Sections[1].FileOffset, align*2)
+}

src/exe/Section.go

@@ -0,0 +1,9 @@
+package exe
+
+// Section represents some data within the executable that will also be loaded into memory.
+type Section struct {
+	Bytes        []byte
+	FileOffset   int
+	Padding      int
+	MemoryOffset int
+}

src/linker/WriteExecutable.go

@@ -0,0 +1,50 @@
+package linker
+
+import (
+	"encoding/binary"
+	"os"
+
+	"git.urbach.dev/cli/q/src/arm"
+	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/core"
+	"git.urbach.dev/cli/q/src/elf"
+	"git.urbach.dev/cli/q/src/x86"
+)
+
+// WriteExecutable writes an executable file to disk.
+func WriteExecutable(b *build.Build, result *core.Environment) error {
+	executable := b.Executable()
+	file, err := os.Create(executable)
+
+	if err != nil {
+		return err
+	}
+
+	code := []byte{}
+	data := []byte{}
+
+	switch b.Arch {
+	case build.ARM:
+		code = arm.MoveRegisterNumber(code, arm.X8, 93)
+		code = arm.MoveRegisterNumber(code, arm.X0, 0)
+		code = binary.LittleEndian.AppendUint32(code, arm.Syscall())
+
+	case build.X86:
+		code = x86.MoveRegisterNumber(code, x86.R0, 60)
+		code = x86.MoveRegisterNumber(code, x86.R7, 0)
+		code = x86.Syscall(code)
+	}
+
+	switch b.OS {
+	case build.Linux:
+		elf.Write(file, b, code, data)
+	}
+
+	err = file.Close()
+
+	if err != nil {
+		return err
+	}
+
+	return os.Chmod(executable, 0755)
+}

src/linker/WriteExecutable_test.go

@@ -0,0 +1,38 @@
+package linker_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"git.urbach.dev/cli/q/src/build"
+	"git.urbach.dev/cli/q/src/compiler"
+	"git.urbach.dev/cli/q/src/linker"
+	"git.urbach.dev/go/assert"
+)
+
+func TestWriteExecutable(t *testing.T) {
+	tmpDir := filepath.Join(os.TempDir(), "q", "tests")
+	err := os.MkdirAll(tmpDir, 0755)
+	assert.Nil(t, err)
+
+	fromPath := "../../examples/hello/hello.q"
+	contents, err := os.ReadFile(fromPath)
+	assert.Nil(t, err)
+
+	toPath := filepath.Join(tmpDir, "hello.q")
+	err = os.WriteFile(toPath, contents, 0755)
+	assert.Nil(t, err)
+
+	b := build.New(toPath)
+	env, err := compiler.Compile(b)
+	assert.Nil(t, err)
+
+	b.SetArch(build.ARM)
+	err = linker.WriteExecutable(b, env)
+	assert.Nil(t, err)
+
+	b.SetArch(build.X86)
+	err = linker.WriteExecutable(b, env)
+	assert.Nil(t, err)
+}