├── TODO ├── doc ├── 0_Introduction.md ├── 11_Processes.md ├── 12_User_Mode.md ├── 13_Process_Memory_Management.md ├── serve ├── retag.sh ├── README.md ├── 7_Multiboot_Data.md ├── 5_Unit_Testing.md ├── 6_Debug_Output.md ├── 9_Memory_Management.md ├── 8_Exceptions.md ├── 4_Higher_Half_Kernel.md ├── 10_Threading.md ├── 2_A_Bootable_Kernel.md ├── 1_Toolchain.md └── 3_Activate_Long_Mode.md ├── .gitignore ├── src └── kernel │ ├── include │ ├── vga.h │ ├── serial.h │ ├── scheduler.h │ ├── ports.h │ ├── acpi.h │ ├── multiboot.h │ ├── process.h │ ├── cpu.h │ ├── debug.h │ ├── interrupts.h │ ├── queue.tt │ ├── memory.h │ └── queue.h │ ├── proc │ ├── scheduler.c │ ├── swtch.S │ ├── procmm.c │ └── process.c │ ├── cpu │ ├── cpu.c │ ├── isr.S.py │ ├── interrupts.tt │ ├── isr_common.S │ ├── registers.S │ ├── interrupts.c │ └── gdt.c │ ├── drivers │ ├── serial.c │ ├── vga.c │ ├── vga.tt │ └── acpi.c │ ├── boot │ ├── multiboot_header.S │ ├── boot_PT.S │ ├── kmain.c │ ├── boot.S │ ├── debug.c │ ├── multiboot.c │ ├── debug.tt │ └── multiboot.tt │ ├── Link.ld │ ├── memory │ ├── pmm.c │ ├── memory.c │ ├── pmm.tt │ ├── string.c │ ├── string.tt │ ├── vmm.c │ └── vmm.tt │ └── Makefile ├── toolchain ├── gdb ├── Dockerfile ├── emul ├── mkiso ├── colortail ├── ttest.tt ├── build-toolchain.sh ├── gdbinit └── ttest.h ├── Makefile ├── ttest ├── d ├── LICENSE └── README.md /TODO: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/0_Introduction.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/11_Processes.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/12_User_Mode.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/13_Process_Memory_Management.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | sysroot/ 2 | *.o 3 | *.d 4 | src/kernel/kernel 5 | -------------------------------------------------------------------------------- /src/kernel/include/vga.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define VGA_MEMORY P2V(0xB8000) 4 | 5 | void vga_init(); 6 | void vga_write(char c); 7 | -------------------------------------------------------------------------------- /src/kernel/include/serial.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #define PORT_COM1 0x3F8 5 | 6 | void serial_init(uint16_t port); 7 | void serial_write(uint16_t port, uint8_t c); 8 | -------------------------------------------------------------------------------- /toolchain/gdb: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z ${MITTOS64+x} ]; then >&2 echo "Unsupported environment! See README"; exit 1; fi 4 | 5 | /usr/bin/gdb -q -x ${BUILDROOT}toolchain/gdbinit -se ${BUILDROOT}sysroot/kernel $@ 6 | -------------------------------------------------------------------------------- /src/kernel/include/scheduler.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #define runQ readyQ, readyQ_next, struct process 5 | QUEUE_DECLARE(runQ); 6 | 7 | void ready(struct process *proc); 8 | struct process *scheduler_next(); 9 | -------------------------------------------------------------------------------- /toolchain/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.6 2 | 3 | ADD build-toolchain.sh /opt/build-toolchain.sh 4 | 5 | RUN /opt/build-toolchain.sh 6 | 7 | ENV PATH "/opt/toolchain:$PATH" 8 | ENV MITTOS64 "true" 9 | ENV BUILDROOT "/opt/" 10 | WORKDIR /opt 11 | -------------------------------------------------------------------------------- /doc/serve: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script serves the documentation folder rendered to HTML using GitHub Markdown on port 5000 4 | 5 | # docker run -d --rm -v $(pwd)/doc:/data -p 5000:6419 kbai/grip 0.0.0.0:6419 6 | docker run -d --rm -v $(pwd)/doc:/data -p 5000:3080 thomsch98/markserv 7 | -------------------------------------------------------------------------------- /src/kernel/proc/scheduler.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | QUEUE_DEFINE(runQ); 6 | 7 | void ready(struct process *proc) 8 | { 9 | queue_add(runQ, proc); 10 | } 11 | 12 | struct process *scheduler_next() 13 | { 14 | return queue_pop(runQ); 15 | } 16 | -------------------------------------------------------------------------------- /toolchain/emul: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z ${MITTOS64+x} ]; then >&2 echo "Unsupported environment! See README"; exit 1; fi 4 | 5 | iso=${BUILDROOT}mittos64.iso 6 | seriallog=${BUILDROOT}serial.log 7 | 8 | ${BUILDROOT}toolchain/mkiso 9 | 10 | qemu-system-x86_64 -s -S -cdrom ${iso} -serial file:${seriallog} $@ 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(MITTOS64),) 2 | $(error Unsupported environment! See README) 3 | endif 4 | 5 | CC = x86_64-elf-gcc 6 | 7 | all: kernel 8 | 9 | kernel: 10 | ifeq ($(shell make -sqC src/kernel || echo 1), 1) 11 | CC=$(CC) $(MAKE) -C src/kernel install 12 | endif 13 | 14 | clean: 15 | $(MAKE) -C src/kernel clean 16 | rm -rf sysroot 17 | 18 | .PHONY: kernel clean 19 | -------------------------------------------------------------------------------- /src/kernel/include/ports.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | static __inline void _outb(uint16_t port, uint8_t value) 5 | { 6 | asm volatile("outb %1, %0" : : "dN" (port), "a" (value)); 7 | } 8 | 9 | static __inline uint8_t _inb(uint16_t port) 10 | { 11 | uint8_t ret; 12 | asm volatile("inb %1, %0" : "=a" (ret) : "dN" (port)); 13 | return ret; 14 | } 15 | 16 | #define outb _outb 17 | #define inb _inb 18 | -------------------------------------------------------------------------------- /src/kernel/proc/swtch.S: -------------------------------------------------------------------------------- 1 | .intel_syntax noprefix 2 | 3 | .global switch_stack 4 | 5 | switch_stack: 6 | push rbp 7 | mov rbp, rsp 8 | push rdi 9 | push r15 10 | push r14 11 | push r13 12 | push r12 13 | push rbx 14 | push rbp 15 | 16 | mov [rdi], rsp 17 | mov rsp, [rsi] 18 | 19 | pop rbp 20 | pop rbx 21 | pop r12 22 | pop r13 23 | pop r14 24 | pop r15 25 | pop rdi 26 | leaveq 27 | ret 28 | -------------------------------------------------------------------------------- /src/kernel/cpu/cpu.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | void gdt_init(struct cpu *cpu); 7 | 8 | struct cpu __seg_gs *cpu = 0; 9 | 10 | void cpu_init() 11 | { 12 | // Set up cpu struct 13 | struct cpu *c = P2V(pmm_calloc()); 14 | c->cpu = c; 15 | write_msr(KERNEL_GS_BASE, (uint64_t)c); 16 | asm("swapgs"); 17 | 18 | interrupt_init(); 19 | gdt_init(cpu->cpu); 20 | } 21 | 22 | -------------------------------------------------------------------------------- /src/kernel/drivers/serial.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void serial_init(uint16_t port) 5 | { 6 | outb(port + 1, 0x00); 7 | outb(port + 3, 0x80); 8 | outb(port + 0, 0x03); 9 | outb(port + 1, 0x00); 10 | outb(port + 3, 0x03); 11 | outb(port + 2, 0xC7); 12 | outb(port + 4, 0x0B); 13 | } 14 | 15 | void serial_write(uint16_t port, uint8_t c) 16 | { 17 | while(!(inb(port + 5)&0x20)); 18 | outb(port, c); 19 | } 20 | -------------------------------------------------------------------------------- /doc/retag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | first_commit=`git rev-list --max-parents=0 master` 6 | 7 | revs=`git rev-list --reverse ${first_commit}..master` 8 | 9 | git tag | xargs git tag -d 10 | 11 | for rev in $revs; do 12 | message=`git log --pretty=tformat:%s -1 $rev` 13 | chapter=`echo $message | sed -e "/^Chapter \(.*\):.*$/!d;s//ch\1/"` 14 | if [[ -n $chapter ]]; then 15 | git tag $chapter $rev 16 | fi 17 | done 18 | 19 | -------------------------------------------------------------------------------- /src/kernel/boot/multiboot_header.S: -------------------------------------------------------------------------------- 1 | #define MBOOT2_MAGIC 0xE85250D6 2 | #define MBOOT2_ARCH 0 3 | #define MBOOT2_LENGTH (Multiboot2HeaderEnd - Multiboot2Header) 4 | #define MBOOT2_CHECKSUM -(MBOOT2_MAGIC + MBOOT2_ARCH + MBOOT2_LENGTH) 5 | 6 | .section .multiboot 7 | 8 | .align 0x8 9 | Multiboot2Header: 10 | .long MBOOT2_MAGIC 11 | .long MBOOT2_ARCH 12 | .long MBOOT2_LENGTH 13 | .long MBOOT2_CHECKSUM 14 | 15 | .short 0 16 | .short 0 17 | .long 8 18 | Multiboot2HeaderEnd: 19 | -------------------------------------------------------------------------------- /src/kernel/include/acpi.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #define MAX_CPUS 16 5 | #define MAX_IOAPIC 4 6 | 7 | struct acpi_info 8 | { 9 | int num_cpus; 10 | struct { 11 | uint8_t id; 12 | uint8_t apic; 13 | } cpu[MAX_CPUS]; 14 | 15 | int num_ioapic; 16 | struct { 17 | uint8_t id; 18 | uint32_t addr; 19 | uint32_t base; 20 | } ioapic[MAX_IOAPIC]; 21 | 22 | uint32_t int_map[256]; 23 | }; 24 | extern struct acpi_info acpi_info; 25 | 26 | void acpi_init(); 27 | int get_cpu(); 28 | -------------------------------------------------------------------------------- /toolchain/mkiso: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | if [ -z ${MITTOS64+x} ]; then >&2 echo "Unsupported environment! See README"; exit 1; fi 4 | 5 | sysroot=${BUILDROOT}sysroot 6 | iso=${BUILDROOT}mittos64.iso 7 | 8 | mkdir -p ${sysroot} 9 | 10 | # Grub configuration file 11 | # set to boot /kernel with minimal wait 12 | mkdir -p ${sysroot}/boot/grub 13 | cat > ${sysroot}/boot/grub/grub.cfg << EOF 14 | set timeout=1 15 | set default=0 16 | 17 | menuentry "mittos64" { 18 | multiboot2 /kernel 19 | } 20 | 21 | EOF 22 | 23 | grub-mkrescue -o ${iso} ${sysroot} 24 | -------------------------------------------------------------------------------- /src/kernel/include/multiboot.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | struct kernel_boot_data_st 5 | { 6 | int multiboot_version; 7 | char *bootloader; 8 | char *commandline; 9 | size_t mmap_size; 10 | unsigned int mmap_len; 11 | void *mmap; 12 | }; 13 | 14 | #define MMAP_FREE 1 15 | 16 | extern struct kernel_boot_data_st kernel_boot_data; 17 | 18 | int multiboot_init(uint64_t magic, void *mboot_info); 19 | int multiboot_get_memory_area(size_t count, uintptr_t *start, uintptr_t *end, uint32_t *type); 20 | int multiboot_page_used(uintptr_t start); 21 | -------------------------------------------------------------------------------- /toolchain/colortail: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | function ce() 4 | { 5 | echo -e "${@}" 6 | } 7 | 8 | red="\x1b[31m" 9 | yellow="\x1b[33m" 10 | green="\x1b[32m" 11 | blue="\x1b[34m" 12 | normal="\x1b[0m" 13 | 14 | function colorize() 15 | { 16 | echo "$1" | sed \ 17 | -e "s/^\[INFO\]/\[$(ce $blue)INFO$(ce $normal)\]/" \ 18 | -e "s/^\[OK\]/\[$(ce $green)OK$(ce $normal)\]/" \ 19 | -e "s/^\[WARNING\]/\[$(ce $yellow)WARNING$(ce $normal)\]/" \ 20 | -e "s/^\[ERROR\]/\[$(ce $red)ERROR$(ce $normal)\]/" 21 | } 22 | 23 | tail -f $1 | while read line; do 24 | colorize "$line" 25 | done 26 | -------------------------------------------------------------------------------- /src/kernel/Link.ld: -------------------------------------------------------------------------------- 1 | ENTRY(_start) 2 | 3 | KERNEL_OFFSET = 0xFFFFFF8000000000; 4 | KERNEL_START = 0x10000; 5 | 6 | SECTIONS 7 | { 8 | . = KERNEL_START + KERNEL_OFFSET; 9 | kernel_start = .; 10 | .text : AT(ADDR(.text) - KERNEL_OFFSET) 11 | { 12 | *(.multiboot) 13 | *(.text) 14 | } 15 | .rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) 16 | { 17 | *(.rodata*) 18 | } 19 | .data : AT(ADDR(.data) - KERNEL_OFFSET) 20 | { 21 | *(.data) 22 | } 23 | .bss : AT(ADDR(.bss) - KERNEL_OFFSET) 24 | { 25 | *(.COMMON) 26 | *(.bss) 27 | } 28 | kernel_end = .; 29 | } 30 | -------------------------------------------------------------------------------- /src/kernel/include/process.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | struct process 9 | { 10 | uint64_t pid; 11 | void *stack_ptr; 12 | uint64_t state; 13 | 14 | uint64_t P4; 15 | uint64_t brk; 16 | uint64_t stack; 17 | QUEUE_SPOT(runQ); 18 | }; 19 | 20 | struct process *new_process(void (*function)(void)); 21 | void yield(); 22 | void start_scheduler(); 23 | 24 | void switch_stack(void *old_ptr, void *new_ptr); 25 | 26 | uint64_t procmm_brk(struct process *p, void *addr); 27 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | # MittOS64 2 | 3 | **Table of Contents** 4 | 5 | [Chapter 0: Introduction](0_Introduction.md)
6 | [Chapter 1: Toolchain](1_Toolchain.md)
7 | [Chapter 2: Booting a Kernel](2_A_Bootable_Kernel.md)
8 | [Chapter 3: Activate Long Mode](3_Activate_Long_Mode.md)
9 | [Chapter 4: "Higher Half" Kernel](4_Higher_Half_Kernel.md)
10 | [Chapter 5: Unit Testing Framework](5_Unit_Testing.md)
11 | [Chapter 6: Debug output](6_Debug_Output.md)
12 | [Chapter 7: Multiboot Data](7_Multiboot_Data.md)
13 | [Chapter 8: Exceptions and Interrupts](8_Exceptions.md)
14 | [Chapter 9: Memory Management](9_Memory_Management.md)
15 | 16 | -------------------------------------------------------------------------------- /src/kernel/memory/pmm.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // Virtual addres of next free page 4 | uint64_t next = 0; 5 | 6 | void pmm_free(uint64_t page) 7 | { 8 | // Write previous free pointer to freed page 9 | *(uint64_t *)P2V(page) = next; 10 | // And update free pointer 11 | next = (uint64_t)P2V(page); 12 | } 13 | 14 | uint64_t pmm_alloc() 15 | { 16 | if(!next) return 0; 17 | uint64_t page = next; 18 | // Read new free pointer from allocated page 19 | next = *(uint64_t *)page; 20 | return (uint64_t)V2P(page); 21 | } 22 | 23 | uint64_t pmm_calloc() 24 | { 25 | uint64_t page = pmm_alloc(); 26 | memset(P2V(page), 0, PAGE_SIZE); 27 | return page; 28 | } 29 | -------------------------------------------------------------------------------- /src/kernel/cpu/isr.S.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import print_function 5 | 6 | num_isr = 256 7 | pushes_error = [8, 10, 11, 12, 13, 14, 17] 8 | 9 | print(''' 10 | .intel_syntax noprefix 11 | .extern isr_common 12 | ''') 13 | 14 | 15 | print('// Interrupt Service Routines') 16 | for i in range(num_isr): 17 | print('''isr{0}: 18 | cli 19 | {1} 20 | push {0} 21 | jmp isr_common '''.format(i, 22 | 'push 0' if i not in pushes_error else 'nop')) 23 | 24 | print('') 25 | print(''' 26 | // Vector table 27 | 28 | .section .data 29 | .global isr_table 30 | isr_table:''') 31 | 32 | for i in range(num_isr): 33 | print(' .quad isr{}'.format(i)) 34 | -------------------------------------------------------------------------------- /src/kernel/include/cpu.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | struct cpu 6 | { 7 | void *cpu; 8 | uint64_t gdt[6]; 9 | uint8_t tss[104]; 10 | struct process *proc; 11 | struct process *scheduler; 12 | }; 13 | 14 | extern struct cpu __seg_gs *cpu; 15 | 16 | void cpu_init(); 17 | 18 | void interrupt_stack(void *rsp0); 19 | 20 | void load_idt(void *); 21 | void load_gdt(void *); 22 | 23 | uint64_t read_cr0(); 24 | uint64_t read_cr2(); 25 | uint64_t read_cr3(); 26 | void write_cr3(uint64_t); 27 | uint64_t read_cr4(); 28 | 29 | // Model Specific Registers 30 | // Functions defined in cpu/registers.S 31 | void write_msr(uint64_t reg, uint64_t value); 32 | uint64_t read_msr(uint64_t reg); 33 | #define KERNEL_GS_BASE 0xC0000102 34 | -------------------------------------------------------------------------------- /src/kernel/cpu/interrupts.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | #include "interrupts.c" 4 | 5 | uintptr_t isr_table[] ={}; 6 | void load_idt(void *_) 7 | { 8 | (void)_; 9 | } 10 | 11 | uint8_t *idt_raw = (uint8_t *)idt; 12 | 13 | TEST(idt_set_gate_correctly_sets_address_L) 14 | { 15 | idt_set_gate(1, 0x1234567890ABCDEF, 0, 0, 0); 16 | 17 | ASSERT_EQ_INT(*(uint16_t *)&idt_raw[16+0], 0xCDEF); 18 | } 19 | TEST(idt_set_gate_correctly_sets_address_M) 20 | { 21 | idt_set_gate(1, 0x1234567890ABCDEF, 0, 0, 0); 22 | 23 | ASSERT_EQ_INT(*(uint16_t *)&idt_raw[16+6], 0x90AB); 24 | } 25 | TEST(idt_set_gate_correctly_sets_address_H) 26 | { 27 | idt_set_gate(1, 0x1234567890ABCDEF, 0, 0, 0); 28 | 29 | ASSERT_EQ_INT(*(uint32_t *)&idt_raw[16+8], 0x12345678); 30 | } 31 | -------------------------------------------------------------------------------- /src/kernel/boot/boot_PT.S: -------------------------------------------------------------------------------- 1 | #include 2 | .intel_syntax noprefix 3 | 4 | .section .data 5 | .align PAGE_SIZE 6 | .global BootP4 7 | BootP4: 8 | .quad offset V2P(BootP3) + (PAGE_PRESENT | PAGE_WRITE) 9 | .rept ENTRIES_PER_PT - 2 10 | .quad 0 11 | .endr 12 | .quad offset V2P(BootP3) + (PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL) 13 | BootP3: 14 | .quad offset V2P(BootP2) + (PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL) 15 | .rept ENTRIES_PER_PT - 1 16 | .quad 0 17 | .endr 18 | BootP2: 19 | .quad offset V2P(BootP1) + (PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL) 20 | .rept ENTRIES_PER_PT - 1 21 | .quad 0 22 | .endr 23 | BootP1: 24 | .set i, 0 25 | .rept ENTRIES_PER_PT 26 | .quad (i << 12) + (PAGE_PRESENT | PAGE_WRITE | PAGE_GLOBAL) 27 | .set i, (i+1) 28 | .endr 29 | -------------------------------------------------------------------------------- /ttest: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | dirs="src/kernel" 4 | 5 | main() 6 | { 7 | local failures=0 8 | for dir in $dirs; do 9 | local files=`find $dir -name "*.tt"` 10 | for suite in $files; do 11 | 12 | test_exec="${suite}est" 13 | compiler_output=`cc -x c $suite -o $test_exec -Wall -Wextra -Werror -I $dir/include -I toolchain -DNDEBUG -DTTEST 2>&1` 14 | compiler_status=$? 15 | 16 | if [[ "$compiler_status" -eq "0" ]]; then 17 | $test_exec $@ || failures=$(($failures + 1)) 18 | else 19 | failures=$(($failures + 1)) 20 | echo 21 | echo -e "\x1b[31mCOMPILATION OF SUITE FAILED\x1b[0m" 22 | echo "$compiler_output" | sed -e 's/\.tt\.c:/\.tt:/' 23 | fi 24 | rm -f $test_exec 25 | done 26 | done 27 | 28 | echo 29 | exit $failures 30 | 31 | } 32 | 33 | main "$@" 34 | -------------------------------------------------------------------------------- /src/kernel/cpu/isr_common.S: -------------------------------------------------------------------------------- 1 | .intel_syntax noprefix 2 | 3 | .extern int_handler 4 | .global isr_common 5 | .global isr_return 6 | 7 | isr_common: 8 | push r15 9 | push r14 10 | push r13 11 | push r12 12 | push r11 13 | push r10 14 | push r9 15 | push r8 16 | push rbp 17 | push rdi 18 | push rsi 19 | push rdx 20 | push rcx 21 | push rbx 22 | push rax 23 | 24 | mov rax, [rsp + 152] 25 | and rax, (3<<12) 26 | jz .kernel_interrupt 27 | swapgs 28 | 29 | .kernel_interrupt: 30 | mov rdi, rsp 31 | call int_handler 32 | 33 | mov rdi, rax 34 | isr_return: 35 | mov rsp, rdi 36 | 37 | mov rax, [rsp + 152] 38 | and rax, (3<<12) 39 | jz .kernel_return 40 | swapgs 41 | 42 | .kernel_return: 43 | pop rax 44 | pop rbx 45 | pop rcx 46 | pop rdx 47 | pop rsi 48 | pop rdi 49 | pop rbp 50 | pop r8 51 | pop r9 52 | pop r10 53 | pop r11 54 | pop r12 55 | pop r13 56 | pop r14 57 | pop r15 58 | add rsp, 0x10 59 | iretq 60 | -------------------------------------------------------------------------------- /src/kernel/cpu/registers.S: -------------------------------------------------------------------------------- 1 | .intel_syntax noprefix 2 | 3 | .global load_idt 4 | load_idt: 5 | lidt [rdi] 6 | ret 7 | 8 | .global load_gdt 9 | load_gdt: 10 | lgdt [rdi] 11 | mov ax, 0x0 12 | mov ss, ax 13 | mov ds, ax 14 | mov es, ax 15 | movabs rax, offset .load_gdt 16 | pushq 0x8 17 | push rax 18 | retfq 19 | .load_gdt: 20 | mov rax, 0x20 21 | ltr ax 22 | ret 23 | 24 | .global read_cr0 25 | read_cr0: 26 | mov rax, cr0 27 | ret 28 | .global read_cr2 29 | read_cr2: 30 | mov rax, cr2 31 | ret 32 | .global read_cr3 33 | read_cr3: 34 | mov rax, cr3 35 | ret 36 | .global write_cr3 37 | write_cr3: 38 | mov cr3, rdi 39 | ret 40 | .global read_cr4 41 | read_cr4: 42 | mov rax, cr4 43 | ret 44 | 45 | .global read_msr 46 | read_msr: 47 | mov rcx, rdi 48 | rdmsr 49 | shl rdx, 32 50 | add rax, rdx 51 | ret 52 | .global write_msr 53 | write_msr: 54 | mov rcx, rdi 55 | mov rax, rsi 56 | mov rdx, rsi 57 | shr rdx, 32 58 | wrmsr 59 | ret 60 | -------------------------------------------------------------------------------- /src/kernel/memory/memory.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | uint64_t kernel_P4; 6 | 7 | void memory_init() 8 | { 9 | kernel_P4 = (uint64_t)&BootP4; 10 | uint64_t start, end; 11 | uint32_t type, i = 0; 12 | debug_info("Parsing memory map\n"); 13 | while(!multiboot_get_memory_area(i++, &start, &end, &type)) 14 | { 15 | debug("0x%016x-0x%016x (%d)\n", start, end, type); 16 | for(uint64_t p = start; p < end; p += PAGE_SIZE) 17 | { 18 | if(p >= V2P(&kernel_start) && p < V2P(&kernel_end)) 19 | continue; 20 | if(multiboot_page_used(p)) 21 | continue; 22 | 23 | uint64_t addr = (uint64_t)P2V(p); 24 | uint64_t page = vmm_get_page(kernel_P4, addr); 25 | if(!PAGE_EXIST(page) || !(page & PAGE_PRESENT)) 26 | { 27 | uint16_t flags = PAGE_GLOBAL | PAGE_WRITE; 28 | vmm_set_page(kernel_P4, addr, p, flags | PAGE_PRESENT); 29 | } 30 | 31 | if(type == MMAP_FREE) 32 | pmm_free(p); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /d: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | imagename=mittos64 4 | 5 | buildroot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" 6 | 7 | # Check if the docker image exists 8 | # If not, ask if it should be built 9 | if [[ ! $(docker image ls -q -f reference=${imagename}) ]]; then 10 | echo "Docker image does not exist." 11 | echo "Do you wish to build it? (This could take a while)" 12 | read -r -p "[y/N]: " response 13 | case "$response" in 14 | [Yy][Ee][Ss]|[Yy]) 15 | docker build -t ${imagename} ${buildroot}/toolchain 16 | ;; 17 | *) 18 | exit 19 | ;; 20 | esac 21 | fi 22 | 23 | 24 | # Check if a container is already running the image 25 | # If so, execute the command in the running container 26 | #if docker ps -f name=${imagename}-run | grep ${imagename}-run ; then 27 | if [[ $(docker ps -q -f name=${imagename}-run) ]]; then 28 | docker exec -it -u $(id -u):$(id -g) ${imagename}-run "$@" 29 | else 30 | docker run -it --rm -v ${buildroot}:/opt --name ${imagename}-run -u $(id -u):$(id -g) ${imagename} "$@" 31 | fi 32 | -------------------------------------------------------------------------------- /src/kernel/include/debug.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #ifndef NDEBUG 4 | #define debug(...) debug_printf(__VA_ARGS__) 5 | #define debug_info(...) \ 6 | do{debug("[INFO] "); debug(__VA_ARGS__);}while(0) 7 | #define debug_ok(...) \ 8 | do{debug("[OK] "); debug(__VA_ARGS__);}while(0) 9 | #define debug_warning(...) \ 10 | do{debug("[WARNING] "); debug(__VA_ARGS__);}while(0) 11 | #define debug_error(...) \ 12 | do{debug("[ERROR] "); debug(__VA_ARGS__);}while(0) 13 | #else 14 | #define debug(...) 15 | #define debug_info(...) 16 | #define debug_ok(...) 17 | #define debug_warning(...) 18 | #define debug_error(...) 19 | #endif 20 | 21 | void debug_printf(char *fmt, ...); 22 | void debug_puts(char *s); 23 | void debug_putsn(char *s, size_t n); 24 | void debug_putch(char c); 25 | 26 | #define S(x) #x 27 | #define S_(x) S(x) 28 | #define S__LINE__ S_(__LINE__) 29 | #define PANIC(...) \ 30 | do{ \ 31 | debug("\n\nKernel panic!\n%s:%d\n", __FILE__, __LINE__); \ 32 | debug(__VA_ARGS__); \ 33 | volatile int _override = 0; \ 34 | while(1){ \ 35 | asm("panic_breakpoint_" S__LINE__ ":"); \ 36 | if(_override) break; \ 37 | } \ 38 | }while(0) 39 | -------------------------------------------------------------------------------- /toolchain/ttest.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | 4 | 5 | TEST(two_equal_ints) 6 | { 7 | ASSERT_EQ_INT(1, 1); 8 | } 9 | TEST(FAIL_two_equal_ints) 10 | { 11 | ASSERT_EQ_INT(2, 3); 12 | } 13 | 14 | TEST(two_different_ints) 15 | { 16 | ASSERT_NEQ_INT(2, 3); 17 | } 18 | TEST(FAIL_two_different_ints) 19 | { 20 | ASSERT_NEQ_INT(1, 1); 21 | } 22 | 23 | TEST(two_equal_chars) 24 | { 25 | ASSERT_EQ_CHR('a', 'a'); 26 | } 27 | TEST(FAIL_two_equal_chars) 28 | { 29 | ASSERT_EQ_CHR('b', 'c'); 30 | } 31 | 32 | TEST(two_different_chars) 33 | { 34 | ASSERT_NEQ_CHR('b', 'c'); 35 | } 36 | TEST(FAIL_two_different_chars) 37 | { 38 | ASSERT_NEQ_CHR('a', 'a'); 39 | } 40 | 41 | TEST(two_equal_strings) 42 | { 43 | ASSERT_EQ_STR("Hello", "Hello", 5); 44 | } 45 | TEST(FAIL_two_equal_strings) 46 | { 47 | ASSERT_EQ_STR("Hello", "World", 5); 48 | } 49 | 50 | int a, b; 51 | 52 | TEST(two_equal_pointers) 53 | { 54 | ASSERT_EQ_PTR(&a, &a); 55 | } 56 | TEST(FAIL_two_equal_pointers) 57 | { 58 | ASSERT_EQ_PTR(&a, &b); 59 | } 60 | 61 | TEST(two_different_pointers) 62 | { 63 | ASSERT_NEQ_PTR(&a, &b); 64 | } 65 | TEST(FAIL_two_different_pointers) 66 | { 67 | ASSERT_NEQ_PTR(&a, &a); 68 | } 69 | -------------------------------------------------------------------------------- /src/kernel/Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(MITTOS64),) 2 | $(error Unsupported environment! See README) 3 | endif 4 | 5 | SRC := $(wildcard **/*.[cS]*) 6 | OBJ := $(patsubst %, %.o, $(basename $(basename $(SRC)))) 7 | 8 | CFLAGS ?= -Wall -Wextra -pedantic 9 | CFLAGS += -ffreestanding -mcmodel=large 10 | ifndef NDEBUG 11 | CFLAGS += -ggdb -O0 12 | ASFLAGS += -ggdb 13 | else 14 | CFLAGS += -O3 -mno-sse 15 | CPPFLAGS += -DNDEBUG 16 | endif 17 | CPPFLAGS += -I include 18 | LDFLAGS := -n -nostdlib -lgcc -T Link.ld 19 | 20 | kernel: $(OBJ) 21 | $(LINK.c) $^ -o $@ 22 | 23 | %.o: %.S.py 24 | python $^ | $(COMPILE.S) $(DEPFLAGS) -x assembler-with-cpp - -o $@ 25 | 26 | # Automatically generate dependency files 27 | # Those keep track of which header files are used by which source files 28 | DEP := $(OBJ:.o=.d) 29 | DEPFLAGS = -MT $@ -MMD -MP -MF $*.d 30 | $(OBJ): CPPFLAGS += $(DEPFLAGS) 31 | %.d: ; 32 | 33 | DESTDIR ?= $(BUILDROOT)sysroot 34 | 35 | # Copy kernel to sysroot 36 | $(DESTDIR)$(PREFIX)/kernel: kernel 37 | install -D kernel $(DESTDIR)$(PREFIX)/kernel 38 | 39 | install: $(DESTDIR)$(PREFIX)/kernel 40 | 41 | clean: 42 | rm -rf $(OBJ) $(DEP) kernel 43 | 44 | .PHONY: install clean 45 | 46 | # Include generated dependency files 47 | include $(DEP) 48 | -------------------------------------------------------------------------------- /src/kernel/include/interrupts.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | typedef struct 5 | { 6 | uint64_t rax; 7 | uint64_t rbx; 8 | uint64_t rcx; 9 | uint64_t rdx; 10 | uint64_t rsi; 11 | uint64_t rdi; 12 | uint64_t rbp; 13 | uint64_t r8; 14 | uint64_t r9; 15 | uint64_t r10; 16 | uint64_t r11; 17 | uint64_t r12; 18 | uint64_t r13; 19 | uint64_t r14; 20 | uint64_t r15; 21 | 22 | uint64_t int_no; 23 | uint64_t err_code; 24 | 25 | uint64_t rip; 26 | uint64_t cs; 27 | uint64_t rflags; 28 | uint64_t rsp; 29 | uint64_t ss; 30 | } registers; 31 | 32 | #define debug_print_registers(r) \ 33 | debug("RAX=%016x RBX=%016x RCX=%016x RDX=%016x\n", r->rax, r->rbx, r->rcx, r->rdx); \ 34 | debug("RSI=%016x RDI=%016x RBP=%016x RSP=%016x\n", r->rsi, r->rdi, r->rbp, r->rsp); \ 35 | debug("R8 =%016x R9 =%016x R10=%016x R11=%016x\n", r->r8, r->r9, r->r10, r->r11); \ 36 | debug("R12=%016x R13=%016x R14=%016x R15=%016x\n", r->r12, r->r13, r->r14, r->r15); \ 37 | debug("RIP=%016x RFL=%08x\n", r->rip, r->rflags); \ 38 | debug("CS=%016x SS=%016x\n", r->cs, r->ss); \ 39 | debug("CR0=%08x CR2=%016x CR3=%016x CR4=%08x\n", read_cr0(), read_cr2(), read_cr3(), read_cr4()); 40 | 41 | 42 | typedef registers *(*int_handler_t)(registers *); 43 | 44 | void interrupt_init(); 45 | int_handler_t bind_interrupt(uint32_t num, int_handler_t fn); 46 | void isr_return(registers *); 47 | -------------------------------------------------------------------------------- /src/kernel/memory/pmm.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | 4 | #include 5 | #undef P2V 6 | #define P2V(addr) ((void *)((uintptr_t)(addr)+1)) 7 | #undef V2P 8 | #define V2P(addr) ((uintptr_t)(addr)-1) 9 | #include "pmm.c" 10 | 11 | struct { 12 | uint8_t data[PAGE_SIZE]; 13 | }__attribute__((packed)) mem[2]; 14 | 15 | TEST(alloc_returns_freed_page) 16 | { 17 | pmm_free(V2P(&mem[0])); 18 | uint64_t a = pmm_alloc(); 19 | ASSERT_EQ_PTR(a, V2P(&mem[0])); 20 | } 21 | TEST(alloc_returns_0_if_no_free_pages) 22 | { 23 | uint64_t a = pmm_alloc(); 24 | ASSERT_EQ_PTR(a, 0); 25 | } 26 | TEST(alloc_zero_after_all_free_pages) 27 | { 28 | pmm_free(V2P(&mem[0])); 29 | pmm_alloc(); 30 | uint64_t a = pmm_alloc(); 31 | ASSERT_EQ_PTR(a, 0); 32 | } 33 | 34 | TEST(alloc_two_pages___first_page_is_not_zero) 35 | { 36 | pmm_free(V2P(&mem[0])); 37 | pmm_free(V2P(&mem[1])); 38 | uint64_t a = pmm_alloc(); 39 | pmm_alloc(); 40 | ASSERT_NEQ_PTR(a, 0); 41 | } 42 | TEST(alloc_two_pages___second_page_is_not_zero) 43 | { 44 | pmm_free(V2P(&mem[0])); 45 | pmm_free(V2P(&mem[1])); 46 | pmm_alloc(); 47 | uint64_t a = pmm_alloc(); 48 | ASSERT_NEQ_PTR(a, 0); 49 | } 50 | TEST(alloc_two_pages___doesnt_return_same_page_twice) 51 | { 52 | pmm_free(V2P(&mem[0])); 53 | pmm_free(V2P(&mem[1])); 54 | uint64_t a = pmm_alloc(); 55 | uint64_t b = pmm_alloc(); 56 | ASSERT_NEQ_PTR(a, b); 57 | } 58 | -------------------------------------------------------------------------------- /src/kernel/boot/kmain.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | void thread_function() 13 | { 14 | char *p = 0xBADC0FFEE; 15 | while(1) 16 | p[0] = 1; 17 | } 18 | 19 | 20 | void kmain(uint64_t multiboot_magic, void *multiboot_data) 21 | { 22 | serial_init(PORT_COM1); 23 | vga_init(); 24 | debug_info("kernel loaded\n"); 25 | 26 | multiboot_init(multiboot_magic, P2V(multiboot_data)); 27 | 28 | debug_info("Kernel was loaded with command line \"%s\", by <%s>\n", kernel_boot_data.commandline, kernel_boot_data.bootloader); 29 | 30 | memory_init(); 31 | 32 | cpu_init(); 33 | 34 | acpi_init(); 35 | 36 | struct process *p1 = new_process((void (*)(void))0x10000); 37 | procmm_brk(p1, (void *)0x10100); 38 | memcpy_to_p4(p1->P4, (void *)0x10000, (void *)(uintptr_t)thread_function, 100); 39 | ready(p1); 40 | 41 | struct process *p2 = new_process((void (*)(void))0x10000); 42 | procmm_brk(p2, (void *)0x10100); 43 | memcpy_to_p4(p2->P4, (void *)0x10000, (void *)(uintptr_t)thread_function, 100); 44 | ready(p2); 45 | 46 | debug_ok("Boot \"Complete\"\n"); 47 | 48 | start_scheduler(); 49 | 50 | PANIC("Reached end of kernel main function\n"); 51 | for(;;); 52 | } 53 | 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Thomas Lovén 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included 14 | in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | ----------------------------------------------------------------------- 25 | 26 | This project will download several other source files which are under 27 | their respective licenses. 28 | 29 | Those project comprise GNU binutils, GNU gcc, GNU gdb and GRUB2. 30 | -------------------------------------------------------------------------------- /toolchain/build-toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | # Tools and libraries required to build binutils and gcc 4 | apk --update add build-base 5 | apk add gmp-dev mpfr-dev mpc1-dev 6 | 7 | # For making a bootable iso 8 | apk add grub-bios xorriso 9 | # For debugging 10 | apk add gdb valgrind 11 | # We need a later version of qemu than included in the default repo 12 | # due to a bug in the qemu gdb server 13 | apk --update add qemu-system-x86_64 --repository http://dl-cdn.alpinelinux.org/alpine/v3.7/main 14 | 15 | rm -rf /var/cache/apk/* 16 | 17 | # For now we'll use the default x86_64-elf target 18 | target=x86_64-elf 19 | binutils=binutils-2.29 20 | gcc=gcc-7.2.0 21 | 22 | 23 | # Configure, make and install binutils 24 | # configured for our target 25 | cd /opt 26 | wget http://ftp.gnu.org/gnu/binutils/${binutils}.tar.gz 27 | tar -xf ${binutils}.tar.gz 28 | mkdir binutils-build && cd binutils-build 29 | ../${binutils}/configure \ 30 | --target=${target} \ 31 | --disable-nls \ 32 | --disable-werror \ 33 | --with-sysroot \ 34 | 35 | make -j 4 36 | make install 37 | 38 | # Configure, make and install gcc and libgcc 39 | cd /opt 40 | wget http://ftp.gnu.org/gnu/gcc/${gcc}/${gcc}.tar.gz 41 | tar -xf ${gcc}.tar.gz 42 | mkdir gcc-build && cd gcc-build 43 | ../${gcc}/configure \ 44 | --target=${target} \ 45 | --disable-nls \ 46 | --enable-languages=c \ 47 | --without-headers \ 48 | 49 | make all-gcc all-target-libgcc -j 4 50 | make install-gcc install-target-libgcc 51 | 52 | cd / 53 | # Remove apk cache 54 | rm -rf /opt 55 | -------------------------------------------------------------------------------- /src/kernel/memory/string.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | // NOTE: Functions in this file have different names during testing, since they 6 | // are otherwise overridden by standard library functions 7 | 8 | #ifdef TTEST 9 | void *my_memcpy(void *dst, const void *src, size_t n) 10 | #else 11 | void *memcpy(void *dst, const void *src, size_t n) 12 | #endif 13 | { 14 | char *dp = dst; 15 | const char *sp = src; 16 | while(n--) *dp++ = *sp++; 17 | return dst; 18 | } 19 | 20 | #ifdef TTEST 21 | void *my_memset(void *s, int c, size_t n) 22 | #else 23 | void *memset(void *s, int c, size_t n) 24 | #endif 25 | { 26 | unsigned char *p = s; 27 | while(n--) *p++ = (unsigned char)c; 28 | return s; 29 | } 30 | 31 | #ifdef TTEST 32 | void *my_memmove(void *dst, const void *src, size_t n) 33 | #else 34 | void *memmove(void *dst, const void *src, size_t n) 35 | #endif 36 | { 37 | if(src == dst) 38 | return dst; 39 | 40 | const void *src_end = (const void *)((uintptr_t)src + n); 41 | if(src < dst && dst < src_end) 42 | { 43 | char *dp = dst; 44 | const char *sp = src; 45 | while(n--) 46 | dp[n] = sp[n]; 47 | return dst; 48 | } 49 | 50 | memcpy(dst, src, n); 51 | return dst; 52 | } 53 | 54 | #ifdef TTEST 55 | int my_memcmp(const void *s1, const void *s2, size_t n) 56 | #else 57 | int memcmp(const void *s1, const void *s2, size_t n) 58 | #endif 59 | { 60 | const unsigned char *p1 = s1, *p2 = s2; 61 | for(; n--; p1++, p2++) 62 | { 63 | if(*p1 != *p2) 64 | return *p1 - *p2; 65 | } 66 | return 0; 67 | } 68 | 69 | #ifdef TTEST 70 | size_t my_strlen(const char *s) 71 | #else 72 | size_t strlen(const char *s) 73 | #endif 74 | { 75 | size_t len = 0; 76 | while(*s++) len++; 77 | return len; 78 | } 79 | -------------------------------------------------------------------------------- /src/kernel/include/queue.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | 4 | #include "queue.h" 5 | 6 | #define TestQ test_queue, test_queue_next, struct item 7 | QUEUE_DECLARE(TestQ); 8 | QUEUE_DEFINE(TestQ); 9 | 10 | struct item 11 | { 12 | int id; 13 | QUEUE_SPOT(TestQ); 14 | }; 15 | 16 | struct item item1 = {1, 0}; 17 | struct item item2 = {2, 0}; 18 | 19 | TEST(peek_returns_queued_item) 20 | { 21 | queue_add(TestQ, &item1); 22 | struct item *i = queue_peek(TestQ); 23 | ASSERT_EQ_INT(i->id, 1); 24 | } 25 | TEST(peek_returns_first_queued_item) 26 | { 27 | queue_add(TestQ, &item1); 28 | queue_add(TestQ, &item2); 29 | struct item *i = queue_peek(TestQ); 30 | ASSERT_EQ_INT(i->id, 1); 31 | } 32 | TEST(peek_returns_zero_for_empty_queue) 33 | { 34 | struct item *i = queue_peek(TestQ); 35 | ASSERT_EQ_PTR(i, 0); 36 | } 37 | 38 | TEST(drop_drops_item_from_queue) 39 | { 40 | queue_add(TestQ, &item1); 41 | queue_add(TestQ, &item2); 42 | queue_drop(TestQ); 43 | struct item *i = queue_peek(TestQ); 44 | ASSERT_EQ_INT(i->id, 2); 45 | } 46 | TEST(drop_empty_queue_does_not_crash) 47 | { 48 | queue_drop(TestQ); 49 | ASSERT_EQ_INT(!!queue_empty(TestQ), 1); 50 | } 51 | 52 | TEST(empty_reports_empty_queue_empty) 53 | { 54 | ASSERT_EQ_INT(!!queue_empty(TestQ), 1); 55 | } 56 | TEST(empty_reports_nonempty_queue_not_empty) 57 | { 58 | queue_add(TestQ, &item1); 59 | ASSERT_EQ_INT(!!queue_empty(TestQ), 0); 60 | } 61 | 62 | TEST(pop_returns_queued_item) 63 | { 64 | queue_add(TestQ, &item1); 65 | struct item *i = queue_pop(TestQ); 66 | ASSERT_EQ_INT(i->id, 1); 67 | } 68 | TEST(pop_removes_item_from_queue) 69 | { 70 | queue_add(TestQ, &item1); 71 | queue_add(TestQ, &item2); 72 | queue_pop(TestQ); 73 | struct item *i = queue_peek(TestQ); 74 | ASSERT_EQ_INT(i->id, 2); 75 | } 76 | -------------------------------------------------------------------------------- /src/kernel/proc/procmm.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | void procmm_init(struct process *p) 10 | { 11 | p->P4 = new_P4(); 12 | p->brk = 0; 13 | p->stack = KERNEL_OFFSET; 14 | 15 | } 16 | 17 | uint64_t procmm_brk(struct process *p, void *addr) 18 | { 19 | while((uint64_t)addr > p->brk) 20 | { 21 | vmm_set_page(p->P4, p->brk, pmm_alloc(), PAGE_USER | PAGE_WRITE | PAGE_PRESENT); 22 | p->brk += PAGE_SIZE; 23 | } 24 | return p->brk; 25 | } 26 | 27 | registers *proc_pagefault(registers *r) 28 | { 29 | if(!(r->rflags & (3<<12))) 30 | { 31 | debug_error("Page fault in kernel!\n"); 32 | debug("Interrupt number: %d Error code: %d\n", r->int_no, r->err_code); 33 | debug_print_registers(r); 34 | PANIC("Page fault in kernel\n"); 35 | } 36 | 37 | uint64_t fault_addr = read_cr2(); 38 | uint64_t stack = cpu->proc->stack; 39 | if(fault_addr + PAGE_SIZE >= stack) 40 | { 41 | // Page fault happened just below stack. Add another page to it. 42 | // Unless it's about to run into brk. 43 | if(stack - PAGE_SIZE <= cpu->proc->brk) 44 | PANIC("Stack overflow in process %d\n", cpu->proc->pid); 45 | 46 | stack -= PAGE_SIZE; 47 | vmm_set_page(cpu->proc->P4, stack, pmm_alloc(), PAGE_USER | PAGE_WRITE | PAGE_PRESENT); 48 | cpu->proc->stack = stack; 49 | 50 | return r; 51 | } 52 | 53 | // Very uggly and very temporary way of signaling the kernel from a user mode process. 54 | // TODO: REMOVE THIS CRAP! 55 | if(read_cr2() == 0xBADC0FFEE) 56 | { 57 | debug("Bad signal from process %d\n", cpu->proc->pid); 58 | yield(); 59 | return r; 60 | } 61 | PANIC("Page fault in process\n"); 62 | return r; 63 | } 64 | -------------------------------------------------------------------------------- /src/kernel/include/memory.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #define KERNEL_OFFSET 0xFFFFFF8000000000 3 | 4 | #ifdef __ASSEMBLER__ 5 | #define V2P(a) ((a) - KERNEL_OFFSET) 6 | #define P2V(a) ((a) + KERNEL_OFFSET) 7 | #else 8 | #include 9 | #define V2P(a) ((uintptr_t)(a) & ~KERNEL_OFFSET) 10 | #define P2V(a) ((void *)((uintptr_t)(a) | KERNEL_OFFSET)) 11 | #define incptr(p, n) ((void *)(((uintptr_t)(p)) + (n))) 12 | #endif 13 | 14 | #define P1_OFFSET(a) (((a)>>12) & 0x1FF) 15 | #define P2_OFFSET(a) (((a)>>21) & 0x1FF) 16 | #define P3_OFFSET(a) (((a)>>30) & 0x1FF) 17 | #define P4_OFFSET(a) (((a)>>39) & 0x1FF) 18 | 19 | #define PAGE_PRESENT 0x001 20 | #define PAGE_WRITE 0x002 21 | #define PAGE_USER 0x004 22 | #define PAGE_HUGE 0x080 23 | #define PAGE_GLOBAL 0x100 24 | 25 | #define PAGE_SIZE 0x1000 26 | #define ENTRIES_PER_PT 512 27 | 28 | #ifndef __ASSEMBLER__ 29 | #include 30 | 31 | extern uint64_t kernel_P4; 32 | 33 | void *memcpy(void *dst, const void *src, size_t n); 34 | void *memset(void *s, int c, size_t n); 35 | void *memmove(void *dest, const void *src, size_t n); 36 | int memcmp(const void *s1, const void *s2, size_t n); 37 | size_t strlen(const char *s); 38 | 39 | void pmm_free(uint64_t page); 40 | uint64_t pmm_alloc(); 41 | uint64_t pmm_calloc(); 42 | 43 | uint64_t new_P4(); 44 | uint64_t vmm_get_page(uint64_t P4, uint64_t addr); 45 | #define PAGE_EXIST(p) ((p) != (uint64_t)-1) 46 | int vmm_set_page(uint64_t P4, uint64_t addr, uint64_t page, uint16_t flags); 47 | void vmm_clear_page(uint64_t P4, uint64_t addr, int free); 48 | size_t memcpy_to_p4(uint64_t P4, void *dst, void *src, size_t n); 49 | size_t memcpy_from_p4(void *dst, uint64_t P4, void *src, size_t n); 50 | 51 | extern union PTE BootP4; 52 | extern int kernel_start, kernel_end; 53 | 54 | void memory_init(); 55 | #endif 56 | -------------------------------------------------------------------------------- /src/kernel/drivers/vga.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #define VGA_COLS 80 6 | #define VGA_ROWS 24 7 | #define VGA_SIZE (VGA_COLS*VGA_ROWS) 8 | 9 | #define VGA_ROW(pos) ((pos)/VGA_COLS) 10 | #define VGA_COL(pos) ((pos)%VGA_COLS) 11 | #define VGA_POS(row, col) ((row)*VGA_COLS + (col)) 12 | 13 | #define VGA_ADDRESS_PORT 0x3D4 14 | #define VGA_DATA_PORT 0x3D5 15 | #define VGA_REGISTER_CURSOR_POS_LOW 0xF 16 | #define VGA_REGISTER_CURSOR_POS_HIGH 0xE 17 | 18 | void *vidmem; 19 | struct vga_cell{ 20 | uint8_t c; 21 | uint8_t f; 22 | }__attribute__((packed)); 23 | 24 | struct vga_cell buffer[VGA_SIZE]; 25 | 26 | uint64_t cursor; 27 | uint8_t format; 28 | 29 | void vga_init() 30 | { 31 | vidmem = VGA_MEMORY; 32 | memset(vidmem, 0, VGA_SIZE*sizeof(struct vga_cell)); 33 | 34 | // White text on black background 35 | format = 0x07; 36 | } 37 | 38 | void movecursor() 39 | { 40 | outb(VGA_ADDRESS_PORT, VGA_REGISTER_CURSOR_POS_LOW); 41 | outb(VGA_DATA_PORT, (uint8_t)(cursor & 0xFF)); 42 | outb(VGA_ADDRESS_PORT, VGA_REGISTER_CURSOR_POS_HIGH); 43 | outb(VGA_DATA_PORT, (uint8_t)((cursor >> 8) & 0xFF)); 44 | } 45 | 46 | void flush() 47 | { 48 | memcpy(vidmem, buffer, sizeof(buffer)); 49 | } 50 | 51 | void scroll() 52 | { 53 | while(cursor >= VGA_SIZE) 54 | { 55 | // Move everything up one row 56 | memmove(buffer, &buffer[VGA_POS(1,0)], VGA_COLS*(VGA_ROWS-1)*sizeof(struct vga_cell)); 57 | // Clear last row 58 | memset(&buffer[VGA_POS(VGA_ROWS-1, 0)], 0, VGA_COLS*sizeof(struct vga_cell)); 59 | cursor -= VGA_COLS; 60 | } 61 | } 62 | 63 | void vga_write(char c) 64 | { 65 | switch(c) 66 | { 67 | case '\n': 68 | cursor += VGA_COLS - VGA_COL(cursor); 69 | break; 70 | default: 71 | buffer[cursor++] = (struct vga_cell){.c = c, .f=format}; 72 | } 73 | scroll(); 74 | flush(); 75 | movecursor(); 76 | } 77 | -------------------------------------------------------------------------------- /src/kernel/cpu/interrupts.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #define IDT_INTERRUPT 0xE 8 | #define IDT_DPL0 0x0 9 | #define IDT_PRESENT 0x80 10 | 11 | #define NUM_INTERRUPTS 256 12 | 13 | struct idt 14 | { 15 | uint16_t base_l; 16 | uint16_t cs; 17 | uint8_t ist; 18 | uint8_t flags; 19 | uint16_t base_m; 20 | uint32_t base_h; 21 | uint32_t ignored; 22 | } __attribute__((packed)) idt[NUM_INTERRUPTS]; 23 | 24 | struct 25 | { 26 | uint16_t len; 27 | struct idt *addr; 28 | } __attribute__((packed)) idtr; 29 | 30 | extern uintptr_t isr_table[]; 31 | int_handler_t int_handlers[NUM_INTERRUPTS]; 32 | 33 | void idt_set_gate(uint32_t num, uintptr_t vector, uint16_t cs, uint8_t ist, uint8_t flags) 34 | { 35 | idt[num].base_l = vector & 0xFFFF; 36 | idt[num].base_m = (vector >> 16) & 0xFFFF; 37 | idt[num].base_h = (vector >> 32) & 0xFFFFFFFF; 38 | idt[num].cs = cs; 39 | idt[num].ist = ist; 40 | idt[num].flags = flags; 41 | } 42 | 43 | void interrupt_init() 44 | { 45 | memset(idt, 0, sizeof(idt)); 46 | memset(int_handlers, 0, sizeof(int_handlers)); 47 | 48 | for(int i=0; i < NUM_INTERRUPTS; i++) 49 | { 50 | idt_set_gate(i, isr_table[i], 0x8, 0, IDT_PRESENT | IDT_DPL0 | IDT_INTERRUPT); 51 | } 52 | 53 | idtr.addr = idt; 54 | idtr.len = sizeof(idt)-1; 55 | load_idt(&idtr); 56 | } 57 | 58 | int_handler_t bind_interrupt(uint32_t num, int_handler_t fn) 59 | { 60 | int_handler_t old = int_handlers[num]; 61 | int_handlers[num] = fn; 62 | return old; 63 | } 64 | 65 | registers *int_handler(registers *r) 66 | { 67 | if(int_handlers[r->int_no]) 68 | return int_handlers[r->int_no](r); 69 | 70 | debug("Unhandled interrupt occurred\n"); 71 | debug("Interrupt number: %d Error code: %d\n", r->int_no, r->err_code); 72 | debug_print_registers(r); 73 | 74 | PANIC("Unhandled interrupt occurred"); 75 | for(;;); 76 | } 77 | -------------------------------------------------------------------------------- /src/kernel/boot/boot.S: -------------------------------------------------------------------------------- 1 | #include 2 | .intel_syntax noprefix 3 | 4 | .section .bss 5 | .align PAGE_SIZE 6 | .skip PAGE_SIZE 7 | BootStack: 8 | 9 | .section .text 10 | 11 | .code32 12 | .global _start 13 | _start: 14 | cli 15 | 16 | mov edi, eax 17 | mov esi, ebx 18 | 19 | //; Set up a known stack 20 | mov esp, offset V2P(BootStack) 21 | 22 | //; Set CR4.PAE 23 | //; enabling Page Address Extension 24 | mov eax, cr4 25 | or eax, 1<<5 26 | mov cr4, eax 27 | 28 | //; Load a P4 page table 29 | mov eax, offset V2P(BootP4) 30 | mov cr3, eax 31 | 32 | //; Set EFER.LME 33 | //; enabling Long Mode 34 | mov ecx, 0x0C0000080 35 | rdmsr 36 | or eax, 1<<8 37 | wrmsr 38 | 39 | //; Set CR0.PG 40 | //; enabling Paging 41 | mov eax, cr0 42 | or eax, 1<<31 43 | mov cr0, eax 44 | 45 | //; Load a new GDT 46 | lgdt [V2P(GDTp)] 47 | 48 | //; and update the code selector by a long jump 49 | jmp 0x8:V2P(long_mode_start) 50 | 51 | .code64 52 | long_mode_start: 53 | 54 | //; Clear out all other selectors 55 | mov eax, 0x0 56 | mov ss, eax 57 | mov ds, eax 58 | mov es, eax 59 | 60 | //; Jump to kernel address space 61 | movabs rax, offset upper_memory 62 | jmp rax 63 | 64 | upper_memory: 65 | 66 | //; Move stack pointer to kernel space 67 | mov rax, KERNEL_OFFSET 68 | add rsp, rax 69 | 70 | //; Remove identity mapping 71 | mov rax, 0 72 | movabs [BootP4], rax 73 | 74 | //; Update page tables 75 | mov rax, cr3 76 | mov cr3, rax 77 | 78 | //; Reload GDT 79 | movabs rax, offset GDTp 80 | lgdt [rax] 81 | mov rax, 0x0 82 | mov ss, rax 83 | mov ds, rax 84 | mov es, rax 85 | 86 | //; Reload CS 87 | movabs rax, offset .reload_cs 88 | pushq 0x8 89 | push rax 90 | retfq 91 | .reload_cs: 92 | 93 | //; Jump to kmain() 94 | .extern kmain 95 | movabs rax, offset kmain 96 | call rax 97 | 98 | hlt 99 | jmp $ 100 | -------------------------------------------------------------------------------- /src/kernel/cpu/gdt.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #define GDT_CODE (0x18<<8) 6 | #define GDT_DATA (0x12<<8) 7 | #define GDT_TSS (0x09<<8) 8 | #define GDT_DPL(lvl) ((lvl)<<13) 9 | #define GDT_PRESENT (1<<15) 10 | #define GDT_LONG (1<<21) 11 | 12 | struct gdt 13 | { 14 | uint32_t addr; 15 | uint32_t flags; 16 | }__attribute__((packed)); 17 | 18 | struct gdtp 19 | { 20 | uint16_t len; 21 | struct gdt *gdt; 22 | }__attribute__((packed)); 23 | 24 | struct tss 25 | { 26 | uint32_t r1; 27 | uint64_t rsp0; 28 | uint64_t rsp1; 29 | uint64_t rsp2; 30 | uint64_t r2; 31 | uint64_t ist1; 32 | uint64_t ist2; 33 | uint64_t ist3; 34 | uint64_t ist4; 35 | uint64_t ist5; 36 | uint64_t ist6; 37 | uint64_t ist7; 38 | uint64_t r3; 39 | uint16_t r4; 40 | uint16_t io_mba; 41 | }__attribute__((packed)); 42 | 43 | 44 | struct gdt BootGDT[] = { 45 | {0, 0}, 46 | {0, GDT_PRESENT | GDT_DPL(0) | GDT_CODE | GDT_LONG}, 47 | {0, GDT_PRESENT | GDT_DPL(3) | GDT_CODE | GDT_LONG}, 48 | {0, GDT_PRESENT | GDT_DPL(3) | GDT_DATA}, 49 | {0, 0}, 50 | {0, 0}, 51 | }; 52 | 53 | struct gdtp GDTp = {2*8-1, BootGDT}; 54 | 55 | void gdt_init(struct cpu *c) 56 | { 57 | struct gdt *gdt = (void *)c->gdt; 58 | memcpy(gdt, BootGDT, sizeof(BootGDT)); 59 | 60 | struct tss *tss = (void *)c->tss; 61 | tss->io_mba = sizeof(struct tss); 62 | 63 | uint32_t tss_limit = sizeof(struct tss); 64 | uint64_t tss_base = (uint64_t)tss; 65 | gdt[4].flags = GDT_PRESENT | GDT_TSS; 66 | gdt[4].flags |= ((tss_base >> 24) & 0xFF) << 24; 67 | gdt[4].flags |= (tss_base >> 16) & 0xFF; 68 | gdt[4].flags |= ((tss_limit >> 16) & 0xF) << 16; 69 | gdt[4].addr = ((tss_base & 0xFFFF) << 16) | (tss_limit & 0xFFFF); 70 | gdt[5].addr = (tss_base >> 32) & 0xFFFFFFFF; 71 | 72 | GDTp.len = 6*8 - 1; 73 | GDTp.gdt = gdt; 74 | 75 | load_gdt(&GDTp); 76 | } 77 | 78 | void interrupt_stack(void *rsp0) 79 | { 80 | struct tss __seg_gs *tss = (void __seg_gs *)(cpu->tss); 81 | tss->rsp0 = (uint64_t)rsp0; 82 | } 83 | -------------------------------------------------------------------------------- /src/kernel/memory/string.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | #include 4 | #include "string.c" 5 | 6 | char *lipsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus interdum eros at enim tempus sed."; 7 | 8 | char dst[100]; 9 | 10 | BEFORE() 11 | { 12 | for(int i = 0; i < 100; dst[i++] = '\0'); 13 | } 14 | 15 | TEST(memcpy_copies_data) 16 | { 17 | my_memcpy(dst, lipsum, 100); 18 | ASSERT_EQ_STR(dst, lipsum, 100); 19 | } 20 | TEST(memcpy_doesnt_copy_too_much) 21 | { 22 | my_memcpy(dst, lipsum, 10); 23 | ASSERT_EQ_CHR(dst[10], 0); 24 | } 25 | 26 | TEST(memset_sets_data) 27 | { 28 | memset(dst, 0, 10); 29 | my_memset(dst, 'a', 5); 30 | ASSERT_EQ_STR(dst, "aaaaa", 100); 31 | } 32 | 33 | TEST(memmove_moves_data) 34 | { 35 | memcpy(&dst[10], "12345", 5); 36 | my_memmove(dst, &dst[10], 5); 37 | ASSERT_EQ_STR(dst, "12345", 5); 38 | } 39 | TEST(memmove_handles_overlap) 40 | { 41 | memcpy(&dst[5], "1234567890", 10); 42 | my_memmove(dst, &dst[5], 10); 43 | ASSERT_EQ_STR(dst, "1234567890", 10); 44 | } 45 | TEST(memmove_handles_overlap_the_other_way) 46 | { 47 | memcpy(dst, "1234567890", 10); 48 | my_memmove(&dst[5], dst, 10); 49 | ASSERT_EQ_STR(&dst[5], "1234567890", 10); 50 | } 51 | TEST(memmove_moves_correct_number_of_chars) 52 | { 53 | memcpy(&dst[5], "1234567890", 10); 54 | my_memmove(dst, &dst[5], 9); 55 | ASSERT_EQ_STR(dst, "123456789567890", 15); 56 | } 57 | TEST(memmove_moves_correct_number_of_chars_the_other_way) 58 | { 59 | memcpy(dst, "1234567890", 10); 60 | my_memmove(&dst[5], dst, 3); 61 | ASSERT_EQ_STR(dst, "1234512390", 10); 62 | } 63 | 64 | TEST(memcmp_returns_zero_for_equal_strings) 65 | { 66 | ASSERT_EQ_INT(my_memcmp("1234", "1234", 4), 0); 67 | } 68 | TEST(memcmp_returns_difference_for_unequal_strings) 69 | { 70 | ASSERT_EQ_INT(my_memcmp("1234", "0234", 4), 1); 71 | } 72 | TEST(memcmp_returns_signed_difference_for_unequal_strings) 73 | { 74 | ASSERT_EQ_INT(my_memcmp("1234", "1236", 4), -2); 75 | } 76 | 77 | TEST(strlen_counts_correctly) 78 | { 79 | ASSERT_EQ_INT(my_strlen("12345"), 5); 80 | } 81 | 82 | -------------------------------------------------------------------------------- /src/kernel/include/queue.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | // This header file contains macros for making FIFO queues from structures 4 | // A queue is set up via a preprocessor definition 5 | // #define QueueName header_name, spot_name, struct type 6 | // where 7 | // QueueName is the queue identifying name 8 | // header_name is a unique name used for the queue head 9 | // spot_name is the name of the queue placeholder in the struct 10 | // struct type is the type of struct the queue is made from 11 | // 12 | // Each queue requires a declaration 13 | // QUEUE_DECLARE(QueueName); 14 | // and a definition 15 | // QUEUE_DEFINE(QueueName); 16 | 17 | #define _QUEUE_DECL(queue, entry, type) \ 18 | struct queue{ \ 19 | type *first; \ 20 | type *last; \ 21 | } queue 22 | #define QUEUE_DECLARE(...) _QUEUE_DECL(__VA_ARGS__) 23 | #define _QUEUE_HEAD(queue, entry, type) \ 24 | struct queue queue = {0, 0} 25 | #define QUEUE_DEFINE(...) _QUEUE_HEAD(__VA_ARGS__) 26 | 27 | #define _QUEUE_SPOT(queue, entry, type) \ 28 | type *entry 29 | #define QUEUE_SPOT(...) _QUEUE_SPOT(__VA_ARGS__) 30 | 31 | #define _QUEUE_EMPTY(queue, entry, type) \ 32 | (!(queue.last)) 33 | #define queue_empty(...) _QUEUE_EMPTY(__VA_ARGS__) 34 | #define queue_not_empty(...) (!(_QUEUE_EMPTY(__VA_ARGS__))) 35 | 36 | #define _QUEUE_ADD(queue, entry, type, item) \ 37 | if(!queue.last) \ 38 | queue.first = (item); \ 39 | else \ 40 | queue.last->entry = (item); \ 41 | queue.last = item; \ 42 | (item)->entry = 0; 43 | #define queue_add(...) _QUEUE_ADD(__VA_ARGS__) 44 | 45 | #define _QUEUE_DROP(queue, entry, type) \ 46 | if(queue.first && !(queue.first = queue.first->entry)) \ 47 | queue.last = 0; 48 | #define queue_drop(...) _QUEUE_DROP(__VA_ARGS__) 49 | 50 | #define _QUEUE_PEEK(queue, entry, type) \ 51 | (queue.first) 52 | #define queue_peek(...) _QUEUE_PEEK(__VA_ARGS__) 53 | 54 | #define _QUEUE_POP(queue, entry, type) \ 55 | __extension__({ \ 56 | type *_ret = _QUEUE_PEEK(queue, entry, type); \ 57 | _QUEUE_DROP(queue, entry, type); \ 58 | _ret; \ 59 | }) 60 | #define queue_pop(...) _QUEUE_POP(__VA_ARGS__) 61 | -------------------------------------------------------------------------------- /src/kernel/proc/process.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | struct swtch_stack 7 | { 8 | uint64_t RBP; 9 | uint64_t RBX; 10 | uint64_t R12; 11 | uint64_t R13; 12 | uint64_t R14; 13 | uint64_t R15; 14 | uint64_t isr_return_arg; 15 | uint64_t RBP2; 16 | uint64_t ret; 17 | registers r; 18 | }__attribute__((packed)); 19 | 20 | void procmm_init(struct process *p); 21 | registers *proc_pagefault(registers *r); 22 | 23 | #define csched cpu->scheduler 24 | #define cproc cpu->proc 25 | 26 | uint64_t next_pid = 1; 27 | struct process *new_process(void (*function)(void)) 28 | { 29 | if(next_pid == 1) 30 | { 31 | bind_interrupt(14, proc_pagefault); 32 | } 33 | 34 | struct process *proc = P2V(pmm_calloc()); 35 | proc->pid = next_pid++; 36 | proc->stack_ptr = incptr(proc, PAGE_SIZE - sizeof(struct swtch_stack)); 37 | procmm_init(proc); 38 | 39 | struct swtch_stack *stk = proc->stack_ptr; 40 | stk->RBP = (uint64_t)&stk->RBP2; 41 | 42 | stk->ret = (uint64_t)isr_return; 43 | stk->isr_return_arg = (uint64_t)&stk->r; 44 | 45 | stk->r.rip = (uint64_t)function; 46 | stk->r.cs = 0x10 | 3; 47 | stk->r.ss = 0x18 | 3; 48 | stk->r.rflags = 3<<12; 49 | stk->r.rsp = KERNEL_OFFSET; 50 | 51 | return proc; 52 | } 53 | 54 | void yield() 55 | { 56 | switch_stack(&cproc->stack_ptr, &csched->stack_ptr); 57 | } 58 | 59 | void scheduler() 60 | { 61 | while(1) 62 | { 63 | struct process *new = 0; 64 | while(!(new = scheduler_next())); 65 | 66 | cproc = new; 67 | write_cr3(new->P4); 68 | interrupt_stack(incptr(new, PAGE_SIZE)); 69 | switch_stack(&csched->stack_ptr, &new->stack_ptr); 70 | 71 | ready(cproc); 72 | cproc = 0; 73 | } 74 | } 75 | 76 | void start_scheduler() 77 | { 78 | struct process *sched = csched = P2V(pmm_calloc()); 79 | sched->pid = (uint64_t)-1; 80 | sched->stack_ptr = incptr(sched, PAGE_SIZE - sizeof(struct swtch_stack) + sizeof(registers)); 81 | sched->P4 = kernel_P4; 82 | 83 | struct swtch_stack *stk = sched->stack_ptr; 84 | stk->RBP = (uint64_t)&stk->RBP2; 85 | 86 | stk->ret = (uint64_t)scheduler; 87 | 88 | uint64_t stack; 89 | switch_stack(&stack, &sched->stack_ptr); 90 | } 91 | -------------------------------------------------------------------------------- /doc/7_Multiboot_Data.md: -------------------------------------------------------------------------------- 1 | # Multiboot Data 2 | 3 | In this chapter we'll parse the multiboot information structure passed 4 | to the kernel from the bootloader. 5 | 6 | ## Passing the data to the kernel 7 | When a multiboot compliant bootloader passes controll to the kernel, the 8 | registers `eax` and `ebx` contains a magic number and a pointer to the 9 | multiboot information structure respectively. 10 | 11 | If we wish to pass those on to the kernel main function, we can do that 12 | according to the System V ABI calling convention. According to this, 13 | the first two arguments to a function should be passed in the `rdi` and 14 | `rsi` registers for x86\_64. Those obviously correspond to `edi` and 15 | `esi` in 32-bit mode, and luckily those registers are unused throughout 16 | the entire long mode transition process, untill we call `kmain`. 17 | 18 | So everything we need to to do preserve the information is to move it 19 | into those registers. Might as well do that as soon as possible. 20 | 21 | `src/kernel/boot/boot.S` 22 | ```asm 23 | ... 24 | _start: 25 | cli 26 | 27 | mov edi, eax 28 | mov esi, ebx 29 | 30 | ... 31 | ``` 32 | 33 | And `kmain` is updated accordingly: 34 | `src/kernel/boot/kmain.c` 35 | ```c 36 | ... 37 | void kmain(uint64_t multiboot_magic, void *multiboot_data) 38 | { 39 | ... 40 | ``` 41 | 42 | ## Reading the multiboot information 43 | 44 | Reading the data passed from the bootloader is actually pretty straight 45 | forward. It's stored as a list of "tags" with a 32 bit type and 32 bit 46 | size followed by the data. 47 | 48 | I just step through the tag list and save the data I want into a global 49 | structure. The only possible pitfall is that tags are 8 byte alligned, 50 | so finding the next tag can look like: 51 | 52 | ```c 53 | int padded_size = tag->size + ((tag->size % 8)?(8-(tag->size%8)):0); 54 | tag = incptr(tag, padded_size); 55 | ``` 56 | 57 | `incptr` is just a macro I made to move a pointer a number of bytes 58 | forward or backward. Quite useful. 59 | 60 | ```c 61 | #define incptr(p, n) ((void *)(((uintptr_t)(p)) + (n))) 62 | ``` 63 | 64 | For now, I only save the bootloader name (type 2) and kernel commandline 65 | (type 1), and that's just for testing purposes. Later we'll also want to 66 | save and parse the memory map (type 6) and any modules (type 3). 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MITTOS64 2 | 3 | This project is my attempt at a modern kernel. 4 | 5 | Differences from my earlier kernels include 6 | - 64 bit 7 | - symmetric multiprocessing 8 | - docker-based build chain 9 | - unit tests 10 | 11 | I meant to write very extensive documentation of the development process, but with two kids I have realized that will probably never happen, so now I'm releasing as is. 12 | 13 | Also take a look at [mittos64-old](https://github.com/thomasloven/mittos64-old/) which is a previous version of this that got a bit further (processes, c-library (JIT-patching of musl), ATA). 14 | 15 | ## Building 16 | This project is set up to be built inside a Docker container. 17 | 18 | The container can be built by running the `d` script in the projects root directory - provided Docker is installed on your computer. 19 | 20 | When the docker container is built, commands can be run inside it through the `d` script. The script will run any arguments passed to it as commands inside the container. 21 | 22 | E.g: 23 | 24 | d ls 25 | 26 | If a previous command is running, the next command will be run inside the same container. Otherwise a new container will be started. 27 | 28 | 29 | ## Generating and iso file 30 | A cdrom ISO file can be built by the command 31 | 32 | d mkiso 33 | 34 | ## Running the emulator 35 | The project can be run in a qemu virtual machine inside the docker container by the command 36 | 37 | d emul 38 | 39 | ## Running the debugger 40 | The gdb debugger can be run inside the docker container by the command 41 | 42 | d dbg 43 | 44 | This assumes that a qemu session is already running. 45 | 46 | ### Debugger commands 47 | Gdb is configured with a few special commands 48 | 49 | (gdb) q 50 | 51 | Stops the qemu emulator and closes the debugger. 52 | 53 | (gdb) reset 54 | 55 | Reboots the qemu virtual machine. 56 | 57 | (gdb) reg 58 | 59 | Displays information about cpu register contents from the emulator. Note that this is slightly different from the built-in gdb command `info registers`. 60 | 61 | 62 | ### Bypassing the docker container 63 | Most scripts and makefiles in the project will check for wether they are run inside the container by looking for the environment variable $MITTOS64. If not present, execution will stop. 64 | 65 | If you know what you're doing, this check can be bypassed by defining the environment variable MITTOS64. I really don't recommend it, though. 66 | -------------------------------------------------------------------------------- /src/kernel/boot/debug.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | // TODO: Temporary declarations 9 | void vga_write(char c); 10 | 11 | 12 | void num2str(char *buf, uint64_t num, uint64_t base) 13 | { 14 | if(num == 0) 15 | { 16 | buf[0] = '0'; 17 | buf[1] = '\0'; 18 | return; 19 | } 20 | uint64_t i=0, j=0; 21 | char chars[] = "0123456789ABCDEF"; 22 | while(num > 0) 23 | { 24 | buf[i++] = chars[num%base]; 25 | num /= base; 26 | } 27 | i--; 28 | while(j= '0' && *fmt <= '9') 81 | { 82 | padwidth *= 10; 83 | padwidth += (int)(*fmt++ - '0'); 84 | } 85 | uint64_t base = 0; 86 | switch(*fmt) 87 | { 88 | case 'b': 89 | base = 2; 90 | break; 91 | case 'o': 92 | base = 8; 93 | break; 94 | case 'd': 95 | base = 10; 96 | break; 97 | case 'x': 98 | base = 16; 99 | break; 100 | case 'c': 101 | debug_putch((char)va_arg(args, uint64_t)); 102 | break; 103 | case 's': 104 | debug_puts(va_arg(args, char*)); 105 | break; 106 | default: 107 | debug_putch('%'); 108 | fmt--; 109 | } 110 | 111 | if(base) 112 | { 113 | uintmax_t number = va_arg(args, uintmax_t); 114 | char buf[128]; 115 | num2str(buf, number, base); 116 | if(padwidth > strlen(buf)) 117 | for(size_t i = 0; i < (padwidth - strlen(buf)); i++) 118 | debug_putch(padchar); 119 | debug_puts(buf); 120 | } 121 | 122 | fmt++; 123 | debug_vprintf(fmt, args); 124 | } 125 | 126 | void debug_printf(char *fmt, ...) 127 | { 128 | va_list args; 129 | va_start(args, fmt); 130 | debug_vprintf(fmt, args); 131 | va_end(args); 132 | } 133 | -------------------------------------------------------------------------------- /src/kernel/boot/multiboot.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #define MBOOT_REPLY 0x36D76289 6 | 7 | struct taglist{ 8 | uint32_t total_size; 9 | uint32_t reserved; 10 | }__attribute__((packed)); 11 | 12 | struct tag { 13 | uint32_t type; 14 | uint32_t size; 15 | uint8_t data[]; 16 | }__attribute__((packed)); 17 | 18 | struct mmap_entry{ 19 | uint64_t base; 20 | uint64_t len; 21 | uint32_t type; 22 | uint32_t reserved; 23 | }__attribute__((packed)); 24 | 25 | struct mmap { 26 | uint32_t entry_size; 27 | uint32_t entry_version; 28 | struct mmap_entry entries[]; 29 | }__attribute__((packed)); 30 | 31 | #define MBOOT2_COMMANDLINE 1 32 | #define MBOOT2_BOOTLOADER 2 33 | #define MBOOT2_MMAP 6 34 | 35 | struct kernel_boot_data_st kernel_boot_data; 36 | 37 | int parse_multiboot2(struct taglist *tags) 38 | { 39 | struct tag *tag = incptr(tags, sizeof(struct taglist)); 40 | struct mmap *mmap; 41 | while(tag->type) 42 | { 43 | switch(tag->type) 44 | { 45 | case MBOOT2_BOOTLOADER: 46 | kernel_boot_data.bootloader = (char *)tag->data; 47 | break; 48 | case MBOOT2_COMMANDLINE: 49 | kernel_boot_data.commandline = (char *)tag->data; 50 | break; 51 | case MBOOT2_MMAP: 52 | mmap = kernel_boot_data.mmap = (void *)tag->data; 53 | kernel_boot_data.mmap_len = (tag->size - 8)/mmap->entry_size; 54 | kernel_boot_data.mmap_size = (tag->size - 8); 55 | break; 56 | default: 57 | debug_warning("Unknown multiboot tag type:%d \n", tag->type); 58 | } 59 | int padded_size = tag->size + ((tag->size % 8)?(8-(tag->size%8)):0); 60 | tag = incptr(tag, padded_size); 61 | } 62 | return 0; 63 | } 64 | 65 | int multiboot_init(uint64_t magic, void *mboot_info) 66 | { 67 | if(magic == MBOOT_REPLY) 68 | { 69 | kernel_boot_data.multiboot_version = 2; 70 | parse_multiboot2(mboot_info); 71 | } 72 | else 73 | return 1; 74 | 75 | return 0; 76 | } 77 | 78 | int multiboot_get_memory_area(size_t count, uintptr_t *start, uintptr_t *end, uint32_t *type) 79 | { 80 | if(count >= kernel_boot_data.mmap_len) return 1; 81 | 82 | struct mmap *mmap = kernel_boot_data.mmap; 83 | struct mmap_entry *entry = mmap->entries; 84 | entry = incptr(entry, count*mmap->entry_size); 85 | 86 | *start = entry->base; 87 | *end = entry->base + entry->len; 88 | *type = entry->type; 89 | return 0; 90 | } 91 | 92 | int multiboot_page_used(uintptr_t start) 93 | { 94 | #define overlap(st, len) ((uintptr_t)st < (start + PAGE_SIZE) && start <= ((uintptr_t)st + len)) 95 | if( 96 | overlap(kernel_boot_data.bootloader, strlen(kernel_boot_data.bootloader)) || 97 | overlap(kernel_boot_data.commandline, strlen(kernel_boot_data.commandline)) || 98 | overlap(kernel_boot_data.mmap, kernel_boot_data.mmap_size) || 99 | 0) 100 | return 1; 101 | 102 | return 0; 103 | } 104 | -------------------------------------------------------------------------------- /toolchain/gdbinit: -------------------------------------------------------------------------------- 1 | set prompt \033[31m(gdb) \033[0m 2 | set disassembly-flavor intel 3 | 4 | python 5 | import os 6 | gdb.execute('file ' + os.environ['BUILDROOT'] + 'sysroot/kernel') 7 | end 8 | 9 | target remote :1234 10 | 11 | set height 0 12 | set width 0 13 | 14 | # The PANIC() macro - defined in src/kernel/include/debug.h - creates 15 | # a label of the form panic_breakpoint_xxx, where xxx is a number. 16 | # Unfortunately, gdb can set breakpoints on FUNCTIONS based on regex, but 17 | # not on LABELS. 18 | # The following piece of python code runs objdump to extract all panic_breakpoint 19 | # labels, and set breakpoints for each. 20 | python 21 | import subprocess 22 | import os 23 | dump = subprocess.Popen(("objdump", "-t", os.environ['BUILDROOT'] + "sysroot/kernel"), stdout=subprocess.PIPE) 24 | lines = subprocess.check_output(('grep', 'panic_breakpoint'), stdin=dump.stdout) 25 | dump.wait() 26 | for line in lines.split('\n'): 27 | name = line.split(' ')[-1] 28 | if name: 29 | gdb.execute('b ' + name, to_string=True) 30 | end 31 | 32 | define q 33 | monitor quit 34 | end 35 | 36 | define reset 37 | monitor system_reset 38 | end 39 | 40 | define mmap 41 | monitor info mem 42 | end 43 | 44 | python 45 | 46 | import re 47 | 48 | class Reg(gdb.Command): 49 | 50 | def __init__(self): 51 | super(Reg, self).__init__("reg", gdb.COMMAND_USER) 52 | 53 | def invoke(self, arg, from_tty): 54 | regs = gdb.execute('monitor info registers', False, True) 55 | 56 | if not arg: 57 | # If no argument was given, print the output from qemu 58 | print regs 59 | return 60 | 61 | if arg.upper() in ['CS', 'DS', 'ES', 'FS', 'GS', 'SS', 'LDT', 'TR']: 62 | # Code selectors may contain equals signs 63 | for l in regs.splitlines(): 64 | if l.startswith(arg.upper()): 65 | print l 66 | elif arg.upper() in ['EFL', 'RFL']: 67 | # The xFLAGS registers contains equals signs 68 | for l in regs.splitlines(): 69 | if arg.upper() in l: 70 | print ' '.join(l.split()[1:]) 71 | # The xFLAGS register is the second one on the line 72 | else: 73 | # Split at any word followed by and equals sign 74 | # Clean up both sides of the split and put into a dictionary 75 | # then print the requested register value 76 | regex = re.compile("[A-Z0-9]+\s?=") 77 | names = [v[:-1].strip() for v in regex.findall(regs)] 78 | values = [v.strip() for v in regex.split(regs)][1:] 79 | regs = dict(zip(names, values)) 80 | print "%s=%s" % (arg.upper(), regs[arg.upper()]) 81 | 82 | 83 | Reg() 84 | 85 | end 86 | 87 | define restore_env 88 | set $name = $arg0 89 | python 90 | 91 | registers = {r: gdb.parse_and_eval('$name->' + r) for r in 92 | ['rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 93 | 'r11', 'r12', 'r13', 'r14', 'r15', 'rip']} 94 | 95 | for r in registers.items(): 96 | gdb.parse_and_eval('$%s=%s' % r) 97 | gdb.execute('frame 0') 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /src/kernel/drivers/vga.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | 4 | #include 5 | #undef outb 6 | #define outb(...) 7 | #include 8 | #undef VGA_MEMORY 9 | #define VGA_MEMORY calloc(80*24,3) 10 | #include "vga.c" 11 | 12 | #include 13 | 14 | struct cell 15 | { 16 | uint8_t c; 17 | uint8_t f; 18 | }__attribute__((packed)); 19 | 20 | struct cell *cells; 21 | 22 | BEFORE() 23 | { 24 | vga_init(); 25 | cells = vidmem; 26 | cells[VGA_COLS*VGA_ROWS].c = '$'; 27 | } 28 | 29 | #define VGA_ASSERT_EQ(pos, ch) ASSERT_EQ_CHR(cells[(pos)].c, (ch)) 30 | 31 | 32 | TEST(vga_starts_with_a_clear_screen) 33 | { 34 | VGA_ASSERT_EQ(VGA_COLS*VGA_ROWS-1, '\0'); 35 | } 36 | 37 | TEST(vga_write_adds_character_to_vidmem) 38 | { 39 | vga_write('a'); 40 | VGA_ASSERT_EQ(0, 'a'); 41 | } 42 | 43 | TEST(vga_output_is_visible) 44 | { 45 | vga_write('a'); 46 | ASSERT_NEQ_INT(cells[0].f, 0); 47 | } 48 | 49 | TEST(vga_write_adds_multiple_characters_to_vidmem) 50 | { 51 | vga_write('a'); 52 | vga_write('b'); 53 | VGA_ASSERT_EQ(1, 'b'); 54 | } 55 | 56 | TEST(vga_writes_entire_screen_full) 57 | { 58 | for(int i = 0; i < (VGA_COLS*VGA_ROWS-1); i++) 59 | vga_write('x'); 60 | 61 | VGA_ASSERT_EQ(VGA_POS(VGA_ROWS-1, VGA_COLS-2), 'x'); 62 | } 63 | TEST(vga_does_not_overflow_memory_area) 64 | { 65 | for(int i = 0; i < (VGA_COLS*VGA_ROWS+1); i++) 66 | vga_write('x'); 67 | VGA_ASSERT_EQ(VGA_COLS*VGA_ROWS, '$'); 68 | } 69 | 70 | TEST(newline_moves_down_one_line_when_at_beginning) 71 | { 72 | vga_write('\n'); 73 | vga_write('a'); 74 | VGA_ASSERT_EQ(VGA_POS(1,0), 'a'); 75 | } 76 | TEST(newline_moves_to_beginning_of_line) 77 | { 78 | vga_write('a'); 79 | vga_write('\n'); 80 | vga_write('b'); 81 | VGA_ASSERT_EQ(VGA_POS(1,0), 'b'); 82 | } 83 | 84 | void vga_write_str(char *str) 85 | { 86 | while(*str) vga_write(*str++); 87 | } 88 | 89 | TEST(writing_past_end_of_screen_scrolls_screen_one_line) 90 | { 91 | vga_write_str("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); 92 | vga_write_str("11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n"); 93 | vga_write_str("21\n22\n23\n24\n"); 94 | 95 | vga_write_str("abcde"); 96 | VGA_ASSERT_EQ(0, '2'); 97 | } 98 | TEST(screen_does_not_scroll_until_write) 99 | { 100 | vga_write_str("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); 101 | vga_write_str("11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n"); 102 | vga_write_str("21\n22\n23\n24"); 103 | vga_write_str("34567890123456789012345678901234567890123456789012345678901234567890123456789"); 104 | 105 | VGA_ASSERT_EQ(0, '1'); 106 | } 107 | TEST(writing_past_end_of_screen_clears_last_line) 108 | { 109 | vga_write_str("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); 110 | vga_write_str("11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n"); 111 | vga_write_str("21\n22\n23\n24\n"); 112 | vga_write('a'); 113 | VGA_ASSERT_EQ(VGA_POS(VGA_ROWS-1, 1), '\0'); 114 | } 115 | TEST(first_character_of_new_line_is_not_cleared) 116 | { 117 | vga_write_str("1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n"); 118 | vga_write_str("11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n"); 119 | vga_write_str("21\n22\n23\n24\n"); 120 | vga_write('a'); 121 | VGA_ASSERT_EQ(VGA_POS(VGA_ROWS-1, 0), 'a'); 122 | } 123 | -------------------------------------------------------------------------------- /src/kernel/boot/debug.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | #include "debug.c" 4 | 5 | #define BUFFER_SIZE 16 6 | char vga_recv[BUFFER_SIZE]; 7 | char serial_recv[BUFFER_SIZE]; 8 | 9 | BEFORE() 10 | { 11 | for(int i = 0; i < BUFFER_SIZE; i++) 12 | { 13 | vga_recv[i] = '\0'; 14 | serial_recv[i] = '\0'; 15 | } 16 | } 17 | 18 | void vga_write(char c) 19 | { 20 | static int i = 0; 21 | vga_recv[i++] = c; 22 | } 23 | void serial_write(uint16_t port, uint8_t c) 24 | { 25 | (void)port; 26 | static int i = 0; 27 | serial_recv[i++] = (char)c; 28 | } 29 | 30 | TEST(putch_sends_character_to_vga) 31 | { 32 | char input = 'a'; 33 | debug_putch(input); 34 | ASSERT_EQ_CHR(vga_recv[0], input); 35 | } 36 | TEST(putch_sends_character_to_serial) 37 | { 38 | char input = 'a'; 39 | debug_putch(input); 40 | ASSERT_EQ_CHR(serial_recv[0], input); 41 | } 42 | 43 | TEST(putsn_writes_string) 44 | { 45 | char *str = "hello"; 46 | debug_putsn(str, 5); 47 | ASSERT_EQ_STR(vga_recv, str, BUFFER_SIZE); 48 | } 49 | TEST(putsn_writes_correct_number_of_characters) 50 | { 51 | char *str = "1234567890"; 52 | debug_putsn(str, 5); 53 | ASSERT_EQ_STR(vga_recv, "12345", BUFFER_SIZE); 54 | } 55 | TEST(puts_writes_string) 56 | { 57 | char *str = "world"; 58 | debug_puts(str); 59 | ASSERT_EQ_STR(vga_recv, str, BUFFER_SIZE); 60 | } 61 | 62 | TEST(printf_prints_string) 63 | { 64 | char *str = "Hello, world!"; 65 | debug_printf(str); 66 | ASSERT_EQ_STR(vga_recv, str, BUFFER_SIZE); 67 | } 68 | TEST(printf_does_not_print_percent) 69 | { 70 | debug_printf("123%d", 45); 71 | ASSERT_NEQ_CHR(vga_recv[3], '%'); 72 | } 73 | TEST(printf_prints_binary_number) 74 | { 75 | debug_printf("%b", 0x55aa); 76 | ASSERT_EQ_STR(vga_recv, "101010110101010", BUFFER_SIZE); 77 | } 78 | TEST(printf_prints_ocal_number) 79 | { 80 | debug_printf("%o", 8); 81 | ASSERT_EQ_STR(vga_recv, "10", BUFFER_SIZE); 82 | } 83 | TEST(printf_prints_decimal_number) 84 | { 85 | debug_printf("%d", 123); 86 | ASSERT_EQ_STR(vga_recv, "123", BUFFER_SIZE); 87 | } 88 | TEST(printf_prints_hexadecimal_number) 89 | { 90 | debug_printf("%x", 42); 91 | ASSERT_EQ_STR(vga_recv, "2A", BUFFER_SIZE); 92 | } 93 | TEST(printf_prints_char) 94 | { 95 | debug_printf("%c", 'X'); 96 | ASSERT_EQ_STR(vga_recv, "X", BUFFER_SIZE); 97 | } 98 | TEST(printf_prints_passed_string) 99 | { 100 | debug_printf("%s", "asdf"); 101 | ASSERT_EQ_STR(vga_recv, "asdf", BUFFER_SIZE); 102 | } 103 | TEST(printf_keeps_printing_after_number) 104 | { 105 | debug_printf("%x123", 0); 106 | ASSERT_EQ_STR(vga_recv, "0123", BUFFER_SIZE); 107 | } 108 | TEST(printf_prints_text_around_number) 109 | { 110 | debug_printf("ABC%dDEF", 0); 111 | ASSERT_EQ_STR(vga_recv, "ABC0DEF", BUFFER_SIZE); 112 | } 113 | TEST(printf_keeps_going_for_unknown_format_specifier) 114 | { 115 | debug_printf("%y"); 116 | ASSERT_EQ_STR(vga_recv, "%y", BUFFER_SIZE); 117 | } 118 | TEST(printf_pads_value) 119 | { 120 | debug_printf("%5d", 123); 121 | ASSERT_EQ_STR(vga_recv, " 123", BUFFER_SIZE); 122 | } 123 | TEST(printf_pads_more_than_9) 124 | { 125 | debug_printf("%16d", 1); 126 | ASSERT_EQ_STR(vga_recv, " 1", BUFFER_SIZE); 127 | } 128 | TEST(printf_zero_pads) 129 | { 130 | debug_printf("%05d", 123); 131 | ASSERT_EQ_STR(vga_recv, "00123", BUFFER_SIZE); 132 | } 133 | -------------------------------------------------------------------------------- /src/kernel/memory/vmm.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #define FLAGS_MASK (PAGE_SIZE-1) 5 | #define MASK_FLAGS(addr) ((uint64_t)addr & ~FLAGS_MASK) 6 | 7 | #define PRESENT(p) (p & PAGE_PRESENT) 8 | 9 | #define PT(ptr) ((uint64_t *)P2V(MASK_FLAGS(ptr))) 10 | // Get the entry correspoding to address addr in page dir P4 11 | // for P4 table (P4E), P3 table (P3E) and so on. 12 | // Note: Those macros requires variables to be named 13 | #define P4E (PT(P4)[P4_OFFSET(addr)]) 14 | #define P3E PT(P4E)[P3_OFFSET(addr)] 15 | #define P2E PT(P3E)[P2_OFFSET(addr)] 16 | #define P1E PT(P2E)[P1_OFFSET(addr)] 17 | 18 | uint64_t new_P4() 19 | { 20 | uint64_t p4 = pmm_alloc(); 21 | memcpy(P2V(p4), (void *)kernel_P4, PAGE_SIZE); 22 | return p4; 23 | } 24 | 25 | static int page_exists(uint64_t P4, uint64_t addr) 26 | { 27 | if(P4 && PRESENT(P4E) && PRESENT(P3E) && PRESENT(P2E)) 28 | return 1; 29 | return 0; 30 | } 31 | 32 | static int touch_page(uint64_t P4, uint64_t addr, uint16_t flags) 33 | { 34 | if(!P4) return -1; 35 | 36 | if((!PRESENT(P4E)) && (!(P4E = pmm_calloc()))) 37 | return -1; 38 | P4E |= flags | PAGE_PRESENT; 39 | 40 | if((!PRESENT(P3E)) && (!(P3E = pmm_calloc()))) 41 | return -1; 42 | P3E |= flags | PAGE_PRESENT; 43 | 44 | if((!PRESENT(P2E)) && (!(P2E = pmm_calloc()))) 45 | return -1; 46 | P2E |= flags | PAGE_PRESENT; 47 | 48 | return 0; 49 | } 50 | 51 | uint64_t vmm_get_page(uint64_t P4, uint64_t addr) 52 | { 53 | if(page_exists(P4, addr)) 54 | { 55 | return P1E; 56 | } 57 | return -1; 58 | } 59 | 60 | int vmm_set_page(uint64_t P4, uint64_t addr, uint64_t page, uint16_t flags) 61 | { 62 | if(!page_exists(P4, addr)) 63 | { 64 | if(touch_page(P4, addr, flags)) 65 | return -1; 66 | } 67 | 68 | P1E = page | flags; 69 | return 0; 70 | } 71 | 72 | void vmm_clear_page(uint64_t P4, uint64_t addr, int free) 73 | { 74 | if(!page_exists(P4, addr)) 75 | return; 76 | 77 | uint64_t *pt; 78 | 79 | P1E = 0; 80 | 81 | if(!free) 82 | return; 83 | 84 | pt = PT(P2E); 85 | for(int i = 0; i < ENTRIES_PER_PT; i++) 86 | if(pt[i]) 87 | return; 88 | pmm_free(MASK_FLAGS(P2E)); 89 | P2E = 0; 90 | 91 | pt = PT(P3E); 92 | for(int i = 0; i < ENTRIES_PER_PT; i++) 93 | if(pt[i]) 94 | return; 95 | pmm_free(MASK_FLAGS(P3E)); 96 | P3E = 0; 97 | 98 | pt = PT(P4E); 99 | for(int i = 0; i < ENTRIES_PER_PT; i++) 100 | if(pt[i]) 101 | return; 102 | pmm_free(MASK_FLAGS(P4E)); 103 | P4E = 0; 104 | } 105 | 106 | #define min(a,b) (((a) < (b))?(a):(b)) 107 | #define offset(p) ((uintptr_t)(p) % PAGE_SIZE) 108 | #define remaining(p) (PAGE_SIZE - offset(p)) 109 | size_t memcpy_to_p4(uint64_t P4, void *dst, void *src, size_t n) 110 | { 111 | size_t copied = 0; 112 | while(n) 113 | { 114 | size_t bytes = min(remaining(dst), n); 115 | uintptr_t page = vmm_get_page(P4, (uintptr_t)dst); 116 | if(!PAGE_EXIST(page)) 117 | return copied; 118 | 119 | void *to = P2V(MASK_FLAGS(page) + offset(dst)); 120 | memcpy(to, src, bytes); 121 | 122 | copied += bytes; 123 | n -= bytes; 124 | dst = incptr(dst, bytes); 125 | src = incptr(src, bytes); 126 | } 127 | return copied; 128 | } 129 | 130 | size_t memcpy_from_p4(void *dst, uint64_t P4, void *src, size_t n) 131 | { 132 | size_t copied = 0; 133 | while(n) 134 | { 135 | size_t bytes = min(remaining(src), n); 136 | uintptr_t page = vmm_get_page(P4, (uintptr_t)src); 137 | if(!PAGE_EXIST(page)) 138 | return copied; 139 | 140 | void *from = P2V(MASK_FLAGS(page) + offset(src)); 141 | memcpy(dst, from, bytes); 142 | 143 | copied += bytes; 144 | n -= bytes; 145 | dst = incptr(dst, bytes); 146 | src = incptr(src, bytes); 147 | } 148 | return copied; 149 | } 150 | -------------------------------------------------------------------------------- /src/kernel/drivers/acpi.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #define packed __attribute__((packed)) 6 | 7 | struct rsdp 8 | { 9 | uint8_t signature[8]; 10 | uint8_t checksum; 11 | uint8_t OEMID[6]; 12 | uint8_t revision; 13 | uint32_t rsdt; 14 | uint32_t length; 15 | uint64_t xsdt; 16 | uint8_t checksum2; 17 | uint8_t _[3]; 18 | }packed; 19 | #define RSDP_SIGNATURE "RSD PTR " 20 | 21 | struct sdt 22 | { 23 | uint8_t signature[4]; 24 | uint32_t len; 25 | uint8_t revision; 26 | uint8_t checksum; 27 | uint8_t OEMID[6]; 28 | uint8_t table_ID[8]; 29 | uint32_t OEM_revision; 30 | uint32_t creator; 31 | uint32_t creator_rev; 32 | uint8_t data[]; 33 | }packed; 34 | #define MADT_SIGNATURE "APIC" 35 | 36 | struct madt 37 | { 38 | uint32_t lic_address; 39 | uint32_t flags; 40 | uint8_t data[]; 41 | }packed; 42 | struct madt_entry 43 | { 44 | uint8_t type; 45 | uint8_t len; 46 | union{ 47 | struct { 48 | uint8_t id; 49 | uint8_t apic; 50 | uint32_t flags; 51 | }packed lapic; 52 | struct { 53 | uint8_t id; 54 | uint8_t _; 55 | uint32_t addr; 56 | uint32_t base; 57 | }packed ioapic; 58 | struct { 59 | uint8_t bus; 60 | uint8_t source; 61 | uint32_t target; 62 | uint16_t flags; 63 | }packed interrupt; 64 | }; 65 | }packed; 66 | 67 | #define MADT_CPU 0 68 | #define MADT_IOAPIC 1 69 | #define MADT_INT 2 70 | 71 | struct acpi_info acpi_info = {0}; 72 | 73 | static void *scan_rsdp(uint64_t start, uint64_t end) 74 | { 75 | void *p = P2V(start); 76 | while(p < P2V(end)) 77 | { 78 | if(!memcmp(p, RSDP_SIGNATURE, 8)) 79 | return p; 80 | p = incptr(p, 16); 81 | } 82 | return 0; 83 | } 84 | 85 | static struct rsdp *find_rsdp() 86 | { 87 | // Scan the Extended BIOS Data Area 88 | uint16_t *ebda_ptr = P2V(0x40e); 89 | uint64_t ebda = *ebda_ptr << 4; 90 | void *p = scan_rsdp(ebda, ebda+1024); 91 | if(p) return p; 92 | 93 | // Scan 0xE0000 - 0xFFFFF 94 | p = scan_rsdp(0xE0000, 0xFFFFF); 95 | if(p) return p; 96 | 97 | return 0; 98 | } 99 | 100 | static void parse_madt(struct madt *madt, uint32_t len) 101 | { 102 | uintptr_t end = (uintptr_t)madt + len; 103 | struct madt_entry *e = (void *)madt->data; 104 | debug_info("Local Interrupt Controller: %x\n", madt->lic_address); 105 | while((uintptr_t)e < end) 106 | { 107 | int i; 108 | switch(e->type) 109 | { 110 | case MADT_CPU: // APIC descriptor (corresponds to unique cpu core) 111 | // Check if cpu is enabled 112 | if(!(e->lapic.id & 1)) break; 113 | // Add to list 114 | i = acpi_info.num_cpus; 115 | acpi_info.cpu[i].id = e->lapic.id; 116 | acpi_info.cpu[i].apic = e->lapic.apic; 117 | acpi_info.num_cpus++; 118 | break; 119 | 120 | case MADT_IOAPIC: // IOAPIC descriptor 121 | i = acpi_info.num_ioapic; 122 | acpi_info.ioapic[i].id = e->ioapic.id; 123 | acpi_info.ioapic[i].addr = e->ioapic.addr; 124 | acpi_info.ioapic[i].base = e->ioapic.base; 125 | acpi_info.num_ioapic++; 126 | break; 127 | 128 | case MADT_INT: // Interrupt remap 129 | acpi_info.int_map[e->interrupt.source] = e->interrupt.target; 130 | break; 131 | } 132 | debug_info(" MADT: type:%d len:%d\n", e->type, e->len); 133 | e = incptr(e, e->len); 134 | } 135 | } 136 | 137 | static void parse_sdt(struct sdt *sdt, uint8_t revision) 138 | { 139 | uint32_t *p32 = (void *)sdt->data; 140 | uint64_t *p64 = (void *)sdt->data; 141 | int entries = (sdt->len - sizeof(struct sdt)) / (revision ? 8 : 4); 142 | for(int i = 0; i < entries; i++) 143 | { 144 | struct sdt *table = P2V(revision ? p64[i] : p32[i]); 145 | 146 | debug_info("Found table: "); 147 | debug_putsn((char *)table->signature, 4); 148 | debug_printf("\n"); 149 | 150 | if(!memcmp(table->signature, MADT_SIGNATURE, 4)) 151 | parse_madt((void *)table->data, table->len); 152 | } 153 | } 154 | 155 | void acpi_init() 156 | { 157 | struct rsdp *rsdp = find_rsdp(); 158 | struct sdt *s = P2V(rsdp->revision ? rsdp->xsdt : rsdp->rsdt); 159 | parse_sdt(s, rsdp->revision); 160 | } 161 | -------------------------------------------------------------------------------- /src/kernel/memory/vmm.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | 4 | #include 5 | #undef P2V 6 | #define P2V(p) ((void *)(p)) 7 | 8 | uint64_t kernel_P4; 9 | uint64_t pmm_alloc() 10 | { 11 | return 0; 12 | } 13 | 14 | #include "vmm.c" 15 | 16 | void *data; 17 | uint64_t *p4, *p3, *p2, *p1; 18 | uint64_t P4; 19 | 20 | #define ADDR1234 ((1UL<<39) + (2UL<<30) + (3UL<<21) + (4UL<<12)) 21 | #define BUILD_PT(o4, o3, o2) \ 22 | p4[(o4)] = (uint64_t)p3; p4[(o4)] |= PAGE_PRESENT; \ 23 | p3[(o3)] = (uint64_t)p2; p3[(o3)] |= PAGE_PRESENT; \ 24 | p2[(o2)] = (uint64_t)p1; p2[(o2)] |= PAGE_PRESENT; 25 | 26 | BEFORE() 27 | { 28 | data = calloc(PAGE_SIZE, 5); 29 | p4 = (void *)(((uintptr_t)data + PAGE_SIZE) & ~(PAGE_SIZE-1)); 30 | p3 = &p4[512]; 31 | p2 = &p4[1024]; 32 | p1 = &p4[1536]; 33 | P4 = (uint64_t)p4; 34 | } 35 | AFTER() 36 | { 37 | free(data); 38 | } 39 | 40 | TEST(get_page_returns_correct_address) 41 | { 42 | BUILD_PT(0,0,0); 43 | p1[0] = 0x1234567890ABC000 | PAGE_PRESENT; 44 | 45 | uint64_t ret = vmm_get_page(P4, 0); 46 | 47 | ASSERT_EQ_PTR(ret, 0x1234567890ABC000 | PAGE_PRESENT); 48 | } 49 | TEST(get_page_ignores_flags) 50 | { 51 | BUILD_PT(0,0,0); 52 | p1[0] = 0x1234567890ABC000 | PAGE_PRESENT; 53 | 54 | uint64_t ret = vmm_get_page(P4, 0); 55 | 56 | ASSERT_EQ_PTR(ret, 0x1234567890ABC000 | PAGE_PRESENT); 57 | } 58 | TEST(get_page_works_for_different_address) 59 | { 60 | BUILD_PT(1,2,3) 61 | p1[4] = 0x34567890ABCDE000 | PAGE_PRESENT; 62 | 63 | uint64_t ret = vmm_get_page(P4, ADDR1234); 64 | 65 | ASSERT_EQ_PTR(ret, 0x34567890ABCDE000 | PAGE_PRESENT); 66 | } 67 | TEST(get_page_fails_if_PTE_not_present) 68 | { 69 | BUILD_PT(0,0,0); 70 | p2[0] = (uint64_t)p1; 71 | p1[0] = 0x1234567890ABC000 | PAGE_PRESENT; 72 | 73 | uint64_t ret = vmm_get_page(P4, 0); 74 | 75 | ASSERT_EQ_PTR(ret, -1); 76 | } 77 | 78 | TEST(set_page_sets_page) 79 | { 80 | BUILD_PT(0,0,0); 81 | 82 | vmm_set_page(P4, 0, 0x1234567890ABC000, PAGE_PRESENT); 83 | 84 | ASSERT_EQ_PTR(p1[0], 0x1234567890ABC000 | PAGE_PRESENT); 85 | } 86 | TEST(set_page_returns_success_if_working) 87 | { 88 | BUILD_PT(0,0,0); 89 | 90 | int retval = vmm_set_page(P4, 0, 0x1234567890ABC000, PAGE_PRESENT); 91 | 92 | ASSERT_EQ_INT(retval, 0); 93 | } 94 | TEST(set_page_does_not_fail_if_PT_missing_but_touch_flag_is_set) 95 | { 96 | BUILD_PT(0,0,0); 97 | p3[0] = (uint64_t)p2; 98 | 99 | int retval = vmm_set_page(P4, 0, 0x1234567890ABC000, PAGE_PRESENT); 100 | 101 | ASSERT_EQ_INT(retval, 0); 102 | } 103 | TEST(set_page_makes_P3_if_PT_missing_but_touch_flag_is_set) 104 | { 105 | BUILD_PT(0,0,0); 106 | p3[0] = (uint64_t)p2; 107 | 108 | vmm_set_page(P4, 0, 0x1234567890ABC000, PAGE_PRESENT); 109 | 110 | ASSERT_EQ_INT(p3[0] & PAGE_PRESENT, PAGE_PRESENT); 111 | } 112 | 113 | uint64_t pmm_calloc() 114 | { 115 | uint64_t *pages[] = {p3, p2, p1}; 116 | static int counter=0; 117 | if(counter >= 3) return 0; 118 | return (uint64_t)pages[counter++]; 119 | } 120 | 121 | TEST(touching_page_adds_P3) 122 | { 123 | vmm_set_page(P4, ADDR1234, 0, 0); 124 | 125 | ASSERT_EQ_PTR(p4[1], (uint64_t)p3 | PAGE_PRESENT); 126 | } 127 | TEST(touching_page_adds_P2) 128 | { 129 | vmm_set_page(P4, ADDR1234, 0, 0); 130 | 131 | ASSERT_EQ_PTR(p3[2], (uint64_t)p2 | PAGE_PRESENT); 132 | } 133 | TEST(touching_page_adds_P1) 134 | { 135 | vmm_set_page(P4, ADDR1234, 0, 0); 136 | 137 | ASSERT_EQ_PTR(p2[3], (uint64_t)p1 | PAGE_PRESENT); 138 | } 139 | TEST(touching_page_sets_flags) 140 | { 141 | vmm_set_page(P4, 0, 0, PAGE_WRITE); 142 | 143 | ASSERT_EQ_PTR(p2[0], (uint64_t)p1 | PAGE_WRITE | PAGE_PRESENT); 144 | } 145 | TEST(touching_page_fails_if_out_of_pages) 146 | { 147 | pmm_calloc(); 148 | int retval = vmm_set_page(P4, ADDR1234, 0, 0); 149 | ASSERT_NEQ_INT(retval, 0); 150 | } 151 | 152 | TEST(clear_page_unsets_page) 153 | { 154 | BUILD_PT(1,2,3); 155 | p1[4] = PAGE_PRESENT; 156 | 157 | vmm_clear_page(P4, ADDR1234, 0); 158 | 159 | ASSERT_EQ_PTR(p1[4], 0); 160 | } 161 | TEST(clear_page_unsets_P2_entry_if_P1_is_empty) 162 | { 163 | BUILD_PT(1,2,3); 164 | p1[4] = PAGE_PRESENT; 165 | 166 | vmm_clear_page(P4, ADDR1234, 1); 167 | 168 | ASSERT_EQ_PTR(p2[3], 0); 169 | } 170 | TEST(clear_page_does_not_unset_P2_entry_if_not_asked_to) 171 | { 172 | BUILD_PT(1,2,3); 173 | p1[4] = PAGE_PRESENT; 174 | 175 | vmm_clear_page(P4, ADDR1234, 0); 176 | 177 | ASSERT_NEQ_PTR(p2[3], 0); 178 | } 179 | TEST(clear_page_does_not_unset_P2_entry_if_P1_is_not_empty) 180 | { 181 | BUILD_PT(1,2,3); 182 | p1[4] = PAGE_PRESENT; 183 | p1[0] = PAGE_PRESENT; 184 | 185 | vmm_clear_page(P4, ADDR1234, 1); 186 | 187 | ASSERT_NEQ_PTR(p2[3], 0); 188 | } 189 | 190 | uint64_t freed[] = {0,0,0,0,0}; 191 | void pmm_free(uint64_t page) 192 | { 193 | static int counter = 0; 194 | freed[counter++] = page; 195 | } 196 | TEST(clear_page_returns_P1_to_PMM_if_empty) 197 | { 198 | BUILD_PT(1,2,3); 199 | p1[4] = PAGE_PRESENT; 200 | 201 | vmm_clear_page(P4, ADDR1234, 1); 202 | 203 | ASSERT_EQ_PTR(freed[0], p1); 204 | } 205 | -------------------------------------------------------------------------------- /src/kernel/boot/multiboot.tt: -------------------------------------------------------------------------------- 1 | // vim: ft=c 2 | #include 3 | 4 | #include "multiboot.c" 5 | 6 | #include 7 | #include 8 | 9 | #define MAGIC 0x36D76289 10 | 11 | struct tag_st 12 | { 13 | uint32_t type; 14 | uint32_t size; 15 | uint8_t data[]; 16 | }__attribute__((packed)); 17 | struct header 18 | { 19 | uint32_t size; 20 | uint32_t _; 21 | }__attribute__((packed)); 22 | 23 | 24 | struct tag_st *generate_tag(uint32_t type, void *data, uint32_t data_size) 25 | { 26 | struct tag_st *tag = calloc(1, sizeof(struct tag_st) + data_size); 27 | tag->type = type; 28 | tag->size = sizeof(struct tag_st) + data_size; 29 | if(data_size) 30 | memcpy(tag->data, data, data_size); 31 | return tag; 32 | } 33 | 34 | #define padded_size(tag) (tag->size + ((tag->size%8)?(8-(tag->size%8)):0)) 35 | 36 | struct header *generate_taglist(int num, ...) 37 | { 38 | va_list args; 39 | va_start(args, num); 40 | uint32_t total_size = sizeof(struct header); 41 | struct tag_st **tags = calloc(num, sizeof(struct tag_st *)); 42 | for(int i = 0; i < num; i++) 43 | { 44 | tags[i] = va_arg(args, struct tag_st *); 45 | total_size += padded_size(tags[i]); 46 | } 47 | 48 | struct header *taglist = calloc(1, total_size); 49 | taglist->size = total_size; 50 | 51 | int pos = 8; 52 | uint8_t *p = (uint8_t *)taglist; 53 | for(int i = 0; i < num; i++) 54 | { 55 | memcpy(&p[pos], tags[i], tags[i]->size); 56 | pos += padded_size(tags[i]); 57 | free(tags[i]); 58 | } 59 | va_end(args); 60 | 61 | return taglist; 62 | } 63 | 64 | TEST(correctly_identifies_multiboot2_magic) 65 | { 66 | uint32_t tags[] = {16, 0, 0, 8}; 67 | multiboot_init(MAGIC, tags); 68 | ASSERT_EQ_INT(kernel_boot_data.multiboot_version, 2); 69 | } 70 | 71 | TEST(reads_boot_loader_name) 72 | { 73 | char *name = "ttest"; 74 | void *tags = generate_taglist(2, 75 | generate_tag(2, name, strlen(name) + 1), 76 | generate_tag(0,0,0)); 77 | 78 | multiboot_init(MAGIC, tags); 79 | ASSERT_EQ_STR(kernel_boot_data.bootloader, name, strlen(name)); 80 | free(tags); 81 | } 82 | TEST(reads_kernel_commandline) 83 | { 84 | char *cmd = "mittos64-kern boot"; 85 | void *tags = generate_taglist(2, 86 | generate_tag(1, cmd, strlen(cmd) + 1), 87 | generate_tag(0,0,0)); 88 | 89 | multiboot_init(MAGIC, tags); 90 | ASSERT_EQ_STR(kernel_boot_data.commandline, cmd, strlen(cmd)); 91 | free(tags); 92 | } 93 | // Here are four very similar tests 94 | // The difference is what values are checked and the order of the tags 95 | // Their existence is based on experience - helped me actually find a bug 96 | TEST(reads_multiple_tags_1) 97 | { 98 | char *cmd = "mittos64-kern boot"; 99 | char *name = "ttest"; 100 | void *tags = generate_taglist(3, 101 | generate_tag(1, cmd, strlen(cmd) + 1), 102 | generate_tag(2, name, strlen(name) + 1), 103 | generate_tag(0,0,0) 104 | ); 105 | 106 | multiboot_init(MAGIC, tags); 107 | ASSERT_EQ_STR(kernel_boot_data.commandline, cmd, strlen(cmd)); 108 | free(tags); 109 | } 110 | TEST(reads_multiple_tags_2) 111 | { 112 | char *cmd = "mittos64-kern boot"; 113 | char *name = "ttest"; 114 | void *tags = generate_taglist(3, 115 | generate_tag(1, cmd, strlen(cmd) + 1), 116 | generate_tag(2, name, strlen(name) + 1), 117 | generate_tag(0,0,0) 118 | ); 119 | 120 | multiboot_init(MAGIC, tags); 121 | ASSERT_EQ_STR(kernel_boot_data.bootloader, name, strlen(name)); 122 | free(tags); 123 | } 124 | TEST(reads_multiple_tags_3) 125 | { 126 | char *cmd = "mittos64-kern boot"; 127 | char *name = "ttest"; 128 | void *tags = generate_taglist(3, 129 | generate_tag(2, name, strlen(name) + 1), 130 | generate_tag(1, cmd, strlen(cmd) + 1), 131 | generate_tag(0,0,0) 132 | ); 133 | 134 | multiboot_init(MAGIC, tags); 135 | ASSERT_EQ_STR(kernel_boot_data.commandline, cmd, strlen(cmd)); 136 | free(tags); 137 | } 138 | TEST(reads_multiple_tags_4) 139 | { 140 | char *cmd = "mittos64-kern boot"; 141 | char *name = "ttest"; 142 | void *tags = generate_taglist(3, 143 | generate_tag(2, name, strlen(name) + 1), 144 | generate_tag(1, cmd, strlen(cmd) + 1), 145 | generate_tag(0,0,0) 146 | ); 147 | 148 | multiboot_init(MAGIC, tags); 149 | ASSERT_EQ_STR(kernel_boot_data.bootloader, name, strlen(name)); 150 | free(tags); 151 | } 152 | 153 | TEST(reads_mmap_tag) 154 | { 155 | uint32_t mmap[] = {24, 0, 156 | 0x90ABCDEF, 0x12345678, 0x00010000, 0x00000000, 0x1, 0x0 157 | }; 158 | void *tags = generate_taglist(2, 159 | generate_tag(6, mmap, 32), 160 | generate_tag(0,0,0) 161 | ); 162 | multiboot_init(MAGIC, tags); 163 | 164 | ASSERT_EQ_INT(kernel_boot_data.mmap_len, 1); 165 | free(tags); 166 | } 167 | void setup_memory_areas() 168 | { 169 | uint32_t mmap[] = {24, 0, 170 | 0x90ABCDEF, 0x12345678, 0x00010000, 0x00000000, 0x1, 0x0, 171 | 0x0ABCDEF1, 0x23456789, 0x00010000, 0x00000000, 0x1, 0x0 172 | }; 173 | void *tags = generate_taglist(2, 174 | generate_tag(6, mmap, 56), 175 | generate_tag(0,0,0) 176 | ); 177 | multiboot_init(MAGIC, tags); 178 | } 179 | TEST(reads_multiple_mmap_tags) 180 | { 181 | setup_memory_areas(); 182 | ASSERT_EQ_INT(kernel_boot_data.mmap_len, 2); 183 | } 184 | TEST(returns_memory_area_start) 185 | { 186 | setup_memory_areas(); 187 | uintptr_t start, end; 188 | uint32_t type; 189 | multiboot_get_memory_area(0, &start, &end, &type); 190 | ASSERT_EQ_PTR(start, 0x1234567890ABCDEF); 191 | } 192 | TEST(returns_memory_area_end) 193 | { 194 | setup_memory_areas(); 195 | uintptr_t start, end; 196 | uint32_t type; 197 | multiboot_get_memory_area(0, &start, &end, &type); 198 | ASSERT_EQ_PTR(end, 0x1234567890ACCDEF); 199 | } 200 | TEST(returns_memory_area_type) 201 | { 202 | setup_memory_areas(); 203 | uintptr_t start, end; 204 | uint32_t type; 205 | multiboot_get_memory_area(0, &start, &end, &type); 206 | ASSERT_EQ_INT(type, 1); 207 | } 208 | TEST(returns_second_memory_area_start) 209 | { 210 | setup_memory_areas(); 211 | uintptr_t start, end; 212 | uint32_t type; 213 | multiboot_get_memory_area(1, &start, &end, &type); 214 | ASSERT_EQ_PTR(start, 0x234567890ABCDEF1); 215 | } 216 | TEST(does_not_return_too_many_areas) 217 | { 218 | setup_memory_areas(); 219 | uintptr_t start, end; 220 | uint32_t type; 221 | int retval = multiboot_get_memory_area(2, &start, &end, &type); 222 | ASSERT_NEQ_INT(retval, 0); 223 | } 224 | -------------------------------------------------------------------------------- /toolchain/ttest.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | 9 | #define TT_FAIL(error, ...) dprintf(tt_pipe[1], "\"%s\" Line %d: %s >> " error "\n", tt_current->filename, __LINE__, tt_current->name, ##__VA_ARGS__); 10 | 11 | 12 | #define ASSERT_EQUAL(type, pf, lhs, rhs) do { \ 13 | type tt_lhs = (type)(lhs); \ 14 | type tt_rhs = (type)(rhs); \ 15 | if(tt_lhs != tt_rhs) { \ 16 | TT_FAIL("Expected <%" pf "> got <%" pf ">", tt_rhs, tt_lhs); \ 17 | return 1; \ 18 | } \ 19 | return 0; \ 20 | }while(0); 21 | 22 | #define ASSERT_NOT_EQUAL(type, pf, lhs, rhs) do { \ 23 | type tt_lhs = (type)(lhs); \ 24 | type tt_rhs = (type)(rhs); \ 25 | if(tt_lhs == tt_rhs) { \ 26 | TT_FAIL("Got <%" pf "> but expected anything else", tt_rhs); \ 27 | return 1; \ 28 | } \ 29 | return 0; \ 30 | }while(0); 31 | 32 | #define ASSERT_STRN(lhs, rhs, n) do { \ 33 | char *tt_lhs = (char *)(lhs); \ 34 | char *tt_rhs = (char *)(rhs); \ 35 | if(!tt_lhs || !tt_rhs) \ 36 | { \ 37 | TT_FAIL("Expected string, got null pointer"); \ 38 | return 1; \ 39 | } \ 40 | size_t tt_n = (size_t)(n); \ 41 | char *tt_lhs_c = malloc(tt_n+1); \ 42 | char *tt_rhs_c = malloc(tt_n+1); \ 43 | memcpy(tt_lhs_c, tt_lhs, tt_n); \ 44 | memcpy(tt_rhs_c, tt_rhs, tt_n); \ 45 | tt_rhs_c[tt_n] = tt_lhs_c[tt_n] = '\0'; \ 46 | if(strncmp(tt_lhs_c, tt_rhs_c, tt_n)) { \ 47 | TT_FAIL("Expected <%s> got <%s>", tt_rhs_c, tt_lhs_c); \ 48 | free(tt_lhs_c); free(tt_rhs_c); \ 49 | return 1; \ 50 | } \ 51 | free(tt_lhs_c); free(tt_rhs_c); \ 52 | return 0; \ 53 | }while(0); 54 | 55 | #define ASSERT_EQ_INT(lhs, rhs) ASSERT_EQUAL(int, "d", lhs, rhs) 56 | #define ASSERT_NEQ_INT(lhs, rhs) ASSERT_NOT_EQUAL(int, "d", lhs, rhs) 57 | #define ASSERT_EQ_CHR(lhs, rhs) ASSERT_EQUAL(char, "c", lhs, rhs) 58 | #define ASSERT_NEQ_CHR(lhs, rhs) ASSERT_NOT_EQUAL(char, "c", lhs, rhs) 59 | #define ASSERT_EQ_STR(lhs, rhs, n) ASSERT_STRN(lhs, rhs, n) 60 | #define ASSERT_EQ_PTR(lhs, rhs) ASSERT_EQUAL(uintptr_t, PRIxPTR, lhs, rhs) 61 | #define ASSERT_NEQ_PTR(lhs, rhs) ASSERT_NOT_EQUAL(uintptr_t, PRIxPTR, lhs, rhs) 62 | 63 | #define TEST(name) \ 64 | int ttt_##name(); \ 65 | __attribute__((constructor)) void tttr_##name() { \ 66 | tt_register(#name, ttt_##name, __FILE__); \ 67 | } \ 68 | int ttt_##name() 69 | 70 | #define BEFORE() void tt_before() 71 | #define AFTER() void tt_after() 72 | 73 | void __attribute__((weak)) tt_before(void); 74 | void __attribute__((weak)) tt_after(void); 75 | 76 | #ifndef TT_BUFFER_SIZE 77 | #define TT_BUFFER_SIZE 512 78 | #endif 79 | 80 | struct tt_test 81 | { 82 | char *filename; 83 | char *name; 84 | int (*test)(void); 85 | int status; 86 | char *output; 87 | }; 88 | 89 | // Global variables 90 | int tt_pipe[2]; 91 | int tt_color = 1; 92 | int tt_verbose = 0; 93 | int tt_silent = 0; 94 | struct tt_test *tt_tests; 95 | struct tt_test *tt_current; 96 | unsigned int tt_max_name_len = 0; 97 | int tt_test_count = 0; 98 | 99 | void tt_register(char *name, int (*fn)(void), char *filename) 100 | { 101 | tt_tests = realloc(tt_tests, (tt_test_count+1)*sizeof(struct tt_test)); 102 | 103 | struct tt_test *t = &tt_tests[tt_test_count++]; 104 | t->filename = filename; 105 | t->name = name; 106 | t->test = fn; 107 | t->status = 1; 108 | t->output = malloc(TT_BUFFER_SIZE); 109 | if(strlen(name) > tt_max_name_len) 110 | tt_max_name_len = strlen(name); 111 | } 112 | 113 | 114 | #define TT_CLR_RED ((tt_color)?"\x1b[31m":"") 115 | #define TT_CLR_GRN ((tt_color)?"\x1b[32m":"") 116 | #define TT_CLR_YEL ((tt_color)?"\x1b[33m":"") 117 | #define TT_CLR_RES ((tt_color)?"\x1b[0m":"") 118 | 119 | int main(int argc, char **argv) 120 | { 121 | if(!isatty(1)) tt_color = 0; 122 | int opt; 123 | while((opt = getopt(argc, argv, "vscn")) != -1) 124 | { 125 | switch(opt) 126 | { 127 | case 'v': 128 | tt_verbose = 1; 129 | tt_silent = 0; 130 | break; 131 | case 's': 132 | tt_silent = 1; 133 | tt_verbose = 0; 134 | break; 135 | case 'c': 136 | tt_color = 1; 137 | break; 138 | case 'n': 139 | tt_color = 0; 140 | break; 141 | } 142 | } 143 | 144 | int ok = 0; 145 | int failed = 0; 146 | int crashed = 0; 147 | 148 | if(!tt_silent) 149 | printf("\n%s\n", tt_tests[0].filename); 150 | 151 | for(int i = 0; i < tt_test_count; i++) 152 | { 153 | tt_current = &tt_tests[i]; 154 | 155 | fflush(stdout); 156 | pipe(tt_pipe); 157 | 158 | int pid; 159 | if(!(pid = fork())) 160 | { 161 | // Run test 162 | close(tt_pipe[0]); 163 | if(tt_before) tt_before(); 164 | int result = tt_current->test(); 165 | if(tt_after) tt_after(); 166 | exit(result); 167 | } 168 | 169 | // Capture test output 170 | close(tt_pipe[1]); 171 | read(tt_pipe[0], tt_current->output, TT_BUFFER_SIZE); 172 | close(tt_pipe[0]); 173 | 174 | // Determine if test passed or not 175 | int status; 176 | waitpid(pid, &status, 0); 177 | if(!WIFEXITED(status)) 178 | { 179 | crashed++; 180 | tt_current->status = -1; 181 | } else if(WEXITSTATUS(status)) { 182 | failed++; 183 | tt_current->status = WEXITSTATUS(status); 184 | } else { 185 | ok++; 186 | tt_current->status = 0; 187 | } 188 | 189 | // Output progress 190 | if(tt_verbose) 191 | printf("%3d/%3d %-*s ", i+1, tt_test_count, 192 | tt_max_name_len, tt_current->name); 193 | switch(tt_current->status) 194 | { 195 | case 0: 196 | if(tt_verbose) 197 | printf("[%sOK%s]\n", TT_CLR_GRN, TT_CLR_RES); 198 | else 199 | printf("%s.%s", TT_CLR_GRN, TT_CLR_RES); 200 | break; 201 | case -1: 202 | if(tt_verbose) 203 | printf("[%sCRASHED%s]\n", TT_CLR_RED, TT_CLR_RES); 204 | else 205 | printf("%sC%s", TT_CLR_RED, TT_CLR_RES); 206 | break; 207 | default: 208 | if(tt_verbose) 209 | printf("[%sFAILED%s]\n", TT_CLR_YEL, TT_CLR_RES); 210 | else 211 | printf("%sF%s", TT_CLR_YEL, TT_CLR_RES); 212 | } 213 | 214 | } 215 | 216 | int retval = failed+crashed; 217 | 218 | // Print summary 219 | if(!tt_silent) 220 | { 221 | printf("\n%s", (retval)?TT_CLR_RED:TT_CLR_GRN); 222 | printf("%d tests, %d failures", tt_test_count, retval); 223 | printf("%s\n", TT_CLR_RES); 224 | } 225 | 226 | if(retval && tt_silent) 227 | printf("\n"); 228 | 229 | // Print any errors 230 | for(int i = 0; i < tt_test_count; i++) 231 | { 232 | if(tt_tests[i].status) 233 | printf("%s", tt_tests[i].output); 234 | free(tt_tests[i].output); 235 | } 236 | 237 | 238 | return retval; 239 | } 240 | -------------------------------------------------------------------------------- /doc/5_Unit_Testing.md: -------------------------------------------------------------------------------- 1 | # Unit Testing Framework 2 | 3 | In this chapter we'll build a unit testing framework that will help us 4 | developing the rest of the kernel. 5 | 6 | ## Why build it yourself? 7 | 8 | As far as I understand it, there's really no good arguments for building 9 | your own unit testing framework. There are lots of finished alternatives 10 | available allready, and using them will let you focus on your main 11 | project instead. 12 | 13 | I'll write my own, simply because I feel like it. If you don't, then use 14 | an existing one. 15 | 16 | ## Requirements 17 | 18 | I don't have very many requirements for the testing framework. But it's important that: 19 | 20 | - It runs fast 21 | - It's clear what failed 22 | - Tests are isolated so that they don't influence each other 23 | 24 | I also want the tests to be "close" to the code. I was thinking of 25 | including them in the actual source file, but instead opted to put them 26 | in a separate file with the same name, but with a `.tt` extension. (tt 27 | for Thomas-Test). 28 | 29 | ## Designing the framework 30 | 31 | The framework will be in two parts; a c source file to include in each 32 | test, and a shell script to run all tests. 33 | 34 | Let's start with the shell script. 35 | 36 | It should scan the supplied directories for `.tt` files, compile them and run them: 37 | 38 | `ttest` 39 | ```sh 40 | #!/bin/sh 41 | 42 | dirs = src/kernel 43 | 44 | main() 45 | { 46 | for dir in $dirs; do 47 | local files=`find $dir -name "*.tt" 48 | for suite in $files; do 49 | test_exec="${suite}"est 50 | cc -x c $suite.c -o $test_exec -I $dir/include -I toolchain -DTTEST` 51 | $test_exec 52 | rm $suite.c $test_exec 53 | done 54 | done 55 | } 56 | ``` 57 | 58 | `$test_exec` contains the name of the compiled test executable. It will get the 59 | extension `.ttest`. Since gcc doesn't know how to compile `.tt` files we tell 60 | it that it contains c code with the `-x` switch. 61 | 62 | Now let's take a look at the c source file (well... it's really a header file 63 | *acting* like a c source file...) 64 | 65 | This should contain the `main()` function of the test which should run each 66 | test in turn and keep track of any failures. But first, we have a struct to 67 | keep track of everything related to a test. 68 | 69 | `toolchain/ttest.h` 70 | ```c 71 | ... 72 | struct tt_test 73 | { 74 | char *name; 75 | int(*test)(void); 76 | int status; 77 | char *output; 78 | }; 79 | 80 | struct tt_test *tt_tests; 81 | struct tt_test *tt_current; 82 | int tt_test_count = 0; 83 | ... 84 | ``` 85 | 86 | Next for the main function. For insolation, each test is forked off and run as 87 | a separate process. Errors are passed back to the main process through a pipe. 88 | 89 | `toolchain/ttest.h` 90 | ```c 91 | ... 92 | int tt_pipe[2]; 93 | 94 | int main(int argc, char **argv) 95 | { 96 | for(int i = 0; i < tt_test_count; i++) 97 | { 98 | tt_current = &tt_tests[i]; 99 | 100 | fflush(stdout); 101 | pipe(tt_pipe); 102 | 103 | int pid; 104 | if(!(pid = fork())) 105 | { 106 | close(tt_pipe[0]); // Close read end of pipe 107 | exit(tt_current->test()); 108 | } 109 | 110 | close(tt_pipe[1]); // Close write end of pipe 111 | read(tt_pipe[0], tt_current->output, TT_BUFFER_SIZE); 112 | close(tt_pipe[0]); 113 | 114 | int status; 115 | waitpid(pid, &status, 0); 116 | if((tt_current->status = WEXITSTATUS(status))) 117 | { 118 | printf("F"); 119 | } else { 120 | printf("."); 121 | } 122 | } 123 | 124 | for(int i = 0; i < tt_test_cound; i++) 125 | { 126 | if(tt_tests[i].status) 127 | printf("%s: %s\n", tt_tests[i].name, tt_tests[i].output) 128 | } 129 | } 130 | ``` 131 | 132 | This is, of course severey simplified. I also have a lot of prettification, 133 | keeping track of the number of failures, checking if the test crashed before 134 | finishing and such... 135 | 136 | Ok, so how is the `tt_tests` array populated? 137 | 138 | Well. There's a macro to define each test in the `.tt`-file: 139 | 140 | `toolchain/ttest.h` 141 | ```c 142 | ... 143 | #define TEST(name) \ 144 | int ttt_##name(); \ 145 | __attribute__((constructor)) void tttr_##name() { \ 146 | tt_register(#name, ttt_##name); \ 147 | } \ 148 | int ttt_#name() 149 | ``` 150 | 151 | This looks... weird... What does it do? Let's try expanding it. 152 | 153 | I'll tell you right away that it's supposed to be used for a function definition, so using it like: 154 | 155 | ```c 156 | TEST(adder_adds_two_numbers) 157 | { 158 | // Test goes here 159 | } 160 | ``` 161 | 162 | will expand to 163 | 164 | ```c 165 | int ttt_adder_adds_two_numbers(); 166 | __attribute__((constructor)) void tttr_adder_adds_two_numbers() 167 | { 168 | tt_register("adder_adds_two_numbers", ttt_adder_adds_two_numbers); 169 | } 170 | int ttt_adder_adds_two_numbers() 171 | { 172 | // Test goes here 173 | } 174 | ``` 175 | 176 | So it declares, and then defines a function. But what's the stuff in between? 177 | 178 | It's magic - that's what. 179 | 180 | The `__attribute__((constructor))` is a gcc specific attribute which tells the 181 | compiler that the function `tttr_adder_adds_two_numbers` should run as soon as 182 | it gets into scope - which is when the test loads and, most importantly, before 183 | `main()` is called. 184 | 185 | So `tt_register` just has to set up an entry into the `tt_tests` array, and 186 | the rest will take care of itself: 187 | 188 | `toolchain/ttest.h` 189 | ```c 190 | ... 191 | void tt_register(char *name, int (*fn)(void)) 192 | { 193 | tt_tests = realloc(tt_tests, (tt_test_count+1)*sizeof(struct tt_test)); 194 | 195 | struct tt_test *t = &tt_tests[tt_test_count++]; 196 | t->name = name; 197 | t->test = fn 198 | t->status = 1 199 | t->output = malloc(TT_BUFFER_SIZE) 200 | } 201 | ... 202 | ``` 203 | The final part needed for a test is an assertion, which is yet another macro: 204 | 205 | `toolchain/ttest.h` 206 | ```c 207 | ... 208 | 209 | #define ASSERT_EQUAL(lhs, rhs) do { \ 210 | int tt_lhs = (int)(lhs); \ 211 | int tt_rhs = (int)(rhs); \ 212 | if(tt_lhs != tt_rhs) { \ 213 | dprintf(tt_pipe[1], "Got <%d> but expected <%d>\n", tt_lhs, tt_rhs); \ 214 | return 1; \ 215 | } \ 216 | } while(0); 217 | ... 218 | ``` 219 | 220 | The arguments of the assertion macro are copied immediately. That way 221 | they are only evaluated once. 222 | 223 | And that's it! 224 | 225 | ## Using the framework 226 | 227 | A typical source file with tests can look like this 228 | 229 | `adder.c` 230 | ```c 231 | int adder(int a, int b) 232 | { 233 | return 5; 234 | } 235 | ``` 236 | 237 | `adder.tt` 238 | ```c 239 | #include 240 | #inclued "adder.c" 241 | 242 | TEST(adder_adds_two_numbers) 243 | { 244 | ASSERT_EQUAL(adder(2,3), 5); 245 | } 246 | 247 | TEST(adder_adds_two_other_numbers) 248 | { 249 | ASSERT_EQUAL(adder(3,4), 7); 250 | } 251 | ``` 252 | 253 | And running it can look like this 254 | 255 | ```bash 256 | $ d ./ttest 257 | .F 258 | adder_adds_two_other_numbers: Got <5> but expected <7> 259 | ``` 260 | 261 | If you have a c compiler installed you should also be able to run the 262 | test framework locally. This speeds it up a bit. 263 | 264 | ## Further improvements 265 | 266 | Of course this is only a very rudimentary test framework, and a lot of 267 | improvements can be made. But this series isn't about designing a test 268 | framework, and I don't know anything about designing a test framework 269 | anyway, so I'll just leave it here. 270 | 271 | I will however note some differences between this version and what's 272 | actually in my code: 273 | 274 | - The `ASSERT_EQUAL` macro has a different signature and should in fact 275 | not be used directly. 276 | - Instead use 277 | - `ASSERT_EQ_INT(lhs, rhs)` for integer comparison 278 | - `ASSERT_EQ_CHR(lhs, rhs)` for char comparison 279 | - `ASSERT_EQ_STR(lhs, rhs, len)` for string comparison 280 | - There's also `ASSERT_NEQ_INT` and `ASSERT_NEQ_CHR` for not-equal 281 | assertions 282 | - Macros `BEFORE()` and `AFTER()` define functions that are run before 283 | and after each test respectively. Use them for setup and teardown of the 284 | objects under test 285 | - The printed errors contain the name of the test file and the faulting line 286 | in a format that can be processed automatically by vim and used in the 287 | quickfix list 288 | - Output to terminal is colorized 289 | - The output is colorized 290 | - The shell script handles compilation errors of the tests 291 | 292 | If you run the tests locally, everything should work if your c compiler 293 | is gcc. 294 | -------------------------------------------------------------------------------- /doc/6_Debug_Output.md: -------------------------------------------------------------------------------- 1 | # Debug Output 2 | 3 | In this chapter we'll be looking at some ways of getting status information out 4 | of the kernel. 5 | 6 | ## The printf function and TDD 7 | 8 | I really like having access to a printf-like function when developing. It just 9 | makes debugging so much easier. 10 | 11 | For further convenience, I want the debug printer to print both to screen and 12 | to a serial port. 13 | 14 | Now, I'm doing this as a learning experience, and one thing I've been meaning 15 | to practice is Test-Driven Development. So let's write the debug printing that 16 | way. 17 | 18 | Let's start very simple. A function that outputs a character to VGA and to 19 | serial. Assuming (slightly prematurely) that we'll have two functions `void 20 | vga_write(char c)` and `void serial_write(uint16_t port, uint8_t c)` The test 21 | could look like this: 22 | 23 | `src/kernel/boot/debug.tt` 24 | ```c 25 | char vga_recv; 26 | char serial_recv; 27 | 28 | void vga_write(char c) 29 | { 30 | vga_recv = c; 31 | } 32 | void serial_write(uint16_t port, uint8_t c) 33 | { 34 | serial_recv = (char)c; 35 | } 36 | 37 | TEST(putch_sends_character_to_vga) 38 | { 39 | debug_putch('a'); 40 | ASSERT_EQ_CHR(vga_recv, 'a'); 41 | } 42 | TEST(putch_sends_character_to_serial) 43 | { 44 | debug_putch('a'); 45 | ASSERT_EQ_CHR(serial_recv, 'a'); 46 | } 47 | ``` 48 | 49 | ... which can be made to pass with a simple function like: 50 | 51 | `src/kernel/boot/debug.c` 52 | ```c 53 | void debug_putch(char c) 54 | { 55 | vga_write(c); 56 | serial_write(PORT_COM1, c); 57 | } 58 | ``` 59 | 60 | ... with some `#define`s and `#include`s and such, of course. 61 | 62 | As I understand it, in efficient TDD, you should write one test at a time, and 63 | then make that pass - so that's what I tried to do, but I'm presenting more 64 | than one test at a time for brevity. 65 | 66 | Next is writing strings, which requires an update to the mock functions: 67 | 68 | `src/kernel/boot/debug.tt` 69 | ```c 70 | char vga_recv[BUFFER_SIZE]; 71 | char serial_recv[BUFFER_SIZE]; 72 | 73 | BEFORE() 74 | { 75 | for(int i= 0; i < BUFFER_SIZE; i++) 76 | { 77 | vga_recv[i] = '\0'; 78 | serial_recv[i] = '\0'; 79 | } 80 | } 81 | 82 | void vga_write(char c) 83 | { 84 | static int i = 0; 85 | vga_recv[i++] = c; 86 | } 87 | void serial_write(uint16_t port, uint8_t c) 88 | { 89 | static int i = 0; 90 | serial_recv[i++] = (char)c; 91 | } 92 | 93 | TEST(putsn_writes_string) 94 | { 95 | char *str = "hello"; 96 | debug_putsn(str, 5); 97 | ASSERT_EQ_STR(vga_recv, str, BUFFER_SIZE); 98 | } 99 | TEST(putsn_writes_correct_number_of_characters) 100 | { 101 | char *str = "1234567890"; 102 | debug_putsn(str, 5); 103 | ASSERT_EQ_STR(vga_recv, "12345", BUFFER_SIZE); 104 | } 105 | ``` 106 | 107 | 108 | `src/kernel/boot/debug.c` 109 | ```c 110 | debug_putsn(char *s, size_t n) 111 | { 112 | while(n--) 113 | debug_putch(*s++); 114 | } 115 | ``` 116 | 117 | Then there's `debug_printf`, which is left as an exercise for the reader - or 118 | you could just look through my code. 119 | 120 | > You'll probably find that my implementation, among many other things, is 121 | > influenced by [musl libc](https://www.musl-libc.org/) which is a very clean 122 | > standard library implementation. We'll take a much closer look at musl later... 123 | 124 | The names of the tests are: 125 | - `printf_prints_string` 126 | - `printf_does_not_print_percent` 127 | - `printf_prints_binary_number` 128 | - `printf_prints_octal_number` 129 | - `printf_prints_decimal_number` 130 | - `printf_prints_hexadecimal_number` 131 | - `printf_prints_char` 132 | - `printf_prints_passed_string` 133 | - `printf_keeps_printing_after_number` 134 | - `printf_prints_text_around_number` 135 | - `printf_keeps_going_for_unknown_format_specifier` 136 | - `printf_pads_value` 137 | - `printf_pads_more_than_9` 138 | - `printf_zero_pads` 139 | 140 | ## Printing to screen 141 | 142 | Next, let's look at `vga_write()`. 143 | 144 | I tried to use TDD for this as well, and that had an unexpected outcome. 145 | 146 | In order to test VGA printing, I had to mock up VGA hardware somehow - or at 147 | least the VGA memory mapping. 148 | 149 | This brought me two realizations. 150 | 151 | First of all, each character on screen is described by a 16 bit value - one 152 | byte for the character and one for the color attribute. 153 | 154 | This can be implemented as a structure: 155 | 156 | `src/kernel/drivers/vga.c` 157 | ```c 158 | struct vga_cell { 159 | uint8_t c; 160 | uint8_t f; 161 | }__attribute__((packed)); 162 | 163 | struct vga_cell buffer[24*80] = (void *)0xB8000; 164 | ... 165 | ``` 166 | 167 | The second realization was that each screen line comes right after the previous 168 | one in memory. 169 | 170 | This seems obvious, of course - but stay with me. 171 | 172 | You've probably gone through some kernel development tutorial at 173 | some point (I'd assume [Brandon Friesen's](http://www.osdever.net/bkerndev/Docs/title.htm), 174 | [James Molloy's](http://www.jamesmolloy.co.uk/tutorial_html/) or 175 | [The osdev.org Bare Bones ](http://wiki.osdev.org/Bare_Bones)). 176 | If so, you may have a `putch()`, `monitor_put()` or 177 | `terminal_putchar()` function which writes a character to screen, 178 | increases a counter, and if it has reached the end of a line, goes 179 | to the next. 180 | 181 | See where I'm going with this? There's no point in keeping track on the line 182 | and column separately. All you need is a single counter. 183 | 184 | That makes my corresponding function very short and simple: 185 | 186 | `src/kernel/drivers/vga.c` 187 | ```c 188 | ... 189 | uint64_t cursor = 0; 190 | 191 | void vga_write(char c) 192 | { 193 | switch(c) 194 | { 195 | case '\n': 196 | cursor += 80 - (cursor%80); 197 | break; 198 | default: 199 | buffer[cursor++] = (struct vga_cell){.c = c, .f=7}; 200 | } 201 | scroll(); 202 | } 203 | ``` 204 | 205 | `scroll()` scrolls the screen if we pass beyond the last line, and is also very simple: 206 | 207 | ```c 208 | ... 209 | void scroll() 210 | { 211 | memmove(buffer, &buffer[80], 80*(24-1)*sizeof(struct vga_cell)); 212 | memset(&buffer[80*(24-1)], 0, 80*sizeof(struct vga_cell)); 213 | cursor -= 80; 214 | } 215 | ... 216 | ``` 217 | 218 | As a final note, the line `struct vga_cell buffer[24*80] = (void *)0xB8000;` is 219 | a really bad one. You should make sure to use double buffering when dealing 220 | with VGA memory. The reason for this is that *reading* from VGA memory (such as 221 | when scrolling the screen) is really really slow. 222 | 223 | ## Bonus: PANIC 224 | 225 | Sometimes, things go wrong. 226 | 227 | When they do, you probably want to know, and perhaps get a chance to find out 228 | why, or even to put things right. 229 | 230 | For this reason, I made a `PANIC()` macro. The definition looks like this: 231 | 232 | `src/kernel/include/debug.h` 233 | ```c 234 | ... 235 | #define S(x) #x 236 | #define S_(x) S(x) 237 | #define S__LINE__ S_(__LINE__) 238 | 239 | #define PANIC(...) \ 240 | do{ \ 241 | debug_printf("\n\nKernel panic!\n%s:%d\n", __FILE__, __LINE__); \ 242 | debug_printf(__VA_ARGS__); \ 243 | volatile int _override = 0; \ 244 | while(1){ \ 245 | asm("panic_breakpoint_" S__LINE__ ":"); \ 246 | if(_override) break; \ 247 | } \ 248 | }while(0) 249 | ... 250 | ``` 251 | 252 | It's pretty simple actually. It just prints an error message, and then loops 253 | infinitely. 254 | 255 | However, the loop can be broken out of by changing the value of `_override` by 256 | e.g. breaking in gdb and running 257 | 258 | ```gdb 259 | (gdb) set _override=1 260 | ``` 261 | 262 | The `asm` line creates a label in the code, which you can add as a breakpoint 263 | in gdb. The `S__LINE__` stuff and related macros is so that the labels will be 264 | unique if you call `PANIC()` twice in the same source file (gdb will happily 265 | break at any of several labels with the same name, but gcc doesn't like two 266 | equal labels in the same file). 267 | 268 | Gdb has a neat command called `rbreak` which sets breakpoints based on a 269 | regular expression. Unfortunately that only works for functions - not labels. 270 | Therefore, I put the following function in gdbinit, to automatically find all 271 | `panic_breakpoint`s and add them to gdb. 272 | 273 | `toolchain/gdbinit` 274 | ```gdb 275 | ... 276 | python 277 | import subprocess 278 | import os 279 | dump = subprocess.Popen(("objdump", "-t", os.environ['BUILDROOT'] + "sysroot/kernel"), stdout=subprocess.PIPE) 280 | lines = subprocess.check_output(('grep', 'panic_breakpoint'), stdin=dump.stdout) 281 | dump.wait() 282 | for line in lines.split('\n'): 283 | name = line.split(' ')[-1] 284 | if name: 285 | gdb.execute('b ' + name, to_string=True) 286 | end 287 | ... 288 | ``` 289 | -------------------------------------------------------------------------------- /doc/9_Memory_Management.md: -------------------------------------------------------------------------------- 1 | # Memory Management 2 | 3 | Normally, I think of the memory management of the kernel as four parts. 4 | 5 | - Physical Memory Manager 6 | - Virtual Memory Manager 7 | - Kernel Heap 8 | - Process Memory Manager 9 | 10 | We'll go through them in order, but first a radical design decision. 11 | 12 | ## Mapping the entire physical memory into kernel space 13 | 14 | A 32-bit processor can address about 4 Gb of memory. Near the end of the 32-bit 15 | era, even home desktop computers were actually running into this limit. 16 | 17 | In chapter 4, I mentioned that te top 16 address bits must be the same due to 18 | hardware limitations, in what's called canonical addressing. This leaves 48 19 | bits, which lets us address 256 Tb. 20 | 21 | I also mentioned that I reserve a 512th of this for the kernel (virtual 22 | addresses above 0xFFFFFF8000000000), which is about 512 Gb. 23 | 24 | 512 Gb - to paraphrase something Bill Gates probably never actually said - 25 | ought to be enough for anybody. 26 | 27 | This means we could - for any forseeable future, and definitely longer than the 28 | lifespan of my hobby kernel designed for running in an emulated environment 29 | with a couple of Mb of RAM - map the entirety of RAM right into the kernel 30 | address space. 31 | 32 | The main advantage of this is that we will never have to map anything into 33 | kernel space temporarily, say to modifiy the page tables of a process. It also 34 | means that we can use unused pages for temporary storage (more on this in a 35 | minute). 36 | 37 | The main disadvantage is that the entirety of memory is mapped into the kernels 38 | virtual memory at all times. At the time of writing this, the Spectre and 39 | Meltdown hardware bugs affecting almost all modern Intel CPUs have recently 40 | proved that anything mapped into memory is insecure. 41 | 42 | There are ways to remedy this, but they come at a performance cost, and - 43 | honestly - I don't feel like implementing them right now. Perhaps I will later, 44 | or perhaps I'll never have to because Intel decides that maybe they should fix 45 | the issue. 46 | 47 | Anyway. The entirety of physical RAM will be mapped into kernel space. That 48 | means a physical page with address `addr` can be accessed at `P2V(addr)`, where 49 | `P2V` is one of the two macros defined in Chapter 4. This explains their name 50 | P2V - Physical to Virtual, and V2P - Virtual to Physical. We'll use that a lot. 51 | 52 | ## The physical memory manager - PMM 53 | 54 | The PMM keeps track of available physical memory and hands out free pages when 55 | requested. 56 | 57 | Thanks to virtual address translation, we almost never need to care where a 58 | physical page is located, and therefor the PMM can be made very simple. 59 | 60 | In reality, we sometimes do need a number of continuous physical pages - for 61 | example when reading from disk using Direct Memory Access, but we'll save that 62 | for another day. 63 | 64 | The entire PMM has only one single state variable. A pointer to the last freed 65 | page. This page, in turn, holds a pointer to the one that was freed before, and 66 | so on, so when we free a page, we write the pointer to it, and update the free 67 | pointer: 68 | 69 | `src/kernel/memory/pmm.c` 70 | ```c 71 | ... 72 | uint64_t next = 0; 73 | 74 | void pmm_free(uint64_t page) 75 | { 76 | *(uint64_t *)P2V(page) = next; 77 | next = (uint64_t)P2V(page); 78 | } 79 | ... 80 | ``` 81 | 82 | That's all. Allocating a page is equally simple: 83 | 84 | ```c 85 | ... 86 | uint64_t pmm_alloc() 87 | { 88 | if(!next) return 0; 89 | uint64_t page = next; 90 | next = *(uint64_t *)page; 91 | return V2P(page); 92 | } 93 | ... 94 | ``` 95 | 96 | Feeling comfortable about dereferencing a pointer cast will help you when 97 | debugging your VMM in gdb, by the way. 98 | 99 | I also define a `pmm_calloc()` function, which allocates and zeros a page. 100 | 101 | ## The virtual memory manager - VMM 102 | 103 | The VMM handles setting up page tables, and separating user and kernel space. 104 | 105 | As mentioned earlier, x86\_64 uses four levels of page tables, each containing 106 | 512 64 bit entries. The 52 most significant bits of each entry is a pointer to 107 | the next level of page table, or the page itself in the case of the last page 108 | table (P1 in my inofficial nomenclature). This pointer is obviously always page 109 | alligned. 110 | 111 | The 12 least significant bits of each entry are for various flags. In order to 112 | find the next level, they need to be masked out. So let's start with some 113 | macros: 114 | 115 | `src/kernel/memory/vmm.c` 116 | ```c 117 | ... 118 | #define FLAGS_MASK (PAGE_SIZE - 1) 119 | #define MASK_FLAGS(addr) ((uint64_t)addr & ~FLAGS_MASK) 120 | ... 121 | ``` 122 | 123 | Now it's easy to go from a page table entry to something we can parse: 124 | 125 | ```c 126 | ... 127 | #define PT(ptr) ((uint64_t *)P2V(MASK_FLAGS(ptr))) 128 | ... 129 | ``` 130 | 131 | Page table entries are physical addresses, so we need to go through `P2V` to 132 | access them. To access a certain page table entry, you just do e.g. 133 | `PT(P4)[num]`.I chose to define some macros to access the entries for a certain 134 | address in each page table (P4 trough P1). They look like this: 135 | 136 | ```c 137 | ... 138 | #define P4E (PT(P4)[P4_OFFSET(addr)]) 139 | #define P3E PT(P4E)[P3_OFFSET(addr)]) 140 | #define P2E PT(P3E)[P2_OFFSET(addr)]) 141 | #define P1E PT(P2E)[P1_OFFSET(addr)]) 142 | ... 143 | ``` 144 | 145 | This assumes that there's a variable named `addr` which 146 | contains the virtual address whose page table entries you 147 | want, and a variable named `P4` which points to the top 148 | level page directory. The `OFFSET` macros finds the 149 | correct entry for an address for each page table level 150 | (`#define P4_OFFSET(a) (((a)>>39 & 0x1FF)` and so on). 151 | 152 | Now getting the physical page for a virtual address is very easy: 153 | 154 | ```c 155 | ... 156 | uint64_t vmm_get_page(uint64_t P4, uint64_t addr) 157 | { 158 | if(P4 && PRESENT(P4E) && PRESENT(P3E) && PRESENT(P2E)) 159 | return P1E; 160 | return -1; 161 | } 162 | ... 163 | ``` 164 | 165 | Where the `PRESENT` macro just checks for the `PAGE_PRESENT` bit being set. 166 | 167 | Setting the physical page for a virtal address is also very easy: 168 | 169 | ```c 170 | ... 171 | int vmm_set_page(uint64_t P4, uint64_t addr, uint64_t page, uint16_t flags) 172 | { 173 | ... 174 | 175 | if(!PRESENT(P4E) && !(P4E = pmm_calloc())) 176 | return -1; 177 | P4E |= flags | PAGE_PRESENT; 178 | 179 | // Do the same thing for P3E and P2E 180 | ... 181 | 182 | P1E = page | flags; 183 | return 0; 184 | } 185 | ... 186 | ``` 187 | 188 | The first three lines checks if the P4 entry is set, and if not, allocates a P3 189 | and sets the entry to point to it. If the allocation fails, `vmm_set_page` 190 | fails as well. The same is then done with P2 and P1, and finally the correct P1 191 | entry is set with the page address and flags. 192 | 193 | I also wrote a function `void vmm_clear_page(uint64_t P4, uint64_t addr, int 194 | free)` which zeros the P1 entry, and - if `free` is true - frees P1, P2 and P3 195 | if they are empty. This is left as an exercise to the reader. 196 | 197 | ## The kernel heap 198 | 199 | The kernel heap keeps track of and hands out small chunks of memory for 200 | temporary storage. This corresponds to the `malloc()`-family of functions. 201 | 202 | In this case, I actually chose to forego the heap, and try to make do with 203 | hard-coded global variables and structures using entire pages. 204 | 205 | For example, when setting up a new process, I might normally do something like 206 | 207 | ```c 208 | struct process *new_process = kmalloc(sizeof(struct process)); 209 | setup_process(new_process); 210 | ``` 211 | 212 | Now, I'll instead do: 213 | 214 | ```c 215 | struct process *new_process = (void *)pmm_alloc(); 216 | setup_process(new_process); 217 | ``` 218 | 219 | This will reqire me to think a bit more carefully of how I define my various 220 | data structures in order to keep wasted space to a minimum. It'll be an 221 | interesting experiment, but we'll see - perhaps I'll end up implementing a heap 222 | later... 223 | 224 | ## The process memory manager - PROCMM 225 | 226 | The final memory related part of the kernel - the procmm - handles user space 227 | memory. Setting up and cloning process memory spaces, replacing them with new 228 | executables, and handling the user stack and `brk()` calls are some of its 229 | tasks. 230 | 231 | Since there are no processes yet, having a process memory manager doesn't really make sense, so I'll save this for later... 232 | 233 | ## A remark about the git history 234 | 235 | For this chapter, I went a bit crazy with the TDD and made one git commit every 236 | time I wrote and passed a test. Perhaps that would make sense if I had a 237 | finished API to conform agains. Now, it just got a bit messy... If you explore 238 | the git history - I'm sorry. 239 | -------------------------------------------------------------------------------- /doc/8_Exceptions.md: -------------------------------------------------------------------------------- 1 | # Exceptions and Interrupts 2 | 3 | Sometimes, things go wrong. When they do, we want to fail gracefully - or even 4 | recover. That's the point of exceptions. 5 | 6 | ## Interrupt Service Routines 7 | 8 | The x86 interrupt handling method is, for historical reasons I assume, messy. 9 | The x86\_64 architecture saw a slight improvement in that the stack pointer and 10 | segment are always pushed, even if the cpu was running in ring 0 when the 11 | interrupt happened. Still, though, some exceptions push an error code, and 12 | others do not. And no data is provided to determine which interrupt occurred, 13 | besides which interrupt service routine was called. 14 | 15 | If all interrupts pushed a dummy error code and an identifying number, a single 16 | ISR would be enough, and the rest could be done in software. 17 | 18 | Anyway. Let's play with the cards we're dealt. 19 | 20 | The most common way of solving this discrepancy is by having a number of short 21 | ISRs in the form 22 | 23 | ```asm 24 | isr1: 25 | push 0 //; Dummy error code 26 | push 1 //; Interrupt number 27 | jmp isr_common //; The rest is the same for all interrupts 28 | ``` 29 | 30 | You may want up to 256 ISRs, so let's do some finger warmup exercises! 31 | 32 | Or rather yet, let's generate the ISRs automatically. With python! 33 | 34 | `src/kernel/cpu/isr.S.py` 35 | ```python 36 | #!/usr/bin/env python2 37 | # -*- coding: utf-8 -*- 38 | 39 | from __future__ import print_function 40 | 41 | num_isr = 256 42 | pushes_error = [8, 10, 11, 12, 13, 14, 17] 43 | 44 | print(''' 45 | .intel_syntax noprefix 46 | .extern isr_common 47 | ''') 48 | 49 | 50 | print('// Interrupt Service Routines') 51 | for i in range(num_isr): 52 | print('''isr{0}: 53 | cli 54 | {1} 55 | push {0} 56 | jmp isr_common '''.format(i, 57 | 'push 0' if i not in pushes_error else 'nop')) 58 | 59 | print('') 60 | print(''' 61 | // Vector table 62 | 63 | .section .data 64 | .global isr_table 65 | isr_table:''') 66 | 67 | for i in range(num_isr): 68 | print(' .quad isr{}'.format(i)) 69 | ``` 70 | 71 | This outputs an assembly file with 256 ISRs like the one above, except numbers 72 | 8, 10, 11, 12, 13, 14 and 17, which has an `nop` instruction instead of pushing 73 | a bogus error code. 74 | 75 | It's written for python 2 because that's what's included in the alpine version 76 | the build docker image is based on - despite it being 2018. The encoding is 77 | utf-8, and I import the print function from \_\_future\_\_, because it's 2018. 78 | 79 | It also makes a table with pointers to each ISR, which makes it easy to set up 80 | the Interrupt Descriptor Table later: 81 | 82 | `src/kernel/cpu/interrupts.c` 83 | ```c 84 | ... 85 | struct idt 86 | { 87 | uint16_t base_l; 88 | uint16_t cs; 89 | uint8_t ist; 90 | uint8_t flags; 91 | uint16_t base_m; 92 | uint32t base_h; 93 | uint32_t _; 94 | }__attribute__((packed)) idt[NUM_INTERRUPTS]; 95 | 96 | extern uintptr_t isr_table[] 97 | 98 | void interrupt_init() 99 | { 100 | memset(idt, 0, sizeof(idt)); 101 | for(int i=0; i < NUM_INTERRUPTS; i++) 102 | { 103 | idt[i].base_l = isr_table[i] & 0xFFFF; 104 | idt[i].base_m = (isr_table[i] >> 16) & 0xFFFF; 105 | idt[i].base_h = (isr_table[i] >> 32) & 0xFFFFFFFF; 106 | idt[i].cs = 0x8; 107 | idt[i].ist = 0; 108 | idt[i].flags = IDT_PRESENT | IDT_DPL0 | IDT_INTERRUPT; 109 | } 110 | ... 111 | ``` 112 | 113 | `isr_common` pushes all registers to the stack (one by one, there's no `pusha` 114 | instruction in x86\_64) and passes controll to a c interrupt handler. Note that 115 | for x86\_64 the arguments to a function is not primarily passed on the stack, 116 | but in registers. So the last thing it does before calling the c function is 117 | move the stack pointer into `rdi`. In case the handler returns, `isr_common` 118 | restores the stack pointer from `rax` - which is the function return value, 119 | pops all values again, and performs an `iretq` instruction, which is pretty 120 | much a backwards interrupt. 121 | 122 | `src/kernel/cpu/isr_common.S` 123 | ```asm 124 | ... 125 | isr_common: 126 | push r15 127 | push r14 128 | ... 129 | push rbx 130 | push rax 131 | mov rdi, rsp 132 | call int_handler 133 | 134 | mov rdi, rax 135 | isr_return: 136 | mov rsp, rdi 137 | pop rax 138 | pop rbx 139 | ... 140 | pop r14 141 | pop r15 142 | add rsp, 0x10 143 | iretq 144 | ``` 145 | 146 | But what's the deal with passing `rax` to `rsp` via `rdi`? Doing it this way 147 | will allow us to call `isr_return` as a function, with a faked interrupt stack. 148 | We'll use this later to get into user mode. 149 | 150 | ## Building isr.S.py 151 | 152 | But back to the ISRs. In order to build this, we need some changes in the 153 | kernel makefile. 154 | First of all, the lines 155 | 156 | ```make 157 | SRC := $(wildcard **/*.[cS]) 158 | OBJ := $(patsubst %, %.o, $(basename $(SRC))) 159 | ``` 160 | 161 | need to be updated to allow more file extensions: 162 | 163 | ```make 164 | SRC := $(wildcard **/*.[cS]*) 165 | OBJ := $(patsubst %, %.o, $(basename $(basename $(SRC)))) 166 | ``` 167 | 168 | We also need a special rule to generate .o files from .S.py: 169 | 170 | `src/kernel/Makefile` 171 | ```asm 172 | %.o: %.S.py 173 | python $^ | $(COMPILE.S) $(DEPFLAGS) -x assembler-with-cpp - -o $@ 174 | ``` 175 | 176 | In theory, it should be enough with a rule of the form 177 | 178 | ```make 179 | %.S: %.S.py 180 | python $^ > $@ 181 | ``` 182 | 183 | However, this generates the dependency tree .o <- .s <- .S <- .py rather than 184 | .o <- .S <- .py, which uses `as` to compile, and causes some other trouble as 185 | well with intermediate files that are removed once, but not if you run make 186 | again, and stuff... 187 | 188 | Some of this can be solved with an `.INTERMEDIATE:` rule, but that's not very 189 | elegant. The big problem's probably with me rather than make. 190 | 191 | 192 | ## The Interrupt Handler 193 | 194 | The c interrupt handler routine is a simple thing. Its default modus operandi 195 | is to print an error message and hang. 196 | 197 | However, before doing this, it checks a table of other interrupt handlers, and 198 | if one exists for the current interrupt, it passes execution over to that. 199 | 200 | `src/kernel/cpu/interrupts.c` 201 | ```c 202 | registers *int_handler(registers *r) 203 | { 204 | if(int_handlers[r->int_no]) 205 | return int_handlers[r->int_no](r); 206 | 207 | debug("Unhandled interrupt occurred\n"); 208 | debug("Interrupt number: %d Error code: %d\n", r->int_no, r->err_code); 209 | debug_print_registers(r); 210 | 211 | PANIC("Unhandled interrupt occurred"); 212 | for(;;); 213 | } 214 | ``` 215 | 216 | ## Final Note 217 | 218 | For tidyness sake, I wrapped the call to `interrupt_init` inside a function 219 | called `cpu_init`, which in turn is called from `kmain`. For now, that's all it 220 | is, but it will soon grow more important. 221 | 222 | ## Bonus: Debugging Interrupts 223 | 224 | There's a small problem with the way interrupts are handled by the processor; 225 | they don't follow the calling convention. 226 | 227 | This means that when an interrupt occurs, and the debugger breaks in the 228 | `PANIC` macro, it has lost all context, and we can't see what happened. 229 | 230 | But wait. The entire context is saved. It was pushed to the stack and passed to 231 | the interrupt handler. And by using gdbs ability to set the value of registers 232 | in qemu, we can bring it back into scope. 233 | 234 | I put the following function in `toolchain/gdbinit` 235 | 236 | ```gdb 237 | define restore_env 238 | set $name = $arg0 239 | python 240 | 241 | registers = {r: gdb.parse_and_eval('$name->' + r) for r in 242 | ['rax', 'rbx', 'rcx', 'rdx', 'rsi', 'rdi', 'rbp', 'rsp', 'r8', 'r9', 'r10', 243 | 'r11', 'r12', 'r13', 'r14', 'r15', 'rip']} 244 | 245 | for r in registers.items(): 246 | gdb.parse_and_eval('$%s=%s' % r) 247 | gdb.execute('frame 0') 248 | end 249 | end 250 | ``` 251 | 252 | And it's used like this: 253 | 254 | ``` 255 | (gdb) c 256 | Continuing. 257 | 258 | Thread 1 hit Breakpoint 2, int_handler (r=0xffffff8000019f10) at cpu/interrupts.c:74 259 | 74 PANIC("Unhandled interrupt occurred"); 260 | (gdb) restore_env r 261 | #0 0xffffff8000010caa in divide_two_numbers (divisor=0, dividend=0) at boot/kmain.c:18 262 | 18 return dividend/divisor; 263 | (gdb) bt 264 | #0 0xffffff8000010caa in divide_two_numbers (divisor=0, dividend=0) at boot/kmain.c:18 265 | #1 0xffffff8000010dbd in kmain (multiboot_magic=920085129, multiboot_data=0x105fa0) at boot/kmain.c:33 266 | #2 0xffffff8000010efd in .reload_cs () at boot/boot.S:96 267 | #3 0x0000000000000007 in ?? () 268 | #4 0x0000000000000730 in ?? () 269 | #5 0x0000000000000000 in ?? () 270 | (gdb) list 271 | 13 for(;;); 272 | 14 } 273 | 15 274 | 16 int divide_two_numbers(int divisor, int dividend) 275 | 17 { 276 | 18 return dividend/divisor; 277 | 19 } 278 | 20 279 | 21 void kmain(uint64_t multiboot_magic, void *multiboot_data) 280 | 22 { 281 | (gdb) p divisor 282 | $1 = 0 283 | (gdb) p divident 284 | $2 = 5 285 | (gdb) frame 1 286 | #1 0xffffff8000010dbd in kmain (multiboot_magic=920085129, multiboot_data=0x105fa0) at boot/kmain.c:33 287 | 33 divide_two_numbers(0,5); // Calculate 0/5 and discard the results 288 | (gdb) 289 | ``` 290 | 291 | By restoring the processor to the state stored in `r`, we can debug from where 292 | the interrupt occurred as normal. By backtracing and inspecting variables we 293 | find that whoever wrote line 33 in `kmain.c` got the divisor and divident mixed 294 | up, which resulted in a divide by zero exception. 295 | -------------------------------------------------------------------------------- /doc/4_Higher_Half_Kernel.md: -------------------------------------------------------------------------------- 1 | # Chapter 4 - "Higher Half" Kernel 2 | 3 | In this chapter we'll make our kernel run in the top of memory - well out of 4 | the way of user programs and memory mapped devices. 5 | 6 | ## What is a higher half kernel 7 | 8 | Some arguments for a higher half kernel can be found at [the osdev 9 | wiki](http://wiki.osdev.org/Higher_Half_Kernel). There are arguments against 10 | as well, such as it being pointless with modern memory management routines. My 11 | main argument for using it is that it makes things simpler. 12 | 13 | I chose to put the split at `0xFFFFFF8000000000`, which corresponds to the last 14 | entry in P4. 15 | 16 | > A note about the address. If you go through the calculations, you'll find 17 | > that the addresses mapped from the last entry in P4 actually starts at 18 | > `0xFF8000000000` or `0x0000FF8000000000`. However, due to limitations in the 19 | > hardware, the X86 architecture requires the most significant 13 bits of an 20 | > address to be equal; thus `0xFFFFFF8000000000`. This address format is called 21 | > 'canonical'. 22 | 23 | ## Higher half linking 24 | 25 | I want to have the kernel running at address `0xFFFFFF8000000000` and above. 26 | However, when we're booted from GRUB, paging is not enabled, so 27 | we're limited to physical RAM. 28 | 29 | The solution to this problem is to tell GRUB to load the kernel into a low 30 | memory position, but make the kernel think it's loaded at a high position. This 31 | can be done with a linker script trick. 32 | 33 | We change the linker script from Chapter 2 to something like this: 34 | 35 | `src/kernel/Link.ld` 36 | ``` 37 | ENTRY(_start) 38 | 39 | KERNEL_OFFSET = 0xFFFFFF8000000000; 40 | KERNEL_START = 0x10000; 41 | 42 | SECTIONS 43 | { 44 | . = KERNEL_START + KERNEL_OFFSET; 45 | .text : AT(ADDR(.text.) - KERNEL_OFFSET) 46 | { 47 | *(.multiboot) 48 | *(.text) 49 | } 50 | } 51 | ``` 52 | 53 | What this does is tell the linker to assume the code starts at 54 | `0xFFFFFF8000010000` when calculating addresses for things like function calls 55 | and jumps and such, but to generate an ELF file with headers saying the `.text` 56 | section should be *loaded* at an address `0xFFFFFF8000000000` below that, i.e. 57 | `0x10000` which is within the physical RAM limits. 58 | 59 | This also means that GRUB will jump to address `0x10000` after loading the 60 | kernel, and from there we can set up paging and jump to above 61 | `0xFFFFFF8000000000`. We just need to take care at all memory references, since 62 | we can't trust the linker to sort them out before paging is setup. 63 | 64 | Oh, and you will also want to do the same with the other default elf sections 65 | in the linker file, such as `.rodata`, `.data` and `.bss`. 66 | 67 | ## Fixing memory references 68 | 69 | In order to make sure all memory references are correct, we'll define some 70 | helpful macros. 71 | 72 | `src/kernel/include/memory.h` 73 | ```c 74 | #define KERNEL_OFFSET 0xFFFFFF8000000000 75 | 76 | #ifdef __ASSEMBLER__ 77 | #define V2P(a) ((a) - KERNEL_OFFSET) 78 | #define P2V(a) ((a) + KERNEL_OFFSET) 79 | #else 80 | #include 81 | #define V2P(a) ((uintptr_t)(a) & ~KERNEL_OFFSET) 82 | #define P2V(a) ((uintptr_t)(a) | KERNEL_OFFSET)) 83 | #endif 84 | ... 85 | ``` 86 | 87 | I define two versions of the macros, one for use in assembly and one for c. 88 | `__ASSEMBLER__` is set by gcc when compiling a `.S` file. The c version uses 89 | bit operations which means you can run `V2P(VP2(address))` without any problems 90 | due to the format of `KERNEL_OFFSET`. The proof of this is left as an exercise 91 | to the reader. 92 | 93 | Note that although we don't have access to a standard c library at this point, 94 | `stdint.h` (which defines `uintptr_t`) can still be used since it's included in 95 | `libgcc`, which we built and installed in the docker image together with the 96 | compiler. 97 | 98 | Then we need to go through our code and make sure all memory references are 99 | corrected. 100 | 101 | `src/kernel/boot/boot.S` 102 | ```asm 103 | #include 104 | .intel_syntax noprefix 105 | 106 | ... 107 | _start: 108 | cli 109 | mov esp, offset V2P(BootStack) 110 | 111 | ... 112 | 113 | mov eax, offset V2P(BootP4) 114 | mov cr3, eax 115 | 116 | ... 117 | 118 | lgdt [V2P(BootGDTp)] 119 | 120 | ... 121 | 122 | jmp 0x8:V2P(long_mode_start) 123 | ... 124 | ``` 125 | 126 | Note that `call` instructions don't have to be modified, since `call` uses 127 | relative addressing. 128 | 129 | And don't forget about the memory references in the page tables: 130 | 131 | `src/kernel/boot/boot_PT.S` 132 | ```asm 133 | ... 134 | BootP4: 135 | .quad offset V2P(BootP3) + (PAGE_PRESENT | PAGE_WRITE) 136 | ... 137 | ``` 138 | 139 | Note also that the GDT pointer does not require to be redirected to `V2P(BootGDT)` as one would assume. But why is that? Because of pure luck and coincidence. 140 | 141 | Before starting long mode the `lgdt` instruction will expect a 32 bit gdt pointer. If there's any more data, the top bits will just be truncated (due to the small-endian nature of the processor). As luck would have it, the only difference between `BootGDT` and `V2P(BootGDT)` lies in the top 32 bits. This also means that when it's time to load a 64 bit GDT, we can use the same pointer. Neat! 142 | 143 | At this point, it would be a good idea to check that the kernel still boots. 144 | However, gdb won't be able to tell you anything about the code since we're 145 | running outside of the linked addresses. You can still use it to inspect 146 | registers and such, though. You can also set breakpoints by modifying the 147 | address manually: `(gdb) break *(long_mode_start - 0xFFFFFF8000000000)`. 148 | 149 | ## Jumping to higher half 150 | 151 | The final piece of setup we need to do before we can start running in the 152 | higher half is update the page table. 153 | 154 | We'll do this by adding a pointer to the same P3 we set up earlier at the end 155 | of the BootP4. 156 | 157 | `src/kernel/boot/boot_PT.S` 158 | ```asm 159 | ... 160 | BootP4: 161 | .quad offset V2P(BootP3) + (PAGE_PRESENT | PAGE_WRITE) 162 | .rept ENTRIES_PER_PT - 2 163 | .quad 0 164 | .endr 165 | .quad offset V2P(BootP3) + (PAGE_PRESENT | PAGE_WRITE) 166 | ... 167 | ``` 168 | 169 | If you start up the emulator now you can check that the higher half is mapped 170 | 171 | ``` 172 | (gdb) mmap 173 | 0000000000000000-0000000040000000 0000000040000000 -rw 174 | 0000ff8000000000-0000ff8040000000 0000000040000000 -rw 175 | ``` 176 | 177 | Note that qemu doesn't report the addresses in canonical mode. 178 | 179 | Anyway. It should now be safe to jump to higher half code: 180 | 181 | `src/kernel/boot/boot.S` 182 | ```asm 183 | ... 184 | .code64 185 | long_mode_start: 186 | mov eax, 0x0 187 | mov ss, eax 188 | mov ds, eax 189 | mov es, eax 190 | mov fs, eax 191 | mov gs, eax 192 | 193 | movabs rax, offset upper_memory 194 | jmp rax 195 | 196 | upper_memory: 197 | 198 | jmp $ 199 | ``` 200 | 201 | By loading the address of the `upper_memory` into a register and jumping to it 202 | we force the assembler to make a non-relative jump. 203 | 204 | If you run this, you'll find that gdb will be able to track where in the code 205 | you are again (after passing `upper_memory:`, or you could check the `RIP` 206 | register. 207 | 208 | ``` 209 | upper_memory () at boot/boot.S:116 210 | 116 jmp $ 211 | (gdb) reg RIP 212 | RIP=ffffff800001019f 213 | ``` 214 | 215 | Great! Now we can do some cleanup 216 | 217 | Move the stack pointer to higher half memory: 218 | ```asm 219 | ... 220 | upper_memory: 221 | mov rax, KERNEL_OFFSET 222 | add rsp, rax 223 | ... 224 | ``` 225 | 226 | and unmap the identity mapping of the first gigabyte and reload the page table: 227 | 228 | ```asm 229 | ... 230 | mov rax, 0 231 | movabs [BootP4], rax 232 | 233 | mov rax, cr3 234 | mov cr3, rax 235 | ... 236 | ``` 237 | 238 | Run it all again, and check that the low memory is unmapped: 239 | 240 | ``` 241 | (gdb) mmap 242 | 0000ff8000000000-0000ff8040000000 0000000040000000 -rw 243 | ``` 244 | 245 | Finally, we also need to reload the GDT. In long mode, the GDT 246 | register points to the physical address of the GDT, and we just 247 | unmapped that... 248 | 249 | So we need to 250 | 251 | - reload the GDT and update the data selectors: 252 | ```asm 253 | ... 254 | lgdt[rax] 255 | mov rax, 0x0 256 | mov ss, rax 257 | mod ds, rax 258 | mov es, rax 259 | ... 260 | ``` 261 | - and reload the code selector. There are no long jumps in long mode, 262 | so instead we'll use the `retfq` instruction which pops a return 263 | address and code segment selector off the stack: 264 | ```asm 265 | ... 266 | movabs rax, offset .reload_cs 267 | pushq 0x8 268 | push rax 269 | retfq 270 | .reload_cs: 271 | ``` 272 | 273 | ## Running c code 274 | 275 | Now that the instruction pointer is safely within our linked memory, we can 276 | trust c code to run. 277 | 278 | Calling a c function is simple enough: 279 | 280 | `src/kernel/boot/boot.S` 281 | ```asm 282 | ... 283 | .reload_cs: 284 | 285 | .extern kmain 286 | movabs rax, offset kmain 287 | call rax 288 | 289 | hlt 290 | jmp $ 291 | ``` 292 | 293 | And the c source file: 294 | `src/kernel/boot/kmain.c` 295 | ```c 296 | #include 297 | 298 | void clear_screen() 299 | { 300 | unsigned char *vidmem = P2V(0xB8000); 301 | for(int i=0; i < 80*24*2; i++) 302 | *vidmem++ = 0; 303 | } 304 | 305 | void print_string(char *str) 306 | { 307 | unsigned char *vidmem = P2V(0xB8000); 308 | while(*str) 309 | { 310 | *vidmem++ = *str++; 311 | *vidmem++ = 0x7; 312 | } 313 | } 314 | 315 | void kmain() 316 | { 317 | clear_screen(); 318 | print_string("Hello from c, world!"); 319 | for(;;); 320 | } 321 | ``` 322 | 323 | ... which will clear the screen and print "Hello from c, world!". 324 | Things are so much simpler in c... 325 | 326 | But what now? This doesn't compile! 327 | 328 | You'll probably get an error about "relocation truncated to fit: R_X86_64_32 329 | against \`.rodata\`" 330 | 331 | This is because gcc assumes your code will be running at a lower memory 332 | address, and optimizes it as such. The solution is to tell gcc to make no 333 | assumption about addresses by adding the switch `-mcmodel=large` to `CFLAGS` in 334 | your makefile. 335 | -------------------------------------------------------------------------------- /doc/10_Threading.md: -------------------------------------------------------------------------------- 1 | # Threading 2 | 3 | In this chapter we'll implement context switching and a simple scheduler. 4 | 5 | ## Context switching 6 | 7 | Switching from one thread to another is actually really easy. All you need to 8 | do is replace everything in every processor register at the same time, and 9 | you're good to go. Ok... that doesn't sound so simple, but we get some help. 10 | 11 | In the conventions used by our gcc cross compiler (known as [System V 12 | ABI](https://wiki.osdev.org/System_V_ABI)), when calling a function only a few 13 | registers are guaranteed to be preserved. Those are `rbx`, `rsp`, `rbp`, `r12`, 14 | `r13`, `r14` and `r15`. The rest can not be assumed to retain the same value. 15 | 16 | So, if we make our context switch out to look like a function call, we only 17 | need to replace those seven registers. This can easily be done with a small asm 18 | routine: 19 | 20 | `src/kernel/proc/swtch.S` 21 | ```asm 22 | .intel_syntax noprefix 23 | 24 | .global switch_stack 25 | switch_stack: 26 | push rbp 27 | mov rbp, rsp 28 | 29 | push r15 30 | push r14 31 | push r13 32 | push r12 33 | push rbx 34 | push rbp 35 | 36 | mov [rdi], rsp 37 | mov rsp, [rsi] 38 | 39 | pop rbp 40 | pop rbx 41 | pop r12 42 | pop r13 43 | pop r14 44 | pop r15 45 | 46 | leaveq 47 | ret 48 | ``` 49 | 50 | This pushes all registers, writes the stack pointer to the address passed as 51 | the first argument, reads a new stack pointer from the address in the second 52 | argument, and pops all registers. Since the return address is on the stack, the 53 | `ret` instruction will return to the new thread. 54 | 55 | > #### A note on credit 56 | > 57 | > Not everything I present here is my own original ideas. In fact, most of it 58 | > probably isn't. I've been itterating my kernel design from the ground up a 59 | > dozen times or more through the last ten years, and where I picked up methods 60 | > and ideas have gotten lost along the way. 61 | > 62 | > This method of switching threads, though, I know I got from XV6, where it may 63 | > or may not have originated. 64 | > 65 | > I'm sorry I can't always give proper and detailed credit to the giants on 66 | > whose shoulders I stand, but for a list of my most significant sources of 67 | > inspiration through the years, see Chapter 0. 68 | 69 | ## Threads 70 | 71 | Ok, so now switching between two threads of execution only requires: 72 | 73 | ```c 74 | switch_stack(&old_stack_ptr, &new_stack_ptr); 75 | ``` 76 | 77 | So the next step would be a structured and reliable way of keeping track of new 78 | and old stack pointers, and to allocate the stacks themselves. We might also 79 | want some extra information relating to each thread. A struct would be ideal 80 | for this. 81 | 82 | `src/kernel/include/thread.h` 83 | ```c 84 | struct thread 85 | { 86 | uint64_t tid; 87 | void *stack_ptr; 88 | uint64_t state; 89 | }; 90 | ``` 91 | 92 | This will grow with a lot of more information later. 93 | 94 | But where should we put this struct? you may remember from an earlier chapter 95 | that we don't have a `malloc` implementation to assign storage. We could use 96 | `pmm_alloc` to get some memory, but that would give us an entire page, and this 97 | struct is only 24 bytes. So what should we do with the rest of the space? 98 | 99 | How about using it for the thread stack? 100 | Allocating a new thread would then look something like this: 101 | 102 | `src/kernel/proc/thread.c` 103 | ```c 104 | uint64_t next_tid = 1; 105 | struct thread *new_thread() 106 | { 107 | struct thread *th = P2V(pmm_calloc()); 108 | th->tid = next_tid++; 109 | th->stack_ptr = incptr(th, PAGE_SIZE); 110 | 111 | return th; 112 | } 113 | ``` 114 | 115 | Of course, this thread can't be run. If we try to switch to it, the 116 | `switch_stack` function won't get a propper stack to start from, so we need to 117 | mock that up first: 118 | 119 | `src/kernel/proc/thread.c` 120 | ```c 121 | struct swtch_stack 122 | { 123 | uint64_t RBP; 124 | uint64_t RBX; 125 | uint64_t R12; 126 | uint64_t R13; 127 | uint64_t R14; 128 | uint64_t R15; 129 | uint64_t RBP2; 130 | uint64_t ret; 131 | }; 132 | 133 | uint64_t next_tid = 1; 134 | struct thread *new_thread(void (*function)(void)) 135 | { 136 | struct thread *th = P2V(pmm_calloc()); 137 | th->tid = next_tid++; 138 | th->stack_ptr = incptr(th, PAGE_SIZE - sizeof(struct swtch_stack)); 139 | 140 | struct swtch_stack *stk = th->stack_ptr; 141 | stk->RBP = (uint64_t)&stk->RBP2; 142 | stk->ret = (uint64_t)function; 143 | 144 | return th; 145 | } 146 | ``` 147 | 148 | That's all you need to do to set up a new thread with it's own stack, which 149 | will start running the function you pass to `new_thread`. 150 | 151 | If you wish, you can try this out now. Here's a quick (untested) mockup on how 152 | it might be done: 153 | 154 | `src/kernel/boot/kmain.c` 155 | ```c 156 | 157 | struct thread *current, *next; 158 | 159 | void thread_function() 160 | { 161 | int thread_id = current->tid; 162 | 163 | while(1) 164 | { 165 | debug("Thread %d\n", thread_id); 166 | struct thread *_next = next; 167 | struct thread *_current = current; 168 | 169 | // Update "scheduler" 170 | next = current; 171 | current = _next; 172 | 173 | // Switch thread 174 | switch_stack(&_current->stack_ptr, &_next->stack_ptr) 175 | } 176 | } 177 | 178 | void kmain(...) 179 | { 180 | ... 181 | 182 | current = new_thread(thread_function); 183 | next = new_thread(thread_function); 184 | 185 | uint64_t dummy_stack_ptr; 186 | switch_stack(&dummy_stack_ptr, ¤t->stack_ptr); 187 | 188 | ... 189 | } 190 | 191 | ``` 192 | 193 | This implements a simple "scheduler" which keeps track of two threads and 194 | switches between them. 195 | 196 | Switching in the first thread requires a dummy variable to store the old stack 197 | pointer. This value is thrown away, because we will never switch back into 198 | `kmain`. 199 | 200 | If you run this, the screen should fill with alternating lines of "Thread 1" 201 | and "Thread 2". Note that the variable `thread_id` is function local, and thus 202 | stored on the stack. If you're not convinced, you can make the threads run 203 | different functions instead. 204 | 205 | I don't get why tutorial writers make this look so hard... 206 | 207 | Now it's time to make it scaleable. 208 | 209 | ## Queueing 210 | 211 | Ok, so now we can create and switch between threads, but we still need some way 212 | of keeping track of them. 213 | 214 | Threads can be in one of several states. They are either *running*, *waiting to 215 | run*, or *waiting for something else*. This might seem obvious, kind of like 216 | how everything is either *a banana* or *not a banana*, but those three states 217 | are kind of important and decide how we keep track of the thread. 218 | 219 | For the running threads it's easy. Only one thread per cpu core can be running 220 | at a time, so it makes sense to keep a global variable which points to the 221 | running thread (one per cpu core). 222 | 223 | Threads that are waiting to run will be kept in the scheduler queue. When a 224 | running thread has finished running for one reason or another, the sheduler 225 | will pick the next one to run from the run queue based on various conditions 226 | and, if necessary, put the previously running thread back in the queue. 227 | 228 | Where the threads waiting for something else are kept depends on what they are 229 | waiting for. For example, a thread waiting for a disk read to finish will 230 | probably be kept track of by the disk driver or simmilar. Once the read 231 | completes, the driver will hand the thread over to the scheduler to be put in 232 | the run queue. 233 | 234 | Either way, almost all waiting threads will be kept in a queue somewhere. 235 | 236 | I won't go into the pros and cons of different queueing methods here, but just 237 | use a simple linked list, where the queue header keeps track of the first and 238 | last item, and each item in the queue keeps track of the next one. Something 239 | like this: 240 | 241 | ```c 242 | struct { 243 | struct thread *first; 244 | struct thread *last; 245 | } run_queue; 246 | 247 | 248 | struct thread 249 | { 250 | ... 251 | struct thread *run_queue_next; 252 | .. 253 | }; 254 | 255 | void init_queue() 256 | { 257 | run_queue.first = 0; 258 | run_queue.last = 0; 259 | } 260 | 261 | void queue_add(struct thread *th) 262 | { 263 | if(!run_queue.last) 264 | run_queue.first = th; 265 | else 266 | run_queue.last->run_queue_next = th; 267 | run_queue.last = th; 268 | th->run_queue_next = 0; 269 | } 270 | 271 | thread *queue_pop() 272 | { 273 | thread *ret = run_queue.first; 274 | if(run_queue.first && !(run_queue.first = run_queue.first->run_queue_next)) 275 | run_queue.last = 0; 276 | return ret; 277 | } 278 | ``` 279 | 280 | That's a full FIFO queue setup. Simple, but not very fun. 281 | 282 | Let's generalize! 283 | 284 | There are three things which are unique for each queue. 285 | 286 | - The name of the queue header struct 287 | - The name of the pointer to the next item in the item struct 288 | - The type of item in the queue 289 | 290 | With some macro magic, we can condense this into a single symbol: 291 | 292 | ```c 293 | #define RunQ run_queue, run_queue_next, struct thread 294 | ``` 295 | 296 | The plan is that `RunQ` should be used every time we want to do or define 297 | something with the run queue. Such as `queue_add(RunQ, my_thread)` or 298 | `queue_pop(RunQ)`, or: 299 | 300 | ```c 301 | struct thread 302 | { 303 | ... 304 | QUEUE_SPOT(RunQ); 305 | ... 306 | }; 307 | ``` 308 | 309 | You probably see by now that `queue_add` would need to be a macro. You can't 310 | pass a type to a function, and the example above would expand to 311 | `queue_add(run_queue, run_queue_next, struct thread, my_thread)`. 312 | 313 | But if `queue_add` is a macro, `RunQ` won't be expanded... 314 | 315 | This can be solved with an extra layer of indirection. We define a variadic 316 | wrapper macro which expands all arguments and pass them to another macro. This 317 | will make the preprocessor realize it needs to make another run of the code. 318 | 319 | ```c 320 | #define _QUEUE_ADD(queue, entry, type, item) \ 321 | if(!queue.last) \ 322 | queue.first = (item) \ 323 | else 324 | queue.last->entry = (item); \ 325 | queue.last = (item); \ 326 | (item)->entry = 0; 327 | #define queue_add(...) _QUEUE_ADD(__VA_ARGS__) 328 | ``` 329 | 330 | Then we do the same thing with any other queue operations we might need, as 331 | well as declaring and defining the queue head and next item pointer. 332 | 333 | ```c 334 | #define _QUEUE_DECL(queue, entry, type) \ 335 | struct queue{ \ 336 | type *first; \ 337 | type *last; \ 338 | } queue; 339 | #define QUEUE_DECLARE(...) _QUEUE_DECL(__VA_ARGS__) 340 | 341 | #define _QUEUE_HEAD(queue, entry, type) \ 342 | struct queue queue = {0, 0}; 343 | #define QUEUE_DEFINE(...) _QUEUE_HEAD(__VA_ARGS__) 344 | 345 | #define _QUEUE_SPOT(queue, entry, type) \ 346 | type *entry 347 | #define QUEUE_SPOT(...) _QUEUE_SPOT(__VA_ARGS__) 348 | 349 | #define _QUEUE_POP(queue, entry, type) \ 350 | __extension__({ \ 351 | type *_ret = _(queue.first); \ 352 | if(queue.first && !(queue.first = queue.first->entry)) \ 353 | queue.last = 0; \ 354 | _ret; \ 355 | }) 356 | #define queue_pop(...) _QUEUE_POP(__VA_ARGS__) 357 | ``` 358 | 359 | > The `__extension__` thing is a workaround to make gcc accept a macro with a 360 | > return value without warnings. For some reason I get warnings that this is 361 | > valid ANSI c even when compiling with `-std=gnu11`... 362 | -------------------------------------------------------------------------------- /doc/2_A_Bootable_Kernel.md: -------------------------------------------------------------------------------- 1 | # Chapter 2 - Booting a Kernel 2 | 3 | In this chapter we'll create a minimal kernel that can be loaded by a Multiboot compatible bootloader. 4 | 5 | ## Makefile 6 | 7 | Gnu Make is a powerful tool. I don't think most people realize just how powerful it is. It's really ridiculously smart, if you trust it. 8 | 9 | As an example, I'm currently going through the [Build Your Own Text Editor](http://viewsourcecode.org/snaptoken/kilo/) booklet. Here's my Makefile for the project: 10 | ```make 11 | CFLAGS := -Wall -Wextra -pedantic -std=c99 12 | 13 | kilo: 14 | ``` 15 | 16 | And Make takes care of the rest. Of course, the makefile for a kernel 17 | will be a bit more complex, but we'll trust Make as far as possible. 18 | 19 | Just for fun, you can run `make -p` to see all the built-in rules 20 | of Make, some of which we will make use of, such as `$(LINK.C)` or 21 | `$(COMPILE.S)` 22 | 23 | But let's get into it. First we create a directory for our sourc files, 24 | and the kernel sources specifically. In it, we place a Makefile: 25 | 26 | `src/kernel/Makefile` 27 | ```make 28 | ifeq ($(MITTOS64),) 29 | $(error Unsupported environment! See README) 30 | endif 31 | 32 | CC := x86_64-elf-gcc 33 | 34 | SRC := $(wildcard **/*.[cS]) 35 | OBJ := $(patsubst %, %.o, $(basename $(SRC))) 36 | 37 | CFLAGS := -Wall -Wextra -pedantic -ffreestanding 38 | CFLAGS += -ggdb -O0 39 | ASFLAGS += -ggdb 40 | CPPFLAGS += -I include 41 | LDFLAGS := -n -nostdlib -lgcc -T Link.ld 42 | 43 | kernel: $(OBJ) 44 | $(LINK.c) $^ -o $@ 45 | 46 | 47 | DEP := $(OBJ:.o=.d) 48 | DEPFLAGS = -MT $@ -MMD -MP -MF $*.d 49 | $(OBJ): CPPFLAGS += $(DEPFLAGS) 50 | %.d: ; 51 | 52 | DESTDIR ?= $(BUILDROOT)sysroot 53 | 54 | # Copy kernel to sysroot 55 | $(DESTDIR)$(PREFIX)/kernel: kernel 56 | install -D kernel $(DESTDIR)$(PREFIX)/kernel 57 | 58 | install: $(DESTDIR)$(PREFIX)/kernel 59 | 60 | clean: 61 | rm -rf $(OBJ) $(DEP) kernel 62 | 63 | .PHONY: install 64 | 65 | include $(DEP) 66 | ``` 67 | 68 | I think there are a few things here that needs explanation. 69 | 70 | First of all, there's the environment check 71 | 72 | ```make 73 | ifeq ($(MITTOS64),) 74 | $(error Unsupported environment! See README) 75 | endif 76 | ``` 77 | 78 | This is just as discussed in the previous chapter, to make sure the 79 | makefile is only run inside the Docker container. 80 | 81 | Then we set `CC := x86_64-elf-gcc` to make sure the kernel is build 82 | using our cross compiler. If you look through the built in Make 83 | rules, you'll see that `CC` is used for compiling .c files, 84 | compiling .cpp files, compiling .S files and linking it all 85 | together. 86 | 87 | The next three lines 88 | 89 | ```make 90 | SRC := $(wildcard **/*.[cS]) 91 | OBJ := $(patsubst %, %.o, $(basename $(SRC))) 92 | ``` 93 | 94 | scan the source directory and all subdirectories for `.c` or `.S` files, and find the names of the corresponding object files. 95 | 96 | Then we set some compiler and linker options 97 | 98 | ```make 99 | CFLAGS := -Wall -Wextra -pedantic -ffreestanding 100 | CFLAGS += -ggdb -O0 101 | ASFLAGS += -ggdb 102 | CPPFLAGS += -I include 103 | LDFLAGS := -n -nostdlib -lgcc -T Link.ld 104 | ``` 105 | 106 | - We want to compile with `-Wall -Wextra and -pedantic` to help keep the 107 | code quality high. 108 | - `-ffreestanding` stops the compiler from assuming that there's a 109 | standard c library present. 110 | - `-ggdb and -O0` simplifies debugging by including gdb specific debug 111 | symbols in the compiled object code and turning off all optimizations 112 | - and we also want `-ggdb` for assembly files. 113 | -The c preprocessor should look for include files in 114 | `src/kernel/include`, that's what the `-I include` option does. 115 | - And finally, there's the linker options. `-nostdlib` because we don't 116 | want any libraries but `libgcc` to be linked with our kernel. `-n` tells 117 | the linker that we don't necessarily want the sections page aligned. 118 | `-T Link.ld` tells the linker that the file `src/kernel/Link.ld` 119 | contains more information about how we want the file structured. 120 | 121 | The next two lines: 122 | 123 | ```make 124 | kernel: $(OBJ) 125 | $(LINK.c) $^ -o $@ 126 | ``` 127 | 128 | tells `make` that the file `kernel` depends on all our objects and how 129 | to build it. Here we're using the built in `$(LINK.c)` rule. 130 | 131 | And that's really all we need to build the kernel. 15 lines. The rest of 132 | the file is just for convenience. 133 | 134 | Like the lines: 135 | 136 | ```make 137 | DEP := $(OBJ:.o=.d) 138 | DEPFLAGS = -MT $@ -MMD -MP -MF $*.d 139 | $(OBJ): CPPFLAGS += $(DEPFLAGS) 140 | %.d: ; 141 | [...] 142 | include $(DEP) 143 | ``` 144 | 145 | which deserve some explanation. 146 | 147 | If you update a source file and run make, it will know which files were 148 | updated and need to be rebuilt. But what about included header files? If 149 | they update, you'd want to rebuild all files that include them. 150 | 151 | The book `Managing Projects with GNU Make` by Robert Meclenburg suggests 152 | a solution that uses the `-M` option of gcc to generate a list of 153 | dependencies from a source file, coupled with a mess of temporary files 154 | and `sed` commands to generate make rules that you can import. I've seen 155 | this method used in lots of projects. 156 | 157 | I am sorry to say I can't remember where I picked this up - I certainly 158 | didn't come up with it myself - but it turns out that through the use 159 | of a bunch of more `gcc` options, the rules can be generated exactly as 160 | we want them. `-MT $@` for example changes the name of the generated 161 | rule to the input file - which is what Meclenburg does with `sed 162 | 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $@.$$$ > $@;`. 163 | 164 | I just think this is a very neat solution. 165 | 166 | The part of the Makefile I skipped is for "installing" the kernel, i.e. 167 | copying it to the sysroot directory and for cleaning up. 168 | 169 | ### Bonus 170 | For convenience, I added a makefile to the project root directory which 171 | contains the following rule 172 | 173 | ```make 174 | kernel: 175 | ifeq ($(shell make -sqC src/kernel || echo 1), 1) 176 | $(MAKE) -C src/kernel install 177 | endif 178 | ``` 179 | 180 | ## Linking 181 | 182 | As described above, the linker gets passed a linker script. This is as 183 | basic as possible for now. 184 | 185 | `src/kernel/Link.ld` 186 | 187 | ```Linker Script 188 | ENTRY(_start) 189 | 190 | SECTIONS 191 | { 192 | .text : 193 | { 194 | *(.multiboot) 195 | *(.text) 196 | } 197 | } 198 | ``` 199 | 200 | What's important here is that the `*(.multiboot)` statement is first 201 | inside the `.text` section. The multiboot headers (described in the next 202 | section) need to be near the beginning of the kernel executable, or the 203 | bootloader won't be able to find them. As the kernel grows, this will 204 | ensure they can still be found. 205 | 206 | Note that you can't do this: 207 | 208 | ```Linker Script 209 | SECTIONS 210 | { 211 | .multiboot : {*(.multiboot)} 212 | .text : {*(.text)} 213 | } 214 | ``` 215 | 216 | Oh, how I wish this worked. But for some reason, the linker will put the 217 | `.text` section first in the output file, no matter what. 218 | 219 | The linker script will soon grow, but for now, this is enough. 220 | 221 | ## Multiboot Headers 222 | 223 | There are currently two actively used versions of the multiboot 224 | specification, "Multiboot 1" (latest version is 0.6.96) and "Multiboot 225 | 2" (latest version is 1.6). Since Multiboot 2 has better support for 64 226 | bit kernels, that's what I'll use. 227 | 228 | In order for the bootloader to be able to recognize a multiboot kernel, 229 | there has to be a special header near the start of the executable. Via 230 | our linker file trick above, we can place the header at the very start 231 | (actually after the program headers, but that's OK) by making sure it's 232 | in the `.multiboot` section. 233 | 234 | `src/kernel/boot/multiboot_header.S` 235 | 236 | ```asm 237 | #include 238 | .section .multiboot 239 | 240 | .align 0x8 241 | Multiboot2Header: 242 | .long MBOOT2_MAGIC 243 | .long MBOOT2_ARCH 244 | .long MBOOT2_LENGTH 245 | .long MBOOT2_CHECKSUM 246 | 247 | .short 0 248 | .short 0 249 | .long 8 250 | Multiboot2HeaderEnd: 251 | ``` 252 | 253 | The Multiboot 2 header is followed by a list of tags to specify further 254 | options. We only have one; the list termination tag. 255 | 256 | The constants above are defined in `multiboot.h`. 257 | 258 | `src/kernel/include/multiboot.h` 259 | 260 | ```c 261 | #pragma once 262 | 263 | #define MBOOT2_MAGIC 0xE85250D6 264 | #define MBOOT2_REPLY 0x36D76289 265 | #define MBOOT2_ARCH 0 266 | #define MBOOT2_LENGTH (Multiboot2HeaderEnd - Multiboot2Header) 267 | #define MBOOT2_CHECKSUM -(MBOOT2_MAGIC + MBOOT2_ARCH + MBOOT2_LENGTH) 268 | ``` 269 | 270 | 271 | This is just for the simplest possible multiboot headers. No frills or extra 272 | functions. 273 | 274 | For more information, see [Multiboot 1 275 | specification](https://www.gnu.org/software/grub/manual/multiboot/multiboot.html) 276 | or [Multiboot 2 277 | specification](https://www.gnu.org/software/grub/manual/multiboot2/multiboot.html). 278 | 279 | ## The kernel code 280 | 281 | Ladies and gentlemen, I present to you: The Simplest Kernel 282 | 283 | `src/kernel/boot/boot.S` 284 | 285 | ```asm 286 | .intel_syntax noprefix 287 | .section .text 288 | .global _start 289 | .code32 290 | _start: 291 | cli 292 | jmp $ 293 | ``` 294 | 295 | That's all you need to make sure the 296 | booting procedure works as it should. 297 | 298 | ## Testing it out 299 | 300 | Let's go. 301 | 302 | ```bash 303 | # compile 304 | $ d make 305 | make -C src/kernel install 306 | make[1]: Entering directory '/opt/src/kernel' 307 | cc -ggdb -I include -MT boot/multiboot_header.o -MMD -MP -MF boot/multiboot_header.d -c -o boot/multiboot_header.o boot/multiboot_header.S 308 | cc -ggdb -I include -MT boot/boot.o -MMD -MP -MF boot/boot.d -c -o boot/boot.o boot/boot.S 309 | cc -Wall -Wextra -pedantic -ffreestanding -ggdb -O0 -I include -n -nostdlib -lgcc -T Link.ld boot/multiboot_header.o boot/boot.o -o kernel 310 | mkdir -p /opt/sysroot 311 | cp kernel /opt/sysroot/kernel 312 | make[1]: Leaving directory '/opt/src/kernel' 313 | ``` 314 | 315 | Then run the emulator as before with `d emul` and also start the debugger in 316 | another terminal window with `d gdb`. 317 | 318 | Now you can load the kernel debug symbols into gdb with the command 319 | 320 | (gdb) file sysroot/kernel 321 | A program is being debugged already 322 | Are you sure you want to change the file? (y or n) y 323 | Reading symbols from sysroot/kernel...done. 324 | 325 | A small problem at this point is that GRUB will start the kernel in 32 bit 326 | protected mode, while gdb assumes we're in 64 bit mode. You can see this by 327 | running 328 | 329 | (gdb) show architecture 330 | The target architecture is set automatically (currently i386:x86-64) 331 | 332 | So, to get the addresses and stuff correct, you can run 333 | 334 | (gdb) set architecture i386 335 | The target architecture is assumed to be i386 336 | 337 | And finally, we're ready to start the emulator 338 | 339 | (gdb) c 340 | Continuing. 341 | 342 | This will make the familliar `grub>` prompt show up in the emulator. 343 | 344 | It's time to load our kernel 345 | 346 | grub> multiboot2 /kernel 347 | grub> boot 348 | 349 | And... nothing should happen, that you can see... 350 | 351 | But switch back to gdb and press ctrl+C to interrupt the emulator and you should see 352 | 353 | Program received signal SIGINT, Interrupt. 354 | _start () at boot/boot.S:9 355 | 9 jmp $ 356 | (gdb) 357 | 358 | So, apparently, the processor is running our "kernel" and is stuck at the 359 | infinite loop on line 9. Just as we expected! 360 | 361 | The multiboot specification says that the bootloader should leave a magic 362 | number in the `EAX` register after boot. Let's check it. 363 | 364 | (gdb) p/x $eax 365 | $1 = 0x36d76289 366 | 367 | Exactly according to the specification. Great! 368 | 369 | And with that everything seems to work! 370 | 371 | ## Some extra fancy stuff 372 | 373 | You really don't want to keep typing all those grub and gdb 374 | instructions in manually each time you test your OS, right? 375 | 376 | The gdb commands can be added to `toolchain/gdbinit`. In the name of 377 | cinsistency, the debug symbols should not be loaded from `sysroot/kernel`, 378 | but from `/opt/sysroot/kernel`, or rather yet `${BUILDROOT}sysroot/kernel`. 379 | Using environment variables in a gdb script is a bit tricky, but can be 380 | done via the `python` function: 381 | 382 | ```gdb 383 | python 384 | import os 385 | gdb.execute('file ' + os.environ['BUILDROOT'] + 'sysroot/kernel') 386 | end 387 | ``` 388 | 389 | In order to make grub load the kernel automatically, you need to add a file 390 | called `grub.cfg` at `/boot/grub` in the sysroot. I suggest you add this 391 | file to `sysroot` while building the iso in `toolchain/mkiso`. 392 | 393 | I did it this way 394 | 395 | ```bash 396 | cat > ${sysroot}/boot/grub/grub.cfg << EOF 397 | set timeout=1 398 | set default=0 399 | 400 | menuentry "mittos64" { 401 | multiboot2 /kernel 402 | } 403 | 404 | EOF 405 | ``` 406 | 407 | See the [grub 0.97 manual](https://www.gnu.org/software/grub/manual/legacy/grub.html) for more information. 408 | -------------------------------------------------------------------------------- /doc/1_Toolchain.md: -------------------------------------------------------------------------------- 1 | # Chapter 1 - Setting up a toolchain 2 | 3 | In this chapter we'll build a docker image which contains all the tools 4 | we need to build and emulate our OS. 5 | 6 | We'll also make some helper scripts to run commands in the Docker 7 | container and for running the emulator and debugger. 8 | 9 | 10 | ## Docker image 11 | 12 | ### Why docker? 13 | 14 | I've heard the name Docker thrown around a lot the last year or two, but 15 | only just recently started to look into it. The idea of using it for 16 | compiling an operating system came to me almost immediately. 17 | 18 | Docker lets you run processes inside a well defined, isolated and 19 | portable, linux-based environment. What's there not to like? 20 | 21 | So, let's build a Docker image for osdeving. 22 | 23 | ### What we want 24 | 25 | For now, I want the following in the image: 26 | 27 | - binutils 28 | - gcc 29 | - make 30 | - grub 31 | - xorriso 32 | - qemu 33 | - gdb 34 | 35 | In order to get a known compiler configuration, we will be building 36 | `binutils` and `gcc` from source. At this point, we'll only use a base 37 | configuration, and could therefore probably use the versions that come 38 | with the docker base linux image. Later, however, we'll patch them to 39 | add new targets for compiling native usermode code for our OS, so we 40 | might as well get the practice of compiling. 41 | 42 | `Make` is just for simplifying the build process. An indispensable tool, 43 | really, but more on that later. 44 | 45 | `Grub` and `xorriso` is used to generate a bootable cdrom ISO with our 46 | kernel. We'll need to make sure we get `grub` with BIOS support, though, 47 | because that's what `qemu` expects. 48 | 49 | `Qemu` for emulating. We won't need all of `qemu`, but most package 50 | managers will let you install just one or a few system emulators. In our 51 | case, we want `qemu-system-x86_64` specifically. 52 | 53 | Finally `gdb` can attach to qemu and be used to inspect and change 54 | memory, variables, code, registers. It has saved me inumerable times 55 | already. 56 | 57 | ### Dockerfile 58 | 59 | I chose to build my image on top of alpine linux, because that seems to 60 | be generally accepted as best practice. 61 | 62 | Alpine also happens to be built on musl c library, which is what I plan 63 | to port to this OS. Not that it matters... 64 | 65 | So... 66 | 67 | `toolchain/Dockerfile` 68 | 69 | ```dockerfile 70 | FROM alpine:3.6 71 | 72 | ADD build-toolchain.sh /opt/build-toolchain.sh 73 | 74 | RUN /opt/build-toolchain.sh 75 | 76 | ENV PATH "/opt/toolchain:$PATH" 77 | ENV MITTOS64 "true" 78 | ENV BUILDROOT "/opt/" 79 | WORKDIR /opt 80 | ``` 81 | 82 | This simply copies over the installation script, runs it and then sets a 83 | few environment variables. 84 | 85 | The installation script looks like this: 86 | 87 | `toolchain/build-toolchain.sh` 88 | 89 | ```bash 90 | #!/bin/sh -e 91 | 92 | apk --update add build-base 93 | apk add gmp-dev mpfr-dev mpc1-dev 94 | 95 | apk add make 96 | apk add grub-bios xorriso 97 | apk add gdb 98 | apk --update add qemu-system-x86_64 --repository http://dl-cdn.alpinelinux.org/alpine/v3.7/main 99 | 100 | rm -rf /var/cache/apk/* 101 | 102 | target=x86_64-elf 103 | binutils=binutils-2.29 104 | gcc=gcc-7.2.0 105 | 106 | 107 | cd /opt 108 | wget http://ftp.gnu.org/gnu/binutils/${binutils}.tar.gz 109 | tar -xf ${binutils}.tar.gz 110 | mkdir binutils-build && cd binutils-build 111 | ../${binutils}/configure \ 112 | --target=${target} \ 113 | --disable-nls \ 114 | --disable-werror \ 115 | --with-sysroot \ 116 | 117 | make -j 4 118 | make install 119 | 120 | cd /opt 121 | wget http://ftp.gnu.org/gnu/gcc/${gcc}/${gcc}.tar.gz 122 | tar -xf ${gcc}.tar.gz 123 | mkdir gcc-build && cd gcc-build 124 | ../${gcc}/configure \ 125 | --target=${target} \ 126 | --disable-nls \ 127 | --enable-languages=c \ 128 | --without-headers \ 129 | 130 | make all-gcc all-target-libgcc -j 4 131 | make install-gcc install-target-libgcc 132 | 133 | apk del build-base 134 | 135 | cd / 136 | rm -rf /opt 137 | ``` 138 | 139 | First we use the alpine package manager `apk` to install the things we need for 140 | compiling, `build-base` - which is compilers and stuff, and some libraries 141 | needed to compile `gcc`. Then we install the packages discussed above and 142 | finally download, configure, make and install binutils and gcc. Note that make 143 | is installed specifically, even though it's included in build-base. This is so 144 | that we can uninstall build-base to save room, and still have make available. 145 | 146 | > #### A note about qemu versions 147 | > You'll note that qemu is installed from a different repository than the rest 148 | > of the packages. This is because of a problem with the gdb server in some 149 | > qemu versions. For some versions, gdb can't follow when qemu switches 150 | > processor mode, e.g. from 32 to 64 bit execution. You will then get some 151 | > error message about "g packet size" and some numbers. The way to solve this 152 | > is to disconnect from the remote debugging, change architecture manually, and 153 | > then reconnect: 154 | 155 | > ``` 156 | > (gdb) disconnect 157 | > (gdb) set architecture i386:x86_64:intel 158 | > (gdb) target remote :1234 159 | > ``` 160 | 161 | > Or you could make sure you're running qemu version 2.10 or later, which seems 162 | > to have fixed this problem. 163 | 164 | > At the time of writing, the lates alpine docker image is version 3.6, which 165 | > installs qemu version 2.8 by default. Therefore, we manually choose the 166 | > repository for alpine 3.7 instead. 167 | 168 | The configuration flags are well described in the [GCC 169 | Cross-Compiler](http://wiki.osdev.org/GCC_Cross-Compiler) article over at 170 | [osdev.org](http://osdev.org), so I recommend you read that if you didn't 171 | already. This also gives us a good base for later adding a custom build target. 172 | 173 | The image can be built with `docker build -t mittos64 toolchain/.` and 174 | when done, you can run a command inside it to test that it works: 175 | 176 | ``` 177 | $ docker run --rm mittos64 x86_64-elf-gcc --version 178 | x86_64-elf-gcc (GCC) 7.2.0 179 | Copyright (C) 2017 Free Software Foundation, Inc. 180 | This is free software; see the source for copying conditions. There is NO 181 | warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 182 | ``` 183 | 184 | ## Docker helper script 185 | 186 | In order to compile our code inside Docker, we need to mount our source 187 | directory to the container. This can be done with the -v flag, but at 188 | this point things are starting to look messy, so let's write a script 189 | for it. 190 | 191 | I simply named it `d` and put it in the project root directory. 192 | 193 | `d` 194 | 195 | ```bash 196 | #!/usr/bin/env bash 197 | imagename=mittos64 198 | buildroot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" 199 | 200 | if [[ $(docker ps -q -f name=${imagename}-run) ]]; then 201 | docker exec -it -u $(id -u):$(id -g) ${imagename}-run "$@" 202 | else 203 | docker run -it --rm -v ${buildroot}:/opt --name ${imagename}-run -u $(id -u):$(id -g) ${imagename} "$@" 204 | fi 205 | ``` 206 | 207 | This will run any command inside the docker container as the calling user: 208 | 209 | $ ./d qemu-system-x86_64 --version 210 | QEMU emulator version 2.8.1 211 | Copyright (c) 2003-2016 Fabrice Bellard and the QEMU Project developers 212 | 213 | Furthermore. If a command is already running in the container, the next 214 | invocation of `d` will not launch a new container, but instead connect 215 | to the currently running one. This means you can e.g. run `qemu` and 216 | `gdb` inside the same container, so that they may talk to each other. 217 | 218 | The command will mount the directory the `d` script resides in to `/opt` 219 | in the container, which also is the default working directory, so we'll 220 | have direct access to all our source. 221 | 222 | 223 | ## Making a bootable ISO file 224 | 225 | When the kernel is compiled, we need to get it to a computer or emulator 226 | somehow. `Qemu` can actually load a MultiBoot 1 compatible kernel 227 | directly, but I believe making a bootable ISO is a more robust way to 228 | go. This is obviously something we will need to do a lot of times, so 229 | it's scripting time. 230 | 231 | `toolchain/mkiso` 232 | 233 | ```bash 234 | #!/bin/sh -e 235 | if [ -z ${MITTOS64+x} ]; then >&2 echo "Unsupported environment! See README"; exit 1; fi 236 | 237 | sysroot=${BUILDROOT}sysroot 238 | iso=${BUILDROOT}mittos64.iso 239 | 240 | mkdir -p ${sysroot} 241 | 242 | grub-mkrescue -o ${iso} ${sysroot} 243 | ``` 244 | 245 | The first two lines need some explanation. First of all, I normally 246 | write all my scripts for `bash` Alpine linux, however, does not include 247 | `ash`by default. So instead we are going for `sh`. 248 | 249 | The second line checks if the $MITTOS64 environment variable is set. If 250 | it's not, execution stops immediately. This will be a feature of most 251 | scripts within the project. This is just to avoid messing anything up 252 | on your own computer. Remember that this variable was defined by the 253 | Dockerfile. 254 | 255 | `BUILDROOT` was also defined in the Dockerfile. 256 | 257 | The rest of the script builds a `sysroot` directory (that is where our 258 | boot filesystem will live, for now it's empty...) and then turns it all 259 | into an ISO with grub installed. 260 | 261 | 262 | ## Running the emulator 263 | 264 | We'll test the kernel using `qemu`. 265 | 266 | `Qemu` has a lot of command line flags. 267 | 268 | We don't want to type those in all the time. 269 | 270 | Script time: 271 | 272 | `toolchain/emul` 273 | 274 | ```bash 275 | #!/bin/sh -e 276 | if [ -z ${MITTOS64+x} ]; then >&2 echo "Unsupported environment! See README"; exit 1; fi 277 | 278 | iso=${BUILDROOT}mittos64.iso 279 | 280 | ${BUILDROOT}toolchain/mkiso 281 | 282 | qemu-system-x86_64 -s -S -cdrom ${iso} -curses 283 | ``` 284 | 285 | This should be simple enough. After the environment check, it runs the 286 | `mkiso` script and then starts `qemu` with the options: 287 | 288 | - -s to start a gdb server at telnet port 1234 289 | - -S to freeze the cpu at startup and wait for a command to continue 290 | - -cdrom ${iso} to mount our ISO as a cd 291 | - -curses to output the screen (in VGA text mode) to the terminal 292 | 293 | Later we'll add stuff like multiple cpus and VNC output to be able to 294 | use graphics modes, but this is good enough for now. 295 | 296 | 297 | ## Debugger 298 | 299 | Finally, we add a script to start `gdb` with some initial settings 300 | 301 | `toolchain/gdb` 302 | 303 | ```bash 304 | #!/bin/sh -e 305 | if [ -z ${MITTOS64+x} ]; then >&2 echo "Unsupported environment! See README"; exit 1; fi 306 | 307 | /usr/bin/gdb -q -x ${BUILDROOT}toolchain/gdbinit 308 | ``` 309 | 310 | This just runs `gdb` and tells it to read and execute 311 | `toolchain/gdbinit`. Normally, you'd probably use a `.gdbinit` either 312 | in your home directory, or in the project root, but I wanted it to be 313 | visible (filenames starting with `.` are hidden in UNIX-like systems) 314 | and together with the rest of the toolchain stuff. Hence the script - 315 | which overloads `gdb` since it will come earlier in $PATH. 316 | 317 | So, the important stuff in this section is actually the `gdbinit` file. 318 | 319 | `toolchain/gdbinit` 320 | 321 | ```gdb 322 | set prompt \033[31m(gdb) \033[0m 323 | set disassembly-flavor intel 324 | 325 | target remote :1234 326 | 327 | define q 328 | monitor quit 329 | end 330 | 331 | define reg 332 | monitor info registers 333 | end 334 | 335 | define reset 336 | monitor system_reset 337 | end 338 | ``` 339 | 340 | This script does the following: 341 | 342 | - Colors the gdb prompt read for improved visibility 343 | - Makes gdb use intel assembly syntax, rather that AT&T. Personal preference. 344 | - Connects to the `qemu` gdb server at port 1234 345 | - redefines the `q` command to stop the emulator (this will kill `gdb` 346 | as well). If you want to, you can still use `(gdb) quit` to quit just 347 | the debugger. 348 | - Defines a `reg` command which pulls in the register information from 349 | `qemu`. This is more detailed than the `gdb` register output. 350 | - Defines a `reset` command to reboot the emulator. 351 | 352 | 353 | ## Trying it all out 354 | 355 | ### Emulator 356 | 357 | To make sure everything works, open up a terminal window and run 358 | 359 | $ d emul 360 | 361 | After a second or two, you should get a blank screen with the text `VGA Blank mode`. 362 | That means qemu is paused and waiting for a command to start. 363 | 364 | You could start it by switching to the qemu monitor with `META+2` (if 365 | you don't have a meta key, press and release `ESC` immediately followed 366 | by `2`) and running 367 | 368 | (qemu) c 369 | 370 | Then you can switch back to the VGA output with `META+1`. 371 | 372 | Once the emulator is running, you should soon see the GRUB start screen: 373 | `GNU GRUB version 2.02` followed by some text about tab completion and 374 | then a grub prompt 375 | 376 | grub> 377 | 378 | You can exit the emulator by going to the monitor and issuing 379 | 380 | (qemu) q 381 | 382 | ### Debugger 383 | 384 | To check that the debugger works, start the emulator again with 385 | 386 | $ d emul 387 | 388 | Then open another terminal window and run 389 | 390 | $ d gdb 391 | 392 | You should se some text and then a `(gdb)` prompt. The emulator window 393 | should still show `VGA Blank mode`. 394 | 395 | Now run 396 | 397 | (gdb) c 398 | 399 | and the emulator should start running and bring you to the `grub` prompt. 400 | When you're there, pause executing with `CTRL+c` (in gdb), which brings 401 | back the prompt. 402 | 403 | You can now inspect the processor registers with 404 | 405 | (gdb) reg 406 | EAX=00000000 EBX=00000000 ECX=07fa0880 EDX=00000031 407 | ESI=00000000 edI=07fa0880 EBX)00001ff0 ESP=00001ff4 408 | [...] 409 | XMM06=00000000000000000000000000000000 XMM07=00000000000000000000000000000000 410 | 411 | Finally, run 412 | 413 | (gdb) q 414 | 415 | and notice that both the emulator and gdb stops. 416 | -------------------------------------------------------------------------------- /doc/3_Activate_Long_Mode.md: -------------------------------------------------------------------------------- 1 | # Chapter 3 - Entering Long Mode 2 | 3 | In this chapter, we'll put the processor in long mode with minimal 4 | possible effort. 5 | 6 | 7 | ## Preparation 8 | 9 | The AMD64 manual volume 4 outlines what needs to be done in order to 10 | actiave long mode (chapter 14). It says we need: 11 | 12 | - An IDT with 64 bit interrupt-gate descriptors *We don't need this as long as 13 | interrupts are disabled* 14 | - 64-bit interrupt and exception handlers *See above* 15 | - A GDT containing: 16 | - Any LDT descriptors *We don't have any* 17 | - A TSS descriptor *Only needed when we want to enter User mode* 18 | - Code descriptors for long mode code *One is enough for now* 19 | - Data-segment descriptors for software running in compatibility mode *We 20 | don't have that* 21 | - FS and GS data-segment descriptors *We won't be using those* 22 | - A 64-bit TSS *See note about TSS descriptor above* 23 | - The 4-level page translation tables 24 | 25 | So if we bring it down to the essentials: 26 | 27 | - A GDT with one entry 28 | - A Page Table 29 | 30 | Shouldn't be too hard. In fact, for now we can actually pretty much 31 | hardcode those... 32 | 33 | ## GDT 34 | 35 | In long mode, segmentation and the GDT doesn't really fill any 36 | purpose... It's still required, for some reason, but if you read the 37 | AMD manual, you'll see that in long mode, almost all fields of the GDT 38 | entries are ignored. 39 | 40 | What's left can be set up like this: 41 | 42 | `src/kernel/boot/boot_GDT.S` 43 | ```asm 44 | #include 45 | .intel_syntax noprefix 46 | 47 | .section .rodata 48 | .global BootGDT 49 | .global BootGDTp 50 | 51 | BootGDT: 52 | .long 0,0 53 | .long 0, (GDT_PRESENT | GDT_CODE | GDT_LONG) 54 | 55 | BootGDTp: 56 | .short 2*8-1 57 | .quad offset BootGDT 58 | ``` 59 | 60 | where 61 | - `GDT_PRESENT = 1<<15` 62 | - `GDT_CODE = 3<<11` 63 | - `GDT_LONG = 1<<21` 64 | 65 | The GDT is page aligned, of course, and the GDT pointer is configured in 66 | the same way as in 32 bit mode. 67 | 68 | 69 | ## Page Tables 70 | 71 | Paging works pretty much exactly the same way in 64 bit mode as in 72 | 32, but with four levels of nested tables instead of two. If you have 73 | trouble wrapping your head around it, chapter 5 *Page Translation and 74 | Protection* of the AMD64 Systems programming manual should help. 75 | 76 | The four levels do have names, "Page-Map Level-4 Table", "Page-Directory 77 | Pointer Table", "Page Directory Table" and "Page Table", but I like to 78 | think of them as P4, P3, P2 and P1. 79 | 80 | We could make use of the 2 Mb page translation feature, which uses only three 81 | levels. I.e. the entries of P2 points directly at the start of a 2 Mb memory 82 | area rather than at a P1. This is indicated by a special flag in the P2 entry. 83 | Doing so would make the memory management a bit more complicated later, though, 84 | so I won't use that for now. 85 | 86 | For now, we'll just identity map the first two megabytes of memory. That should 87 | be enough to get the kernel started. So we just need a P4 where the first 88 | entry points to a P3 where the first entry points to a P2 where the first entry 89 | points to a P1 filled with 512 entries ranging from 0 to 2 mb. 90 | 91 | `src/kernel/boot/boot_PT.S` 92 | ```asm 93 | .#include 94 | .intel_syntax noprefix 95 | 96 | .section .data 97 | .align PAGE_SIZE 98 | .global BootP4 99 | 100 | BootP4: 101 | .quad offset BootP3 + (PAGE_PRESENT | PAGE_WRITE) 102 | .rept ENTRIES_PER_PT - 1 103 | .quad 0 104 | .endr 105 | BootP3: 106 | .quad offset BootP2 + (PAGE_PRESENT | PAGE_WRITE) 107 | .rept ENTRIES_PER_PT - 1 108 | .quad 0 109 | .endr 110 | BootP2: 111 | .quad offset BootP1 + (PAGE_PRESENT | PAGE_WRITE) 112 | .rept ENTRIES_PER_PT - 1 113 | .quad 0 114 | .endr 115 | BootP1: 116 | .set i, 0 117 | .rept ENTRIES_PER_PT 118 | .quad (i << 12) + (PAGE_PRESENT | PAGE_WRITE) 119 | .set i, (i+1) 120 | .endr 121 | ``` 122 | 123 | where 124 | - `PAGE_PRESENT = 0x001` 125 | - `PAGE_WRITE = 0x002` 126 | - `ENTRIES_PER_PT = 512` 127 | 128 | 129 | ## Activating Long Mode 130 | 131 | Again, consulting the AMD64 manual we find the following steps to 132 | activate long mode: 133 | 134 | 1. Disable paging *Paging isn't enabled by GRUB, so we're good to go* 135 | 2. In any order: 136 | - Enable PAE by setting CR4.PAE to 1 137 | - Load CR3 with the address of P4 138 | - Enable long mode by setting EFER.LME to 1 139 | 3. Enable paging 140 | 141 | We should then reload the system tables (in our case only GDT) with 64 142 | bit descriptors. 143 | 144 | The manual is even kind enough to supply us with some sample code which 145 | also performs some checks to ensure that long mode is available. So 146 | let's go. 147 | 148 | `src/kernel/boot/boot.S` 149 | 150 | ```asm 151 | ... 152 | .code32 153 | .global _start 154 | _start: 155 | cli 156 | mov esp, offset BootStack 157 | ... 158 | ``` 159 | 160 | First we set up a temporary stack for booting. The label BootStack is 161 | defined earlier: 162 | 163 | ```asm 164 | .section .bss 165 | .align PAGE_SIZE 166 | .skip PAGE_SIZE 167 | BootStack: 168 | ``` 169 | 170 | Note that the label is after the reserved memory, since the stack grows upwards. 171 | 172 | If you wish to make things The Right Way, you should probably check if the 173 | processor supports long mode before going further. This can be done through 174 | the `cpuid` instruction and the process is described in the AMD64 manual. I 175 | opted to skip this check, and just fail in an uncontrolled manner in the 176 | unlikely event that the code is run on 32 bit processor. 177 | 178 | Ok. Let's get to the meat of it 179 | 180 | `src/kernel/boot/boot.S` 181 | ```asm 182 | ... 183 | //; Set CR4.PAE 184 | //; enabling Page Address Extension 185 | mov eax, cr4 186 | or eax, 1<<5 187 | mov cr4, eax 188 | 189 | //; Load a P4 page table 190 | mov eax, offset BootP4 191 | mov cr3, eax 192 | 193 | //; Set EFER.LME 194 | //; enabling Long Mode 195 | mov ecx, 0x0C0000080 196 | rdmsr 197 | or eax, 1<<8 198 | wrmsr 199 | 200 | //; Set CR0.PG 201 | //; enabling Paging 202 | mov eax, cr0 203 | or eax, 1<<31 204 | mov cr0, eax 205 | ... 206 | ``` 207 | 208 | I think the comments explain this well enough. It's just following the 209 | list of actions from the AMD manual anyway. 210 | 211 | > Speaking of comments, I apologize for the unconventional comment style `//;`. 212 | > Normally GAS assembly is commented by a `;`, but I run all my files through 213 | > the gcc preprocessor, which interprets semicolon as the end of a line. 214 | > Instead, I have to use c-style comments (`//` or `/* */`). Those are, 215 | > however, not recognized by the github markdown syntax coloring engine, and 216 | > the results look messy with weird colors all over the place. That's why I use 217 | > the combination. 218 | 219 | The only step that's left is reloading the system tables. This is done 220 | in exactly the same way as when going to protected mode, by loading a 221 | GDT, loading selectors, and performing a long jump to load CS. 222 | 223 | `src/kernel/boot/boot.S` 224 | ```asm 225 | ... 226 | //; Load a new GDT 227 | lgdt [BootGDTp] 228 | 229 | //; and update the code selector by a long jump 230 | jmp 0x8:long_mode_start 231 | 232 | .code64 233 | long_mode_start: 234 | 235 | //; Clear out all other selectors 236 | mov eax, 0x0 237 | mov ss, eax 238 | mov ds, eax 239 | mov es, eax 240 | 241 | //; Loop infinitely 242 | jmp $ 243 | ``` 244 | 245 | And that's all! 246 | 247 | ## Testing it out 248 | 249 | Fire up the emulator, make sure the kernel is loaded into gdb, and let's go! 250 | 251 | Let's step through the entire boot process 252 | 253 | (gdb) b _start 254 | Breakpoint 1 at 0x91: file boot/boot.S, line 63 255 | (gdb) c 256 | Continuing. 257 | 258 | Breakpoint 1, _start () at boot/boot.S:63 259 | 64 cli 260 | (gdb) 261 | 262 | The first thing that happens is that we set the stack pointer. You can 263 | see that this happens by printing `esp`. 264 | 265 | (gdb)p/x $esp 266 | $1 = 0x7ff00 267 | (gdb)si 268 | 65 mov esp, offset BootStack 269 | (gdb)si 270 | 67 call check_cpuid 271 | (gdb)p/x $esp 272 | $2 = 0x5000 273 | 274 | So things seem to work so far. 275 | 276 | The next thing that happens is that the two functions are called to 277 | check cpuid and long mode availability. You can step through those and 278 | inspect values as you wish. I'll just skip to after returning from 279 | check\_longmode. 280 | 281 | You can't print the contents of `CR4` in gdb, but you can read it from 282 | the qemu monitor command `info registers` which can be called from gdb 283 | by the `monitor command`. 284 | 285 | _start () at boot/boot.S:72 286 | 72 mov eax, cr4 287 | (gdb) monitor info registers 288 | EAX=00000664 EBX=00000000 ECX=00000005 EDX=2193fbfd 289 | ... 290 | CR0=00000011 CR2=00000000 CR3=00000000 CR4=00000000 291 | ... 292 | XMM06=00000000000000000000000000000000 XMM07=00000000000000000000000000000000 293 | (gdb) 294 | 295 | To simplify things, I wrote a python function to extract individual 296 | registers and put it in `toolchain/gdbinit` - see next section. This 297 | let's you run `reg cr4` to se the register contents. 298 | 299 | (gdb) reg cr4 300 | CR4=00000000 301 | (gdb) n 302 | 73 or eax, 1<<5 303 | (gdb) 304 | 74 mov cr4, eax 305 | (gdb) 306 | 77 mov eax, offset BootP4 307 | (gdb) reg cr4 308 | CR4=00000020 309 | (gdb) 310 | 311 | No surprises there, really. Page address extension should now be enabled. 312 | 313 | The next step is loading the page table. This shouldn't actually matter, 314 | since paging is disabled, so just step through it and make sure the 315 | value loaded into `CR3` is page aligned. 316 | 317 | 82 mov ecx, 0x8C000080 318 | (gdb) reg cr3 319 | CR3=00002000 320 | 321 | The same goes for setting the Long Mode Enable bit of the EFER register. 322 | Make sure to remember the value of EFER, though ... 323 | 324 | 89 mov eax, cr0 325 | (gdb) reg efer 326 | EFER=0000000000000100 327 | (gdb) monitor info mem 328 | PG disabled 329 | (gdb) 330 | 331 | ... because after we enable paging by setting the Paging bit in CR0 ... 332 | 333 | (gdb) reg cr0 334 | CR0=00000011 335 | (gdb) n 336 | 90 or eax, 1<<31 337 | (gdb) 338 | 91 mov cr0, eax 339 | (gdb) 340 | 94 lgdt [BootGDTp] 341 | (gdb) reg cr0 342 | CR0=80000011 343 | (gdb) 344 | 345 | ... the Long Mode Active bit (bit 10) should also be set. 346 | 347 | (gdb) reg efer 348 | EFER=0000000000000500 349 | (gdb) monitor info mem 350 | 0000000000000000-0000000040000000 0000000040000000 -rw 351 | (gdb) 352 | 353 | This means the processor is in Long Mode! 354 | 355 | You'll also see that the command `monitor info mem` (which I mapped to 356 | `mmap` in my gdbinit - see next section) show that paging is enabled 357 | and that the first Gb of virtual memory is mapped. Also note that the 358 | virtual address space expects addresses of 64 bits now. 359 | 360 | But we're still running code in Legacy Mode. That's why we load the GDT 361 | and reload the segment selectors next. 362 | 363 | (gdb) n 364 | 97 jmp 0x8:long_mode_start 365 | (gdb) 366 | long_mode_start () at boot/boot.S:103 367 | 103 mov eax, 0x0 368 | (gdb) reg 369 | RAX=0000000080000011 RBX=0000000000000000 RCX=00000000c0000080 RDX=0000000000000000 370 | ... 371 | CS =0008 0000000000000000 00000000 00209800 DPL=0 CS64 [---] 372 | ... 373 | (gdb) 374 | 375 | You'll note that the `reg` command now outputs registers with the R 376 | prefix (`RAX` instead of `EAX` etc.) and that they are twice as big. 377 | 378 | We are now running 64 bit code! 379 | 380 | ## Bonus 381 | 382 | I mentioned two custom gdb commands in the previous section. 383 | 384 | `mmap` which shows the memory map from qemu, and `reg` which prints the 385 | value of a register. Those are defined in `toolchain/gdbinit`: 386 | 387 | ``` 388 | define mmap 389 | monitor info mem 390 | end 391 | 392 | python 393 | 394 | import re 395 | 396 | class Reg(gdb.Command): 397 | 398 | def __init__(self): 399 | super(Reg, self).__init__("reg", gdb.COMMAND_USER) 400 | 401 | def invoke(self, arg, from_tty): 402 | regs = gdb.execute('monitor info registers', False, True) 403 | 404 | if not arg: 405 | # If no argument was given, print the output from qemu 406 | print regs 407 | return 408 | 409 | if arg.upper() in ['CS', 'DS', 'ES', 'FS', 'GS', 'SS']: 410 | # Code selectors may contain equals signs 411 | for l in regs.splitlines(): 412 | if l.startswith(arg.upper()): 413 | print l 414 | elif arg.upper() in ['EFL', 'RFL']: 415 | # The xFLAGS registers contains equals signs 416 | for l in regs.splitlines(): 417 | if arg.upper() in l: 418 | print ' '.join(l.split()[1:]) 419 | # The xFLAGS register is the second one on the line 420 | else: 421 | # Split at any word followed by and equals sign 422 | # Clean up both sides of the split and put into a dictionary 423 | # then print the requested register value 424 | regex = re.compile("[A-Z0-9]+\s?=") 425 | names = [v[:-1].strip() for v in regex.findall(regs)] 426 | values = [v.strip() for v in regex.split(regs)][1:] 427 | regs = dict(zip(names, values)) 428 | print "%s=%s" % (arg.upper(), regs[arg.upper()]) 429 | 430 | 431 | Reg() 432 | 433 | end 434 | ``` 435 | 436 | The `mmap` command is obvious enough, but the `reg` one is a bit tougher. 437 | A bit of information on the syntax of python commands in gdb can be 438 | found [here](https://sourceware.org/gdb/onlinedocs/gdb/Python-API.html). 439 | 440 | The rest is some rather messy python, but the basic flow is this 441 | 442 | - Get the register output from grub by running `monitor info registers` 443 | - If no argument was given, print the output we got and end 444 | - Look for text in the format `SOMETHING=SOMETHINGELSE` and split it into `SOMETHING` and `SOMETHINGELSE` 445 | - Put `SOMETHING` and `SOMETHINGELSE` back together in a way that's more useful for python 446 | - Print the value we want 447 | 448 | Then there are some special cases for things like `EFL` which contains 449 | equals signs in the output. E.g. displaying bit flags `EFL=0000002 450 | [-------] CPL=0 II=0 A20=1 SMM=0 HLT=0`. Note that I'm not catching all 451 | such cases, but only the ones I think might be interesting. 452 | 453 | Note also that gdb doesn't require the entire command name, but only enough to 454 | make it unambiguous. As such, you can run `(gdb) mm` instead of `(gdb) mmap` if 455 | you'd like. Just a heads up... 456 | 457 | --------------------------------------------------------------------------------