├── .gdbinit ├── .gitattributes ├── .gitignore ├── .gitmodules ├── LICENSE.md ├── Makefile ├── README.md ├── TODO.md ├── include ├── addr.h ├── asm.h ├── atomic.h ├── drivers │ ├── acpi.h │ ├── apic.h │ ├── cansid.h │ ├── pit.h │ ├── ps2.h │ ├── serial.h │ ├── smbios.h │ └── vga_tmode.h ├── ds │ ├── bitmap.h │ ├── generic.h │ ├── hash.h │ ├── linked.h │ └── rbtree.h ├── err.h ├── fs.h ├── fs │ └── initrd.h ├── include.asm ├── interrupts.h ├── libk.h ├── limits.h ├── mm.h ├── mm_types.h ├── multiboot2.h ├── mutex.h ├── percpu.h ├── proc.h ├── smp.h ├── sync.h ├── syscall.h ├── types.h └── util.h ├── iso └── boot │ └── grub │ └── grub.cfg ├── kernel ├── boot.asm ├── cpu │ ├── entry.asm │ ├── exceptions.c │ ├── flush.asm │ ├── idt.c │ ├── interrupts.c │ ├── isr.c │ ├── isr_stubs.asm │ ├── percpu.c │ └── simd.asm ├── crt │ ├── crtbegin.o │ ├── crtend.o │ ├── crti.asm │ └── crtn.asm ├── drivers │ ├── acpi │ │ └── acpi.c │ ├── apic │ │ ├── apic.c │ │ ├── ioapic.c │ │ └── lapic.c │ ├── pit │ │ └── pit.c │ ├── ps2 │ │ ├── keyboard.c │ │ └── ps2.c │ ├── serial │ │ └── serial.c │ ├── smbios │ │ └── smbios.c │ └── vga_tmode │ │ ├── cansid.c │ │ └── vga_tmode.c ├── fs │ ├── initrd │ │ ├── dir.c │ │ ├── file.c │ │ ├── super.c │ │ └── ustar.c │ ├── inode_cache.c │ └── vfs.c ├── kmain.c ├── mm │ ├── area.c │ ├── cow.c │ ├── mmap.c │ ├── mmu.c │ ├── pmm.c │ ├── slob.c │ ├── tlb.c │ └── vmm.c ├── proc │ ├── fork.asm │ ├── idle.asm │ ├── init.c │ ├── runq.c │ ├── sched.c │ ├── switch_to.asm │ └── task.c ├── smp │ ├── cpuset.c │ ├── ipi.c │ ├── smp.c │ ├── spin.asm │ └── trampoline.asm └── syscall │ ├── defs.c │ └── syscall.asm ├── libk ├── abort.c ├── ds │ ├── bitmap.c │ └── rbtree.c ├── kprintf.c ├── ssp.c ├── string.c ├── tests │ ├── tmain.c │ └── tstring.c └── ubsan.c ├── linker.ld └── util ├── err_gen.py └── syscall_gen.py /.gdbinit: -------------------------------------------------------------------------------- 1 | set architecture i386:x86-64 2 | set disassembly-flavor intel 3 | set pagination off 4 | symbol-file build/byteos.sym 5 | target remote localhost:1234 6 | hbreak long_mode_entry 7 | continue 8 | define hook-stop 9 | list *$pc 10 | end 11 | define myn 12 | next 13 | refresh 14 | end 15 | define mys 16 | step 17 | refresh 18 | end 19 | define myc 20 | continue 21 | refresh 22 | end 23 | tui enable 24 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.h linguist-language=C 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | include/gen/* 3 | *.elf 4 | *.sym 5 | *.iso 6 | *.d 7 | *.o 8 | *.a 9 | *.orig 10 | *.gch 11 | bochsout.txt 12 | libk/tests/snow.h 13 | tags 14 | 15 | .* 16 | !.git* 17 | !.gdbinit 18 | !kernel/crt/crtbegin.o 19 | !kernel/crt/crtend.o 20 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/ds"] 2 | path = vendor/ds 3 | url = https://github.com/64/ds 4 | [submodule "vendor/snow"] 5 | path = vendor/snow 6 | url = https://github.com/mortie/snow 7 | [submodule "vendor/cansid"] 8 | path = vendor/cansid 9 | url = https://github.com/64/cansid 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Thank you https://github.com/no92 for cleaning this up a lot! 2 | ISO := build/byteos.iso 3 | KERNEL := build/byteos.elf 4 | OS ?= $(shell uname -s) 5 | 6 | AS := nasm 7 | QEMU ?= qemu-system-x86_64 8 | BOCHS ?= bochs 9 | AR := x86_64-elf-ar 10 | CC ?= cc 11 | TEST_CC := $(CC) 12 | CC := x86_64-elf-gcc 13 | OBJDUMP := x86_64-elf-objdump 14 | READELF := x86_64-elf-readelf 15 | OBJCOPY := x86_64-elf-objcopy 16 | GDB ?= gdb 17 | PYTHON ?= python3 18 | 19 | CFLAGS += -ffreestanding -mno-red-zone -mcmodel=kernel -Iinclude -std=gnu11 20 | CFLAGS += -Wall -Werror -Wextra -Wparentheses -Wmissing-declarations -Wunreachable-code -Wunused 21 | CFLAGS += -Wmissing-field-initializers -Wmissing-prototypes -Wpointer-arith -Wswitch-enum 22 | CFLAGS += -Wredundant-decls -Wshadow -Wstrict-prototypes -Wswitch-default -Wuninitialized 23 | CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-sse3 -mno-ssse3 -mno-sse4 -mno-sse4.1 -mno-sse4.2 -mno-avx -mno-sse4a 24 | DEBUG_CFLAGS ?= -fsanitize=undefined -Og -g -DDEBUG -fstack-protector 25 | RELEASE_CFLAGS ?= -O3 -flto 26 | ASFLAGS ?= -f elf64 -F dwarf -g -w+all -Werror -i$(shell pwd)/include/ 27 | QEMUFLAGS ?= -net none -smp sockets=1,cores=4,threads=1 -serial stdio -cdrom $(ISO) 28 | ASTYLEFLAGS := --style=linux -z2 -k3 -H -xg -p -T8 -S 29 | BOCHSFLAGS ?= -f .bochsrc -q 30 | 31 | CRTI_OBJ := kernel/crt/crti.asm.o 32 | CRTBEGIN_OBJ := kernel/crt/crtbegin.o 33 | CRTEND_OBJ := kernel/crt/crtend.o 34 | CRTN_OBJ := kernel/crt/crtn.asm.o 35 | LIBK_OBJ := $(addsuffix .o,$(shell find libk -not -path "*tests*" -name '*.c' -o -name '*.asm')) 36 | KERNEL_OBJ_RAW := $(addsuffix .o,$(shell find kernel -path kernel/crt -prune -type f -o -name '*.c' -o -name '*.asm')) 37 | KERNEL_OBJ_ALL := $(CRTI_OBJ) $(CRTN_OBJ) $(KERNEL_OBJ_RAW) $(LIBK_OBJ) 38 | KERNEL_OBJ := $(CRTI_OBJ) $(CRTBEGIN_OBJ) $(KERNEL_OBJ_RAW) $(LIBK_OBJ) $(CRTEND_OBJ) $(CRTN_OBJ) 39 | KERNEL_SRC_DEPS := include/gen/syscall_gen.h include/gen/err_gen.h include/gen/syscall_gen.c include/gen/syscall_gen.asm 40 | 41 | DEPFILES := $(patsubst %.o,%.d,$(KERNEL_OBJ_ALL)) 42 | 43 | LIBK_TESTABLE := $(addprefix libk/,string.c) 44 | DEPFILES += $(patsubst %.o,%.d,$(LIBK_OBJ)) 45 | TIME_START := $(shell date +"%s.%N") 46 | 47 | MOD_DS := vendor/ds 48 | MOD_CANSID := vendor/cansid 49 | MOD_SNOW := vendor/snow 50 | SUBMODULES := ds cansid snow 51 | 52 | ifeq ($(OS),Linux) 53 | QEMUFLAGS += -M accel=kvm:tcg 54 | endif 55 | 56 | ifeq ($(DEBUG),0) 57 | CFLAGS += $(RELEASE_CFLAGS) 58 | else 59 | CFLAGS += $(DEBUG_CFLAGS) 60 | endif 61 | 62 | ifeq ($(VERBOSE),1) 63 | CFLAGS += -DVERBOSE 64 | endif 65 | 66 | .PHONY: all clean run vbox bochs gdb disassemble update-modules copy-all copy-snow copy-ds copy-cansid loc tidy test ctags 67 | .SUFFIXES: .o .c .asm 68 | 69 | all: $(ISO) 70 | 71 | run: $(ISO) 72 | @$(QEMU) $(QEMUFLAGS) 73 | 74 | bochs: $(ISO) 75 | @$(BOCHS) $(BOCHSFLAGS) 76 | 77 | vbox: $(ISO) 78 | @virtualbox --startvm "ByteOS" --dbg 79 | 80 | clean: 81 | @$(RM) -r build 82 | @$(RM) -r include/gen 83 | @$(RM) iso/boot/byteos.elf 84 | @$(RM) -v $(shell find kernel libk -name "*.orig") 85 | @$(RM) $(KERNEL_OBJ_ALL) $(LIBK_OBJ) 86 | @$(RM) $(DEPFILES) 87 | 88 | gdb: $(ISO) 89 | @$(QEMU) $(QEMUFLAGS) -no-reboot -s -S & 90 | @sleep 0.5 91 | @$(GDB) 92 | @pkill qemu 93 | 94 | ctags: 95 | ctags -R 96 | 97 | disassemble: build/ $(KERNEL) 98 | @$(OBJDUMP) --no-show-raw-insn -d -Mintel $(KERNEL) | source-highlight -s asm -f esc256 | less -eRiMX 99 | 100 | symbols: build/ $(KERNEL) 101 | @$(READELF) build/byteos.sym -s | less 102 | 103 | update-modules: 104 | git submodule update --init --recursive --remote 105 | 106 | $(MOD_SNOW) $(MOD_DS) $(MOD_CANSID): update-modules 107 | 108 | copy-all: copy-ds copy-cansid copy-snow 109 | 110 | copy-snow: $(MOD_SNOW) 111 | @cp $(MOD_SNOW)/snow/snow.h ./libk/tests/snow.h 112 | 113 | copy-ds: $(MOD_DS) 114 | @cp $(MOD_DS)/include/ds/*.h ./include/ds/ 115 | @cp $(MOD_DS)/src/*.c ./libk/ds/ 116 | 117 | copy-cansid: $(MOD_CANSID) 118 | @cp $(MOD_CANSID)/cansid.c ./kernel/drivers/vga_tmode 119 | @cp $(MOD_CANSID)/cansid.h ./include/drivers/ 120 | @cat ./kernel/drivers/vga_tmode/cansid.c | sed 's/cansid.h/drivers\/cansid.h/g' > temp.c 121 | @mv temp.c ./kernel/drivers/vga_tmode/cansid.c 122 | 123 | loc: 124 | @cloc --vcs=git --force-lang="C",h --exclude-lang="Markdown" 125 | 126 | tidy: 127 | @astyle $(filter-out libk/tests/snow.h,$(shell find kernel libk -name "*.[ch]")) $(ASTYLEFLAGS) 128 | 129 | build/tmain: build/ libk/tests/tstring.c 130 | $(TEST_CC) libk/tests/tmain.c -Iinclude -Wall -std=gnu11 -o $@ 131 | 132 | test: build/tmain 133 | @./build/tmain 134 | 135 | iso/boot/byteos.elf: $(KERNEL) 136 | @cp $< $@ 137 | 138 | build/: 139 | @mkdir build 140 | @mkdir -p include/gen 141 | 142 | $(ISO): build/ iso/boot/byteos.elf 143 | @printf "\t\e[32;1mCreating\e[0m $(ISO)\n" 144 | @grub-mkrescue -o $@ iso 2> /dev/null 145 | @printf "\t\e[32;1;4mDone\e[24m in $(shell date +%s.%3N --date='$(TIME_START) seconds ago')s\e[0m\n" 146 | 147 | $(KERNEL): $(KERNEL_SRC_DEPS) $(KERNEL_OBJ_ALL) 148 | @printf "\t\e[32;1mLinking\e[0m $(KERNEL)\n" 149 | @$(CC) -T linker.ld -o $@ $(KERNEL_OBJ) $(LDFLAGS) -n -nostdlib -lgcc 150 | @$(OBJCOPY) --only-keep-debug $(KERNEL) build/byteos.sym 151 | @$(OBJCOPY) --strip-debug $(KERNEL) 152 | @grub-file --is-x86-multiboot2 $@ 153 | 154 | kernel/%.asm.o: kernel/%.asm 155 | @printf "\t\e[32;1mAssembling\e[0m $<\n" 156 | @$(AS) $(ASFLAGS) -MD $(addsuffix .d,$<) $< -o $@ 157 | 158 | kernel/%.c.o: kernel/%.c 159 | @printf "\t\e[32;1mCompiling\e[0m $<\n" 160 | @$(CC) -c $< -o $@ -MMD $(CFLAGS) 161 | 162 | libk/%.c.o: libk/%.c 163 | @printf "\t\e[32;1mCompiling\e[0m $<\n" 164 | @$(CC) -c $< -o $@ -MMD $(CFLAGS) 165 | 166 | # Syscall generation code 167 | include/gen/syscall_gen.h: 168 | @$(PYTHON) util/syscall_gen.py h 169 | include/gen/syscall_gen.c: 170 | @$(PYTHON) util/syscall_gen.py c 171 | include/gen/syscall_gen.asm: util/syscall_gen.py 172 | @$(PYTHON) util/syscall_gen.py asm 173 | # Error generation code 174 | include/gen/err_gen.h: util/err_gen.py 175 | @$(PYTHON) util/err_gen.py 176 | 177 | -include $(DEPFILES) 178 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ByteOS 2 | 3 | A simple hobby operating system for the x86_64 architecture, written in C. 4 | 5 | ![Screenshot](https://i.imgur.com/EXdOfS4.jpg) 6 | 7 | The design is heavily influenced by the Linux kernel, and as such you may find many similarities between the two. 8 | 9 | This repository is licensed under the GPLv3. See `LICENSE.md` for more details. 10 | 11 | ## Historical Note 12 | 13 | I wrote most of the code here when I was 16-17 and didn't know better. There are some parts (mainly the bits involving atomics) that I'm not particularly proud of, so copy at your own peril. If you find a problem, feel free to open an issue or send a PR. 14 | 15 | ## Building 16 | 17 | You will need: 18 | * GNU `binutils` for `x86_64-elf` 19 | * `gcc` for `x86_64-elf` with `-mno-red-zone` and C11 capabilities ([instructions](http://wiki.osdev.org/Libgcc_without_red_zone)) 20 | * Make 21 | * NASM 22 | * GRUB 2.02 (Multiboot2 compatible - may need to build from source) 23 | 24 | Then, to build `byteos.iso`, simply run: 25 | ```sh 26 | make 27 | ``` 28 | 29 | ## Running 30 | 31 | You will need (in addition to the above dependencies): 32 | * QEMU `x86_64` 33 | 34 | Then, to run in QEMU: 35 | ```sh 36 | make run 37 | ``` 38 | 39 | ## Debugging 40 | 41 | Debugging requires a [specially patched version of GDB](http://wiki.osdev.org/QEMU_and_GDB_in_long_mode#Workaround_2%3A_Patching_GDB) for interoperability with QEMU. Once you have this installed, simply run the `debug` make target. 42 | 43 | ## Testing 44 | 45 | First run the `update-modules` make target to download the necessary files. Then, simply run `make test`. 46 | 47 | See [`Makefile`](https://github.com/64/ByteOS/blob/master/Makefile) for more details. 48 | 49 | ## Contributing 50 | 51 | Feel free to open an issue if you have any questions/concerns or a pull-request if you would like to contribute some code. 52 | 53 | Special thanks to [no92](https://github.com/no92) for massively cleaning up the build system. 54 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | - [ ] Use `const ptr*` types for function parameters 4 | - [ ] Finish `slob` code - merge blocks 5 | - [ ] Calibrate local APIC timer, RTC (and HPET?) 6 | - [ ] Lazy TLB shootdown and IPIs (https://forum.osdev.org/viewtopic.php?f=15&t=23919, http://archive.is/KnVr6) 7 | - [ ] Use init data section 8 | - [ ] Save caller preserved registers on syscall? 9 | - [ ] Separate IST stacks per CPU 10 | - [ ] Reap dead processes (stack, struct task etc) 11 | - [ ] Implement out of memory checks (killing a process appropriately) 12 | - [ ] Use unions in struct page 13 | - [ ] Per-CPU IDT 14 | - [ ] RAII style mutex macro 15 | - [ ] Rewrite the shitty bits 16 | - [ ] Inode locking 17 | - [ ] Proper mutex, rename rwspin 18 | - [ ] Process table 19 | -------------------------------------------------------------------------------- /include/addr.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "atomic.h" 6 | #include "sync.h" 7 | #include "limits.h" 8 | #include "ds/linked.h" 9 | 10 | typedef uintptr_t physaddr_t; 11 | typedef void *virtaddr_t; 12 | typedef void *kernaddr_t; 13 | 14 | // One of these for each page of available physical memory. 15 | // This should be kept as small as possible (for obvious reasons). 16 | // Initial entries are zeroed out by memset. 17 | struct page { 18 | // TODO: Add this union (will require a flags variable) 19 | //union { 20 | // PMM information 21 | struct { 22 | struct dlist_node list; 23 | int8_t order; 24 | }; 25 | struct { 26 | kref_t refcount; 27 | spinlock_t lock; 28 | }; 29 | //}; 30 | }; 31 | 32 | extern struct page *const page_data; 33 | 34 | static inline physaddr_t virt_to_phys(virtaddr_t v) 35 | { 36 | if (v == NULL) 37 | return (physaddr_t)NULL; 38 | return (physaddr_t)v - KERNEL_LOGICAL_BASE; 39 | } 40 | 41 | static inline virtaddr_t phys_to_virt(physaddr_t p) 42 | { 43 | if (p == (physaddr_t)NULL) 44 | return NULL; 45 | return (virtaddr_t)(p + KERNEL_LOGICAL_BASE); 46 | } 47 | 48 | static inline physaddr_t kern_to_phys(kernaddr_t k) 49 | { 50 | if (k == NULL) 51 | return (physaddr_t)NULL; 52 | return (physaddr_t)k - KERNEL_TEXT_BASE; 53 | } 54 | 55 | static inline kernaddr_t phys_to_kern(physaddr_t p) 56 | { 57 | if (p == (physaddr_t)NULL) 58 | return NULL; 59 | return (kernaddr_t)(p + KERNEL_TEXT_BASE); 60 | } 61 | 62 | static inline struct page *phys_to_page(physaddr_t p) 63 | { 64 | if (p == (physaddr_t)NULL) 65 | return NULL; 66 | return (struct page *)(page_data + (p / PAGE_SIZE)); 67 | } 68 | 69 | static inline physaddr_t page_to_phys(struct page *p) 70 | { 71 | if (p == NULL) 72 | return (physaddr_t)NULL; 73 | return (physaddr_t)((p - page_data) * PAGE_SIZE); 74 | } 75 | 76 | static inline struct page *virt_to_page(virtaddr_t p) 77 | { 78 | return phys_to_page(virt_to_phys(p)); 79 | } 80 | 81 | static inline virtaddr_t page_to_virt(struct page *p) 82 | { 83 | return phys_to_virt(page_to_phys(p)); 84 | } 85 | 86 | -------------------------------------------------------------------------------- /include/asm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | static inline uint64_t msr_read(uint64_t msr) 6 | { 7 | uint32_t low, high; 8 | asm volatile ( 9 | "rdmsr" 10 | : "=a"(low), "=d"(high) 11 | : "c"(msr) 12 | ); 13 | return ((uint64_t)high << 32) | low; 14 | } 15 | 16 | static inline void msr_write(uint64_t msr, uint64_t value) 17 | { 18 | uint32_t low = value & 0xFFFFFFFF; 19 | uint32_t high = value >> 32; 20 | asm volatile ( 21 | "wrmsr" 22 | : 23 | : "c"(msr), "a"(low), "d"(high) 24 | ); 25 | } 26 | 27 | static inline void invlpg(uint64_t addr) 28 | { 29 | asm volatile ( 30 | "invlpg (%0)" 31 | : 32 | : "b"(addr) 33 | : "memory" 34 | ); 35 | } 36 | 37 | static inline void outb(uint16_t port, uint8_t val) 38 | { 39 | asm volatile ( 40 | "outb %0, %1" 41 | : 42 | : "a"(val), "Nd"(port) 43 | ); 44 | } 45 | 46 | static inline uint8_t inb(uint16_t port) 47 | { 48 | uint8_t ret; 49 | asm volatile ( 50 | "inb %1, %0" 51 | : "=a"(ret) 52 | : "Nd"(port) 53 | ); 54 | return ret; 55 | } 56 | 57 | static inline void cli(void) 58 | { 59 | asm volatile ( 60 | "cli" 61 | : 62 | : 63 | : "cc" 64 | ); 65 | } 66 | 67 | static inline void sti(void) 68 | { 69 | asm volatile ( 70 | "sti" 71 | : 72 | : 73 | : "cc" 74 | ); 75 | } 76 | 77 | static inline void irq_enable(void) 78 | { 79 | sti(); 80 | } 81 | 82 | static inline void irq_disable(void) 83 | { 84 | cli(); 85 | } 86 | 87 | static inline void barrier(void) 88 | { 89 | asm volatile ( 90 | "" 91 | : 92 | : 93 | : "memory" 94 | ); 95 | } 96 | 97 | static inline void pause(void) 98 | { 99 | __builtin_ia32_pause(); 100 | } 101 | 102 | static inline uintptr_t read_cr3(void) 103 | { 104 | uintptr_t rv; 105 | asm volatile ( 106 | "mov %%cr3, %0" 107 | : "=a"(rv) 108 | : 109 | ); 110 | return rv; 111 | } 112 | 113 | static inline void write_cr3(uintptr_t cr3) 114 | { 115 | asm volatile ( 116 | "mov %0, %%cr3" 117 | : 118 | : "a"(cr3) 119 | : "memory" 120 | ); 121 | } 122 | 123 | static inline void bochs_magic(void) 124 | { 125 | asm volatile ( 126 | "xchg %%bx, %%bx" 127 | : 128 | : 129 | : "rbx" 130 | ); 131 | } 132 | 133 | static inline uint64_t read_rflags(void) 134 | { 135 | return __builtin_ia32_readeflags_u64(); 136 | } 137 | 138 | static inline void write_rflags(uint64_t rflags) 139 | { 140 | __builtin_ia32_writeeflags_u64(rflags); 141 | } 142 | 143 | // TODO: I don't think these need to be locked 144 | static inline void preempt_inc(void) 145 | { 146 | asm volatile ( 147 | "lock incl %%gs:0x18" 148 | : 149 | : 150 | ); 151 | } 152 | 153 | static inline void preempt_dec(void) 154 | { 155 | asm volatile ( 156 | "lock decl %%gs:0x18" 157 | : 158 | : 159 | ); 160 | } 161 | 162 | -------------------------------------------------------------------------------- /include/atomic.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "asm.h" 6 | 7 | typedef volatile struct { 8 | volatile uint32_t var; 9 | } atomic32_t; 10 | 11 | typedef volatile struct { 12 | volatile uint64_t var; 13 | } atomic64_t; 14 | 15 | typedef atomic32_t kref_t; 16 | 17 | static inline uint32_t atomic_read32(atomic32_t *a) 18 | { 19 | return __atomic_load_n(&a->var, __ATOMIC_SEQ_CST); 20 | } 21 | 22 | static inline uint32_t atomic_inc_read32(atomic32_t *a) 23 | { 24 | return __atomic_add_fetch(&a->var, 1, __ATOMIC_SEQ_CST); 25 | } 26 | 27 | static inline uint32_t atomic_dec_read32(atomic32_t *a) 28 | { 29 | return __atomic_sub_fetch(&a->var, 1, __ATOMIC_SEQ_CST); 30 | } 31 | 32 | static inline void atomic_write32(atomic32_t *a, uint32_t val) 33 | { 34 | __atomic_store_n(&a->var, val, __ATOMIC_SEQ_CST); 35 | } 36 | 37 | 38 | static inline uint64_t atomic_read64(atomic64_t *a) 39 | { 40 | return __atomic_load_n(&a->var, __ATOMIC_SEQ_CST); 41 | } 42 | 43 | static inline uint64_t atomic_inc_read64(atomic64_t *a) 44 | { 45 | return __atomic_add_fetch(&a->var, 1, __ATOMIC_SEQ_CST); 46 | } 47 | 48 | static inline uint64_t atomic_dec_read64(atomic64_t *a) 49 | { 50 | return __atomic_sub_fetch(&a->var, 1, __ATOMIC_SEQ_CST); 51 | } 52 | 53 | static inline void atomic_write64(atomic64_t *a, uint32_t val) 54 | { 55 | __atomic_store_n(&a->var, val, __ATOMIC_SEQ_CST); 56 | } 57 | 58 | static inline void kref_inc(kref_t *k) 59 | { 60 | atomic_inc_read32(k); 61 | } 62 | 63 | static inline void kref_dec(kref_t *k) 64 | { 65 | atomic_dec_read32(k); 66 | } 67 | 68 | static inline uint32_t kref_inc_read(kref_t *k) 69 | { 70 | return atomic_inc_read32(k); 71 | } 72 | 73 | static inline uint32_t kref_dec_read(kref_t *k) 74 | { 75 | return atomic_dec_read32(k); 76 | } 77 | 78 | static inline uint32_t kref_read(kref_t *k) 79 | { 80 | return atomic_read32(k); 81 | } 82 | -------------------------------------------------------------------------------- /include/drivers/acpi.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "addr.h" 6 | 7 | void acpi_init(void); 8 | virtaddr_t acpi_find_table(char *signature); 9 | 10 | struct acpi_header { 11 | char signature[4]; 12 | uint32_t length; 13 | uint8_t revision; 14 | uint8_t checksum; 15 | char oem_id[6]; 16 | char oem_table_id[8]; 17 | uint32_t oem_revision; 18 | uint8_t creator_id[4]; 19 | uint32_t creator_revision; 20 | } __attribute__((packed)); 21 | 22 | struct acpi_madt { 23 | struct acpi_header header; 24 | uint32_t lapic_address; 25 | uint32_t flags; 26 | uint8_t entries[]; 27 | } __attribute__((packed)); 28 | 29 | struct madt_entry_header { 30 | uint8_t type; 31 | uint8_t length; 32 | } __attribute__((packed)); 33 | 34 | #define MADT_LAPIC 0 35 | struct madt_entry_lapic { 36 | struct madt_entry_header header; 37 | uint8_t acpi_id; 38 | uint8_t apic_id; 39 | uint32_t flags; 40 | } __attribute__((packed)); 41 | 42 | #define MADT_IOAPIC 1 43 | struct madt_entry_ioapic { 44 | struct madt_entry_header header; 45 | uint8_t apic_id; 46 | uint8_t __zero; 47 | uint32_t phys_addr; 48 | uint32_t gsi_base; 49 | } __attribute__((packed)); 50 | 51 | #define MADT_OVERRIDE 2 52 | struct madt_entry_override { 53 | struct madt_entry_header header; 54 | uint8_t bus; // Constant, set to 0 55 | uint8_t source; 56 | uint32_t gsi; 57 | uint16_t flags; 58 | } __attribute__((packed)); 59 | 60 | #define MADT_NMI 4 61 | struct madt_entry_nmi { 62 | struct madt_entry_header header; 63 | uint8_t acpi_id; 64 | uint16_t flags; 65 | uint8_t lint_num; 66 | } __attribute__((packed)); 67 | 68 | #define MADT_LAPIC_ADDR 5 69 | struct madt_entry_lapic_addr { 70 | struct madt_entry_header header; 71 | uint16_t __zero; 72 | uint64_t lapic_addr; 73 | } __attribute__((packed)); 74 | 75 | extern struct rsdp *acpi_rsdp; 76 | -------------------------------------------------------------------------------- /include/drivers/apic.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "limits.h" 5 | #include "addr.h" 6 | #include "drivers/acpi.h" 7 | 8 | #define MAX_LAPICS MAX_CORES 9 | #define MAX_IOAPICS 16 10 | #define MAX_NMIS (2 * MAX_LAPICS) 11 | #define MAX_OVERRIDES 48 12 | 13 | #define IPI_INIT 0x4500 14 | #define IPI_START_UP 0x4600 15 | #define IPI_FIXED 0x4000 16 | 17 | #define IPI_BROADCAST (0b11 << 18) 18 | 19 | struct lapic_info { 20 | uint8_t id; 21 | uint8_t acpi_id; 22 | uint8_t present; 23 | uint32_t ticks_per_10ms; 24 | }; 25 | 26 | void apic_init(void); 27 | 28 | void lapic_enable(void); 29 | void lapic_eoi(uint8_t); 30 | uint8_t lapic_id(void); 31 | void lapic_send_ipi(uint8_t target, uint32_t flags); 32 | uint32_t lapic_timer_prepare(void); 33 | void lapic_timer_enable(void); 34 | 35 | void ioapic_init(void); 36 | void ioapic_redirect(uint32_t gsi, uint8_t source, uint16_t flags, uint8_t target_apic); 37 | void ioapic_mask(uint32_t gsi); 38 | void ioapic_unmask(uint32_t gsi); 39 | uint8_t ioapic_isa_to_gsi(uint8_t isa); 40 | uint8_t ioapic_gsi_to_isa(uint8_t gsi); 41 | 42 | extern struct lapic_info lapic_list[MAX_LAPICS]; 43 | extern struct madt_entry_ioapic *ioapic_list[MAX_IOAPICS]; 44 | extern struct madt_entry_override *override_list[MAX_OVERRIDES]; 45 | extern struct madt_entry_nmi *nmi_list[MAX_NMIS]; 46 | extern size_t lapic_list_size; 47 | extern size_t ioapic_list_size; 48 | extern size_t override_list_size; 49 | extern size_t nmi_list_size; 50 | extern virtaddr_t lapic_base; 51 | -------------------------------------------------------------------------------- /include/drivers/cansid.h: -------------------------------------------------------------------------------- 1 | #ifndef CANSID_H 2 | #define CANSID_H 3 | 4 | struct cansid_state { 5 | enum { 6 | CANSID_ESC, 7 | CANSID_BRACKET, 8 | CANSID_PARSE, 9 | CANSID_BGCOLOR, 10 | CANSID_FGCOLOR, 11 | CANSID_EQUALS, 12 | CANSID_ENDVAL, 13 | } state; 14 | unsigned char style; 15 | unsigned char next_style; 16 | }; 17 | 18 | struct color_char { 19 | unsigned char style; 20 | unsigned char ascii; 21 | }; 22 | 23 | struct cansid_state cansid_init(void); 24 | struct color_char cansid_process(struct cansid_state *state, char x); 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /include/drivers/pit.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | void pit_init(void); 7 | void pit_sleep_ms(uint64_t); 8 | void pit_sleep_watch_flag(uint64_t, volatile bool *, bool); 9 | -------------------------------------------------------------------------------- /include/drivers/ps2.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "asm.h" 5 | #include "libk.h" 6 | 7 | #define PS2_STATUS 0x64 8 | #define PS2_COMMAND 0x64 9 | #define PS2_DATA 0x60 10 | 11 | void ps2kbd_init(void); 12 | 13 | void ps2_init(void); 14 | 15 | static inline uint8_t ps2_read_data(void) 16 | { 17 | while ((inb(PS2_STATUS) & 1) == 0) 18 | ; 19 | return inb(PS2_DATA); 20 | } 21 | 22 | static inline uint8_t ps2_read_status(void) 23 | { 24 | return inb(PS2_STATUS); 25 | } 26 | 27 | static inline void ps2_write_cmd(uint8_t data) 28 | { 29 | while ((inb(PS2_STATUS) & 2) != 0) 30 | ; 31 | outb(PS2_COMMAND, data); 32 | } 33 | 34 | static inline void ps2_write_data(uint8_t data) 35 | { 36 | while ((inb(PS2_STATUS) & 2) != 0) 37 | ; 38 | outb(PS2_DATA, data); 39 | } 40 | 41 | static inline uint8_t ps2_read_config(void) 42 | { 43 | ps2_write_cmd(0x20); 44 | uint8_t config_byte = ps2_read_data(); 45 | kassert_dbg((config_byte & (1 << 7)) == 0); 46 | return config_byte; 47 | } 48 | 49 | static inline void ps2_write_config(uint8_t config_byte) 50 | { 51 | ps2_write_cmd(0x60); 52 | ps2_write_data(config_byte); 53 | } 54 | 55 | -------------------------------------------------------------------------------- /include/drivers/serial.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | void serial_init(void); 4 | void serial_write_com(int, unsigned char); 5 | -------------------------------------------------------------------------------- /include/drivers/smbios.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | void smbios_init(void); 4 | -------------------------------------------------------------------------------- /include/drivers/vga_tmode.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | void vga_tmode_putchar(char); 7 | void vga_tmode_raw_putchar(char, unsigned char); 8 | void vga_tmode_puts(char *); 9 | void vga_tmode_setcursor(size_t x, size_t y); 10 | -------------------------------------------------------------------------------- /include/ds/bitmap.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #define bitmap_test(map, which) (map[(which) >> 3] & (1 << ((which) & 7))) 6 | 7 | #define bitmap_set(map, which) map[(which) >> 3] |= (1 << ((which) & 7)) 8 | 9 | #define bitmap_clear(map, which) map[(which) >> 3] &= ~(1 << ((which) & 7)) 10 | 11 | #define bitmap_toggle(map, which) map[(which) >> 3] ^= (1 << ((which) & 7)) 12 | 13 | //https://graphics.stanford.edu/~seander/bithacks.html#ConditionalSetOrClearBitsWithoutBranching 14 | #define bitmap_write(map, which, state) map[(which) >> 3] ^= (-(state) ^ map[(which) >> 3]) & (1 << ((which) & 7)) 15 | 16 | int bitmap_find_hole(unsigned char *map, size_t size, size_t hole_size); 17 | -------------------------------------------------------------------------------- /include/ds/generic.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | // Compiler magic to find the original struct (taken from linux kernel) 6 | #define container_of(ptr, type, member) ({ \ 7 | const typeof(((type *)0)->member) *__mptr = (ptr); \ 8 | type *__t = (type *)((char *)__mptr - offsetof(type, member)); \ 9 | __mptr == NULL ? NULL : __t; }) 10 | -------------------------------------------------------------------------------- /include/ds/hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ds/linked.h" 4 | 5 | struct hlist_bucket { 6 | struct dlist_node *head; 7 | }; 8 | 9 | #define hlist_foreach(cur, type, member, bucket) \ 10 | for (type *cur = container_of((bucket)->head, type, member); \ 11 | (cur) != NULL; (cur) = dlist_get_next(cur, member)) 12 | 13 | static inline void hlist_add(struct hlist_bucket *bucket, struct dlist_node *node) 14 | { 15 | // Insert at front 16 | node->prev = NULL; 17 | node->next = bucket->head; 18 | 19 | if (bucket->head != NULL) 20 | bucket->head->prev = node; 21 | 22 | bucket->head = node; 23 | } 24 | 25 | static inline void hlist_remove(struct hlist_bucket *bucket, struct dlist_node *node) 26 | { 27 | if (node->prev == NULL) 28 | bucket->head = node->next; 29 | else 30 | node->prev->next = node->next; 31 | 32 | // If we're the last element in the list 33 | if (node->next != NULL) 34 | node->next->prev = node->prev; 35 | } 36 | -------------------------------------------------------------------------------- /include/ds/linked.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "generic.h" 6 | 7 | struct slist_node { 8 | struct slist_node *next; 9 | }; 10 | 11 | struct dlist_node { 12 | struct dlist_node *next, *prev; 13 | }; 14 | 15 | #define slist_entry(ptr, type, member) container_of(ptr, type, member) 16 | 17 | #define dlist_entry(ptr, type, member) container_of(ptr, type, member) 18 | 19 | #define slist_get_next(ptr, member) ({ \ 20 | const struct slist_node *__next = (ptr)->member.next; \ 21 | __next == NULL ? NULL : slist_entry(__next, typeof(*(ptr)), member); }) 22 | 23 | #define dlist_get_next(ptr, member) ({ \ 24 | const struct dlist_node *__next = (ptr)->member.next; \ 25 | __next == NULL ? NULL : dlist_entry(__next, typeof(*(ptr)), member); }) 26 | 27 | #define dlist_get_prev(ptr, member) ({ \ 28 | const struct dlist_node *__prev = (ptr)->member.prev; \ 29 | __prev == NULL ? NULL : dlist_entry(__prev, typeof(*(ptr)), member); }) 30 | 31 | #define slist_set_next(ptr, member, next_entry) ({ \ 32 | const typeof(next_entry) __next_entry = (next_entry); \ 33 | struct slist_node *__head = &(ptr)->member; \ 34 | struct slist_node *__next = (__next_entry == NULL) ? NULL : &__next_entry->member; \ 35 | __head->next = __next; }) 36 | 37 | #define dlist_set_next(ptr, member, next_entry) ({ \ 38 | const typeof(next_entry) __next_entry = (next_entry); \ 39 | struct dlist_node *__head = &(ptr)->member; \ 40 | struct dlist_node *__next = (__next_entry == NULL) ? NULL : &__next_entry->member; \ 41 | __head->next = __next; \ 42 | if (__next != NULL) __next->prev = __head; }) 43 | 44 | #define dlist_set_prev(ptr, member, prev_entry) ({ \ 45 | const typeof(prev_entry) __prev_entry = (prev_entry); \ 46 | struct dlist_node *__head = &(ptr)->member; \ 47 | struct dlist_node *__prev = (__prev_entry == NULL) ? NULL : &__prev_entry->member; \ 48 | __head->prev = __prev; \ 49 | if (__prev != NULL) __prev->next = __head; }) 50 | 51 | #define slist_foreach(head, member, start) \ 52 | for (typeof(start) (head) = start; (head) != NULL; \ 53 | (head) = slist_get_next((head), member)) 54 | 55 | #define dlist_foreach(head, member, start) \ 56 | for (typeof(start) (head) = start; (head) != NULL; \ 57 | (head) = dlist_get_next((head), member)) 58 | 59 | #define slist_append(head, member, node) ({ \ 60 | struct slist_node *__head = &(head)->member; \ 61 | struct slist_node *__node = &(node)->member; \ 62 | __slist_append(__head, __node); }) 63 | 64 | #define dlist_append(head, member, node) ({ \ 65 | struct dlist_node *__head = &(head)->member; \ 66 | struct dlist_node *__node = &(node)->member; \ 67 | __dlist_append(__head, __node); }) 68 | 69 | #define dlist_remove(head, member) ({ \ 70 | struct dlist_node *__head = &(head)->member; \ 71 | __head->prev->next = __head->next; }) 72 | 73 | static inline void __slist_append(struct slist_node *head, struct slist_node *node) { 74 | struct slist_node *prev = head; 75 | while (head != NULL) 76 | prev = head, head = head->next; 77 | prev->next = node; 78 | } 79 | 80 | static inline void __dlist_append(struct dlist_node *head, struct dlist_node *node) { 81 | struct dlist_node *prev = head; 82 | while (head != NULL) 83 | prev = head, head = head->next; 84 | prev->next = node; 85 | if (node != NULL) 86 | node->prev = prev; 87 | } 88 | -------------------------------------------------------------------------------- /include/ds/rbtree.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "generic.h" 8 | 9 | struct rb_node { 10 | uintptr_t parent_color; 11 | struct rb_node *left; 12 | struct rb_node *right; 13 | } __attribute__((aligned((sizeof(long))))); 14 | 15 | struct rbtree { 16 | struct rb_node *root; // Root of the tree 17 | struct rb_node *most_left; // The furthest left node in the tree 18 | }; 19 | 20 | #define rb_entry(ptr, type, member) \ 21 | ({ const struct rb_node *__ptr = (ptr); \ 22 | __ptr != NULL ? container_of(__ptr, type, member) : NULL; }) 23 | 24 | #define RB_PARENT_MASK (~1) 25 | #define RB_COLOR_MASK (~RB_PARENT_MASK) 26 | #define RB_RED 0 27 | #define RB_BLACK 1 28 | 29 | #define rb_mask_parent(ptr) \ 30 | ((uintptr_t)(ptr) & RB_PARENT_MASK) 31 | #define rb_mask_color(ptr) \ 32 | ((uintptr_t)(ptr) & RB_COLOR_MASK) 33 | 34 | #define rb_is_red(node) \ 35 | (rb_color(node) == RB_RED) 36 | #define rb_is_black(node) \ 37 | (rb_color(node) == RB_BLACK) 38 | 39 | #define rb_first_cached(tree) (tree)->most_left 40 | 41 | #define rb_is_leaf(node) \ 42 | ((node) == NULL || ((node)->left == NULL && (node)->right == NULL)) 43 | 44 | static inline struct rb_node *rb_parent(struct rb_node *node) 45 | { 46 | return (struct rb_node *)rb_mask_parent(node->parent_color); 47 | } 48 | 49 | static inline unsigned int rb_color(struct rb_node *node) 50 | { 51 | return node ? rb_mask_color(node->parent_color) : RB_BLACK; 52 | } 53 | 54 | static inline struct rb_node *rb_grandparent(struct rb_node *node) 55 | { 56 | struct rb_node *parent = rb_parent(node); 57 | if (parent == NULL) 58 | return NULL; 59 | return rb_parent(parent); 60 | } 61 | 62 | static inline struct rb_node *rb_sibling(struct rb_node *node) 63 | { 64 | struct rb_node *parent = rb_parent(node); 65 | if (parent == NULL) 66 | return NULL; 67 | else if (node == parent->left) 68 | return parent->right; 69 | else 70 | return parent->left; 71 | } 72 | 73 | static inline struct rb_node *rb_uncle(struct rb_node *node) 74 | { 75 | struct rb_node *parent = rb_parent(node); 76 | struct rb_node *grandparent = rb_grandparent(node); 77 | if (grandparent == NULL) 78 | return NULL; 79 | return rb_sibling(parent); 80 | } 81 | 82 | static inline void rb_set_color(struct rb_node *node, unsigned int color) 83 | { 84 | if (color == 1) 85 | node->parent_color |= 1; 86 | else if (color == 0) 87 | node->parent_color &= ~1; 88 | } 89 | 90 | static inline void rb_set_parent(struct rb_node *node, struct rb_node *parent) 91 | { 92 | unsigned int color = rb_color(node); 93 | node->parent_color = (uintptr_t)parent | color; 94 | } 95 | 96 | static inline void rb_link_node(struct rb_node *node, struct rb_node *parent, 97 | struct rb_node **link) 98 | { 99 | rb_set_parent(node, parent); 100 | rb_set_color(node, RB_RED); 101 | 102 | node->left = node->right = NULL; 103 | *link = node; 104 | } 105 | 106 | static inline struct rb_node *rb_first_uncached(struct rbtree *tree) 107 | { 108 | struct rb_node *node = tree->root; 109 | 110 | if (node == NULL) 111 | return NULL; 112 | 113 | while (node->left) 114 | node = node->left; 115 | 116 | return node; 117 | } 118 | 119 | static inline struct rb_node *rb_next(struct rb_node *node) 120 | { 121 | struct rb_node *parent; 122 | 123 | if (node == NULL) 124 | return NULL; 125 | 126 | if (node->right) { 127 | node = node->right; 128 | while (node->left) 129 | node = node->left; 130 | return node; 131 | } 132 | 133 | while ((parent = rb_parent(node)) && node == parent->right) 134 | node = parent; 135 | 136 | return parent; 137 | } 138 | 139 | void rb_insert(struct rbtree *tree, struct rb_node *node, bool most_left); 140 | void rb_erase(struct rbtree *tree, struct rb_node *node); 141 | void rb_replace(struct rbtree *tree, struct rb_node *victim, struct rb_node *target); 142 | -------------------------------------------------------------------------------- /include/err.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #define RETURNS_ERROR err_t __attribute__((warn_unused_result)) 6 | 7 | typedef int64_t err_t; 8 | 9 | #include "gen/err_gen.h" 10 | -------------------------------------------------------------------------------- /include/fs.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "types.h" 4 | #include "atomic.h" 5 | #include "limits.h" 6 | #include "err.h" 7 | #include "sync.h" 8 | #include "ds/linked.h" 9 | 10 | #define HASOP(value, op) ((value)->ops && (value)->ops->op) 11 | 12 | struct super_block { 13 | dev_t dev; 14 | struct inode *root; 15 | struct super_operations *ops; 16 | }; 17 | 18 | struct super_operations { 19 | struct inode *(*alloc_inode)(struct super_block *); 20 | err_t (*write_inode)(struct super_block *, struct inode *); 21 | err_t (*read_inode)(struct super_block *, struct inode *); 22 | void (*dealloc_inode)(struct inode *); 23 | }; 24 | 25 | #define I_UNKNOWN 0 26 | #define I_REGULAR 1 27 | #define I_DIRECTORY 2 28 | #define I_SYMLINK 3 29 | #define I_MOUNT 4 30 | 31 | #define I_ISREG(inode) ((inode)->mode == I_REGULAR) 32 | #define I_ISDIR(inode) ((inode)->mode == I_DIRECTORY) 33 | #define I_ISSYM(inode) ((inode)->mode == I_SYMLINK) 34 | #define I_ISMNT(inode) ((inode)->mode == I_MOUNT) 35 | 36 | struct inode { 37 | ino_t ino; 38 | mode_t mode; 39 | 40 | kref_t rc; 41 | 42 | struct super_block *sb; 43 | struct inode_operations *ops; 44 | 45 | struct inode *mounted; // When mode is I_MOUNT, this points to the mounted inode 46 | struct dlist_node node; // Used by inode_cache 47 | }; 48 | 49 | struct inode_operations { 50 | err_t (*lookup)(struct inode *dir, const char *name, size_t len, struct inode **out); 51 | /*err_t (*follow_symlink)(struct inode *node, const char **out); 52 | err_t (*link)(struct inode *dir, struct inode *node, const char *name, size_t len); 53 | err_t (*unlink)(struct inode *dir, struct inode *node, const char *name, size_t len);*/ 54 | }; 55 | 56 | extern struct inode vfs_root; 57 | 58 | struct inode *inode_get(struct super_block *, ino_t); 59 | void inode_put(struct inode *); 60 | 61 | void vfs_init(void); 62 | RETURNS_ERROR vfs_mount(struct inode *mount_point, dev_t dev); 63 | RETURNS_ERROR vfs_lookup(struct inode *dir, const char *path, size_t len, struct inode **result); 64 | -------------------------------------------------------------------------------- /include/fs/initrd.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "fs.h" 6 | #include "mm_types.h" 7 | 8 | #define INITRD_ROOT_INO 1 9 | 10 | struct initrd_inode { 11 | struct ustar_header *hdr; 12 | struct inode i; 13 | }; 14 | 15 | struct initrd_fs { 16 | struct super_block super; 17 | struct ustar_header *root_hdr; 18 | struct inode *root; 19 | }; 20 | 21 | struct ustar_header { 22 | char filename[100]; 23 | char mode[8]; 24 | char uid[8]; 25 | char gid[8]; 26 | char size[12]; 27 | char mtime[12]; 28 | char checksum[8]; 29 | char type; 30 | char __padding[512 - 157]; // Other fields 31 | } __attribute__((packed)); 32 | 33 | static inline struct initrd_inode *initrd_inode_container(struct inode *inode) 34 | { 35 | return inode == NULL ? NULL : container_of(inode, struct initrd_inode, i); 36 | } 37 | 38 | static inline struct initrd_fs *initrd_sb_container(struct super_block *super) 39 | { 40 | return super == NULL ? NULL : container_of(super, struct initrd_fs, super); 41 | } 42 | 43 | extern struct inode_operations initrd_inode_file_ops; 44 | extern struct inode_operations initrd_inode_dir_ops; 45 | 46 | struct super_block *initrd_get_super(void); 47 | 48 | uint32_t ustar_oct_to_bin(const char *str, size_t len); 49 | struct ustar_header *ustar_nth_from_entry(struct ustar_header *root_hdr, uint32_t n); 50 | virtaddr_t ustar_file_start(struct ustar_header *hdr); 51 | -------------------------------------------------------------------------------- /include/include.asm: -------------------------------------------------------------------------------- 1 | %include "gen/syscall_gen.asm" 2 | 3 | %define PAGE_SIZE 0x1000 4 | 5 | %define KERNEL_TEXT_BASE 0xFFFFFFFF80000000 6 | %define KERNEL_PHYS_MAP_END 0x1000000 7 | 8 | %define GDT_KERNEL_CODE 0x8 9 | %define GDT_KERNEL_DATA 0x10 10 | %define GDT_USER_DATA 0x20 11 | %define GDT_USER_CODE 0x28 12 | %define GDT_TSS 0x30 ; Change this in percpu.c too 13 | 14 | %define KERNEL_CPU_STACK_SIZE (PAGE_SIZE * 4) 15 | 16 | %define PERCPU_CURRENT gs:0x0 17 | %define PERCPU_RSP_SCRATCH gs:0x8 18 | %define PERCPU_TSS gs:0x10 19 | %define PERCPU_PREEMPT_COUNT gs:0x18 20 | %define PERCPU_ID gs:0x1C 21 | %define PERCPU_RUN_QUEUE gs:0x20 22 | 23 | ; outb(port, val) 24 | %macro outb 2 25 | mov al, %2 26 | mov dx, %1 27 | out dx, al 28 | %endmacro 29 | 30 | %macro preempt_inc 0 31 | lock inc dword [gs:0x18] 32 | %endmacro 33 | 34 | %macro preempt_dec 0 35 | lock dec dword [gs:0x18] 36 | %endmacro 37 | 38 | %macro bochs_magic 0 39 | xchg bx, bx 40 | %endmacro 41 | 42 | %macro debug_hlt 0 43 | .debug_hlt: 44 | cli 45 | hlt 46 | jmp .debug_hlt 47 | %endmacro 48 | 49 | -------------------------------------------------------------------------------- /include/interrupts.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include "asm.h" 5 | #include "drivers/apic.h" 6 | 7 | #define IST_NONE 0 8 | #define IST_NMI 1 9 | #define IST_DOUBLE_FAULT 2 10 | 11 | #define IRQ_APIC_SPURIOUS 0xFF 12 | #define IRQ_APIC_BASE 0x30 13 | #define IRQ_LINT_BASE (IRQ_APIC_SPURIOUS - 3) // One for each LINT pin, one for timer 14 | #define IRQ_LINT_TIMER (IRQ_LINT_BASE + 2) 15 | #define IRQ_NMI (IRQ_LINT_BASE - 1) 16 | 17 | #define IRQ_IPI_TOP (IRQ_NMI - 1) 18 | #define IRQ_IPI_ABORT (IRQ_IPI_TOP - 0) 19 | #define IRQ_IPI_TLB_SHOOTDOWN (IRQ_IPI_TOP - 1) 20 | #define IRQ_IPI_SCHED_HINT (IRQ_IPI_TOP - 2) 21 | 22 | #define ISA_TO_INTERRUPT(x) (ioapic_isa_to_gsi(x) + IRQ_APIC_BASE) 23 | 24 | struct isr_ctx { 25 | uint64_t r11; 26 | uint64_t r10; 27 | uint64_t r9; 28 | uint64_t r8; 29 | uint64_t rax; 30 | uint64_t rcx; 31 | uint64_t rdx; 32 | uint64_t rsi; 33 | uint64_t rdi; 34 | // Contains error code and interrupt number for exceptions 35 | // Contains syscall number for syscalls 36 | // Contains just the interrupt number otherwise 37 | uint64_t info; 38 | // Interrupt stack frame 39 | uint64_t rip; 40 | uint64_t cs; 41 | uint64_t rflags; 42 | uint64_t rsp; 43 | uint64_t ss; 44 | }; 45 | 46 | typedef void (*isr_handler_t)(struct isr_ctx *); 47 | 48 | struct isr_info { 49 | enum isr_type { 50 | ISR_IRQ, // Normal IRQs that require EOI 51 | ISR_EXCEPTION, // CPU exceptions like page faults 52 | ISR_IPI, // IPIs like TLB shootdown 53 | ISR_NOP, // NOP for spurious interrupts 54 | } type; 55 | isr_handler_t handler; 56 | }; 57 | 58 | struct idt_entry { 59 | uint16_t offset_low; 60 | uint16_t selector; 61 | uint8_t ist_index; 62 | uint8_t type_attr; 63 | uint16_t offset_mid; 64 | uint32_t offset_high; 65 | uint32_t __zero; 66 | }; 67 | 68 | void exceptions_init(void); 69 | 70 | void idt_init(void); 71 | void idt_load(void); // Defined in isr_stubs.asm (TODO: should really be in flush.asm) 72 | void idt_set_gate(uint8_t index, virtaddr_t entry, uint8_t ist, uint8_t type_attr); 73 | 74 | void isr_init(void); 75 | void isr_set_info(uint8_t, struct isr_info *); 76 | void isr_global_handler(struct isr_ctx *); 77 | void isr_irq_eoi(uint8_t); 78 | void isr_irq_mask(uint8_t); 79 | void isr_irq_unmask(uint8_t); 80 | -------------------------------------------------------------------------------- /include/libk.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #ifdef LIBK_TEST 6 | #define LIBK_FN(name) __libk_ ## name 7 | #else 8 | #include "asm.h" 9 | #include "smp.h" 10 | #include "sync.h" 11 | #define LIBK_FN(name) name 12 | #endif 13 | 14 | int LIBK_FN(memcmp)(const void *, const void *, size_t); 15 | void *LIBK_FN(memcpy)(void *, const void *, size_t); 16 | void *LIBK_FN(memmove)(void *, const void *, size_t); 17 | void *LIBK_FN(memset)(void *, int, size_t); 18 | void *LIBK_FN(memchr)(const void *, int, size_t); 19 | size_t LIBK_FN(strlen)(const char *); 20 | 21 | __attribute__((format (printf, 1, 2))) int kprintf(const char *, ...); 22 | __attribute__((format (printf, 1, 2))) int kprintf_nolock(const char *, ...); 23 | 24 | __attribute__((noreturn)) void abort(void); // Sends an IPI to abort all other CPUs 25 | __attribute__((noreturn)) void abort_self(void); 26 | 27 | #ifndef LIBK_TEST 28 | __attribute__((noreturn)) void __stack_chk_fail(void); 29 | extern spinlock_t kprintf_lock; 30 | #endif 31 | 32 | #define panic_nolock(...) ({ \ 33 | irq_disable(); \ 34 | cli(); \ 35 | kprintf_nolock( \ 36 | "\n\x1B[0m--------------------------------------------------------------------------------\x1B[0m" \ 37 | "\x1B[1;41;37mpanic at %s:%s:%u CPU %d\x1B[0m\n", \ 38 | __FILE__, __func__, __LINE__, smp_cpu_id() \ 39 | ); \ 40 | kprintf_nolock(__VA_ARGS__); \ 41 | abort(); \ 42 | }) 43 | 44 | #define panic(...) ({ \ 45 | spin_lock(&kprintf_lock); \ 46 | panic_nolock(__VA_ARGS__); \ 47 | }) 48 | 49 | #define kassert(condition) do { \ 50 | if (!(condition)) \ 51 | panic("kassert condition failed at %s:%u: %s\n", __FILE__, __LINE__, #condition); \ 52 | } while(0) 53 | 54 | #define klog(mod, msg, ...) ({ \ 55 | uint64_t time = 0; \ 56 | if (time != 0) \ 57 | kprintf("[%lu] ", time); \ 58 | kprintf(mod ": " msg, ##__VA_ARGS__); }) 59 | 60 | #define klog_warn(mod, msg, ...) klog(mod, "\e[33m**WARN**\e[0m " msg, ##__VA_ARGS__) 61 | 62 | #ifdef DEBUG 63 | #define kassert_dbg(x) kassert(x) 64 | #else 65 | #define kassert_dbg(x) do {} while (0) 66 | #endif 67 | 68 | #ifdef VERBOSE 69 | #define klog_verbose(...) klog(__VA_ARGS__) 70 | #else 71 | #define klog_verbose(...) do {} while (0) 72 | #endif 73 | -------------------------------------------------------------------------------- /include/limits.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define MAX_CORES 20 4 | #define MAX_ORDER 12 5 | 6 | #define NAME_MAX 32 7 | 8 | #define PAGE_SIZE 0x1000 9 | 10 | #define KERNEL_TEXT_BASE 0xFFFFFFFF80000000 11 | #define KERNEL_PHYS_MAP_END 0x1000000 12 | #define KERNEL_LOGICAL_BASE 0xFFFF800000000000 13 | -------------------------------------------------------------------------------- /include/mm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include "multiboot2.h" 7 | #include "smp.h" 8 | #include "sync.h" 9 | #include "atomic.h" 10 | #include "mm_types.h" 11 | #include "ds/linked.h" 12 | 13 | #define KERNEL_PAGE_DATA (KERNEL_TEXT_BASE + KERNEL_PHYS_MAP_END) 14 | 15 | #define PAGE_PRESENT (1ULL << 0) 16 | #define PAGE_WRITABLE (1ULL << 1) 17 | #define PAGE_USER_ACCESSIBLE (1ULL << 2) 18 | #define PAGE_WRITE_THROUGH (1ULL << 3) 19 | #define PAGE_DISABLE_CACHE (1ULL << 4) 20 | #define PAGE_ACCESSED (1ULL << 5) 21 | #define PAGE_DIRTY (1ULL << 6) 22 | #define PAGE_HUGE (1ULL << 7) 23 | #define PAGE_GLOBAL (1ULL << 8) 24 | #define PAGE_COW (1ULL << 9) 25 | #define PAGE_EXECUTABLE (1ULL << 63) 26 | 27 | #define VMM_ALLOC_MMAP (1 << 15) 28 | #define VMM_UNMAP (1 << 16) 29 | 30 | #define PAGE_SHIFT 12 31 | #define PTE_ADDR_MASK (~(0xFFF0000000000FFFUL)) 32 | 33 | #define MMAP_ALLOC_PA (1 << 0) 34 | #define MMAP_MAX_REGIONS 128 35 | 36 | #define GFP_CAN_FAIL (1 << 0) 37 | 38 | #define GFP_NONE 0 39 | #define VMM_NONE 0 40 | #define KM_NONE 0 41 | 42 | 43 | extern const uint8_t zero_page[PAGE_SIZE]; 44 | extern struct mmu_info kernel_mmu; 45 | extern const uintptr_t _kernel_end_phys; 46 | 47 | extern struct tlb_op * volatile tlb_op_location; 48 | extern atomic32_t tlb_remaining_cpus; 49 | 50 | void vmm_init(void); 51 | void vmm_map_all(struct mmap *); 52 | bool vmm_has_flags(struct mmu_info *, void *, uint64_t flags); 53 | pte_t *vmm_get_pte(struct mmu_info *, const void *); 54 | void vmm_map_page(struct mmu_info *, physaddr_t, virtaddr_t, unsigned long); 55 | void vmm_dump_tables(struct mmu_info *); 56 | 57 | struct mmap *mmap_init(struct multiboot_info *); 58 | void mmap_dump_info(void); 59 | struct mmap_region mmap_alloc_low(size_t n, unsigned int alloc_flags); 60 | 61 | void pmm_init(struct mmap *); 62 | struct page *pmm_alloc_order(unsigned int order, unsigned int alloc_flags) __attribute__((warn_unused_result)); 63 | void pmm_free_order(struct page *page, unsigned int order); 64 | 65 | #define get_free_page() page_to_virt(pmm_alloc_order(0, GFP_NONE)) 66 | #define free_page_at(addr) pmm_free_order(virt_to_page(addr), 0) 67 | 68 | void *kmalloc(size_t, unsigned int) __attribute__((malloc)); 69 | void kfree(void *); 70 | 71 | void cow_copy_pte(pte_t *dest, pte_t *src); 72 | bool cow_handle_write(pte_t *pte, virtaddr_t virt); 73 | void cow_handle_free(pte_t *pte); 74 | 75 | struct mmu_info *mmu_alloc(void); 76 | void mmu_free(struct mmu_info *mmu); 77 | void mmu_init(struct mmu_info *mmu); 78 | void mmu_switch(struct mmu_info *next, struct mmu_info *prev); 79 | void mmu_inc_users(struct mmu_info *mmu); 80 | void mmu_dec_users(struct mmu_info *mmu); 81 | void mmu_update_cpuset(struct mmu_info *mmu, cpuid_t id, bool val); 82 | void mmu_reap(struct mmu_info *mmu); 83 | void mmu_clone_cow(struct mmu_info *dest, struct mmu_info *mmu); 84 | 85 | void area_add(struct mmu_info *mmu, struct vm_area *area); 86 | 87 | void tlb_shootdown(struct mmu_info *mmu, virtaddr_t start, virtaddr_t end); 88 | 89 | static inline void tlb_flush_single(virtaddr_t addr) 90 | { 91 | invlpg((uintptr_t)addr); 92 | } 93 | 94 | static inline void tlb_flush_all(void) 95 | { 96 | write_cr3(read_cr3()); 97 | } 98 | 99 | static inline struct page_table *pgtab_extract_virt_addr(struct page_table *pgtab, uint16_t index) 100 | { 101 | pte_t entry = pgtab->pages[index]; 102 | if ((entry & PAGE_PRESENT) == 0) 103 | return NULL; 104 | return phys_to_virt((entry & PTE_ADDR_MASK)); 105 | } 106 | -------------------------------------------------------------------------------- /include/mm_types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "limits.h" 7 | #include "smp.h" 8 | #include "sync.h" 9 | #include "atomic.h" 10 | #include "addr.h" 11 | #include "ds/linked.h" 12 | 13 | typedef uint64_t pte_t; 14 | 15 | struct page_table { 16 | pte_t pages[512]; 17 | } __attribute__((packed, aligned(PAGE_SIZE))); 18 | 19 | struct mmap_region { 20 | uintptr_t base; 21 | size_t len; 22 | // TODO: Don't name this flags, it's an enum 23 | enum mmap_region_flags { 24 | MMAP_NONE, 25 | MMAP_NOMAP 26 | } flags; 27 | }; 28 | 29 | struct mmap_type { 30 | uint32_t count; 31 | struct mmap_region *regions; 32 | }; 33 | 34 | struct mmap { 35 | physaddr_t highest_mapped; 36 | struct mmap_type available; 37 | struct mmap_type reserved; 38 | }; 39 | 40 | // Describes a contiguous block of memory 41 | struct zone { 42 | struct slist_node list; 43 | physaddr_t pa_start; // Start of available memory in zone 44 | size_t len; // Length of available memory in zone 45 | struct page *free_lists[MAX_ORDER]; 46 | }; 47 | 48 | // Describes a region of virtual memory 49 | struct vm_area { 50 | struct slist_node list; 51 | uintptr_t base; 52 | size_t len; 53 | uint32_t type; 54 | #define VM_AREA_OTHER 0 55 | #define VM_AREA_STACK 1 56 | #define VM_AREA_TEXT 2 57 | #define VM_AREA_BSS 3 58 | uint32_t flags; 59 | #define VM_AREA_WRITABLE (1 << 0) 60 | #define VM_AREA_EXECUTABLE (1 << 1) 61 | }; 62 | 63 | struct mmu_info { 64 | struct page_table *p4; // This should stay as the first member 65 | rwspinlock_t pgtab_lock; 66 | 67 | // users: Number of threads with this mmu_info set as active 68 | kref_t users; 69 | 70 | spinlock_t cpu_lock; 71 | cpuset_t cpus; 72 | 73 | spinlock_t area_lock; 74 | struct vm_area *areas; 75 | }; 76 | 77 | struct tlb_op { 78 | virtaddr_t start, end; 79 | }; 80 | 81 | -------------------------------------------------------------------------------- /include/mutex.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "sync.h" 4 | 5 | // TODO: Add sleeping 6 | 7 | typedef struct { 8 | spinlock_t lock; 9 | } mutex_t; 10 | 11 | static inline void mutex_lock(mutex_t *mutex) 12 | { 13 | spin_lock(&mutex->lock); 14 | } 15 | 16 | static inline void mutex_unlock(mutex_t *mutex) 17 | { 18 | spin_unlock(&mutex->lock); 19 | } 20 | -------------------------------------------------------------------------------- /include/percpu.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include "mm_types.h" 7 | #include "smp.h" 8 | #include "util.h" 9 | 10 | struct percpu { 11 | // Be careful changing these four variables as they are referenced in asm 12 | struct task *current_task; // Current task 13 | virtaddr_t rsp_scratch; // Temporary rsp used for syscalls 14 | virtaddr_t tss; // Holds the address of the TSS (needed for context switches) 15 | uint64_t preempt_count; // Incremented for every lock held and decremented for every lock released 16 | 17 | atomic32_t int_depth; 18 | bool reschedule; 19 | 20 | cpuid_t id; 21 | uint8_t apic_id; 22 | struct runq *run_queue; 23 | }; 24 | 25 | void percpu_init_ap(void); 26 | void percpu_init_bsp(void); 27 | void percpu_set_addr(struct percpu *); 28 | 29 | #define __percpu(var) (((struct percpu *)NULL)->var) 30 | #define __percpu_type(var) typeof(__percpu(var)) 31 | #define __percpu_marker(var) ((volatile __percpu_type(var) *)&__percpu(var)) 32 | 33 | #define percpu_get(var) ({ \ 34 | __percpu_type(var) res; \ 35 | asm ("mov %%gs:%1, %0" \ 36 | : "=r" (res) \ 37 | : "m" (*__percpu_marker(var)) \ 38 | ); \ 39 | res; }) 40 | 41 | #define __percpu_set(suffix, var, val) \ 42 | ({ \ 43 | asm ("mov" suffix " %1, %%gs:%0" \ 44 | : "=m" (*__percpu_marker(var)) \ 45 | : "ir" (val)); \ 46 | }) 47 | 48 | #define percpu_set(var, val) \ 49 | ({ \ 50 | switch (sizeof(__percpu_type(var))) { \ 51 | case 1: __percpu_set("b", var, val); break; \ 52 | case 2: __percpu_set("w", var, val); break; \ 53 | case 4: __percpu_set("l", var, val); break; \ 54 | case 8: __percpu_set("q", var, val); break; \ 55 | default: _static_assert(false); \ 56 | } \ 57 | }) 58 | 59 | #define current (percpu_get(current_task)) 60 | 61 | extern struct percpu *percpu_table[]; 62 | 63 | static inline struct percpu *percpu_get_id(cpuid_t id) 64 | { 65 | return percpu_table[id]; 66 | } 67 | 68 | static inline struct percpu *percpu_self(void) 69 | { 70 | return percpu_table[percpu_get(id)]; 71 | } 72 | 73 | -------------------------------------------------------------------------------- /include/proc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "mm.h" 6 | #include "smp.h" 7 | #include "fs.h" 8 | #include "ds/rbtree.h" 9 | 10 | #define TASK_KTHREAD (1 << 0) 11 | #define TASK_RUNNING (1 << 1) 12 | #define TASK_PREEMPTED (1 << 2) 13 | #define TASK_NONE 0 14 | 15 | #define FORK_KTHREAD (1 << 0) 16 | #define FORK_UTHREAD (1 << 1) 17 | 18 | #define TID_IDLE 0 19 | 20 | // Preempt a program after 5 time slices 21 | #define MAX_SLICES 5 22 | 23 | typedef int32_t tgid_t; 24 | typedef int32_t tid_t; 25 | 26 | struct sched_entity { 27 | struct rb_node node; 28 | uint64_t vruntime; // Key for red-black tree 29 | uint32_t num_slices; // Number of time slices since was last preempted 30 | }; 31 | 32 | struct task { 33 | // Careful not to move these as they are referenced in asm 34 | virtaddr_t rsp_top; 35 | virtaddr_t rsp_original; 36 | struct mmu_info *mmu; // For kernel threads, this is &kernel_mmu 37 | uint64_t flags; 38 | 39 | // Process state 40 | enum task_state { 41 | TASK_S_NOT_STARTED, // Not started yet 42 | TASK_S_RUNNABLE, // Either running or ready to be run (running tasks have TASK_RUNNING flag set) 43 | TASK_S_WAITING, // On a wait queue somewhere (e.g has acquired a mutex) 44 | TASK_S_SLEEPING, // To be woken up at a specific time 45 | TASK_S_ZOMBIE, // Waiting to be reaped by parent 46 | TASK_S_DEAD // Totally dead and ought to be removed 47 | } state; 48 | 49 | // Task identifiers 50 | tid_t tid; 51 | tgid_t tgid; 52 | 53 | // VFS information 54 | struct path *root, *cwd; 55 | 56 | cpuset_t affinity; // Defines which processors this task can run on 57 | 58 | struct sched_entity sched; 59 | }; 60 | 61 | // Per-CPU task list 62 | struct runq { 63 | spinlock_t lock; 64 | struct task *idle; // Idle thread for this CPU 65 | struct rbtree tree; // List of threads on the run queue 66 | uint64_t num_threads; // Number of threads running on this CPU 67 | }; 68 | 69 | struct callee_regs { 70 | uint64_t rsp; 71 | uint64_t rbp; 72 | uint64_t rbx; 73 | uint64_t r12; 74 | uint64_t r13; 75 | uint64_t r14; 76 | uint64_t r15; 77 | }; 78 | 79 | void switch_to(struct task *); 80 | 81 | void schedule(void); 82 | void sched_init(void); 83 | void sched_run_bsp(void); 84 | void sched_run_ap(void); 85 | void sched_yield(void); // Called when a task voluntarily gives up control of the CPU 86 | void sched_yield_preempt(void); // Called when a task is preempted 87 | void sched_add(struct task *t); 88 | 89 | struct task *task_fork(struct task *parent, virtaddr_t entry, uint64_t flags, const struct callee_regs *regs); 90 | void task_execve(virtaddr_t function, char *argv[], unsigned int flags); 91 | void task_wakeup(struct task *t); 92 | void task_exit(struct task *t, int code); 93 | 94 | // Arguments to kthreads are passed in rbx 95 | #define create_kthread(entry, arg) ({ \ 96 | struct callee_regs tmp = { \ 97 | .rbx = (uint64_t)(arg) \ 98 | }; \ 99 | task_fork(current, (entry), FORK_KTHREAD, &tmp); }) 100 | 101 | void runq_init(struct task *); 102 | void runq_add(struct task *); 103 | void runq_remove(struct task *); 104 | void runq_balance_pull(void); 105 | struct task *runq_next(void); 106 | 107 | void idle_task(void); 108 | 109 | void init_kernel(void); 110 | -------------------------------------------------------------------------------- /include/smp.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | typedef uint32_t cpuset_t; 7 | typedef uint8_t cpuid_t; 8 | 9 | #include "interrupts.h" 10 | #include "atomic.h" 11 | #include "limits.h" 12 | #include "percpu.h" 13 | 14 | cpuid_t smp_cpu_id_full(void); 15 | void smp_init(void); 16 | void smp_ap_kmain(void); 17 | 18 | 19 | void ipi_send_fixed(cpuid_t, uint8_t vec); 20 | void ipi_sched_hint(struct isr_ctx *regs); 21 | void ipi_abort(struct isr_ctx *regs); 22 | void ipi_tlb_shootdown(struct isr_ctx *regs); 23 | 24 | void cpuset_init(cpuset_t *cpus); 25 | void cpuset_clear(cpuset_t *cpus); 26 | void cpuset_pin(cpuset_t *cpus); 27 | void cpuset_unpin(cpuset_t *cpus); 28 | bool cpuset_is_pinned(cpuset_t *cpus); 29 | void cpuset_copy(cpuset_t *dest, cpuset_t *src); 30 | bool cpuset_query_id(cpuset_t *cpus, cpuid_t id); 31 | void cpuset_set_id(cpuset_t *cpus, cpuid_t id, bool val); 32 | 33 | #ifdef VERBOSE 34 | void cpuset_dump(cpuset_t *cpus); 35 | #endif 36 | 37 | extern atomic32_t smp_nr_cpus_ready; 38 | 39 | static inline uint32_t smp_nr_cpus(void) 40 | { 41 | return atomic_read32(&smp_nr_cpus_ready); 42 | } 43 | 44 | static inline cpuid_t smp_cpu_id(void) 45 | { 46 | return percpu_get(id); 47 | } 48 | -------------------------------------------------------------------------------- /include/sync.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "asm.h" 6 | #include "atomic.h" 7 | 8 | typedef volatile uint64_t spinlock_t; 9 | 10 | typedef volatile struct { 11 | atomic64_t readers; 12 | spinlock_t rd_lock; 13 | spinlock_t wr_lock; 14 | } rwspinlock_t; 15 | 16 | void spin_lock(volatile spinlock_t *lock); 17 | void spin_unlock(volatile spinlock_t *lock); 18 | bool spin_try_lock(volatile spinlock_t *lock); 19 | 20 | #define spin_lock_irqsave(lock, rflags) ({ \ 21 | rflags = read_rflags(); \ 22 | cli(); \ 23 | spin_lock(lock); }) 24 | 25 | #define spin_unlock_irqsave(lock, rflags) ({ \ 26 | spin_unlock(lock); \ 27 | write_rflags(rflags); }) 28 | 29 | static inline void spin_init(volatile spinlock_t *lock) 30 | { 31 | *lock = 0; 32 | } 33 | 34 | static inline void read_spin_lock(volatile rwspinlock_t *lock) 35 | { 36 | spin_lock(&lock->rd_lock); 37 | uint64_t readers = atomic_inc_read64(&lock->readers); 38 | if (readers == 1) 39 | spin_lock(&lock->wr_lock); 40 | preempt_inc(); 41 | spin_unlock(&lock->rd_lock); 42 | } 43 | 44 | static inline void read_spin_unlock(volatile rwspinlock_t *lock) 45 | { 46 | spin_lock(&lock->rd_lock); 47 | uint64_t readers = atomic_dec_read64(&lock->readers); 48 | if (readers == 0) 49 | spin_unlock(&lock->wr_lock); 50 | preempt_dec(); 51 | spin_unlock(&lock->rd_lock); 52 | } 53 | 54 | static inline void write_spin_lock(volatile rwspinlock_t *lock) 55 | { 56 | spin_lock(&lock->wr_lock); 57 | } 58 | 59 | static inline void write_spin_unlock(volatile rwspinlock_t *lock) 60 | { 61 | spin_unlock(&lock->wr_lock); 62 | } 63 | -------------------------------------------------------------------------------- /include/syscall.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | void syscall_enable(void); 4 | 5 | typedef int64_t (*syscall_t)(void); 6 | 7 | #include "gen/syscall_gen.h" 8 | 9 | static inline int64_t execute_syscall(uint64_t num, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) 10 | { 11 | // Use dummy output variables because we need to clobber the input ones 12 | // https://stackoverflow.com/questions/48381184/can-i-modify-input-operands-in-gcc-inline-assembly 13 | int64_t rv; 14 | uint64_t dummy_rdi, dummy_rsi, dummy_rdx, dummy_rcx; 15 | asm volatile ( 16 | "syscall\n" 17 | : "=a"(rv), "=D"(dummy_rdi), "=S"(dummy_rsi), "=d"(dummy_rdx), "=c"(dummy_rcx) 18 | : "a"(num), "D"(arg1), "S"(arg2), "d"(arg3), "c"(arg4) 19 | : "r8", "r9", "r10", "r11" 20 | ); 21 | return rv; 22 | } 23 | -------------------------------------------------------------------------------- /include/types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | typedef uint32_t dev_t; 6 | typedef uint64_t ino_t; 7 | typedef uint32_t mode_t; 8 | -------------------------------------------------------------------------------- /include/util.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define MIN(x, y) ({ \ 4 | const typeof((x)) _x = (x); \ 5 | const typeof((y)) _y = (y); \ 6 | _x > _y ? _y : _x; }) 7 | 8 | #define MAX(x, y) ({ \ 9 | const typeof((x)) _x = (x); \ 10 | const typeof((y)) _y = (y); \ 11 | _x > _y ? _x : _y; }) 12 | 13 | #define ALIGNUP(val, align) ({ \ 14 | const typeof((val)) _val = (val); \ 15 | const typeof((align)) _align = (align); \ 16 | (_val + (_align - 1)) & -_align; }) 17 | 18 | #define ALIGNDOWN(val, align) ({ \ 19 | const typeof((val)) _val = (val); \ 20 | const typeof((align)) _align = (align); \ 21 | _val & ~(_align - 1); }) 22 | 23 | #define DIV_ROUND_UP(val, div) (((val) + (div) - 1) / (div)) 24 | 25 | #define ISALIGN_POW2(val, align) ({ \ 26 | const typeof((val)) _val = (val); \ 27 | const typeof((align)) _align = (align); \ 28 | ((uintptr_t)_val & ((uintptr_t)_align - 1)) == 0; }) 29 | 30 | #define UNUSED(x) x __attribute__((unused)) 31 | #define WARN_UNUSED __attribute__((warn_unused_result)) 32 | 33 | void __attribute__((error("Static assertion failed"))) __error_fn(void); 34 | #define _static_assert(cond) ({ \ 35 | do { \ 36 | if (!(cond)) \ 37 | __error_fn(); \ 38 | } while (0); }) 39 | -------------------------------------------------------------------------------- /iso/boot/grub/grub.cfg: -------------------------------------------------------------------------------- 1 | set timeout=0 2 | set default=0 3 | 4 | menuentry "Byte OS" { 5 | multiboot2 /boot/byteos.elf 6 | boot 7 | } 8 | -------------------------------------------------------------------------------- /kernel/cpu/entry.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | section .text 4 | global long_mode_entry 5 | long_mode_entry: 6 | ; Long mode doesn't care about most segments. 7 | ; GS and FS base addresses can be set with MSRs. 8 | mov ax, 0 9 | mov ss, ax 10 | mov ds, ax 11 | mov es, ax 12 | mov fs, ax 13 | mov gs, ax 14 | 15 | ; Align the stack to 16 bytes 16 | and rsp, ~0xF 17 | 18 | ; Multiboot structure (physical address) 19 | push rbx 20 | sub rsp, 8 21 | 22 | ; Initialise VGA textmode driver 23 | extern vga_tmode_init 24 | call vga_tmode_init 25 | 26 | ; Set up BSP per-CPU data area 27 | extern percpu_init_bsp 28 | call percpu_init_bsp 29 | 30 | ; Initialise scheduler (set dummy task) 31 | extern sched_init 32 | call sched_init 33 | 34 | ; Load interrupt descriptor table 35 | extern interrupts_init 36 | call interrupts_init 37 | 38 | ; Load TSS 39 | mov ax, GDT_TSS 40 | ltr ax 41 | 42 | ; Enable SSE, AVX, AVX-512 43 | extern simd_init 44 | call simd_init 45 | 46 | ; Enable syscall/sysret instruction 47 | extern syscall_enable 48 | call syscall_enable 49 | 50 | ; Call global constructors 51 | extern _init 52 | call _init 53 | 54 | ; Pass multiboot information to kmain 55 | add rsp, 8 56 | pop rdi 57 | extern kmain 58 | call kmain 59 | 60 | ; Don't call global destructors - we should never get here 61 | sti 62 | .end: 63 | hlt 64 | jmp .end 65 | 66 | -------------------------------------------------------------------------------- /kernel/cpu/exceptions.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | #include "proc.h" 3 | #include "percpu.h" 4 | #include "interrupts.h" 5 | #include "drivers/apic.h" 6 | 7 | #define INT_PAGE_FAULT 14 8 | 9 | #define PAGE_FAULT_RW (1 << 1) 10 | 11 | static const char *const exception_messages[32] = { 12 | "Division by zero", 13 | "Debug", 14 | "Non-maskable interrupt", 15 | "Breakpoint", 16 | "Overflow", 17 | "Bound range exceeded", 18 | "Invalid opcode", 19 | "Device not available", 20 | "Double fault", 21 | "(reserved exception 9)", 22 | "Invalid TSS", 23 | "Segment not present", 24 | "Stack segment fault", 25 | "General protection fault", 26 | "Page fault", 27 | "(reserved exception 15)", 28 | "x87 floating-point exception", 29 | "Alignment check", 30 | "Machine check", 31 | "SIMD floating-point exception", 32 | "Virtualization exception", 33 | "(reserved exception 21)", 34 | "(reserved exception 22)", 35 | "(reserved exception 23)", 36 | "(reserved exception 24)", 37 | "(reserved exception 25)", 38 | "(reserved exception 26)", 39 | "(reserved exception 27)", 40 | "(reserved exception 28)", 41 | "(reserved exception 29)", 42 | "(reserved exception 30)", 43 | "(reserved exception 31)" 44 | }; 45 | 46 | static void page_fault_handler(struct isr_ctx *regs) 47 | { 48 | uintptr_t faulting_address; 49 | asm volatile("mov %%cr2, %0" : "=r" (faulting_address)); 50 | 51 | // The fault was likely due to an access in kernel space, so give up 52 | if (faulting_address & (1ULL << 63)) 53 | goto kernel_panic; 54 | 55 | // If interrupts were enabled, we are safe to enable them again 56 | if (regs->rflags & 0x200) 57 | sti(); 58 | 59 | if (current != NULL) { 60 | virtaddr_t aligned_addr = (virtaddr_t)(faulting_address & ~(PAGE_SIZE - 1)); 61 | 62 | write_spin_lock(¤t->mmu->pgtab_lock); 63 | 64 | pte_t *pte = vmm_get_pte(current->mmu, aligned_addr); 65 | bool done = regs->info & PAGE_FAULT_RW && cow_handle_write(pte, aligned_addr); 66 | 67 | write_spin_unlock(¤t->mmu->pgtab_lock); 68 | 69 | if (done) 70 | return; 71 | } 72 | 73 | // TODO: Kill process 74 | 75 | kernel_panic: 76 | // Otherwise, the write was not allowed, so we panic 77 | panic( 78 | "Page fault:\n" 79 | "\tfaulting address: %p\n" 80 | "\trip: %p, rsp: %p\n" 81 | "\terr_code: %lx", 82 | (void *)faulting_address, 83 | (void *)regs->rip, (void *)regs->rsp, 84 | (regs->info & 0xFFFFFFFF) 85 | ); 86 | } 87 | 88 | static void exception_handler(struct isr_ctx *regs) 89 | { 90 | uint8_t int_no = (uint8_t)(regs->info >> 32); 91 | switch (int_no) { 92 | case INT_PAGE_FAULT: 93 | __builtin_unreachable(); 94 | break; 95 | case IRQ_NMI: 96 | int_no = 2; 97 | // fallthrough 98 | default: 99 | panic( 100 | "%s:\n" 101 | "\trip: %p, rsp: %p\n" 102 | "\tint_no: %u, err_code: %lu", 103 | exception_messages[int_no], 104 | (void *)regs->rip, (void *)regs->rsp, 105 | int_no, (regs->info & 0xFFFFFFFF) 106 | ); 107 | } 108 | } 109 | 110 | void exceptions_init(void) 111 | { 112 | for (uint8_t i = 0; i < 32; i++) { 113 | struct isr_info info = { 114 | .type = ISR_EXCEPTION, 115 | .handler = i == INT_PAGE_FAULT ? exception_handler : page_fault_handler, 116 | }; 117 | 118 | isr_set_info(i, &info); 119 | } 120 | 121 | // Set handler for NMI 122 | struct isr_info info = { 123 | .type = ISR_EXCEPTION, 124 | .handler = exception_handler, 125 | }; 126 | 127 | isr_set_info(IRQ_NMI, &info); 128 | } 129 | 130 | -------------------------------------------------------------------------------- /kernel/cpu/flush.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | section .text 4 | global flush_gdt_tss 5 | flush_gdt_tss: 6 | ; rdi: gdt 7 | ; rsi: GDT_SIZE 8 | ; rdx: tss 9 | 10 | sub rsp, 10 11 | mov word [rsp], si 12 | mov qword [rsp + 2], rdi 13 | 14 | lgdt [rsp] 15 | mov ax, GDT_TSS 16 | ltr ax 17 | 18 | add rsp, 10 19 | ret 20 | 21 | 22 | -------------------------------------------------------------------------------- /kernel/cpu/interrupts.c: -------------------------------------------------------------------------------- 1 | #include "interrupts.h" 2 | 3 | void interrupts_init(void); 4 | 5 | void interrupts_init(void) 6 | { 7 | // Create and fill the IDT 8 | idt_init(); 9 | 10 | // Initialise ISR table and mask IRQs 11 | isr_init(); 12 | 13 | // Write exception handlers into ISR table 14 | exceptions_init(); 15 | 16 | // Execute 'lidt' instruction 17 | idt_load(); 18 | } 19 | -------------------------------------------------------------------------------- /kernel/cpu/isr.c: -------------------------------------------------------------------------------- 1 | #include "interrupts.h" 2 | #include "drivers/apic.h" 3 | #include "proc.h" 4 | #include "libk.h" 5 | #include "asm.h" 6 | #include "util.h" 7 | 8 | static struct isr_info isr_table[256]; 9 | 10 | void isr_init(void) 11 | { 12 | // Remap the PIC to interrupts 0x20-0x2F 13 | outb(0x20, 0x11); 14 | outb(0xA0, 0x11); 15 | outb(0x21, 0x20); 16 | outb(0xA1, 0x28); 17 | outb(0x21, 0x04); 18 | outb(0xA1, 0x02); 19 | outb(0x21, 0x01); 20 | outb(0xA1, 0x01); 21 | outb(0x21, 0x00); 22 | outb(0xA1, 0x00); 23 | 24 | // Disable the PIC by masking all interrupts 25 | outb(0xA1, 0xFF); 26 | outb(0x21, 0xFF); 27 | 28 | memset(isr_table, 0, sizeof isr_table); 29 | 30 | irq_disable(); 31 | } 32 | 33 | void isr_set_info(uint8_t vec, struct isr_info *info) 34 | { 35 | memcpy(&isr_table[vec], info, sizeof(struct isr_info)); 36 | } 37 | 38 | void isr_irq_mask(uint8_t vec) 39 | { 40 | kassert_dbg(vec >= IRQ_APIC_BASE); 41 | ioapic_mask(vec - IRQ_APIC_BASE); 42 | } 43 | 44 | void isr_irq_unmask(uint8_t vec) 45 | { 46 | kassert_dbg(vec >= IRQ_APIC_BASE); 47 | ioapic_unmask(vec - IRQ_APIC_BASE); 48 | } 49 | 50 | void isr_global_handler(struct isr_ctx *regs) 51 | { 52 | uint8_t int_no = regs->info & 0xFF; 53 | struct isr_info *info = &isr_table[int_no]; 54 | uint32_t int_depth = 0; 55 | 56 | if (info->type == ISR_IRQ) { 57 | atomic_inc_read32(&percpu_self()->int_depth); 58 | lapic_eoi(int_no); 59 | } 60 | 61 | if (info->handler != NULL) 62 | info->handler(regs); 63 | 64 | if (info->type == ISR_IRQ) 65 | int_depth = atomic_dec_read32(&percpu_self()->int_depth); 66 | 67 | // TODO: Check if task was killed 68 | 69 | // Only preempt if we are about to return to a non-IRQ 70 | if (int_depth == 0 && percpu_get(reschedule)) 71 | sched_yield_preempt(); 72 | 73 | // TODO: Check if task was killed 74 | } 75 | -------------------------------------------------------------------------------- /kernel/cpu/isr_stubs.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | ; Interrupt descriptor table 4 | section .data 5 | align 16 6 | global idt64 7 | idt64: 8 | times 256 dq 0, 0 9 | .pointer: 10 | dw $ - idt64 - 1 11 | dq idt64 12 | 13 | section .text 14 | global idt_load 15 | idt_load: 16 | mov rax, idt64.pointer 17 | lidt [rax] 18 | ret 19 | 20 | isr_common: 21 | push rdi 22 | push rsi 23 | push rdx 24 | push rcx 25 | push rax 26 | push r8 27 | push r9 28 | push r10 29 | push r11 30 | 31 | ; Call the ISR handler 32 | mov rdi, rsp 33 | extern isr_global_handler 34 | call isr_global_handler 35 | 36 | pop r11 37 | pop r10 38 | pop r9 39 | pop r8 40 | pop rax 41 | pop rcx 42 | pop rdx 43 | pop rsi 44 | pop rdi 45 | add rsp, 8 ; Info field 46 | iretq 47 | 48 | %macro isr_stub_err 1 49 | global isr_stub_%1 50 | isr_stub_%1: 51 | ; Store the interrupt number in the highest four bytes of the error code 52 | ; This way we can always increment rsp by 8 before iretq and no memory is wasted. 53 | mov dword [rsp + 4], %1 54 | jmp isr_common 55 | %endmacro 56 | 57 | %macro isr_stub_noerr 1 58 | global isr_stub_%1 59 | isr_stub_%1: 60 | ; For consistency with the err variant 61 | push qword 0 62 | mov dword [rsp + 4], %1 63 | jmp isr_common 64 | %endmacro 65 | 66 | %macro isr_stub_irq 1 67 | global isr_stub_%1 68 | isr_stub_%1: 69 | push qword %1 70 | jmp isr_common 71 | %endmacro 72 | 73 | %macro isr_stub_nop 1 74 | global isr_stub_%1 75 | isr_stub_%1: 76 | ; TODO: Should we call the handler anyway? 77 | iretq 78 | %endmacro 79 | 80 | %macro isr_stub_ipi 2 81 | global isr_stub_%1 82 | isr_stub_%1: 83 | push qword %1 84 | jmp isr_common 85 | %endmacro 86 | 87 | ; Exceptions 88 | isr_stub_noerr 0 89 | isr_stub_noerr 1 90 | isr_stub_noerr 2 91 | isr_stub_noerr 3 92 | isr_stub_noerr 4 93 | isr_stub_noerr 5 94 | isr_stub_noerr 6 95 | isr_stub_noerr 7 96 | isr_stub_err 8 97 | isr_stub_noerr 9 98 | isr_stub_err 10 99 | isr_stub_err 11 100 | isr_stub_err 12 101 | isr_stub_err 13 102 | isr_stub_err 14 103 | isr_stub_noerr 15 104 | isr_stub_noerr 16 105 | isr_stub_err 17 106 | isr_stub_noerr 18 107 | isr_stub_noerr 19 108 | isr_stub_noerr 20 109 | isr_stub_noerr 21 110 | isr_stub_noerr 22 111 | isr_stub_noerr 23 112 | isr_stub_noerr 24 113 | isr_stub_noerr 25 114 | isr_stub_noerr 26 115 | isr_stub_noerr 27 116 | isr_stub_noerr 28 117 | isr_stub_noerr 29 118 | isr_stub_err 30 119 | isr_stub_noerr 31 120 | 121 | ; Unused PIC interrupts 122 | isr_stub_nop 32 123 | isr_stub_nop 33 124 | isr_stub_nop 34 125 | isr_stub_nop 35 126 | isr_stub_nop 36 127 | isr_stub_nop 37 128 | isr_stub_nop 38 129 | isr_stub_nop 39 130 | isr_stub_nop 40 131 | isr_stub_nop 41 132 | isr_stub_nop 42 133 | isr_stub_nop 43 134 | isr_stub_nop 44 135 | isr_stub_nop 45 136 | isr_stub_nop 46 137 | isr_stub_nop 47 138 | 139 | ; Other (IRQs) 140 | isr_stub_irq 48 141 | isr_stub_irq 49 142 | isr_stub_irq 50 143 | isr_stub_irq 51 144 | isr_stub_irq 52 145 | isr_stub_irq 53 146 | isr_stub_irq 54 147 | isr_stub_irq 55 148 | isr_stub_irq 56 149 | isr_stub_irq 57 150 | isr_stub_irq 58 151 | isr_stub_irq 59 152 | isr_stub_irq 60 153 | isr_stub_irq 61 154 | isr_stub_irq 62 155 | isr_stub_irq 63 156 | isr_stub_irq 64 157 | isr_stub_irq 65 158 | isr_stub_irq 66 159 | isr_stub_irq 67 160 | isr_stub_irq 68 161 | isr_stub_irq 69 162 | isr_stub_irq 70 163 | isr_stub_irq 71 164 | isr_stub_irq 72 165 | isr_stub_irq 73 166 | isr_stub_irq 74 167 | isr_stub_irq 75 168 | isr_stub_irq 76 169 | isr_stub_irq 77 170 | isr_stub_irq 78 171 | isr_stub_irq 79 172 | isr_stub_irq 80 173 | isr_stub_irq 81 174 | isr_stub_irq 82 175 | isr_stub_irq 83 176 | isr_stub_irq 84 177 | isr_stub_irq 85 178 | isr_stub_irq 86 179 | isr_stub_irq 87 180 | isr_stub_irq 88 181 | isr_stub_irq 89 182 | isr_stub_irq 90 183 | isr_stub_irq 91 184 | isr_stub_irq 92 185 | isr_stub_irq 93 186 | isr_stub_irq 94 187 | isr_stub_irq 95 188 | isr_stub_irq 96 189 | isr_stub_irq 97 190 | isr_stub_irq 98 191 | isr_stub_irq 99 192 | isr_stub_irq 100 193 | isr_stub_irq 101 194 | isr_stub_irq 102 195 | isr_stub_irq 103 196 | isr_stub_irq 104 197 | isr_stub_irq 105 198 | isr_stub_irq 106 199 | isr_stub_irq 107 200 | isr_stub_irq 108 201 | isr_stub_irq 109 202 | isr_stub_irq 110 203 | isr_stub_irq 111 204 | isr_stub_irq 112 205 | isr_stub_irq 113 206 | isr_stub_irq 114 207 | isr_stub_irq 115 208 | isr_stub_irq 116 209 | isr_stub_irq 117 210 | isr_stub_irq 118 211 | isr_stub_irq 119 212 | isr_stub_irq 120 213 | isr_stub_irq 121 214 | isr_stub_irq 122 215 | isr_stub_irq 123 216 | isr_stub_irq 124 217 | isr_stub_irq 125 218 | isr_stub_irq 126 219 | isr_stub_irq 127 220 | isr_stub_irq 128 221 | isr_stub_irq 129 222 | isr_stub_irq 130 223 | isr_stub_irq 131 224 | isr_stub_irq 132 225 | isr_stub_irq 133 226 | isr_stub_irq 134 227 | isr_stub_irq 135 228 | isr_stub_irq 136 229 | isr_stub_irq 137 230 | isr_stub_irq 138 231 | isr_stub_irq 139 232 | isr_stub_irq 140 233 | isr_stub_irq 141 234 | isr_stub_irq 142 235 | isr_stub_irq 143 236 | isr_stub_irq 144 237 | isr_stub_irq 145 238 | isr_stub_irq 146 239 | isr_stub_irq 147 240 | isr_stub_irq 148 241 | isr_stub_irq 149 242 | isr_stub_irq 150 243 | isr_stub_irq 151 244 | isr_stub_irq 152 245 | isr_stub_irq 153 246 | isr_stub_irq 154 247 | isr_stub_irq 155 248 | isr_stub_irq 156 249 | isr_stub_irq 157 250 | isr_stub_irq 158 251 | isr_stub_irq 159 252 | isr_stub_irq 160 253 | isr_stub_irq 161 254 | isr_stub_irq 162 255 | isr_stub_irq 163 256 | isr_stub_irq 164 257 | isr_stub_irq 165 258 | isr_stub_irq 166 259 | isr_stub_irq 167 260 | isr_stub_irq 168 261 | isr_stub_irq 169 262 | isr_stub_irq 170 263 | isr_stub_irq 171 264 | isr_stub_irq 172 265 | isr_stub_irq 173 266 | isr_stub_irq 174 267 | isr_stub_irq 175 268 | isr_stub_irq 176 269 | isr_stub_irq 177 270 | isr_stub_irq 178 271 | isr_stub_irq 179 272 | isr_stub_irq 180 273 | isr_stub_irq 181 274 | isr_stub_irq 182 275 | isr_stub_irq 183 276 | isr_stub_irq 184 277 | isr_stub_irq 185 278 | isr_stub_irq 186 279 | isr_stub_irq 187 280 | isr_stub_irq 188 281 | isr_stub_irq 189 282 | isr_stub_irq 190 283 | isr_stub_irq 191 284 | isr_stub_irq 192 285 | isr_stub_irq 193 286 | isr_stub_irq 194 287 | isr_stub_irq 195 288 | isr_stub_irq 196 289 | isr_stub_irq 197 290 | isr_stub_irq 198 291 | isr_stub_irq 199 292 | isr_stub_irq 200 293 | isr_stub_irq 201 294 | isr_stub_irq 202 295 | isr_stub_irq 203 296 | isr_stub_irq 204 297 | isr_stub_irq 205 298 | isr_stub_irq 206 299 | isr_stub_irq 207 300 | isr_stub_irq 208 301 | isr_stub_irq 209 302 | isr_stub_irq 210 303 | isr_stub_irq 211 304 | isr_stub_irq 212 305 | isr_stub_irq 213 306 | isr_stub_irq 214 307 | isr_stub_irq 215 308 | isr_stub_irq 216 309 | isr_stub_irq 217 310 | isr_stub_irq 218 311 | isr_stub_irq 219 312 | isr_stub_irq 220 313 | isr_stub_irq 221 314 | isr_stub_irq 222 315 | isr_stub_irq 223 316 | isr_stub_irq 224 317 | isr_stub_irq 225 318 | isr_stub_irq 226 319 | isr_stub_irq 227 320 | isr_stub_irq 228 321 | isr_stub_irq 229 322 | isr_stub_irq 230 323 | isr_stub_irq 231 324 | isr_stub_irq 232 325 | isr_stub_irq 233 326 | isr_stub_irq 234 327 | isr_stub_irq 235 328 | isr_stub_irq 236 329 | isr_stub_irq 237 330 | isr_stub_irq 238 331 | isr_stub_irq 239 332 | isr_stub_irq 240 333 | isr_stub_irq 241 334 | isr_stub_irq 242 335 | isr_stub_irq 243 336 | isr_stub_irq 244 337 | isr_stub_irq 245 338 | isr_stub_irq 246 339 | isr_stub_irq 247 340 | 341 | ; Careful changing these! They must match the values in interrupts.h 342 | isr_stub_ipi 248, sched_hint 343 | isr_stub_ipi 249, tlb_shootdown 344 | isr_stub_ipi 250, abort 345 | isr_stub_noerr 251 ; NMI 346 | isr_stub_irq 252 ; LINT0 347 | isr_stub_irq 253 ; LINT1 348 | isr_stub_irq 254 ; LAPIC timer 349 | isr_stub_nop 255 ; APIC Spurious interrupt vector 350 | -------------------------------------------------------------------------------- /kernel/cpu/percpu.c: -------------------------------------------------------------------------------- 1 | #include "asm.h" 2 | #include "mm.h" 3 | #include "smp.h" 4 | #include "percpu.h" 5 | #include "libk.h" 6 | 7 | extern uint64_t tss64[]; 8 | 9 | struct percpu *percpu_table[MAX_CORES] = { 0 }; 10 | 11 | static struct percpu bsp_percpu = { 12 | .tss = tss64 13 | }; 14 | 15 | static virtaddr_t load_gdt_tss(void) 16 | { 17 | extern virtaddr_t gdt64; 18 | 19 | // Load GDT 20 | const size_t GDT_SIZE = 0x40; 21 | virtaddr_t gdt = kmalloc(GDT_SIZE, KM_NONE); 22 | memcpy(gdt, &gdt64, GDT_SIZE); 23 | 24 | const size_t TSS_SIZE = 0x68; 25 | virtaddr_t tss = kmalloc(TSS_SIZE, KM_NONE); 26 | memset(tss, 0, TSS_SIZE); 27 | // TODO: Create IST entries for NMI, double fault 28 | 29 | // Set TSS descriptor in GDT 30 | uint8_t tmp[8]; memcpy(tmp, &tss, sizeof(virtaddr_t)); 31 | uint8_t *gdt_tss_desc = (uint8_t *)gdt + 0x30; 32 | gdt_tss_desc[2] = tmp[0]; 33 | gdt_tss_desc[3] = tmp[1]; 34 | gdt_tss_desc[4] = tmp[2]; 35 | gdt_tss_desc[5] = 0x89; // Type 36 | gdt_tss_desc[7] = tmp[3]; 37 | gdt_tss_desc[8] = tmp[4]; 38 | gdt_tss_desc[9] = tmp[5]; 39 | gdt_tss_desc[10] = tmp[6]; 40 | gdt_tss_desc[11] = tmp[7]; 41 | 42 | // Load GDT and TSS 43 | extern void flush_gdt_tss(virtaddr_t, uint16_t, virtaddr_t); 44 | flush_gdt_tss(gdt, GDT_SIZE - 1, tss); 45 | 46 | return tss; 47 | } 48 | 49 | void percpu_init_ap(void) 50 | { 51 | // Initialise a temp struct just for the kmalloc call (since it will lock) 52 | struct percpu tmp = { 0 }; 53 | percpu_set_addr(&tmp); 54 | 55 | struct percpu *cpu = kmalloc(sizeof *cpu, KM_NONE); 56 | cpu->id = smp_cpu_id_full(); 57 | cpu->current_task = NULL; 58 | cpu->rsp_scratch = NULL; 59 | cpu->preempt_count = 0; 60 | cpu->tss = load_gdt_tss(); 61 | percpu_table[cpu->id] = cpu; 62 | percpu_set_addr(cpu); 63 | } 64 | 65 | void percpu_init_bsp(void) 66 | { 67 | percpu_table[0] = &bsp_percpu; 68 | percpu_set_addr(&bsp_percpu); 69 | } 70 | 71 | void percpu_set_addr(struct percpu *p) 72 | { 73 | // MSR_GS_BASE 74 | msr_write(0xC0000101, (uint64_t)p); 75 | } 76 | -------------------------------------------------------------------------------- /kernel/cpu/simd.asm: -------------------------------------------------------------------------------- 1 | ; Initialises SSE, AVX and AVX-512 (if available) 2 | ; TODO: Test this 3 | global simd_init 4 | simd_init: 5 | push rax 6 | push rbx 7 | push rcx 8 | push rdx 9 | 10 | ; Detect XSAVE and AVX 11 | mov rax, 1 12 | xor rcx, rcx 13 | cpuid 14 | and ecx, (1 << 26) | (1 << 28) 15 | cmp ecx, (1 << 26) | (1 << 28) 16 | jne .done 17 | 18 | ; Enable XSAVE 19 | mov rax, cr4 20 | or eax, (1 << 18) 21 | mov cr4, rax 22 | 23 | ; Enable AVX 24 | mov rcx, 0 25 | xgetbv 26 | or eax, (1 << 1) | (1 << 2) ; SSE, AVX bits 27 | mov rcx, 0 28 | xsetbv 29 | 30 | ; TODO: AVX-512 31 | .done: 32 | pop rdx 33 | pop rcx 34 | pop rbx 35 | pop rax 36 | ret 37 | -------------------------------------------------------------------------------- /kernel/crt/crtbegin.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/64/ByteOS/0d88696beee91e512b710f86aaf551bdc3aa00d0/kernel/crt/crtbegin.o -------------------------------------------------------------------------------- /kernel/crt/crtend.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/64/ByteOS/0d88696beee91e512b710f86aaf551bdc3aa00d0/kernel/crt/crtend.o -------------------------------------------------------------------------------- /kernel/crt/crti.asm: -------------------------------------------------------------------------------- 1 | bits 64 2 | section .init 3 | global _init 4 | _init: 5 | push rbp 6 | mov rbp, rsp 7 | 8 | section .fini 9 | global _fini 10 | _fini: 11 | push rbp 12 | mov rbp, rsp 13 | -------------------------------------------------------------------------------- /kernel/crt/crtn.asm: -------------------------------------------------------------------------------- 1 | bits 64 2 | section .init 3 | pop rbp 4 | ret 5 | 6 | section .fini 7 | pop rbp 8 | ret 9 | -------------------------------------------------------------------------------- /kernel/drivers/acpi/acpi.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "libk.h" 4 | #include "mm.h" 5 | #include "drivers/acpi.h" 6 | 7 | #define RSDP_SEARCH_START 0xE0000 8 | #define RSDP_SEARCH_END 0xFFFFF 9 | 10 | struct rsdp { 11 | char signature[8]; 12 | uint8_t checksum; 13 | char oem_id[6]; 14 | uint8_t revision; 15 | uint32_t rsdt_address; 16 | uint32_t length; 17 | uint64_t xsdt_address; 18 | uint8_t ext_checksum; 19 | uint8_t reserved[3]; 20 | } __attribute__((packed)); 21 | 22 | struct rsdt { 23 | struct acpi_header header; 24 | uint32_t tables[]; 25 | } __attribute__((packed)); 26 | 27 | struct xsdt { 28 | struct acpi_header header; 29 | physaddr_t tables[]; 30 | } __attribute__((packed)); 31 | 32 | struct rsdp *acpi_rsdp; 33 | 34 | static uint8_t calc_checksum(uint8_t *bytes, size_t len) 35 | { 36 | uint8_t sum = 0; 37 | for (size_t i = 0; i < len; i++) 38 | sum += bytes[i]; 39 | return sum; 40 | } 41 | 42 | static struct rsdp *find_rsdp(void) 43 | { 44 | for (physaddr_t phys = RSDP_SEARCH_START; phys < RSDP_SEARCH_END; phys += 16) { 45 | virtaddr_t virt = phys_to_virt(phys); 46 | if (memcmp("RSD PTR ", virt, 8) == 0) { 47 | struct rsdp *rsdp = virt; 48 | if (rsdp->revision == 0) { // Version 1 49 | if (calc_checksum(virt, sizeof(struct rsdp) - 16) != 0) 50 | continue; 51 | } else if (rsdp->revision == 2) { // Version 2+ 52 | if (calc_checksum(virt, sizeof(struct rsdp)) != 0) 53 | continue; 54 | } else 55 | continue; 56 | return rsdp; 57 | } 58 | } 59 | return NULL; 60 | } 61 | 62 | // Every table we access needs to be called here first 63 | static void map_table(struct acpi_header *hd) 64 | { 65 | // Map the first two pages for safety, then map the rest of the table 66 | // The header could cross a page boundary and generate a page fault. 67 | vmm_map_page(&kernel_mmu, virt_to_phys(hd), hd, VMM_ALLOC_MMAP | PAGE_GLOBAL); 68 | vmm_map_page(&kernel_mmu, virt_to_phys(hd) + sizeof(struct acpi_header), 69 | (virtaddr_t)((uintptr_t)hd + sizeof(struct acpi_header)), VMM_ALLOC_MMAP | PAGE_GLOBAL); 70 | 71 | // Safe to dereference header now 72 | 73 | // Map the rest of the table 74 | for (size_t i = sizeof(struct acpi_header) + PAGE_SIZE; i < hd->length; i += PAGE_SIZE) { 75 | virtaddr_t virt = (virtaddr_t)((uintptr_t)hd + i); 76 | physaddr_t phys = virt_to_phys(virt); 77 | vmm_map_page(&kernel_mmu, phys, virt, VMM_ALLOC_MMAP | PAGE_GLOBAL); 78 | } 79 | 80 | // Safe to access the whole table now 81 | } 82 | 83 | static inline bool table_checksum(struct acpi_header *hd) 84 | { 85 | return calc_checksum((uint8_t *)hd, hd->length) == 0; 86 | } 87 | 88 | static inline void print_table(struct acpi_header *hd) 89 | { 90 | klog("acpi", "%c%c%c%c at %p, OEM %c%c%c%c%c%c\n", 91 | hd->signature[0], hd->signature[1], hd->signature[2], hd->signature[3], (virtaddr_t)virt_to_phys(hd), 92 | hd->oem_id[0], hd->oem_id[1], hd->oem_id[2], hd->oem_id[3], hd->oem_id[4], hd->oem_id[5]); 93 | klog_verbose("acpi", " Length: %u\n", hd->length); 94 | klog_verbose("acpi", " Revision: %u\n", hd->revision); 95 | klog_verbose("acpi", " OEM Table ID: %c%c%c%c%c%c%c%c\n", hd->oem_table_id[0], hd->oem_table_id[1], hd->oem_table_id[2], 96 | hd->oem_table_id[3], hd->oem_table_id[4], hd->oem_table_id[5], hd->oem_table_id[6], hd->oem_table_id[7]); 97 | klog_verbose("acpi", " OEM Revision: %u\n", hd->oem_revision); 98 | klog_verbose("acpi", " Creator ID: %c%c%c%c\n", hd->creator_id[0], hd->creator_id[1], hd->creator_id[2], hd->creator_id[3]); 99 | klog_verbose("acpi", " Creator Revision: %u\n", hd->creator_revision); 100 | } 101 | 102 | static void acpi_dump_tables(void) 103 | { 104 | if (acpi_rsdp->revision == 0) { 105 | // Use ACPI v1 RSDT 106 | struct rsdt *rsdt = phys_to_virt(acpi_rsdp->rsdt_address); 107 | map_table(&rsdt->header); 108 | if (!table_checksum(&rsdt->header)) 109 | panic("acpi: RSDT checksum failed"); 110 | size_t num_entries = (rsdt->header.length - sizeof(struct acpi_header)) / sizeof(uint32_t); 111 | for (size_t i = 0; i < num_entries; i++) { 112 | struct acpi_header *hd = phys_to_virt(rsdt->tables[i]); 113 | map_table(hd); 114 | print_table(hd); 115 | } 116 | } else if (acpi_rsdp->revision == 2) { 117 | // Use ACPI v2+ XSDT 118 | struct xsdt *xsdt = phys_to_virt(acpi_rsdp->xsdt_address); 119 | map_table(&xsdt->header); 120 | if (!table_checksum(&xsdt->header)) 121 | panic("acpi: XSDT checksum failed"); 122 | size_t num_entries = (xsdt->header.length - sizeof(struct acpi_header)) / sizeof(physaddr_t); 123 | for (size_t i = 0; i < num_entries; i++) { 124 | struct acpi_header *hd = phys_to_virt(xsdt->tables[i]); 125 | map_table(hd); 126 | print_table(hd); 127 | } 128 | } 129 | } 130 | 131 | virtaddr_t acpi_find_table(char *signature) 132 | { 133 | // No need for map and checksum, we already did that above 134 | if (acpi_rsdp->revision == 0) { 135 | // Use ACPI v1 RSDT 136 | struct rsdt *rsdt = phys_to_virt(acpi_rsdp->rsdt_address); 137 | size_t num_entries = (rsdt->header.length - sizeof(struct acpi_header)) / sizeof(uint32_t); 138 | for (size_t i = 0; i < num_entries; i++) { 139 | struct acpi_header *hd = phys_to_virt(rsdt->tables[i]); 140 | if (memcmp(hd->signature, signature, 4) == 0 && table_checksum(hd)) 141 | return hd; 142 | } 143 | return NULL; 144 | } else if (acpi_rsdp->revision == 2) { 145 | // Use ACPI v2+ XSDT 146 | struct xsdt *xsdt = phys_to_virt(acpi_rsdp->xsdt_address); 147 | size_t num_entries = (xsdt->header.length - sizeof(struct acpi_header)) / sizeof(physaddr_t); 148 | for (size_t i = 0; i < num_entries; i++) { 149 | struct acpi_header *hd = phys_to_virt(xsdt->tables[i]); 150 | if (memcmp(hd->signature, signature, 4) == 0 && table_checksum(hd)) 151 | return hd; 152 | } 153 | return NULL; 154 | } else 155 | return NULL; 156 | } 157 | 158 | void acpi_init(void) 159 | { 160 | acpi_rsdp = find_rsdp(); 161 | if (acpi_rsdp == NULL) 162 | panic("ACPI: no RSDP found"); 163 | acpi_dump_tables(); 164 | } 165 | -------------------------------------------------------------------------------- /kernel/drivers/apic/apic.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "libk.h" 3 | #include "util.h" 4 | #include "interrupts.h" 5 | #include "mm.h" 6 | #include "drivers/apic.h" 7 | 8 | // Thanks to https://nemez.net/osdev/index.html for the information and sample code 9 | 10 | struct lapic_info lapic_list[MAX_LAPICS]; 11 | struct madt_entry_ioapic *ioapic_list[MAX_IOAPICS]; 12 | struct madt_entry_override *override_list[MAX_OVERRIDES]; 13 | struct madt_entry_nmi *nmi_list[MAX_NMIS]; 14 | size_t lapic_list_size; 15 | size_t ioapic_list_size; 16 | size_t override_list_size; 17 | size_t nmi_list_size; 18 | 19 | static void add_lapic(struct madt_entry_lapic *entry) 20 | { 21 | if (lapic_list_size >= MAX_LAPICS) 22 | return; 23 | lapic_list[lapic_list_size].present = (entry->flags != 0); 24 | lapic_list[lapic_list_size].id = entry->apic_id; 25 | lapic_list[lapic_list_size].acpi_id = entry->acpi_id; 26 | lapic_list_size++; 27 | klog_verbose("apic", "Detected local APIC, id %d\n", entry->apic_id); 28 | } 29 | 30 | static void add_ioapic(struct madt_entry_ioapic *entry) 31 | { 32 | if (ioapic_list_size >= MAX_IOAPICS) 33 | return; 34 | // Map the I/O APIC base so we can access it 35 | vmm_map_page(&kernel_mmu, entry->phys_addr, phys_to_virt(entry->phys_addr), VMM_ALLOC_MMAP | PAGE_GLOBAL | PAGE_DISABLE_CACHE | PAGE_WRITABLE); 36 | ioapic_list[ioapic_list_size++] = entry; 37 | klog_verbose("apic", "Detected I/O APIC at %p, id %d\n", (void *)(uintptr_t)entry->phys_addr, entry->apic_id); 38 | } 39 | 40 | static void add_override(struct madt_entry_override *entry) 41 | { 42 | if (override_list_size >= MAX_OVERRIDES) 43 | return; 44 | override_list[override_list_size++] = entry; 45 | klog_verbose("apic", "GSI %d overrides IRQ %u, flags %x\n", entry->gsi, entry->source, entry->flags); 46 | } 47 | 48 | static void add_nmi(struct madt_entry_nmi *entry) 49 | { 50 | if (nmi_list_size >= MAX_NMIS) 51 | return; 52 | nmi_list[nmi_list_size++] = entry; 53 | if (entry->acpi_id == 0xFF) 54 | klog_verbose("apic", "NMI for all CPUs, LINT%d\n", entry->lint_num); 55 | else 56 | klog_verbose("apic", "NMI for CPU %d, LINT%d\n", entry->acpi_id, entry->lint_num); 57 | 58 | } 59 | 60 | static void parse_madt(struct acpi_madt *madt) 61 | { 62 | // Default LAPIC address (might be overridden by entry type 5) 63 | virtaddr_t tmp_lapic_base = phys_to_virt(madt->lapic_address); 64 | 65 | // Parse all the other entries 66 | struct madt_entry_header *hd = (struct madt_entry_header *)&madt->entries; 67 | struct madt_entry_header *end = (struct madt_entry_header *)((uintptr_t)madt + madt->header.length); 68 | 69 | while (hd < end) { 70 | switch (hd->type) { 71 | case MADT_LAPIC: 72 | add_lapic((struct madt_entry_lapic *)hd); 73 | break; 74 | case MADT_IOAPIC: 75 | add_ioapic((struct madt_entry_ioapic *)hd); 76 | break; 77 | case MADT_OVERRIDE: 78 | add_override((struct madt_entry_override *)hd); 79 | break; 80 | case MADT_NMI: 81 | add_nmi((struct madt_entry_nmi *)hd); 82 | break; 83 | case MADT_LAPIC_ADDR: 84 | tmp_lapic_base = phys_to_virt(((struct madt_entry_lapic_addr *)hd)->lapic_addr); 85 | break; 86 | default: 87 | klog_warn("apic", "Unrecognised entry type %d in MADT\n", hd->type); 88 | break; 89 | } 90 | hd = (struct madt_entry_header *)((uintptr_t)hd + hd->length); 91 | } 92 | 93 | lapic_base = tmp_lapic_base; 94 | klog_verbose("apic", "Local APIC base at %p\n", lapic_base); 95 | klog("apic", "Detected %zu CPUs\n", lapic_list_size); 96 | 97 | // Map the APIC base so we can access it 98 | vmm_map_page(&kernel_mmu, virt_to_phys(lapic_base), lapic_base, VMM_ALLOC_MMAP | PAGE_GLOBAL | PAGE_DISABLE_CACHE | PAGE_WRITABLE); 99 | } 100 | 101 | void apic_init(void) 102 | { 103 | struct acpi_madt *madt = acpi_find_table("APIC"); 104 | if (madt == NULL) 105 | panic("no MADT found in ACPI tables"); 106 | 107 | parse_madt(madt); 108 | } 109 | -------------------------------------------------------------------------------- /kernel/drivers/apic/ioapic.c: -------------------------------------------------------------------------------- 1 | #include "mm.h" 2 | #include "libk.h" 3 | #include "util.h" 4 | #include "interrupts.h" 5 | #include "drivers/acpi.h" 6 | #include "drivers/apic.h" 7 | 8 | #define APIC_REG_ID 0 9 | #define APIC_REG_VER 1 10 | #define APIC_REG_REDTBL 16 11 | 12 | #define APIC_IRQ_MASK 0x10000 13 | 14 | static spinlock_t ioapic_lock; 15 | 16 | // Careful: will race 17 | static inline uint32_t ioapic_read(volatile uint32_t *ioapic, uint8_t reg) 18 | { 19 | ioapic[0] = (reg & 0xFF); 20 | return ioapic[4]; 21 | } 22 | 23 | // Careful: will race 24 | static inline void ioapic_write(volatile uint32_t *ioapic, uint8_t reg, uint32_t data) 25 | { 26 | ioapic[0] = (reg & 0xFF); 27 | ioapic[4] = data; 28 | } 29 | 30 | static inline uint32_t get_max_redirs(size_t ioapic_id) 31 | { 32 | virtaddr_t base = phys_to_virt(ioapic_list[ioapic_id]->phys_addr); 33 | return (ioapic_read(base, APIC_REG_VER) & 0xFF0000) >> 16; 34 | } 35 | 36 | static struct madt_entry_ioapic *gsi_to_ioapic(uint32_t gsi) 37 | { 38 | for (size_t i = 0; i < ioapic_list_size; i++) { 39 | uint32_t max_redirs = get_max_redirs(i); 40 | if (ioapic_list[i]->gsi_base <= gsi && ioapic_list[i]->gsi_base + max_redirs > gsi) 41 | return ioapic_list[i]; 42 | } 43 | panic("I/O APIC not found for GSI %u", gsi); 44 | } 45 | 46 | static uint64_t ioapic_redtbl_read(virtaddr_t ioapic_base, uint8_t irq_line) 47 | { 48 | uint32_t ioredtbl = (irq_line * 2) + APIC_REG_REDTBL; 49 | return ioapic_read(ioapic_base, ioredtbl) | ((uint64_t)ioapic_read(ioapic_base, ioredtbl + 1) << 32); 50 | } 51 | 52 | static void ioapic_redtbl_write(virtaddr_t ioapic_base, uint8_t irq_line, uint64_t value) 53 | { 54 | uint32_t ioredtbl = (irq_line * 2) + APIC_REG_REDTBL; 55 | ioapic_write(ioapic_base, ioredtbl + 0, (uint32_t)value); 56 | ioapic_write(ioapic_base, ioredtbl + 1, (uint32_t)(value >> 32)); 57 | } 58 | 59 | void ioapic_redirect(uint32_t gsi, uint8_t UNUSED(source), uint16_t flags, uint8_t target_apic) 60 | { 61 | uint8_t target_apic_id = lapic_list[target_apic].id; 62 | struct madt_entry_ioapic *ioapic = gsi_to_ioapic(gsi); 63 | virtaddr_t ioapic_base = phys_to_virt(ioapic->phys_addr); 64 | 65 | uint64_t redirection = gsi + IRQ_APIC_BASE; 66 | if (flags & 2) 67 | redirection |= (1 << 13); 68 | if (flags & 8) 69 | redirection |= (1 << 15); 70 | redirection |= ((uint64_t)target_apic_id) << 56; 71 | 72 | ioapic_redtbl_write(ioapic_base, gsi - ioapic->gsi_base, redirection); 73 | } 74 | 75 | void ioapic_mask(uint32_t gsi) 76 | { 77 | spin_lock(&ioapic_lock); 78 | struct madt_entry_ioapic *ioapic = gsi_to_ioapic(gsi); 79 | virtaddr_t ioapic_base = phys_to_virt(ioapic->phys_addr); 80 | uint8_t irq_line = gsi - ioapic->gsi_base; 81 | uint64_t prev = ioapic_redtbl_read(ioapic_base, irq_line); 82 | ioapic_redtbl_write(ioapic_base, irq_line, prev | APIC_IRQ_MASK); 83 | spin_unlock(&ioapic_lock); 84 | } 85 | 86 | void ioapic_unmask(uint32_t gsi) 87 | { 88 | spin_lock(&ioapic_lock); 89 | struct madt_entry_ioapic *ioapic = gsi_to_ioapic(gsi); 90 | virtaddr_t ioapic_base = phys_to_virt(ioapic->phys_addr); 91 | uint8_t irq_line = gsi - ioapic->gsi_base; 92 | uint64_t prev = ioapic_redtbl_read(ioapic_base, irq_line); 93 | ioapic_redtbl_write(ioapic_base, irq_line, prev & (~APIC_IRQ_MASK)); 94 | spin_unlock(&ioapic_lock); 95 | } 96 | 97 | uint8_t ioapic_isa_to_gsi(uint8_t isa) 98 | { 99 | kassert_dbg(isa < 16); 100 | for (size_t i = 0; i < override_list_size; i++) 101 | if (override_list[i]->source == isa) 102 | return override_list[i]->gsi; 103 | return isa; 104 | } 105 | 106 | uint8_t ioapic_gsi_to_isa(uint8_t gsi) 107 | { 108 | for (size_t i = 0; i < override_list_size; i++) 109 | if (override_list[i]->gsi == gsi) 110 | return override_list[i]->source; 111 | return gsi; 112 | } 113 | 114 | void ioapic_init(void) 115 | { 116 | // Initialised the (assumed) wirings for the legacy PIC IRQs 117 | // Send all IRQs to the BSP for simplicity 118 | for (uint8_t i = 0; i < 16; i++) { 119 | ioapic_redirect(i, i, 0, 0); 120 | ioapic_mask(i); 121 | } 122 | 123 | // Setup the actual overrides 124 | for (size_t i = 0; i < override_list_size; i++) { 125 | ioapic_redirect(override_list[i]->gsi, override_list[i]->source, override_list[i]->flags, 0); 126 | ioapic_mask(override_list[i]->gsi); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /kernel/drivers/apic/lapic.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "libk.h" 5 | #include "mm.h" 6 | #include "percpu.h" 7 | #include "util.h" 8 | #include "proc.h" 9 | #include "interrupts.h" 10 | #include "drivers/acpi.h" 11 | #include "drivers/apic.h" 12 | #include "drivers/pit.h" 13 | 14 | #define APIC_CPUID_BIT (1 << 9) 15 | #define APIC_DISABLE 0x10000U // Not exactly sure what this is, found it on the wiki 16 | #define APIC_TIMER_PERIODIC 0x20000U 17 | 18 | #define APIC_REG_ID 0x20 19 | #define APIC_REG_VERSION 0x30 20 | #define APIC_REG_EOI 0xB0 21 | #define APIC_REG_SPURIOUS 0xF0U 22 | #define APIC_REG_LINT0 0x350U 23 | #define APIC_REG_LINT1 0x360U 24 | #define APIC_REG_ICR0 0x300U 25 | #define APIC_REG_ICR1 0x310U 26 | #define APIC_REG_TIMER_LVT 0x320U 27 | #define APIC_REG_TIMER_INITIAL 0x380U 28 | #define APIC_REG_TIMER_CURRENT 0x390U 29 | #define APIC_REG_TIMER_DIVIDE 0x3E0U 30 | #define APIC_REG_ISR_BASE 0x100U 31 | 32 | virtaddr_t lapic_base; // Shared by all CPUs 33 | 34 | static bool has_lapic(void) 35 | { 36 | uint32_t eax, ebx, ecx, edx = 0; 37 | __get_cpuid(1, &eax, &ebx, &ecx, &edx); 38 | return (edx & APIC_CPUID_BIT) != 0; 39 | } 40 | 41 | static inline void lapic_write(uint32_t reg_offset, uint32_t data) 42 | { 43 | *(volatile uint32_t *)((uintptr_t)lapic_base + reg_offset) = data; 44 | } 45 | 46 | static inline uint32_t lapic_read(uint32_t reg_offset) 47 | { 48 | return *(volatile uint32_t *)((uintptr_t)lapic_base + reg_offset); 49 | } 50 | 51 | static inline struct lapic_info *find_lapic(uint8_t id) 52 | { 53 | for (size_t i = 0; i < lapic_list_size; i++) 54 | if (lapic_list[i].id == id) 55 | return &lapic_list[i]; 56 | return NULL; 57 | } 58 | 59 | static inline void lapic_set_nmi(uint8_t vec, struct madt_entry_nmi *nmi_info) 60 | { 61 | kassert_dbg(vec == IRQ_NMI); 62 | uint32_t nmi = 800 | vec; 63 | if (nmi_info->flags & 2) 64 | nmi |= (1 << 13); 65 | if (nmi_info->flags & 8) 66 | nmi |= (1 << 15); 67 | if (nmi_info->lint_num == 0) 68 | lapic_write(APIC_REG_LINT0, nmi); 69 | else if (nmi_info->lint_num == 1) 70 | lapic_write(APIC_REG_LINT1, nmi); 71 | } 72 | 73 | uint8_t lapic_id(void) 74 | { 75 | return lapic_read(APIC_REG_ID) >> 24; 76 | } 77 | 78 | void lapic_eoi(uint8_t vec) 79 | { 80 | // Check the corresponding bit in the LAPIC's ISR 81 | kassert_dbg(vec >= IRQ_APIC_BASE); 82 | uint32_t reg = APIC_REG_ISR_BASE + 0x10 * (vec / 32); 83 | if (lapic_read(reg) & (1 << (vec % 32))) 84 | lapic_write(APIC_REG_EOI, 0); 85 | } 86 | 87 | void lapic_send_ipi(uint8_t target, uint32_t flags) 88 | { 89 | if (lapic_base == NULL) 90 | panic("Tried to send IPI before LAPIC was initialized"); 91 | 92 | // Getting preempted could mean a race condition here 93 | preempt_inc(); 94 | if (!(flags & IPI_BROADCAST)) 95 | lapic_write(APIC_REG_ICR1, (uint32_t)target << 24); 96 | lapic_write(APIC_REG_ICR0, flags); 97 | preempt_dec(); 98 | } 99 | 100 | // Returns the number of ticks in 10ms 101 | uint32_t lapic_timer_prepare(void) 102 | { 103 | const uint32_t test_ms = 30; 104 | const uint32_t ticks_initial = 0xFFFFFFFF; 105 | 106 | lapic_write(APIC_REG_TIMER_DIVIDE, 0x3); 107 | lapic_write(APIC_REG_TIMER_INITIAL, ticks_initial); 108 | 109 | pit_sleep_ms(test_ms); // TODO: Use interrupts for better accuracy (might be tricky getting interrupts with SMP) 110 | 111 | lapic_write(APIC_REG_TIMER_LVT, APIC_DISABLE); 112 | 113 | uint32_t ticks_per_10ms = (ticks_initial - lapic_read(APIC_REG_TIMER_CURRENT)) / (test_ms / 10); 114 | klog_verbose("lapic", "CPU%u timer: %u/10ms\n", smp_cpu_id(), ticks_per_10ms); 115 | 116 | return ticks_per_10ms; 117 | } 118 | 119 | static void lapic_timer_callback(struct isr_ctx *UNUSED(regs)) 120 | { 121 | struct task *t = current; 122 | 123 | // Preempt a task after it's ran for 5 time slices 124 | if (!percpu_get(reschedule)) { 125 | t->sched.num_slices++; 126 | 127 | if (t->sched.num_slices >= MAX_SLICES) 128 | percpu_set(reschedule, true); 129 | } 130 | } 131 | 132 | // Enables the lapic timer with interrupts every 10ms 133 | void lapic_timer_enable(void) 134 | { 135 | uint8_t id = lapic_id(); 136 | struct lapic_info *lapic = find_lapic(id); 137 | 138 | #ifdef DEBUG 139 | const uint32_t period = lapic->ticks_per_10ms * 10; 140 | #else 141 | const uint32_t period = lapic->ticks_per_10ms; 142 | #endif 143 | 144 | lapic_write(APIC_REG_TIMER_DIVIDE, 0x3); 145 | lapic_write(APIC_REG_TIMER_LVT, IRQ_LINT_TIMER | APIC_TIMER_PERIODIC); 146 | lapic_write(APIC_REG_TIMER_INITIAL, period); 147 | } 148 | 149 | void lapic_enable(void) 150 | { 151 | // TODO: Fallback to PIC 152 | if (!has_lapic()) 153 | panic("No local APIC found for current CPU"); 154 | 155 | uint8_t id = lapic_id(); 156 | struct lapic_info *lapic = find_lapic(id); 157 | percpu_set(apic_id, id); 158 | 159 | // Make sure APIC is globally enabled 160 | kassert((msr_read(0x1B) & (1 << 11)) > 0); 161 | 162 | klog_verbose("lapic", "LAPIC Version: %u\n", lapic_read(APIC_REG_VERSION) & 0xFF); 163 | 164 | for (size_t i = 0; i < nmi_list_size; i++) 165 | if (nmi_list[i]->acpi_id == lapic->acpi_id || nmi_list[i]->acpi_id == 0xFF) 166 | lapic_set_nmi(IRQ_NMI, nmi_list[i]); 167 | 168 | // Enable the LAPIC via the spurious interrupt register 169 | lapic_write(APIC_REG_SPURIOUS, lapic_read(APIC_REG_SPURIOUS) | (1 << 8) | IRQ_APIC_SPURIOUS); 170 | 171 | // Calibrate timer 172 | lapic->ticks_per_10ms = lapic_timer_prepare(); 173 | 174 | // Register timer callback (only do this once) 175 | if (id == 0) { 176 | struct isr_info timer_info = { 177 | .type = ISR_IRQ, 178 | .handler = lapic_timer_callback, 179 | }; 180 | 181 | isr_set_info(IRQ_LINT_TIMER, &timer_info); 182 | } 183 | 184 | // Clear any pending interrupts 185 | lapic_write(APIC_REG_EOI, 0); 186 | } 187 | 188 | -------------------------------------------------------------------------------- /kernel/drivers/pit/pit.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | #include "util.h" 3 | #include "asm.h" 4 | #include "sync.h" 5 | #include "interrupts.h" 6 | #include "drivers/pit.h" 7 | #include "drivers/apic.h" 8 | 9 | static spinlock_t pit_lock; 10 | 11 | void pit_init(void) 12 | { 13 | } 14 | 15 | void pit_sleep_ms(uint64_t ms) 16 | { 17 | spin_lock(&pit_lock); 18 | uint64_t total_count = 0x4A9 * ms; 19 | do { 20 | uint16_t count = MIN(total_count, 0xFFFFU); 21 | outb(0x43, 0x30); 22 | outb(0x40, count & 0xFF); 23 | outb(0x40, count >> 8); 24 | do { 25 | pause(); 26 | outb(0x43, 0xE2); 27 | } while ((inb(0x40) & (1 << 7)) == 0); 28 | total_count -= count; 29 | } while ((total_count & ~0xFFFF) != 0); 30 | spin_unlock(&pit_lock); 31 | } 32 | 33 | void pit_sleep_watch_flag(uint64_t ms, volatile bool *flag, bool original) 34 | { 35 | spin_lock(&pit_lock); 36 | uint64_t total_count = 0x4A9 * ms; 37 | do { 38 | uint16_t count = MIN(total_count, 0xFFFFU); 39 | outb(0x43, 0x30); 40 | outb(0x40, count & 0xFF); 41 | outb(0x40, count >> 8); 42 | do { 43 | pause(); 44 | if (__atomic_load_n(flag, __ATOMIC_RELAXED) != original) 45 | goto end; 46 | outb(0x43, 0xE2); 47 | } while ((inb(0x40) & (1 << 7)) == 0); 48 | total_count -= count; 49 | } while ((total_count & ~0xFFFF) != 0); 50 | end: 51 | spin_unlock(&pit_lock); 52 | } 53 | -------------------------------------------------------------------------------- /kernel/drivers/ps2/keyboard.c: -------------------------------------------------------------------------------- 1 | #include "interrupts.h" 2 | #include "libk.h" 3 | #include "util.h" 4 | #include "drivers/ps2.h" 5 | 6 | static void ps2kbd_irq_handler(struct isr_ctx *); 7 | 8 | static __attribute__((unused)) uint8_t ps2kbd_cmd(uint8_t data) 9 | { 10 | // TODO: Add a timeout 11 | uint8_t response; 12 | do { 13 | ps2_write_data(data); 14 | response = ps2_read_data(); 15 | } while (response == 0xFE); 16 | return response; 17 | } 18 | 19 | void ps2kbd_init(void) 20 | { 21 | /* TODO: Why does this not work? 22 | * Bochs and VirtualBox seem to only respond with 0xAA (and then maybe 0xFA). 23 | * QEMU seems to respond with 0xFA, then 0xAA. 24 | * On my PC it doesn't respond at all to the command. 25 | */ 26 | /* uint8_t self_test_status = ps2kbd_cmd(0xFF); 27 | klog("ps2", "Self-test and reset: %x\n", self_test_status); 28 | kassert(self_test_status == 0xFA || self_test_status == 0xAA); 29 | while ((ps2_read_status() & 1) == 1) 30 | ps2_read_data();*/ 31 | 32 | #ifdef VERBOSE 33 | klog("ps2", "Status byte is %x\n", ps2_read_status()); 34 | klog("ps2", "Configuration byte is %x\n", ps2_read_config()); 35 | #endif 36 | 37 | // Don't send any commands after this point, since the interrupt handler will eat the data 38 | uint8_t vec = ISA_TO_INTERRUPT(1); 39 | 40 | struct isr_info ps2kbd_info = { 41 | .type = ISR_IRQ, 42 | .handler = ps2kbd_irq_handler, 43 | }; 44 | 45 | isr_set_info(vec, &ps2kbd_info); 46 | isr_irq_unmask(vec); 47 | 48 | klog("ps2kbd", "Initialised keyboard on IRQ %u\n", vec); 49 | 50 | // Flush the output buffer (again) 51 | while ((inb(PS2_STATUS) & 1) != 0) 52 | (void)inb(PS2_DATA); 53 | } 54 | 55 | static void ps2kbd_irq_handler(struct isr_ctx *UNUSED(regs)) 56 | { 57 | (void)ps2_read_data(); 58 | #ifdef VERBOSE 59 | kprintf("Keyboard handler fired\n"); 60 | #endif 61 | } 62 | -------------------------------------------------------------------------------- /kernel/drivers/ps2/ps2.c: -------------------------------------------------------------------------------- 1 | #include "asm.h" 2 | #include "libk.h" 3 | #include "util.h" 4 | #include "drivers/ps2.h" 5 | 6 | #define PS2_INT0 (1 << 0) 7 | #define PS2_INT1 (1 << 1) 8 | #define PS2_SYS_FLAG (1 << 2) 9 | #define PS2_CLOCK0 (1 << 4) 10 | #define PS2_CLOCK1 (1 << 5) 11 | #define PS2_TRANSLATE (1 << 6) 12 | 13 | void ps2_init(void) 14 | { 15 | // TODO: Initialise USB controllers 16 | // TODO: Determine if the PS/2 controller exists 17 | 18 | // Disable devices 19 | ps2_write_cmd(0xAD); 20 | ps2_write_cmd(0xA7); 21 | 22 | // Flush the output buffer 23 | while ((inb(PS2_STATUS) & 1) != 0) 24 | (void)inb(PS2_DATA); 25 | 26 | // Set the controller configuration byte 27 | uint8_t config_byte = ps2_read_config(); 28 | kassert(config_byte & PS2_CLOCK1); 29 | config_byte &= ~(PS2_INT0 | PS2_INT1 | PS2_TRANSLATE); 30 | ps2_write_config(config_byte); 31 | 32 | // Perform controller self test 33 | ps2_write_cmd(0xAA); 34 | kassert(ps2_read_data() == 0x55); 35 | 36 | // Determine if there are 2 channels 37 | ps2_write_cmd(0xA8); 38 | ps2_write_cmd(0x20); 39 | config_byte = ps2_read_data(); 40 | kassert((config_byte & PS2_CLOCK1) == 0); 41 | ps2_write_cmd(0xA7); 42 | 43 | // Perform interface tests 44 | ps2_write_cmd(0xAB); 45 | kassert(ps2_read_data() == 0x00); 46 | ps2_write_cmd(0xA9); 47 | kassert(ps2_read_data() == 0x00); 48 | 49 | // Enable devices 50 | config_byte = ps2_read_config(); 51 | config_byte |= (PS2_INT0 | PS2_INT1); 52 | ps2_write_config(config_byte); 53 | ps2_write_cmd(0xAE); 54 | ps2_write_cmd(0xA8); 55 | 56 | // Reset devices 57 | ps2kbd_init(); 58 | } 59 | -------------------------------------------------------------------------------- /kernel/drivers/serial/serial.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "drivers/serial.h" 4 | #include "asm.h" 5 | 6 | #define COM1_PORT 0x3F8 7 | #define COM2_PORT 0x2F8 8 | 9 | #ifdef SERIAL_DISABLE 10 | #include "util.h" 11 | void serial_init(void) {} 12 | void serial_write_com(int UNUSED(com), unsigned char UNUSED(data)) {} 13 | #else 14 | 15 | void serial_init(void) 16 | { 17 | // Init COM1 and COM2 18 | outb(COM1_PORT + 1, 0x00); // Disable all interrupts 19 | outb(COM1_PORT + 3, 0x80); // Enable DLAB (set baud rate divisor) 20 | outb(COM1_PORT + 0, 0x03); // Set divisor to 3 (lo byte) 38400 baud 21 | outb(COM1_PORT + 1, 0x00); // (hi byte) 22 | outb(COM1_PORT + 3, 0x02); // 7 bits, no parity, one stop bit 23 | outb(COM1_PORT + 2, 0xC7); // Enable FIFO, clear them, with 14-byte threshold 24 | outb(COM1_PORT + 4, 0x0B); // IRQs enabled, RTS/DSR set 25 | 26 | outb(COM2_PORT + 1, 0x00); // Disable all interrupts 27 | outb(COM2_PORT + 3, 0x80); // Enable DLAB (set baud rate divisor) 28 | outb(COM2_PORT + 0, 0x03); // Set divisor to 3 (lo byte) 38400 baud 29 | outb(COM2_PORT + 1, 0x00); // (hi byte) 30 | outb(COM2_PORT + 3, 0x02); // 7 bits, no parity, one stop bit 31 | outb(COM2_PORT + 2, 0xC7); // Enable FIFO, clear them, with 14-byte threshold 32 | outb(COM2_PORT + 4, 0x0B); // IRQs enabled, RTS/DSR set 33 | 34 | // Print a small separator so we can see output clearly 35 | const char *msg = "-----------------------------------\n"; 36 | while (*msg) 37 | serial_write_com(1, *msg++); 38 | } 39 | 40 | static inline int serial_transmit_empty(uint16_t port) 41 | { 42 | return inb(port + 5) & 0x20; 43 | } 44 | 45 | void serial_write_com(int com, unsigned char data) 46 | { 47 | uint16_t port; 48 | switch (com) { 49 | case 1: 50 | port = COM1_PORT; 51 | break; 52 | case 2: 53 | port = COM2_PORT; 54 | break; 55 | // TODO: COM3 and COM4 56 | default: 57 | return; 58 | } 59 | 60 | while (serial_transmit_empty(port) == 0) 61 | pause(); 62 | 63 | outb(port, data); 64 | } 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /kernel/drivers/smbios/smbios.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "drivers/smbios.h" 4 | #include "libk.h" 5 | #include "util.h" 6 | #include "mm.h" 7 | 8 | #define SMBIOS_PHYS_START 0xF0000 9 | #define SMBIOS_PHYS_END 0xFFFFF 10 | #define SMBIOS3_ANCHOR "_SM3_" 11 | #define SMBIOS2_ANCHOR "_SM_" 12 | 13 | #define SMBIOS_TYPE_PROCESSOR 4 14 | #define SMBIOS_TYPE_END 127 15 | 16 | struct smbios2_entry { 17 | char anchor[4]; 18 | uint8_t checksum; 19 | uint8_t ep_length; 20 | uint8_t ver_major; 21 | uint8_t ver_minor; 22 | uint16_t max_struct; 23 | uint8_t ep_revision; 24 | uint8_t formatted_area[5]; 25 | char intermediate_anchor[5]; 26 | uint8_t intermediate_checksum; 27 | uint16_t struct_table_len; 28 | uint32_t struct_table_addr; 29 | uint16_t nstructs; 30 | uint8_t bcd_revision; 31 | } __attribute__((packed)); 32 | 33 | struct smbios3_entry { 34 | char anchor[5]; 35 | uint8_t checksum; 36 | uint8_t ep_length; 37 | uint8_t ver_major; 38 | uint8_t ver_minor; 39 | uint8_t docrev; 40 | uint8_t ep_revision; 41 | uint8_t __reserved; 42 | uint32_t struct_table_max; 43 | physaddr_t struct_table_addr; 44 | } __attribute__((packed)); 45 | 46 | struct smbios_header { 47 | uint8_t type; 48 | uint8_t length; 49 | uint16_t handle; 50 | } __attribute__((packed)); 51 | 52 | struct smbios_processor { 53 | struct smbios_header hd; 54 | uint8_t socket_designation; 55 | uint8_t processor_type; 56 | uint8_t processor_family; 57 | uint8_t processor_manufacturer; 58 | uint64_t processor_id; 59 | uint8_t processor_version; 60 | uint8_t voltage; 61 | uint16_t external_clock; 62 | uint16_t max_speed; 63 | uint16_t current_speed; 64 | uint8_t status; 65 | uint8_t processor_upgrade; 66 | uint16_t l1_cache_handle; 67 | uint16_t l2_cache_handle; 68 | uint16_t l3_cache_handle; 69 | uint8_t serial_number; 70 | uint8_t asset_tag; 71 | uint8_t part_number; 72 | uint8_t core_count; 73 | uint8_t core_enabled; 74 | uint8_t thread_count; 75 | uint16_t processor_characteristics; 76 | uint16_t processor_family_2; 77 | uint16_t core_count_2; 78 | uint16_t core_enabled_2; 79 | uint16_t thread_count_2; 80 | } __attribute__((packed)); 81 | 82 | static struct smbios_info { 83 | union { 84 | struct smbios2_entry *entry2; 85 | struct smbios3_entry *entry3; 86 | void *entry; 87 | }; 88 | uint8_t major; 89 | uint8_t minor; 90 | } smbios_info; 91 | 92 | static bool calc_checksum(const virtaddr_t ventry, size_t len) 93 | { 94 | uint8_t sum = 0; 95 | const uint8_t *entry = ventry; 96 | for (size_t i = 0; i < len; i++) 97 | sum += entry[i]; 98 | return sum == 0; 99 | } 100 | 101 | static struct smbios_info find_smbios(void) 102 | { 103 | for (uintptr_t addr = SMBIOS_PHYS_START; addr < SMBIOS_PHYS_END; addr += 16) { 104 | void *smbios = phys_to_virt(addr); 105 | 106 | if (memcmp(smbios, SMBIOS3_ANCHOR, 5) == 0) { 107 | 108 | struct smbios3_entry *eps = smbios; 109 | if (!calc_checksum(eps, eps->ep_length)) 110 | continue; 111 | 112 | kassert_dbg(eps->__reserved == 0); 113 | return (struct smbios_info){ 114 | .entry3 = eps, 115 | .major = eps->ver_major, 116 | .minor = eps->ver_minor 117 | }; 118 | } else if (memcmp(smbios, SMBIOS2_ANCHOR, 4) == 0) { 119 | 120 | struct smbios2_entry *eps = smbios; 121 | if (!calc_checksum(eps, eps->ep_length)) 122 | continue; 123 | 124 | // Calculate intermediate checksum 125 | size_t ieps_offset = offsetof(struct smbios2_entry, intermediate_anchor); 126 | if (!calc_checksum(&eps->intermediate_anchor, sizeof(struct smbios2_entry) - ieps_offset)) 127 | continue; 128 | 129 | kassert_dbg(eps->ep_revision != 0 || memcmp(&eps->formatted_area, "\0\0\0\0", 5) == 0); 130 | return (struct smbios_info){ 131 | .entry2 = eps, 132 | .major = eps->ver_major, 133 | .minor = eps->ver_minor 134 | }; 135 | } 136 | } 137 | 138 | return (struct smbios_info){ .major = 0 }; 139 | } 140 | 141 | #ifdef VERBOSE 142 | static const char *table_get_str(struct smbios_header *hd, size_t index) 143 | { 144 | if (index == 0) 145 | return NULL; 146 | 147 | const char *p = (char *)hd + hd->length; 148 | while (--index) { 149 | while (*p++ != '\0') 150 | ; 151 | } 152 | return p; 153 | } 154 | #endif 155 | 156 | #define info(msg, ...) klog_verbose("smbios", " " msg,##__VA_ARGS__) 157 | static void dump_smbios_tables(struct smbios_header *start, size_t nstructs) 158 | { 159 | #ifdef VERBOSE 160 | kassert_dbg(start->type < 128 && (start->type == SMBIOS_TYPE_END || start->type <= 32)); 161 | size_t num = 0; 162 | for (struct smbios_header *cur = start; cur->type != SMBIOS_TYPE_END; num++) { 163 | switch (cur->type) { 164 | case SMBIOS_TYPE_PROCESSOR: { 165 | klog("smbios", "Detected processor information table\n"); 166 | struct smbios_processor *proc = (struct smbios_processor *)cur; 167 | info("Max speed: %u MHz\n", proc->max_speed); 168 | info("Current speed: %u MHz\n", proc->current_speed); 169 | info("External clock frequency: %u MHz\n", proc->external_clock); 170 | info("Processor family: %x\n", proc->processor_family); 171 | info("Processor manufacturer: %s\n", table_get_str(&proc->hd, proc->processor_manufacturer)); 172 | break; 173 | } 174 | default: 175 | break; 176 | } 177 | 178 | // Seek to the end of the string table (ends in a double null) 179 | const char *strtab = (char *)cur + cur->length; 180 | size_t i; 181 | for (i = 1; strtab[i - 1] != '\0' || strtab[i] != '\0'; i++) 182 | ; 183 | 184 | size_t total = cur->length + i + 1; 185 | cur = (struct smbios_header *)((char *)cur + total); 186 | } 187 | klog("smbios", "Scanned %zu SMBIOS tables.\n", num); 188 | kassert(nstructs == 0 || num + 1 == nstructs); 189 | #else 190 | (void)start; 191 | (void)nstructs; 192 | #endif 193 | } 194 | 195 | static void dump_smbios_info(struct smbios_info info) 196 | { 197 | klog("smbios", "Using SMBIOS version %u.%u\n", info.major, info.minor); 198 | info("EP address: %p\n", info.entry); 199 | if (info.major == 3) { 200 | struct smbios3_entry UNUSED(*eps) = info.entry3; 201 | info("EP revision: %u\n", eps->ep_revision); 202 | info("Docrev: %u\n", eps->docrev); 203 | info("Structure table maximum size: %u\n", eps->struct_table_max); 204 | info("Structure table address: %p\n", (virtaddr_t)(uintptr_t)eps->struct_table_addr); 205 | dump_smbios_tables(phys_to_virt(eps->struct_table_addr), 0); 206 | } else if (info.major == 2) { 207 | struct smbios2_entry UNUSED(*eps) = info.entry2; 208 | info("EP revision: %u\n", eps->ep_revision); 209 | info("BCD revision: %u\n", eps->bcd_revision); 210 | info("Maximum structure size: %u\n", eps->max_struct); 211 | info("Number of structures: %u\n", eps->nstructs); 212 | info("Structure table length: %u\n", eps->struct_table_len); 213 | info("Structure table address: %p\n", (virtaddr_t)(uintptr_t)eps->struct_table_addr); 214 | dump_smbios_tables(phys_to_virt(eps->struct_table_addr), eps->nstructs); 215 | } 216 | } 217 | #undef info 218 | 219 | void smbios_init(void) 220 | { 221 | smbios_info = find_smbios(); 222 | if (smbios_info.major != 2 && smbios_info.major != 3) 223 | panic("No SMBIOS entry point found"); 224 | dump_smbios_info(smbios_info); 225 | } 226 | 227 | -------------------------------------------------------------------------------- /kernel/drivers/vga_tmode/cansid.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "drivers/cansid.h" 5 | 6 | #define ESC '\x1B' 7 | 8 | struct cansid_state cansid_init(void) 9 | { 10 | struct cansid_state rv = { 11 | .state = CANSID_ESC, 12 | .style = 0x0F, 13 | .next_style = 0x0F 14 | }; 15 | return rv; 16 | } 17 | 18 | static inline unsigned char cansid_convert_color(unsigned char color) 19 | { 20 | const unsigned char lookup_table[8] = { 0, 4, 2, 6, 1, 5, 3, 7 }; 21 | return lookup_table[(int)color]; 22 | } 23 | 24 | struct color_char cansid_process(struct cansid_state *state, char x) 25 | { 26 | struct color_char rv = { 27 | .style = state->style, 28 | .ascii = '\0' 29 | }; 30 | switch (state->state) { 31 | case CANSID_ESC: 32 | if (x == ESC) 33 | state->state = CANSID_BRACKET; 34 | else { 35 | rv.ascii = x; 36 | } 37 | break; 38 | case CANSID_BRACKET: 39 | if (x == '[') 40 | state->state = CANSID_PARSE; 41 | else { 42 | state->state = CANSID_ESC; 43 | rv.ascii = x; 44 | } 45 | break; 46 | case CANSID_PARSE: 47 | if (x == '3') { 48 | state->state = CANSID_FGCOLOR; 49 | } else if (x == '4') { 50 | state->state = CANSID_BGCOLOR; 51 | } else if (x == '0') { 52 | state->state = CANSID_ENDVAL; 53 | state->next_style = 0x0F; 54 | } else if (x == '1') { 55 | state->state = CANSID_ENDVAL; 56 | state->next_style |= (1 << 3); 57 | } else if (x == '=') { 58 | state->state = CANSID_EQUALS; 59 | } else { 60 | state->state = CANSID_ESC; 61 | state->next_style = state->style; 62 | rv.ascii = x; 63 | } 64 | break; 65 | case CANSID_BGCOLOR: 66 | if (x >= '0' && x <= '7') { 67 | state->state = CANSID_ENDVAL; 68 | state->next_style &= 0x1F; 69 | state->next_style |= cansid_convert_color(x - '0') << 4; 70 | } else { 71 | state->state = CANSID_ESC; 72 | state->next_style = state->style; 73 | rv.ascii = x; 74 | } 75 | break; 76 | case CANSID_FGCOLOR: 77 | if (x >= '0' && x <= '7') { 78 | state->state = CANSID_ENDVAL; 79 | state->next_style &= 0xF8; 80 | state->next_style |= cansid_convert_color(x - '0'); 81 | } else { 82 | state->state = CANSID_ESC; 83 | state->next_style = state->style; 84 | rv.ascii = x; 85 | } 86 | break; 87 | case CANSID_EQUALS: 88 | if (x == '1') { 89 | state->state = CANSID_ENDVAL; 90 | state->next_style &= ~(1 << 3); 91 | } else { 92 | state->state = CANSID_ESC; 93 | state->next_style = state->style; 94 | rv.ascii = x; 95 | } 96 | break; 97 | case CANSID_ENDVAL: 98 | if (x == ';') { 99 | state->state = CANSID_PARSE; 100 | } else if (x == 'm') { 101 | // Finish and apply styles 102 | state->state = CANSID_ESC; 103 | state->style = state->next_style; 104 | } else { 105 | state->state = CANSID_ESC; 106 | state->next_style = state->style; 107 | rv.ascii = x; 108 | } 109 | break; 110 | default: 111 | break; 112 | } 113 | return rv; 114 | } 115 | -------------------------------------------------------------------------------- /kernel/drivers/vga_tmode/vga_tmode.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "drivers/vga_tmode.h" 4 | #include "drivers/cansid.h" 5 | #include "drivers/serial.h" 6 | #include "asm.h" 7 | #include "sync.h" 8 | #include "libk.h" 9 | 10 | // No spinlock here. All writes are serialised by kprintf. 11 | // This allows us to use this code in an NMI handler. 12 | 13 | const size_t VGAWIDTH = 80; 14 | const size_t VGAHEIGHT = 25; 15 | 16 | static uint16_t *const VGABUF = (uint16_t *)0xFFFFFFFF800B8000; 17 | static size_t x_pos, y_pos; 18 | static struct cansid_state cansid_state; 19 | 20 | void vga_tmode_init(void); 21 | 22 | void vga_tmode_init(void) 23 | { 24 | cansid_state = cansid_init(); 25 | serial_init(); 26 | vga_tmode_puts("\x1B[=1;30;47m ByteOS"); 27 | vga_tmode_puts(" \x1B[0m"); 28 | } 29 | 30 | static void vga_tmode_newline(void) 31 | { 32 | if (y_pos == VGAHEIGHT - 1) { 33 | // Shift every line of input upwards 34 | for (size_t i = 2; i < VGAHEIGHT; i++) { 35 | memcpy(VGABUF + ((i - 1) * VGAWIDTH), VGABUF + (i * VGAWIDTH), VGAWIDTH * 2); 36 | } 37 | // Clear bottom line 38 | size_t offset = VGAWIDTH * (VGAHEIGHT - 1); 39 | for (size_t i = 0; i < VGAWIDTH; i++) 40 | VGABUF[offset + i] = (0x0F << 8) | ' '; 41 | x_pos = 0; 42 | } else { 43 | x_pos = 0; 44 | y_pos++; 45 | } 46 | vga_tmode_setcursor(x_pos, y_pos); 47 | } 48 | 49 | static inline void vga_tmode_write_char(char c, unsigned char style, size_t x, size_t y) 50 | { 51 | volatile uint16_t *p = VGABUF + ((y * 80) + x); 52 | *p = (style << 8) | c; 53 | } 54 | 55 | void vga_tmode_putchar(char c) 56 | { 57 | struct color_char ch = cansid_process(&cansid_state, c); 58 | vga_tmode_raw_putchar(ch.ascii, ch.style); 59 | } 60 | 61 | void vga_tmode_raw_putchar(char c, unsigned char style) 62 | { 63 | switch (c) { 64 | case '\0': 65 | break; 66 | case '\n': 67 | vga_tmode_newline(); 68 | break; 69 | case '\t': 70 | vga_tmode_putchar(' '); 71 | vga_tmode_putchar(' '); 72 | vga_tmode_putchar(' '); 73 | vga_tmode_putchar(' '); 74 | break; 75 | default: { 76 | vga_tmode_write_char(c, style, x_pos, y_pos); 77 | if (x_pos == VGAWIDTH - 1) 78 | vga_tmode_newline(); 79 | else 80 | x_pos++; 81 | }; 82 | } 83 | vga_tmode_setcursor(x_pos, y_pos); 84 | } 85 | 86 | void vga_tmode_puts(char *s) 87 | { 88 | while (*s) 89 | vga_tmode_putchar(*s++); 90 | } 91 | 92 | void vga_tmode_setcursor(size_t x, size_t y) 93 | { 94 | x_pos = x; 95 | y_pos = y; 96 | 97 | // Update the position of code on the screen 98 | unsigned short position = (y * 80) + x; 99 | outb(0x3D4, 0x0F); 100 | outb(0x3D5, (unsigned char)(position & 0xFF)); 101 | outb(0x3D4, 0x0E); 102 | outb(0x3D5, (unsigned char)((position >> 8) & 0xFF)); 103 | } 104 | -------------------------------------------------------------------------------- /kernel/fs/initrd/dir.c: -------------------------------------------------------------------------------- 1 | #include "fs.h" 2 | #include "fs/initrd.h" 3 | #include "libk.h" 4 | 5 | static err_t initrd_dir_lookup_inode(struct inode *dir_inode, const char *name, size_t len, struct inode **out) 6 | { 7 | kassert_dbg(dir_inode && I_ISDIR(dir_inode) && dir_inode->sb != NULL); 8 | struct initrd_fs *fs = initrd_sb_container(dir_inode->sb); 9 | 10 | const char *dir_name; 11 | if (dir_inode == fs->root) 12 | dir_name = "/"; 13 | else 14 | dir_name = initrd_inode_container(dir_inode)->hdr->filename; 15 | 16 | size_t dir_len = strlen(dir_name); 17 | 18 | // Scan through the headers, comparing file names against dir->hdr->filename 19 | for (struct ustar_header *cur = fs->root_hdr; cur != NULL; cur = ustar_nth_from_entry(cur, 1)) { 20 | size_t target_len = strlen(cur->filename); 21 | 22 | if (target_len < dir_len || target_len > dir_len + len + 1) 23 | continue; 24 | 25 | // If both path components match, we're done 26 | if (memcmp(dir_name, cur->filename, dir_len) == 0 27 | && memcmp(name, cur->filename + dir_len, len) == 0) { 28 | return 0; 29 | } 30 | } 31 | 32 | // Lookup failure is not an error (or, as I like to call it, LFINAE) 33 | *out = NULL; 34 | return 0; 35 | } 36 | 37 | struct inode_operations initrd_inode_dir_ops = { 38 | .lookup = initrd_dir_lookup_inode 39 | }; 40 | -------------------------------------------------------------------------------- /kernel/fs/initrd/file.c: -------------------------------------------------------------------------------- 1 | #include "fs.h" 2 | #include "fs/initrd.h" 3 | 4 | struct inode_operations initrd_inode_file_ops = { 5 | .lookup = NULL // This is only for directories (see initrd/dir.c) 6 | }; 7 | 8 | -------------------------------------------------------------------------------- /kernel/fs/initrd/super.c: -------------------------------------------------------------------------------- 1 | #include "fs.h" 2 | #include "fs/initrd.h" 3 | #include "mm.h" 4 | #include "libk.h" 5 | #include "multiboot2.h" 6 | 7 | static struct inode *initrd_alloc_inode(struct super_block *super) 8 | { 9 | struct initrd_inode *rd_inode = kmalloc(sizeof *rd_inode, KM_NONE); 10 | memset(rd_inode, 0, sizeof *rd_inode); 11 | rd_inode->i.sb = super; 12 | return &rd_inode->i; 13 | } 14 | 15 | static void initrd_dealloc_inode(struct inode *inode) 16 | { 17 | struct initrd_inode *rd_inode = container_of(inode, struct initrd_inode, i); 18 | if (rd_inode != NULL) 19 | kfree(rd_inode); 20 | } 21 | 22 | static err_t initrd_read_inode(struct super_block *sb, struct inode *inode) 23 | { 24 | if (inode->ino < INITRD_ROOT_INO) 25 | return ENOENT; 26 | else if (inode->ino == INITRD_ROOT_INO) { 27 | inode->ops = &initrd_inode_dir_ops; 28 | inode->mode = I_DIRECTORY; 29 | inode->sb = sb; 30 | return 0; 31 | } 32 | 33 | 34 | struct initrd_fs *fs = initrd_sb_container(sb); 35 | kassert_dbg(inode->sb == &fs->super); 36 | 37 | // This inode is a file/directory/symlink, parse the ustar memory 38 | struct ustar_header *hdr = ustar_nth_from_entry(fs->root_hdr, inode->ino - INITRD_ROOT_INO); 39 | 40 | if (hdr == NULL) 41 | return ENOENT; 42 | 43 | const mode_t mode_table[] = { I_REGULAR, 0, I_SYMLINK, 0, 0, I_DIRECTORY, 0 }; 44 | kassert_dbg(hdr->type >= '0' && hdr->type <= '6'); 45 | mode_t mode = mode_table[hdr->type - '0']; 46 | kassert(mode != 0); 47 | 48 | // Write to the inode 49 | inode->mode = mode; 50 | inode->ops = (mode == I_DIRECTORY) ? &initrd_inode_dir_ops : &initrd_inode_file_ops; 51 | 52 | return 0; 53 | } 54 | 55 | static struct super_operations initrd_super_ops = { 56 | .alloc_inode = initrd_alloc_inode, 57 | .read_inode = initrd_read_inode, 58 | .write_inode = NULL, 59 | .dealloc_inode = initrd_dealloc_inode 60 | }; 61 | 62 | struct super_block *initrd_get_super(void) 63 | { 64 | kassert_dbg(sizeof(struct ustar_header) == 512); 65 | 66 | struct initrd_fs *fs = kmalloc(sizeof *fs, KM_NONE); 67 | memset(fs, 0, sizeof *fs); 68 | 69 | struct inode *inode = kmalloc(sizeof *inode, KM_NONE); 70 | memset(inode, 0, sizeof *inode); 71 | 72 | fs->super.dev = 1; 73 | fs->super.root = inode; 74 | fs->super.ops = &initrd_super_ops; 75 | 76 | // Create the root directory 77 | fs->root = inode_get(&fs->super, INITRD_ROOT_INO); 78 | 79 | return &fs->super; 80 | } 81 | -------------------------------------------------------------------------------- /kernel/fs/initrd/ustar.c: -------------------------------------------------------------------------------- 1 | #include "fs/initrd.h" 2 | #include "util.h" 3 | 4 | #define USTAR_BLOCK 512 5 | 6 | // https://wiki.osdev.org/USTAR 7 | uint32_t ustar_oct_to_bin(const char *str, size_t len) 8 | { 9 | uint32_t n = 0; 10 | const uint8_t *c = (const uint8_t *)str; 11 | while (len-- > 0) { 12 | n *= 8; 13 | n += *c - '0'; 14 | c++; 15 | } 16 | return n; 17 | } 18 | 19 | struct ustar_header *ustar_nth_from_entry(struct ustar_header *root_hdr, uint32_t n) 20 | { 21 | struct ustar_header *cur = root_hdr; 22 | 23 | for (uint32_t i = 0; i < n; i++) { 24 | if (cur->filename[0] == '\0') 25 | return NULL; // EOF 26 | 27 | uint32_t size = ustar_oct_to_bin(cur->size, sizeof cur->size - 1); 28 | 29 | // Get next header 30 | uintptr_t next = ALIGNUP((uintptr_t)cur + size, USTAR_BLOCK); 31 | cur = (struct ustar_header *)next; 32 | } 33 | 34 | return cur; 35 | } 36 | 37 | virtaddr_t ustar_file_start(struct ustar_header *hdr) 38 | { 39 | return (virtaddr_t)((uintptr_t)hdr + USTAR_BLOCK); 40 | } 41 | -------------------------------------------------------------------------------- /kernel/fs/inode_cache.c: -------------------------------------------------------------------------------- 1 | #include "fs.h" 2 | #include "libk.h" 3 | #include "mutex.h" 4 | #include "ds/hash.h" 5 | 6 | #define TAB_SHIFT 8 7 | #define TAB_SIZE (1 << TAB_SHIFT) 8 | 9 | static struct { 10 | mutex_t mutex; 11 | 12 | struct hlist_bucket buckets[TAB_SIZE]; 13 | } inode_cache; 14 | 15 | // https://gist.github.com/lh3/974ced188be2f90422cc 16 | static inline uint64_t hash_inode(ino_t ino, dev_t dev) 17 | { 18 | uint32_t key = (ino + dev), mask = TAB_SIZE - 1; 19 | key = (~key + (key << 21)) & mask; 20 | key = key ^ key >> 24; 21 | key = ((key + (key << 3)) + (key << 8)) & mask; 22 | key = key ^ key >> 14; 23 | key = ((key + (key << 2)) + (key << 4)) & mask; 24 | key = key ^ key >> 28; 25 | key = (key + (key << 31)) & mask; 26 | return key; 27 | } 28 | 29 | struct inode *inode_get(struct super_block *sb, ino_t ino) 30 | { 31 | uint64_t hash = hash_inode(ino, sb->dev); 32 | struct inode *inode = NULL; 33 | struct hlist_bucket *bucket = &inode_cache.buckets[hash]; 34 | 35 | mutex_lock(&inode_cache.mutex); 36 | 37 | // If in list, return pointer 38 | hlist_foreach(cur, struct inode, node, bucket) { 39 | if (cur->ino == ino && cur->sb->dev == sb->dev) { 40 | inode = cur; 41 | break; 42 | } 43 | } 44 | 45 | // Else alloc and read 46 | if (inode == NULL) { 47 | kassert_dbg(HASOP(sb, alloc_inode)); 48 | inode = sb->ops->alloc_inode(sb); 49 | kassert(inode != NULL); // TODO: Add kmalloc can't fail requirement 50 | inode->ino = ino; 51 | 52 | kassert(HASOP(sb, read_inode)); 53 | err_t rv = sb->ops->read_inode(sb, inode); 54 | if (rv) { 55 | klog_warn("vfs", "read_inode fail: %ld\n", rv); 56 | 57 | kassert_dbg(HASOP(sb, dealloc_inode)); 58 | sb->ops->dealloc_inode(inode); 59 | 60 | inode = NULL; 61 | } else { 62 | // Add to list 63 | kassert_dbg(kref_read(&inode->rc) == 0); 64 | hlist_add(bucket, &inode->node); 65 | } 66 | } 67 | 68 | kref_inc(&inode->rc); 69 | 70 | mutex_unlock(&inode_cache.mutex); 71 | 72 | return inode; 73 | } 74 | 75 | void inode_put(struct inode *inode) 76 | { 77 | kprintf("%p\n", inode); 78 | uint64_t hash = hash_inode(inode->ino, inode->sb->dev); 79 | 80 | mutex_lock(&inode_cache.mutex); 81 | 82 | if (kref_dec_read(&inode->rc) == 0) { 83 | // No references left, delete this from the cache safely 84 | hlist_remove(&inode_cache.buckets[hash], &inode->node); 85 | 86 | kassert_dbg(HASOP(inode->sb, dealloc_inode)); 87 | inode->sb->ops->dealloc_inode(inode); 88 | } 89 | 90 | mutex_unlock(&inode_cache.mutex); 91 | 92 | // TODO: Delete inode from underlying superblock 93 | } 94 | -------------------------------------------------------------------------------- /kernel/fs/vfs.c: -------------------------------------------------------------------------------- 1 | #include "err.h" 2 | #include "fs.h" 3 | #include "fs/initrd.h" 4 | #include "libk.h" 5 | 6 | struct inode vfs_root; 7 | 8 | void vfs_init(void) 9 | { 10 | memset(&vfs_root, 0, sizeof vfs_root); 11 | err_t e = vfs_mount(&vfs_root, 1); 12 | if (e) 13 | panic("Failed to mount root fs"); 14 | } 15 | 16 | // Overwrites the inode to mount the filesystem at this point 17 | err_t vfs_mount(struct inode *mount_point, dev_t dev) 18 | { 19 | if (I_ISMNT(mount_point)) 20 | return EBUSY; 21 | 22 | // Get superblock from device number 23 | struct super_block *sb; 24 | if (dev == 1) { // TODO: Don't hardcode 25 | // Use initrd 26 | sb = initrd_get_super(); 27 | } else 28 | panic("Unknown device %d", dev); 29 | 30 | mount_point->sb = sb; 31 | mount_point->mounted = sb->root; 32 | mount_point->mode = I_MOUNT; 33 | 34 | return 0; 35 | } 36 | 37 | // Must call inode_put after the result is used 38 | err_t vfs_lookup(struct inode *dir, const char *path, size_t len, struct inode **result) 39 | { 40 | *result = NULL; 41 | 42 | if (!I_ISDIR(dir)) 43 | return ENOTDIR; 44 | else if (!HASOP(dir, lookup)) 45 | return ENOTSUP; 46 | return dir->ops->lookup(dir, path, len, result); 47 | } 48 | -------------------------------------------------------------------------------- /kernel/kmain.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | #include "multiboot2.h" 3 | #include "mm.h" 4 | #include "smp.h" 5 | #include "sync.h" 6 | #include "interrupts.h" 7 | #include "proc.h" 8 | #include "util.h" 9 | #include "percpu.h" 10 | #include "drivers/apic.h" 11 | #include "drivers/ps2.h" 12 | #include "drivers/pit.h" 13 | #include "drivers/acpi.h" 14 | #include "drivers/smbios.h" 15 | 16 | struct multiboot_info *mboot_info; 17 | 18 | void kmain(physaddr_t); 19 | 20 | void kmain(physaddr_t mboot_info_phys) 21 | { 22 | // Get the virtual address of the multiboot info structure 23 | mboot_info = phys_to_kern(mboot_info_phys); 24 | 25 | // Initialise our per-CPU data area 26 | percpu_init_bsp(); 27 | 28 | // Initialise paging 29 | vmm_init(); 30 | 31 | // Create the bootstrapping memory allocator 32 | struct mmap *mem_map = mmap_init(mboot_info); 33 | 34 | // Linearly map all physical memory 35 | vmm_map_all(mem_map); 36 | mmap_dump_info(); 37 | 38 | // Find ACPI tables 39 | acpi_init(); 40 | 41 | // Find SMBIOS tables 42 | smbios_init(); 43 | 44 | // Gather info from the MADT 45 | apic_init(); 46 | 47 | // Start the physical memory manager 48 | // After this point, we have access to pmm_alloc_order and kmalloc 49 | pmm_init(mem_map); 50 | 51 | // Enable the LAPIC for the BSP 52 | lapic_enable(); 53 | 54 | // Initialise all I/O APICs 55 | ioapic_init(); 56 | 57 | // Initialise mouse and keyboard 58 | ps2_init(); 59 | 60 | // Initialise the PIT 61 | pit_init(); 62 | 63 | // Turn on IRQs 64 | irq_enable(); 65 | 66 | // Boot all the cores 67 | smp_init(); 68 | 69 | // Run the scheduler 70 | sched_run_bsp(); 71 | } 72 | -------------------------------------------------------------------------------- /kernel/mm/area.c: -------------------------------------------------------------------------------- 1 | #include "mm.h" 2 | #include "libk.h" 3 | #include "ds/linked.h" 4 | 5 | void area_add(struct mmu_info *mmu, struct vm_area *area) 6 | { 7 | kassert_dbg(mmu != &kernel_mmu); 8 | if (mmu->areas == NULL) { 9 | mmu->areas = area; 10 | } else { 11 | struct vm_area *prev = NULL; 12 | slist_foreach(cur, list, mmu->areas) { 13 | kassert_dbg(cur->base != area->base); 14 | if (cur->base > area->base) { 15 | kassert_dbg(area->base + area->len < cur->base); 16 | slist_set_next(area, list, cur); 17 | if (prev != NULL) 18 | slist_set_next(prev, list, area); 19 | else { 20 | mmu->areas = area; 21 | slist_set_next(area, list, cur); 22 | } 23 | return; 24 | } 25 | prev = cur; 26 | } 27 | kassert_dbg(prev->base + prev->len < area->base); 28 | slist_set_next(prev, list, area); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /kernel/mm/cow.c: -------------------------------------------------------------------------------- 1 | #include "mm.h" 2 | #include "proc.h" 3 | #include "percpu.h" 4 | #include "libk.h" 5 | #include "asm.h" 6 | 7 | // Note: caller is expected to flush TLB and disable preemption 8 | void cow_copy_pte(pte_t *dest, pte_t *src) 9 | { 10 | struct page *page = phys_to_page(*src & PTE_ADDR_MASK); 11 | 12 | // Make page read-only and CoW if it isn't already 13 | if (!(*src & PAGE_COW)) { 14 | *src &= ~PAGE_WRITABLE; 15 | *src |= PAGE_COW; 16 | kref_inc(&page->refcount); 17 | } else 18 | kassert_dbg(!(*src & PAGE_WRITABLE)); 19 | 20 | if (kref_inc_read(&page->refcount) == 0) 21 | panic("Tried to copy free'd page"); 22 | 23 | // Copy the PTE 24 | *dest = *src; 25 | } 26 | 27 | // Returns true if the write is allowed and should be retried 28 | bool cow_handle_write(pte_t *pte, virtaddr_t virt) 29 | { 30 | if (pte == NULL || !(*pte & PAGE_COW)) 31 | return false; 32 | 33 | struct page *page = phys_to_page(*pte & PTE_ADDR_MASK); 34 | spin_lock(&page->lock); 35 | 36 | //kprintf_nolock("Handle CoW write to address %p\n", (void *)page_to_phys(page)); 37 | 38 | uint32_t next_count = kref_dec_read(&page->refcount); 39 | if (next_count > 0) { 40 | // Allocate another page, and copy the memory into it 41 | physaddr_t dest = page_to_phys(pmm_alloc_order(0, GFP_NONE)); 42 | physaddr_t src = *pte & PTE_ADDR_MASK; 43 | memcpy(phys_to_virt(dest), phys_to_virt(src), PAGE_SIZE); 44 | 45 | // Write the new address into the PTE 46 | *pte &= ~PTE_ADDR_MASK; 47 | *pte |= (dest & PTE_ADDR_MASK); 48 | } 49 | 50 | // Otherwise, this was the last reference and we are free to reuse this memory 51 | spin_unlock(&page->lock); 52 | *pte &= ~PAGE_COW; 53 | *pte |= PAGE_WRITABLE; 54 | tlb_flush_single(virt); 55 | return true; 56 | } 57 | 58 | // TODO: This might be a race condition (either now or in the future) if another 59 | // process tries to copy a PTE at the same time as we're freeing it. 60 | void cow_handle_free(pte_t *pte) 61 | { 62 | kassert_dbg(pte != NULL); 63 | kassert_dbg(*pte & PAGE_COW); 64 | 65 | physaddr_t phys = *pte & PTE_ADDR_MASK; 66 | if (phys < (uintptr_t)&_kernel_end_phys) 67 | return; // This page is probably mapped to the zero page or some kernel code 68 | 69 | struct page *page = phys_to_page(phys); 70 | 71 | uint64_t next_count = kref_dec_read(&page->refcount); 72 | if (next_count == 0) { 73 | pmm_free_order(page, 0); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /kernel/mm/mmap.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "libk.h" 4 | #include "mm.h" 5 | #include "multiboot2.h" 6 | #include "util.h" 7 | 8 | #define WITHIN(x, y, len) ((x) <= (y) && ((x) + (len) > (y))) 9 | 10 | // These could be dynamically resizable in future 11 | struct mmap_region available[MMAP_MAX_REGIONS]; 12 | struct mmap_region reserved[MMAP_MAX_REGIONS]; 13 | 14 | static struct mmap mem_map = { 15 | .available = { .count = 0, .regions = available }, 16 | .reserved = { .count = 0, .regions = reserved } 17 | }; 18 | 19 | static void mmap_delete_region(struct mmap_type *type, size_t i) 20 | { 21 | kassert_dbg(i < type->count); 22 | memmove(type->regions + i, type->regions + i + 1, (type->count - i - 1) * sizeof(struct mmap_region)); 23 | type->count--; 24 | } 25 | 26 | static void mmap_insert_region(struct mmap_type *type, uintptr_t addr, size_t len, enum mmap_region_flags flags) 27 | { 28 | kassert(type->count <= MMAP_MAX_REGIONS); 29 | if (len == 0) 30 | return; 31 | // Search for correct position in region list 32 | for (size_t i = 0; i < type->count; i++) { 33 | uintptr_t rg_addr = type->regions[i].base; 34 | size_t rg_len = type->regions[i].len; 35 | // If the region is directly below, merge 36 | if (addr + len == rg_addr) { 37 | type->regions[i].base = addr; 38 | type->regions[i].len += len; 39 | } 40 | // If the region is below, insert 41 | else if (addr + len < rg_addr) { 42 | if (type->count == MMAP_MAX_REGIONS) 43 | panic("No space left in mmap"); 44 | // Shift everything to the right 45 | memmove(type->regions + i + 1, type->regions + i, (type->count - i) * sizeof(struct mmap_region)); 46 | const struct mmap_region new_region = { 47 | .base = addr, 48 | .len = len, 49 | .flags = flags 50 | }; 51 | type->regions[i] = new_region; 52 | type->count++; 53 | return; 54 | } 55 | // If the region is directly above, merge 56 | else if (rg_addr + rg_len == addr) { 57 | type->regions[i].len += len; 58 | // Check if we can merge the next block 59 | if (i < type->count - 1 && addr + len == type->regions[i + 1].base) { 60 | type->regions[i].len += type->regions[i + 1].len; 61 | mmap_delete_region(type, i + 1); 62 | } 63 | return; 64 | } 65 | // If the memory overlaps, something bad happened 66 | kassert_dbg(!WITHIN(addr, rg_addr, len) && !WITHIN(rg_addr, addr, rg_len)); 67 | } 68 | 69 | // Special case: we are inserting a region after all other regions 70 | if (type->count == MMAP_MAX_REGIONS) 71 | panic("No space left in mmap"); 72 | const struct mmap_region new_region = { 73 | .base = addr, 74 | .len = len, 75 | .flags = flags 76 | }; 77 | type->regions[type->count++] = new_region; 78 | } 79 | 80 | void mmap_dump_info(void) 81 | { 82 | klog("mmap", "Highest mapped physical page is %p\n", (void *)mem_map.highest_mapped); 83 | #ifdef VERBOSE 84 | klog("mmap", "Available:\n"); 85 | for (size_t i = 0; i < mem_map.available.count; i++) 86 | klog("mmap", " Region base %p, len %zu\n", (void *)mem_map.available.regions[i].base, mem_map.available.regions[i].len); 87 | klog("mmap", "Reserved:\n"); 88 | for (size_t i = 0; i < mem_map.reserved.count; i++) 89 | klog("mmap", " Region base %p, len %zu\n", (void *)mem_map.reserved.regions[i].base, mem_map.reserved.regions[i].len); 90 | #endif 91 | } 92 | 93 | struct mmap_region mmap_alloc_low(size_t n, unsigned int alloc_flags) 94 | { 95 | for (size_t i = 0; i < mem_map.available.count; i++) { 96 | struct mmap_region *rg = &mem_map.available.regions[i]; 97 | if (rg->base >= mem_map.highest_mapped) 98 | break; // This memory is above what is mapped, so give up 99 | 100 | size_t available_bytes, effective_len = rg->len; 101 | uintptr_t effective_base = rg->base; 102 | 103 | if (alloc_flags & MMAP_ALLOC_PA) { 104 | effective_base = ALIGNUP(rg->base, PAGE_SIZE); 105 | effective_len -= (effective_base - rg->base); 106 | } 107 | 108 | // If we can satisfy the allocation by splitting into a new block 109 | available_bytes = MIN(mem_map.highest_mapped - effective_base, effective_len); 110 | if (n <= available_bytes) { 111 | size_t remaining = available_bytes - n; 112 | uintptr_t new_base = effective_base + n; 113 | struct mmap_region old_rg = { .base = effective_base, .len = n, .flags = rg->flags }; 114 | if (alloc_flags & MMAP_ALLOC_PA) { 115 | uintptr_t padded_base = rg->base; 116 | size_t padded_len = effective_base - rg->base; 117 | mmap_delete_region(&mem_map.available, i); 118 | mmap_insert_region(&mem_map.available, padded_base, padded_len, rg->flags); 119 | } else 120 | mmap_delete_region(&mem_map.available, i); 121 | mmap_insert_region(&mem_map.reserved, old_rg.base, old_rg.len, old_rg.flags); 122 | if (remaining > 0) 123 | mmap_insert_region(&mem_map.available, new_base, remaining, rg->flags); 124 | kassert_dbg(old_rg.base > 0 && old_rg.len > 0); 125 | // TODO: Memset zero it all out 126 | return old_rg; 127 | } 128 | } 129 | // Allocation not satisfied 130 | panic("Cannot satisfy mmap allocation of size %zu", n); 131 | } 132 | 133 | // TODO: Cleanup 134 | struct mmap *mmap_init(struct multiboot_info *mboot) 135 | { 136 | // Find memory map 137 | struct multiboot_tag_mmap *p = (struct multiboot_tag_mmap *)mboot->tags; 138 | for (; p->type != MULTIBOOT_TAG_TYPE_END && p->type != MULTIBOOT_TAG_TYPE_MMAP; 139 | p = (struct multiboot_tag_mmap *)ALIGNUP((uintptr_t)p + p->size, 8)) 140 | ; 141 | if (p->type != MULTIBOOT_TAG_TYPE_MMAP) 142 | panic("No multiboot memory map tag found"); 143 | 144 | uint32_t entry_size = p->entry_size; 145 | uint32_t total_size = p->size; 146 | uintptr_t kern_end = (uintptr_t)&_kernel_end_phys; 147 | 148 | struct multiboot_mmap_entry *mmap; 149 | // Loop over memory map 150 | for (mmap = p->entries; (uint8_t *)mmap < (uint8_t *)p + total_size; 151 | mmap = (struct multiboot_mmap_entry *)((uintptr_t)mmap + entry_size)) { 152 | // Insert a new region for each node in the memory map 153 | struct mmap_type *target; 154 | if (mmap->type == MULTIBOOT_MEMORY_AVAILABLE) { 155 | // Here we impose some further restrictions. 156 | // Must be above or equal to where the kernel is mapped. 157 | // Try and squeeze any available memory into the mmap. 158 | if (mmap->addr + mmap->len < kern_end) 159 | target = &mem_map.reserved; 160 | else if (mmap->addr < kern_end) { 161 | mmap_insert_region(&mem_map.reserved, mmap->addr, kern_end - mmap->addr, MMAP_NONE); 162 | mmap_insert_region(&mem_map.available, kern_end, mmap->len - (kern_end - mmap->addr), MMAP_NONE); 163 | continue; 164 | } else 165 | target = &mem_map.available; 166 | } else { 167 | target = &mem_map.reserved; 168 | } 169 | mmap_insert_region(target, mmap->addr, mmap->len, MMAP_NONE); 170 | } 171 | 172 | mem_map.highest_mapped = KERNEL_PHYS_MAP_END; 173 | return &mem_map; 174 | } 175 | -------------------------------------------------------------------------------- /kernel/mm/mmu.c: -------------------------------------------------------------------------------- 1 | #include "mm.h" 2 | #include "util.h" 3 | #include "libk.h" 4 | #include "proc.h" 5 | #include "percpu.h" 6 | 7 | static inline void copy_kernel_mappings(struct page_table *p4) 8 | { 9 | memcpy(p4, kernel_mmu.p4, PAGE_SIZE); 10 | } 11 | 12 | struct mmu_info *mmu_alloc(void) 13 | { 14 | struct mmu_info *rv = kmalloc(sizeof(struct mmu_info), KM_NONE); 15 | memset(rv, 0, sizeof *rv); 16 | mmu_inc_users(rv); 17 | return rv; 18 | } 19 | 20 | void mmu_free(struct mmu_info *mmu) 21 | { 22 | kassert_dbg(mmu != &kernel_mmu); 23 | kassert_dbg(kref_read(&mmu->users) == 0); 24 | mmu_reap(mmu); 25 | //kfree(mmu); // Causes use after free issues with mmu_switch if called before switching out 26 | } 27 | 28 | void mmu_init(struct mmu_info *mmu) 29 | { 30 | cpuset_clear(&mmu->cpus); 31 | mmu->p4 = page_to_virt(pmm_alloc_order(0, GFP_NONE)); 32 | copy_kernel_mappings(mmu->p4); 33 | } 34 | 35 | void mmu_update_cpuset(struct mmu_info *mmu, cpuid_t id, bool val) 36 | { 37 | if (mmu != &kernel_mmu) { 38 | spin_lock(&mmu->cpu_lock); 39 | cpuset_set_id(&mmu->cpus, id, val); 40 | spin_unlock(&mmu->cpu_lock); 41 | } 42 | } 43 | 44 | // Important: Be careful if you access current here, as it refers to the new process, not the old one 45 | void mmu_switch(struct mmu_info *next, struct mmu_info *prev) 46 | { 47 | kassert_dbg(next != NULL && prev != NULL); 48 | if (next != prev) { 49 | // Keep track of which CPUs are currently using the MMU. 50 | // This is important so that we know which CPUs we have to send IPIs to 51 | // when we need to do a TLB shootdown. 52 | preempt_inc(); 53 | mmu_update_cpuset(next, percpu_get(id), 0); 54 | mmu_update_cpuset(prev, percpu_get(id), 1); 55 | preempt_dec(); 56 | 57 | // Swap the paging structures 58 | kassert_dbg(next->p4 != prev->p4); 59 | write_cr3(virt_to_phys(next->p4)); 60 | } else 61 | kassert_dbg(next->p4 == prev->p4); 62 | } 63 | 64 | void mmu_inc_users(struct mmu_info *mmu) 65 | { 66 | if (mmu != &kernel_mmu) { 67 | kref_inc(&mmu->users); 68 | } 69 | } 70 | 71 | void mmu_dec_users(struct mmu_info *mmu) 72 | { 73 | if (mmu != &kernel_mmu) { 74 | uint32_t num = kref_dec_read(&mmu->users); 75 | if (num == 0) 76 | mmu_free(mmu); 77 | } 78 | } 79 | 80 | static void free_single_page(pte_t *pte) 81 | { 82 | physaddr_t phys = *pte & PTE_ADDR_MASK; 83 | if (*pte & PAGE_COW) { 84 | cow_handle_free(pte); 85 | } else if (phys > (physaddr_t)&_kernel_end_phys) { 86 | pmm_free_order(phys_to_page(phys), 0); 87 | } 88 | *pte = 0; 89 | } 90 | 91 | void mmu_reap(struct mmu_info *mmu) 92 | { 93 | kassert_dbg(mmu != &kernel_mmu); 94 | kassert_dbg(atomic_read32(&mmu->users) == 0); 95 | 96 | // Don't bother locking since there are guaranteed no other users at this point 97 | 98 | // Only loop over the userspace mappings 99 | for (size_t p4_index = 0; p4_index < (1 << 7); p4_index++) { 100 | struct page_table *p3 = pgtab_extract_virt_addr(mmu->p4, p4_index); 101 | if (p3 == NULL) 102 | continue; 103 | for (size_t p3_index = 0; p3_index < 512; p3_index++) { 104 | struct page_table *p2 = pgtab_extract_virt_addr(p3, p3_index); 105 | if (p2 == NULL) 106 | continue; 107 | for (size_t p2_index = 0; p2_index < 512; p2_index++) { 108 | struct page_table *p1 = pgtab_extract_virt_addr(p2, p2_index); 109 | if (p1 == NULL) 110 | continue; 111 | for (size_t p1_index = 0; p1_index < 512; p1_index++) { 112 | if (p1->pages[p1_index] & PAGE_PRESENT) { 113 | free_single_page(&p1->pages[p1_index]); 114 | } 115 | } 116 | pmm_free_order(virt_to_page(p1), 0); 117 | p2->pages[p2_index] = 0; 118 | } 119 | pmm_free_order(virt_to_page(p2), 0); 120 | p3->pages[p3_index] = 0; 121 | } 122 | pmm_free_order(virt_to_page(p3), 0); 123 | mmu->p4->pages[p4_index] = 0; 124 | } 125 | 126 | // Prevent freeing the PML4 from underneath us 127 | write_cr3(virt_to_phys(kernel_mmu.p4)); 128 | 129 | free_page_at(mmu->p4); 130 | mmu->p4 = NULL; 131 | 132 | // TODO: Might need a TLB shootdown here. 133 | } 134 | 135 | static inline pte_t clone_single_page(pte_t *pte) 136 | { 137 | pte_t dest; 138 | cow_copy_pte(&dest, pte); 139 | return dest; 140 | } 141 | 142 | static struct page_table *clone_pgtab(struct page_table *pgtab, size_t level) 143 | { 144 | struct page_table *rv = page_to_virt(pmm_alloc_order(0, GFP_NONE)); 145 | size_t end_index = 512; 146 | memset(rv, 0, sizeof *rv); 147 | 148 | if (level == 4) { 149 | copy_kernel_mappings(rv); 150 | end_index = (1 << 7); 151 | } 152 | 153 | for (size_t i = 0; i < end_index; i++) { 154 | if (pgtab->pages[i] & PAGE_PRESENT) { 155 | if (level == 1) { 156 | rv->pages[i] = clone_single_page(&pgtab->pages[i]); 157 | } else { 158 | uint64_t flags = pgtab->pages[i] & ~PTE_ADDR_MASK; 159 | physaddr_t pgtab_phys = (physaddr_t)(pgtab->pages[i] & PTE_ADDR_MASK); 160 | virtaddr_t pgtab_virt = phys_to_virt(pgtab_phys); 161 | kassert_dbg(ISALIGN_POW2((uintptr_t)pgtab_virt, PAGE_SIZE)); 162 | rv->pages[i] = virt_to_phys(clone_pgtab(pgtab_virt, level - 1)) | flags; 163 | } 164 | } 165 | } 166 | 167 | return rv; 168 | } 169 | 170 | void mmu_clone_cow(struct mmu_info *dest, struct mmu_info *mmu) 171 | { 172 | kassert_dbg(dest->p4 == NULL); 173 | 174 | // Copy mappings from the parent 175 | write_spin_lock(&mmu->pgtab_lock); 176 | dest->p4 = clone_pgtab(mmu->p4, 4); 177 | write_spin_unlock(&mmu->pgtab_lock); 178 | 179 | // Since the entire address space got mapped as read only, we need to invalidate all of it 180 | tlb_flush_all(); 181 | } 182 | -------------------------------------------------------------------------------- /kernel/mm/slob.c: -------------------------------------------------------------------------------- 1 | #include "mm.h" 2 | #include "libk.h" 3 | #include "util.h" 4 | #include "sync.h" 5 | 6 | #define UNIT sizeof(struct slob_header) 7 | 8 | // Anything more than (4096 - UNIT) bytes will get it's own large_alloc slot 9 | #define BIGALLOC (PAGE_SIZE - UNIT) 10 | 11 | /* Slob Allocator 12 | * 13 | * This is a simple K&R style first-fit linked list allocator for general 14 | * purpose kernel allocations. One list is maintained for small allocations 15 | * and another is used for large allocations (roughly greater than a page). 16 | * 17 | * Inspired by Linux's SLOB allocator. */ 18 | 19 | static spinlock_t slob_lock; 20 | 21 | struct slob_large_alloc { 22 | struct slist_node list; 23 | virtaddr_t addr; 24 | uint8_t order; 25 | }; 26 | 27 | struct slob_header { 28 | struct slist_node list; // Pointer to next 29 | size_t units; // bytes = units * sizeof(struct slob_header) 30 | }; 31 | 32 | struct slob_header *head; 33 | struct slob_large_alloc *large_head; 34 | 35 | static void morecore(void) 36 | { 37 | // Grab a single page 38 | virtaddr_t p = page_to_virt(pmm_alloc_order(0, GFP_NONE)); // TODO: Check for fail 39 | struct slob_header *hd = (struct slob_header *)p; 40 | hd->units = (PAGE_SIZE - UNIT) / UNIT; 41 | 42 | slist_set_next(hd, list, head); 43 | head = hd; 44 | 45 | klog("slob", "Morecore allocated a page at %p\n", p); 46 | } 47 | 48 | // TODO: Worry about alignment 49 | static virtaddr_t slob_alloc(size_t bytes, unsigned int alloc_flags) 50 | { 51 | (void)alloc_flags; 52 | size_t units = DIV_ROUND_UP(bytes, UNIT); 53 | if (bytes < BIGALLOC) { 54 | // This should only loop at most once since our allocation is guaranteed to succeed 55 | // if morecore() is called. This is only true because morecore allocates at least 56 | // BIGALLOC bytes, and also no other allocations can occur inbetween. 57 | spin_lock(&slob_lock); 58 | while (1) { 59 | struct slob_header *prev = NULL; 60 | slist_foreach(block, list, head) { 61 | if (block->units == units) { 62 | // Exact match 63 | if (prev == NULL) 64 | head = block; 65 | else 66 | slist_set_next(prev, list, slist_get_next(block, list)); 67 | spin_unlock(&slob_lock); 68 | return (virtaddr_t)(block + 1); 69 | } else if (block->units > units) { 70 | // Split a block 71 | struct slob_header *new_block = block + units + 1; 72 | new_block->units = block->units - units - 1; 73 | block->units = units; 74 | slist_set_next(new_block, list, slist_get_next(block, list)); 75 | // Remove allocated block from list 76 | if (prev == NULL) 77 | head = new_block; 78 | else 79 | slist_set_next(prev, list, new_block); 80 | spin_unlock(&slob_lock); 81 | return (virtaddr_t)(block + 1); 82 | } 83 | prev = block; 84 | } 85 | morecore(); 86 | } 87 | } else { 88 | panic("unimplemented"); 89 | } 90 | return NULL; 91 | } 92 | 93 | // TODO: Defragment 94 | static void slob_free(virtaddr_t p) 95 | { 96 | if (ISALIGN_POW2((uintptr_t)p, PAGE_SIZE)) { 97 | // BIGALLOC bytes or more 98 | panic("unimplemented"); 99 | } else { 100 | struct slob_header *header = (struct slob_header *)p - 1; 101 | spin_lock(&slob_lock); 102 | slist_set_next(header, list, head); 103 | head = header; 104 | #ifdef DEBUG 105 | memset(p, 0xBB, header->units * UNIT); 106 | #endif 107 | spin_unlock(&slob_lock); 108 | } 109 | } 110 | 111 | #if 0 112 | static void slob_debug(void) 113 | { 114 | slist_foreach(block, list, head) { 115 | kprintf("Block: %p, %zu units, %p next\n", (void *)(block + 1), block->units, (void *)(slist_get_next(block, list) + 1)); 116 | } 117 | } 118 | #endif 119 | 120 | void *kmalloc(size_t n, unsigned int alloc_flags) 121 | { 122 | #ifdef DEBUG 123 | void *p = slob_alloc(n, alloc_flags); 124 | memset(p, 0xBB, n); 125 | return p; 126 | #else 127 | return slob_alloc(n, alloc_flags); 128 | #endif 129 | } 130 | 131 | void kfree(void *p) 132 | { 133 | slob_free(p); 134 | } 135 | -------------------------------------------------------------------------------- /kernel/mm/tlb.c: -------------------------------------------------------------------------------- 1 | #include "mm.h" 2 | #include "percpu.h" 3 | #include "proc.h" 4 | #include "smp.h" 5 | #include "libk.h" 6 | 7 | #define MAX_TLB_OPS MAX_CORES 8 | 9 | struct tlb_op *volatile tlb_op_location; 10 | static spinlock_t shootdown_lock; 11 | atomic32_t tlb_remaining_cpus; 12 | 13 | void tlb_shootdown(struct mmu_info *mmu, virtaddr_t start, virtaddr_t end) 14 | { 15 | struct tlb_op op = { 16 | .start = start, 17 | .end = end, 18 | }; 19 | 20 | klog_verbose("tlb", "TLB shootdown range %p - %p\n", start, end); 21 | spin_lock(&shootdown_lock); 22 | spin_lock(&mmu->cpu_lock); 23 | 24 | tlb_op_location = &op; 25 | 26 | // Send the IPI 27 | // TODO: Find a more efficient method than sending IPIs one by one (logical destination mode?) 28 | for (size_t i = 0; i < smp_nr_cpus(); i++) { 29 | if (i != percpu_get(id) && cpuset_query_id(&mmu->cpus, i)) { 30 | atomic_inc_read32(&tlb_remaining_cpus); 31 | ipi_send_fixed(i, IRQ_IPI_TLB_SHOOTDOWN); 32 | } 33 | } 34 | 35 | // Wait for the other CPUs to finish 36 | while (atomic_read32(&tlb_remaining_cpus) != 0) { 37 | pause(); 38 | } 39 | 40 | spin_unlock(&mmu->cpu_lock); 41 | spin_unlock(&shootdown_lock); 42 | } 43 | 44 | // TODO: Functions for lazy TLB invalidation 45 | 46 | -------------------------------------------------------------------------------- /kernel/mm/vmm.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "libk.h" 3 | #include "mm.h" 4 | #include "asm.h" 5 | #include "util.h" 6 | #include "smp.h" 7 | #include "percpu.h" 8 | #include "drivers/pit.h" 9 | 10 | #define P4_ADDR_SHIFT 39 11 | #define P3_ADDR_SHIFT 30 12 | #define P2_ADDR_SHIFT 21 13 | #define P1_ADDR_SHIFT 12 14 | #define P1_ADDR_MASK (0x1FFUL << P1_ADDR_SHIFT) 15 | #define P4_ADDR_MASK (0x1FFUL << P4_ADDR_SHIFT) 16 | #define P3_ADDR_MASK (0x1FFUL << P3_ADDR_SHIFT) 17 | #define P2_ADDR_MASK (0x1FFUL << P2_ADDR_SHIFT) 18 | #define PAGE_OFFSET_MASK 0xFFFF 19 | 20 | static void dump_page_tables_p2(struct page_table *, uintptr_t); 21 | static void dump_page_tables_p1(struct page_table *, uintptr_t); 22 | static void dump_page_tables_p0(struct page_table *, uintptr_t); 23 | 24 | extern struct page_table p4_table; // Initial kernel p4 table 25 | struct mmu_info kernel_mmu; 26 | 27 | __attribute__((aligned(PAGE_SIZE))) const uint8_t zero_page[PAGE_SIZE] = { 0 }; 28 | 29 | void vmm_init(void) 30 | { 31 | kernel_mmu.p4 = phys_to_kern((physaddr_t)&p4_table); 32 | kernel_mmu.areas = NULL; 33 | klog("vmm", "Kernel P4 at %p\n", kernel_mmu.p4); 34 | // vmm_dump_tables(); 35 | } 36 | 37 | void vmm_map_all(struct mmap *mmap) 38 | { 39 | // First map all regions marked "available", updating highest_mapped 40 | for (size_t i = 0; i < mmap->available.count; i++) { 41 | physaddr_t start = ALIGNUP(mmap->available.regions[i].base, PAGE_SIZE); 42 | physaddr_t end = mmap->available.regions[i].base + mmap->available.regions[i].len; 43 | for (physaddr_t j = start; j <= (end - PAGE_SIZE); j += PAGE_SIZE) { 44 | vmm_map_page(&kernel_mmu, j, phys_to_virt(j), VMM_ALLOC_MMAP | PAGE_GLOBAL | PAGE_WRITABLE); 45 | mmap->highest_mapped = MAX(j, mmap->highest_mapped); 46 | } 47 | } 48 | 49 | // Mark the zero page as read only so we fault if we access it 50 | *vmm_get_pte(&kernel_mmu, zero_page) = 0; 51 | 52 | kernel_mmu.p4 = phys_to_virt((physaddr_t)&p4_table); 53 | } 54 | 55 | static inline pte_t alloc_pgtab(unsigned int alloc_flags) 56 | { 57 | uint64_t flags = PAGE_PRESENT | PAGE_WRITABLE | (alloc_flags & (PAGE_USER_ACCESSIBLE)); 58 | if (alloc_flags & VMM_ALLOC_MMAP) { 59 | // Mmap low allocs are guaranteed to be mapped 60 | physaddr_t pgtab_phys = mmap_alloc_low(PAGE_SIZE, MMAP_ALLOC_PA).base; 61 | kassert_dbg((pgtab_phys & 0xFFF) == 0); // Must be aligned 62 | return (pgtab_phys & PTE_ADDR_MASK) | flags; 63 | } else { 64 | struct page *p = pmm_alloc_order(0, GFP_NONE); 65 | kassert_dbg(p != NULL); 66 | physaddr_t pgtab_phys = page_to_phys(p); 67 | return (pgtab_phys & PTE_ADDR_MASK) | flags; 68 | } 69 | } 70 | 71 | void vmm_dump_tables(struct mmu_info *mmu) 72 | { 73 | // Careful of deadlocks here 74 | read_spin_lock(&mmu->pgtab_lock); 75 | for (size_t i = 0; i < (1 << 7); i++) { 76 | struct page_table *pgtab = pgtab_extract_virt_addr(mmu->p4, i); 77 | if (pgtab == NULL) 78 | continue; 79 | kprintf("P3: %lx\n", mmu->p4->pages[i] & ~PTE_ADDR_MASK); 80 | dump_page_tables_p2(pgtab, 0xFFFF000000000000 | (i << 39)); 81 | } 82 | read_spin_unlock(&mmu->pgtab_lock); 83 | } 84 | 85 | static void dump_page_tables_p2(struct page_table *p3, uintptr_t addr_bits) 86 | { 87 | for (size_t i = 0; i < 512; i++) { 88 | struct page_table *pgtab = pgtab_extract_virt_addr(p3, i); 89 | if (pgtab == NULL) 90 | continue; 91 | kprintf("\tP2: %lx\n", p3->pages[i] & ~PTE_ADDR_MASK); 92 | dump_page_tables_p1(pgtab, addr_bits | (i << 30)); 93 | } 94 | } 95 | 96 | static void dump_page_tables_p1(struct page_table *p2, uintptr_t addr_bits) 97 | { 98 | for (size_t i = 0; i < 512; i++) { 99 | struct page_table *pgtab = pgtab_extract_virt_addr(p2, i); 100 | if (pgtab == NULL) 101 | continue; 102 | virtaddr_t first_virt = (virtaddr_t)(addr_bits | (i << 21)); 103 | physaddr_t first_phys = pgtab->pages[0] & PTE_ADDR_MASK; 104 | kprintf("\t\tP1: %p -> %p, %lx\n", first_virt, (void *)first_phys, p2->pages[i] & ~PTE_ADDR_MASK); 105 | //dump_page_tables_p0(pgtab, addr_bits | (i << 21)); 106 | //pit_sleep_ms(300); 107 | } 108 | } 109 | 110 | static void __attribute__((unused)) dump_page_tables_p0(struct page_table *p0, uintptr_t addr_bits) 111 | { 112 | for (size_t i = 0; i < 512; i++) { 113 | physaddr_t addr = p0->pages[i] & PTE_ADDR_MASK; 114 | if (addr == 0) 115 | continue; 116 | kprintf("\t\t\tPage: %p -> %p, %lx\n", 117 | (virtaddr_t)(addr_bits | (i << 12)), 118 | (void *)(p0->pages[i] & PTE_ADDR_MASK), 119 | p0->pages[i] & ~PTE_ADDR_MASK); 120 | } 121 | } 122 | 123 | // IMPORTANT: The page table lock must be held during this function 124 | pte_t *vmm_get_pte(struct mmu_info *mmu, const void *addr) 125 | { 126 | const uintptr_t va = (uintptr_t)addr; 127 | const uint16_t p4_index = (va & P4_ADDR_MASK) >> P4_ADDR_SHIFT; 128 | const uint16_t p3_index = (va & P3_ADDR_MASK) >> P3_ADDR_SHIFT; 129 | const uint16_t p2_index = (va & P2_ADDR_MASK) >> P2_ADDR_SHIFT; 130 | const uint16_t p1_index = (va & P1_ADDR_MASK) >> P1_ADDR_SHIFT; 131 | 132 | struct page_table *p3_table = pgtab_extract_virt_addr(mmu->p4, p4_index); 133 | if (p3_table == NULL) 134 | return NULL; 135 | 136 | struct page_table *p2_table = pgtab_extract_virt_addr(p3_table, p3_index); 137 | if (p2_table == NULL) 138 | return NULL; 139 | 140 | struct page_table *p1_table = pgtab_extract_virt_addr(p2_table, p2_index); 141 | if (p1_table == NULL) 142 | return NULL; 143 | 144 | return &p1_table->pages[p1_index]; 145 | } 146 | 147 | bool vmm_has_flags(struct mmu_info *mmu, void *addr, uint64_t flags) 148 | { 149 | kassert_dbg(addr != NULL); 150 | read_spin_lock(&mmu->pgtab_lock); 151 | pte_t *pte = vmm_get_pte(mmu, addr); 152 | if (pte == NULL) { 153 | read_spin_unlock(&mmu->pgtab_lock); 154 | return false; 155 | } 156 | 157 | bool rv = (*pte & flags) != 0; 158 | read_spin_unlock(&mmu->pgtab_lock); 159 | return rv; 160 | } 161 | 162 | // TODO: Think about deadlocks that might occur as a result of this function. 163 | void vmm_map_page(struct mmu_info *mmu, physaddr_t phys, virtaddr_t virt, unsigned long flags) 164 | { 165 | const uintptr_t va = (uintptr_t)virt & ~(PAGE_SIZE - 1); 166 | const uint16_t p4_index = (va & P4_ADDR_MASK) >> P4_ADDR_SHIFT; 167 | const uint16_t p3_index = (va & P3_ADDR_MASK) >> P3_ADDR_SHIFT; 168 | const uint16_t p2_index = (va & P2_ADDR_MASK) >> P2_ADDR_SHIFT; 169 | const uint16_t p1_index = (va & P1_ADDR_MASK) >> P1_ADDR_SHIFT; 170 | 171 | bool unmap = flags & VMM_UNMAP; 172 | bool lowmem = virt < (virtaddr_t)KERNEL_LOGICAL_BASE; 173 | unsigned int pgtab_flags = flags | (lowmem ? PAGE_USER_ACCESSIBLE : 0); 174 | 175 | // When the bit is on, execution is disabled, so we need to toggle the bit 176 | flags ^= PAGE_EXECUTABLE; 177 | 178 | write_spin_lock(&mmu->pgtab_lock); 179 | 180 | struct page_table *p3_table = pgtab_extract_virt_addr(mmu->p4, p4_index); 181 | if (p3_table == NULL) { 182 | if (unmap) goto release_and_ret; 183 | mmu->p4->pages[p4_index] = alloc_pgtab(pgtab_flags); 184 | p3_table = pgtab_extract_virt_addr(mmu->p4, p4_index); 185 | kassert_dbg(p3_table != NULL); 186 | memset(p3_table, 0, sizeof(struct page_table)); 187 | } 188 | 189 | struct page_table *p2_table = pgtab_extract_virt_addr(p3_table, p3_index); 190 | if (p2_table == NULL) { 191 | if (unmap) goto release_and_ret; 192 | p3_table->pages[p3_index] = alloc_pgtab(pgtab_flags); 193 | p2_table = pgtab_extract_virt_addr(p3_table, p3_index); 194 | kassert_dbg(p2_table != NULL); 195 | memset(p2_table, 0, sizeof(struct page_table)); 196 | } 197 | 198 | struct page_table *p1_table = pgtab_extract_virt_addr(p2_table, p2_index); 199 | if (p1_table == NULL) { 200 | if (unmap) goto release_and_ret; 201 | p2_table->pages[p2_index] = alloc_pgtab(pgtab_flags); 202 | p1_table = pgtab_extract_virt_addr(p2_table, p2_index); 203 | kassert_dbg(p1_table != NULL); 204 | memset(p1_table, 0, sizeof(struct page_table)); 205 | } 206 | 207 | #ifdef DEBUG 208 | // We should really be doing a TLB shootdown in this case. 209 | if (smp_nr_cpus() > 1) { 210 | kassert((p1_table->pages[p1_index] & PAGE_GLOBAL) == 0); 211 | kassert((flags & PAGE_GLOBAL) == 0); 212 | // TODO: Only shootdown the CPUs that actually need it (look at cpuset in MMU struct) 213 | tlb_shootdown(mmu, virt, (virtaddr_t)((physaddr_t)virt + PAGE_SIZE)); 214 | } 215 | #endif 216 | 217 | if (unmap) { 218 | // TODO: Implement page table freeing 219 | klog_verbose("vmm", "Unmapped page at %p\n", virt); 220 | p1_table->pages[p1_index] = 0; 221 | } else { 222 | flags &= ~(VMM_ALLOC_MMAP); // Don't care about these flags anymore 223 | p1_table->pages[p1_index] = (phys & PTE_ADDR_MASK) | PAGE_PRESENT | flags; 224 | } 225 | 226 | // TODO: Check if we actually need to do this 227 | tlb_flush_single(virt); 228 | 229 | release_and_ret: 230 | write_spin_unlock(&mmu->pgtab_lock); 231 | return; 232 | } 233 | -------------------------------------------------------------------------------- /kernel/proc/fork.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | section .text 4 | global ret_from_ufork 5 | ; Final work before returning to a child user space thread 6 | ret_from_ufork: 7 | mov rax, 0 8 | iretq 9 | 10 | global ret_from_kfork 11 | ; Final work before returning to a kernel thread 12 | ret_from_kfork: 13 | pop rdi 14 | pop rax 15 | call rax 16 | ; Thread finished, now exit 17 | mov rdi, [PERCPU_CURRENT] 18 | mov rsi, 0 19 | extern task_exit 20 | call task_exit 21 | ; Doesn't return 22 | 23 | global ret_from_execve 24 | ; Final work after an execve was called, returning to userspace 25 | ; rdi: Entry point of user task 26 | ; rsi: Stack pointer of user task 27 | ret_from_execve: 28 | pushfq 29 | pop rax 30 | or rax, 0x200 ; Make sure interrupts are enabled 31 | 32 | push qword (GDT_USER_DATA | 0x3) 33 | push qword rsi 34 | push qword rax 35 | push qword (GDT_USER_CODE | 0x3) 36 | push qword rdi 37 | iretq 38 | 39 | -------------------------------------------------------------------------------- /kernel/proc/idle.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | bits 64 3 | 4 | section .rodata 5 | msg: db 'Idle', 0xA, 0 6 | 7 | section .text 8 | global idle_task 9 | idle_task: 10 | .loop: 11 | ;extern kprintf 12 | ;mov rdi, msg 13 | ;xor rax, rax 14 | ;call kprintf 15 | sti 16 | hlt 17 | jmp .loop 18 | -------------------------------------------------------------------------------- /kernel/proc/init.c: -------------------------------------------------------------------------------- 1 | #include "proc.h" 2 | #include "syscall.h" 3 | #include "libk.h" 4 | #include "fs.h" 5 | 6 | static void init_user(void); 7 | 8 | // The first kernel thread. Perform advanced initialisation (e.g forking) from here. 9 | // Ends with a call to execve, beginning the first user process. 10 | void init_kernel(void) 11 | { 12 | vfs_init(); 13 | 14 | /*struct inode *out; 15 | err_t e; 16 | if ((e = vfs_lookup(&vfs_root, "/", 2, &out))) 17 | panic("Failed to find root directory: %s", strerror(e)); 18 | 19 | inode_put(out);*/ 20 | 21 | task_execve(init_user, NULL, 0); 22 | } 23 | 24 | static void init_user(void) 25 | { 26 | int x = 0; 27 | if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 28 | while (1) { 29 | execute_syscall(SYSCALL_WRITE, 'A' + (x++ % 26), 0, 0, 0); 30 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 31 | } 32 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 33 | while (1) { 34 | execute_syscall(SYSCALL_WRITE, 'a' + (x++ % 26), 0, 0, 0); 35 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 36 | } 37 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 38 | while (1) { 39 | execute_syscall(SYSCALL_WRITE, '0' + (x++ % 10), 0, 0, 0); 40 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 41 | } 42 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 43 | while (1) { 44 | execute_syscall(SYSCALL_WRITE, 'A' + (x++ % 26), 0, 0, 0); 45 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 46 | } 47 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 48 | while (1) { 49 | execute_syscall(SYSCALL_WRITE, 'a' + (x++ % 26), 0, 0, 0); 50 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 51 | } 52 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 53 | while (1) { 54 | execute_syscall(SYSCALL_WRITE, '0' + (x++ % 10), 0, 0, 0); 55 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 56 | } 57 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 58 | while (1) { 59 | execute_syscall(SYSCALL_WRITE, 'A' + (x++ % 26), 0, 0, 0); 60 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 61 | } 62 | } else if (execute_syscall(SYSCALL_FORK, 0, 0, 0, 0) == 0) { 63 | while (1) { 64 | execute_syscall(SYSCALL_WRITE, 'a' + (x++ % 26), 0, 0, 0); 65 | //execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 66 | } 67 | } else { 68 | while (1) 69 | ;//execute_syscall(SYSCALL_SCHED_YIELD, 0, 0, 0, 0); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /kernel/proc/runq.c: -------------------------------------------------------------------------------- 1 | #include "proc.h" 2 | #include "smp.h" 3 | #include "percpu.h" 4 | #include "libk.h" 5 | 6 | #define SCHED_ENTITY(n) rb_entry((n), struct sched_entity, node) 7 | 8 | static inline uint64_t min_vruntime(struct runq *r) 9 | { 10 | if (r->tree.most_left == NULL) 11 | return 0; 12 | return SCHED_ENTITY(r->tree.most_left)->vruntime; 13 | } 14 | 15 | static inline struct task *node_to_task(struct rb_node *node) 16 | { 17 | struct sched_entity *sched = rb_entry(node, struct sched_entity, node); 18 | struct task *t = sched == NULL ? NULL : container_of(sched, struct task, sched); 19 | return t; 20 | } 21 | 22 | // Initialises the run queue for the current CPU 23 | void runq_init(struct task *initial_parent) 24 | { 25 | struct runq *rq = kmalloc(sizeof(struct runq), KM_NONE); 26 | memset(rq, 0, sizeof *rq); 27 | percpu_set(run_queue, rq); 28 | 29 | // Create the idle task 30 | struct task *idle = task_fork(initial_parent, idle_task, TASK_KTHREAD, NULL); 31 | 32 | // Pin to this CPU 33 | cpuset_clear(&idle->affinity); 34 | cpuset_set_id(&idle->affinity, percpu_get(id), 1); 35 | cpuset_pin(&idle->affinity); 36 | 37 | // Set flags 38 | idle->state = TASK_S_RUNNABLE; 39 | idle->tid = TID_IDLE; // Idle task always has ID 0 40 | 41 | rq->idle = idle; 42 | } 43 | 44 | static inline void tree_insert(struct runq *rq, struct task *t) 45 | { 46 | struct sched_entity *first = rb_entry(rb_first_cached(&rq->tree), struct sched_entity, node); 47 | 48 | // Find the right nodes to link with 49 | struct rb_node **link = &rq->tree.root, *parent = NULL; 50 | while (*link) { 51 | struct sched_entity *this = rb_entry(*link, struct sched_entity, node); 52 | parent = *link; 53 | 54 | if (t->sched.vruntime < this->vruntime) 55 | link = &(*link)->left; 56 | else 57 | link = &(*link)->right; 58 | } 59 | 60 | // Add to the rbtree 61 | rb_link_node(&t->sched.node, parent, link); 62 | rb_insert(&rq->tree, &t->sched.node, first == NULL || t->sched.vruntime < first->vruntime); 63 | rq->num_threads++; 64 | } 65 | 66 | static inline void tree_remove(struct runq *rq, struct task *t) 67 | { 68 | rb_erase(&rq->tree, &t->sched.node); 69 | rq->num_threads--; 70 | } 71 | 72 | void runq_add(struct task *t) 73 | { 74 | struct runq *rq = percpu_get(run_queue); 75 | spin_lock(&rq->lock); 76 | 77 | // If this hasn't run in a while, set the vruntime to the min_vruntime 78 | if (t->state != TASK_S_RUNNABLE) 79 | t->sched.vruntime = min_vruntime(rq); 80 | 81 | tree_insert(rq, t); 82 | 83 | spin_unlock(&rq->lock); 84 | } 85 | 86 | // Pops the next task from the rbtree 87 | struct task *runq_next(void) 88 | { 89 | struct runq *rq = percpu_get(run_queue); 90 | 91 | spin_lock(&rq->lock); 92 | struct rb_node *node = rb_first_cached(&rq->tree); 93 | struct task *next = node_to_task(node); 94 | if (next != NULL) 95 | tree_remove(rq, next); 96 | spin_unlock(&rq->lock); 97 | 98 | 99 | if (next == NULL) 100 | next = rq->idle; 101 | 102 | return next; 103 | } 104 | 105 | void runq_remove(struct task *t) 106 | { 107 | struct runq *rq = percpu_get(run_queue); 108 | spin_lock(&rq->lock); 109 | tree_remove(rq, t); 110 | spin_unlock(&rq->lock); 111 | 112 | // TODO: Panic if not in tree 113 | //panic("Tried to remove task at %p that isn't in run queue", t); 114 | } 115 | 116 | void runq_balance_pull(void) 117 | { 118 | struct runq *rq = percpu_get(run_queue); 119 | 120 | for (size_t i = 0; i < smp_nr_cpus(); i++) { 121 | if (i != smp_cpu_id()) { 122 | struct runq *other = percpu_table[i]->run_queue; 123 | bool should_exit = false; 124 | 125 | // Prevent deadlocks by always acquiring the lowest ID's lock first 126 | bool this_lower_id = smp_cpu_id() < i; 127 | if (this_lower_id) { 128 | spin_lock(&rq->lock); 129 | spin_lock(&other->lock); 130 | } else { 131 | spin_lock(&other->lock); 132 | spin_lock(&rq->lock); 133 | } 134 | 135 | if (other->num_threads > 0) { 136 | // Pull thread 137 | // TODO: Don't always pull the first thread 138 | // TODO: Take into account affinity 139 | struct task *pulled = node_to_task(rb_first_cached(&other->tree)); 140 | tree_remove(other, pulled); 141 | tree_insert(rq, pulled); 142 | should_exit = true; 143 | } 144 | 145 | // Release locks in correct order 146 | if (this_lower_id) { 147 | spin_unlock(&other->lock); 148 | spin_unlock(&rq->lock); 149 | } else { 150 | spin_unlock(&rq->lock); 151 | spin_unlock(&other->lock); 152 | } 153 | 154 | if (should_exit) 155 | break; 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /kernel/proc/sched.c: -------------------------------------------------------------------------------- 1 | #include "syscall.h" 2 | #include "libk.h" 3 | #include "proc.h" 4 | #include "percpu.h" 5 | #include "mm.h" 6 | 7 | static struct task dummy; 8 | static atomic32_t schedulers_waiting; 9 | 10 | void schedule(void) 11 | { 12 | // Disable preemption 13 | preempt_inc(); 14 | 15 | // Remove 'need preempt' flag 16 | percpu_set(reschedule, false); 17 | 18 | // Update scheduler statistics for the previous task 19 | current->sched.vruntime++; // Note that current may be the idle task here 20 | 21 | // Reinsert current into the rbtree 22 | if (current->tid != TID_IDLE) 23 | runq_add(current); 24 | 25 | // Get next task 26 | struct task *next = runq_next(); 27 | if (next->tid == TID_IDLE) { 28 | // We're about to go idle, run the balancer and try again 29 | runq_balance_pull(); 30 | next = runq_next(); 31 | } 32 | 33 | kassert_dbg(next->state == TASK_S_RUNNABLE); 34 | 35 | // Swap the states of the tasks 36 | current->flags &= ~TASK_RUNNING; 37 | next->flags |= TASK_RUNNING; 38 | next->flags &= ~TASK_PREEMPTED; 39 | 40 | // Pre-emption is re-enabled in switch_to 41 | // Do the context switch 42 | switch_to(next); 43 | } 44 | 45 | void sched_add(struct task *t) 46 | { 47 | runq_add(t); 48 | } 49 | 50 | void sched_yield(void) 51 | { 52 | schedule(); 53 | } 54 | 55 | void sched_yield_preempt(void) 56 | { 57 | preempt_inc(); 58 | current->flags |= TASK_PREEMPTED; 59 | preempt_dec(); 60 | 61 | schedule(); 62 | } 63 | 64 | void sched_init(void) 65 | { 66 | // Initialise the dummy task since every task switch requires a 'current' task to switch from 67 | memset(&dummy, 0, sizeof dummy); 68 | dummy.mmu = &kernel_mmu; 69 | dummy.tid = TID_IDLE; 70 | percpu_set(current_task, &dummy); 71 | } 72 | 73 | static inline void wait_for_schedulers(void) 74 | { 75 | atomic_inc_read32(&schedulers_waiting); 76 | while (atomic_read32(&schedulers_waiting) < smp_nr_cpus()) 77 | ; 78 | 79 | klog("sched", "Starting scheduler for CPU %d...\n", smp_cpu_id()); 80 | } 81 | 82 | void sched_run_bsp(void) 83 | { 84 | // Create the init task 85 | runq_init(&dummy); 86 | 87 | struct task *t = task_fork(&dummy, init_kernel, FORK_KTHREAD, NULL); 88 | task_wakeup(t); 89 | 90 | wait_for_schedulers(); 91 | 92 | lapic_timer_enable(); 93 | // TODO: Possible race condition here, if we get rescheduled before sched_yield happens 94 | sched_yield(); 95 | } 96 | 97 | void sched_run_ap(void) 98 | { 99 | percpu_set(current_task, &dummy); 100 | 101 | runq_init(&dummy); 102 | 103 | wait_for_schedulers(); 104 | 105 | lapic_timer_enable(); 106 | // TODO: Possible race condition here, if we get rescheduled before sched_yield happens 107 | sched_yield(); 108 | } 109 | -------------------------------------------------------------------------------- /kernel/proc/switch_to.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | bits 64 4 | section .text 5 | ; rdi: Pointer to next task 6 | global switch_to 7 | switch_to: 8 | push rbx 9 | push rbp 10 | push r12 11 | push r13 12 | push r14 13 | push r15 14 | 15 | ; Save current rsp 16 | mov rax, [PERCPU_CURRENT] 17 | mov [rax], rsp 18 | ; Load next rsp 19 | mov rsp, [rdi] 20 | 21 | ; Set current in per-cpu data 22 | mov rsi, [PERCPU_CURRENT] 23 | mov [PERCPU_CURRENT], rdi 24 | 25 | ; Set RSP0 in TSS 26 | mov rax, [PERCPU_TSS] 27 | mov rcx, [rdi + 0x8] 28 | mov [rax + 4], rcx 29 | 30 | ; Set the target MMU 31 | mov rdi, [rdi + 0x10] ; next 32 | mov rsi, [rsi + 0x10] ; previous 33 | extern mmu_switch 34 | call mmu_switch 35 | 36 | ; Re-enable preemption 37 | lock dec qword [PERCPU_PREEMPT_COUNT] 38 | 39 | pop r15 40 | pop r14 41 | pop r13 42 | pop r12 43 | pop rbp 44 | pop rbx 45 | ret 46 | -------------------------------------------------------------------------------- /kernel/proc/task.c: -------------------------------------------------------------------------------- 1 | #include "proc.h" 2 | #include "mm.h" 3 | #include "libk.h" 4 | #include "percpu.h" 5 | #include "util.h" 6 | 7 | extern void ret_from_ufork(void); 8 | extern void ret_from_kfork(void); 9 | extern void __attribute__((noreturn)) ret_from_execve(virtaddr_t entry, uint64_t rsp); 10 | 11 | #define TASK_KSTACK_ORDER 1 12 | #define TASK_KSTACK_PAGES (1 << TASK_KSTACK_ORDER) 13 | #define TASK_KSTACK_SIZE (TASK_KSTACK_PAGES * PAGE_SIZE) 14 | 15 | static atomic32_t next_tid = { 0 }; 16 | 17 | static const struct callee_regs default_regs = { 0 }; 18 | 19 | struct task *task_fork(struct task *parent, virtaddr_t entry, uint64_t clone_flags, const struct callee_regs *regs) 20 | { 21 | if (parent == NULL) 22 | parent = current; 23 | 24 | kassert_dbg(!((clone_flags & FORK_KTHREAD) && (clone_flags & FORK_UTHREAD))); 25 | 26 | struct task *t = kmalloc(sizeof(struct task), KM_NONE); 27 | memset(t, 0, sizeof *t); 28 | t->flags = parent->flags; 29 | t->tid = atomic_inc_read32(&next_tid); 30 | t->tgid = t->tid; // TODO 31 | 32 | klog_verbose("task", "Forked PID %d to create PID %d\n", parent->tid, t->tid); 33 | 34 | // Allocate a kernel stack 35 | uintptr_t kstack = TASK_KSTACK_SIZE + (uintptr_t)page_to_virt(pmm_alloc_order(TASK_KSTACK_ORDER, GFP_NONE)); 36 | uint64_t *stack = (uint64_t *)kstack; 37 | // TODO: Remove this variable. We can work out the stack top by masking rsp 38 | // given that the kernel stack size is fixed at compile time, and allocs are aligned. 39 | t->rsp_original = (virtaddr_t)kstack; 40 | 41 | // Copy MMU information and set up the kernel stack 42 | if (clone_flags & FORK_KTHREAD) { 43 | if (regs == NULL) 44 | regs = &default_regs; 45 | t->mmu = &kernel_mmu; 46 | t->flags |= TASK_KTHREAD; 47 | *--stack = (uint64_t)entry; 48 | *--stack = regs->rbx; // Argument passed to the thread 49 | *--stack = (uint64_t)ret_from_kfork; // Where switch_to will return 50 | } else { 51 | kassert_dbg(regs != NULL); 52 | t->flags &= ~(TASK_KTHREAD); 53 | 54 | if (clone_flags & FORK_UTHREAD) { 55 | mmu_inc_users(parent->mmu); 56 | t->mmu = parent->mmu; 57 | } else { 58 | t->mmu = mmu_alloc(); 59 | mmu_clone_cow(t->mmu, parent->mmu); 60 | } 61 | 62 | // Set up simulated iret frame 63 | *--stack = 0x20 | 0x3; // ss 64 | *--stack = regs->rsp; // rsp 65 | *--stack = read_rflags() | 0x200; // rflags with interrupts enabled 66 | *--stack = 0x28 | 0x3; // cs 67 | *--stack = (uint64_t)entry; // rip 68 | *--stack = (uint64_t)ret_from_ufork; // Where switch_to will return 69 | } 70 | 71 | *--stack = regs->rbx; 72 | *--stack = regs->rbp; 73 | *--stack = regs->r12; 74 | *--stack = regs->r13; 75 | *--stack = regs->r14; 76 | *--stack = regs->r15; 77 | t->rsp_top = (virtaddr_t)stack; 78 | 79 | cpuset_copy(&t->affinity, &parent->affinity); 80 | 81 | // Add the task to the scheduler 82 | t->state = TASK_S_NOT_STARTED; 83 | return t; 84 | } 85 | 86 | void task_wakeup(struct task *t) 87 | { 88 | if (t->state != TASK_S_RUNNABLE) { 89 | t->state = TASK_S_RUNNABLE; 90 | sched_add(t); 91 | } 92 | } 93 | 94 | void task_exit(struct task *t, int UNUSED(code)) 95 | { 96 | if (t->state == TASK_S_RUNNABLE) 97 | runq_remove(t); 98 | 99 | t->state = TASK_S_ZOMBIE; 100 | 101 | mmu_dec_users(t->mmu); 102 | 103 | if (t == current) { 104 | // TODO: Put ourselves on scheduler cleanup list if necessary 105 | sched_yield(); 106 | panic("Schedule returned at the end of task_exit"); 107 | } 108 | } 109 | 110 | void __attribute__((noreturn)) task_execve(virtaddr_t function, char UNUSED(*argv[]), unsigned int UNUSED(flags)) 111 | { 112 | struct task *self = current; 113 | if (self->mmu == &kernel_mmu) 114 | self->mmu = mmu_alloc(); 115 | else 116 | mmu_dec_users(self->mmu); 117 | 118 | mmu_init(self->mmu); 119 | 120 | #define UTASK_ENTRY 0x1000 121 | #define UTASK_STACK_BOTTOM 0x3000 122 | #define UTASK_STACK_TOP 0x5000 123 | 124 | // Set up the entry point 125 | uintptr_t entry = UTASK_ENTRY; 126 | vmm_map_page(self->mmu, kern_to_phys(function), (virtaddr_t)entry, PAGE_EXECUTABLE | PAGE_USER_ACCESSIBLE); 127 | vmm_map_page(self->mmu, kern_to_phys(function) + PAGE_SIZE, (virtaddr_t)(entry + PAGE_SIZE), PAGE_EXECUTABLE | PAGE_USER_ACCESSIBLE); 128 | entry += ((uintptr_t)function & 0xFFF); 129 | 130 | for (size_t i = 0; i < 2; i++) { 131 | size_t off = i * PAGE_SIZE; 132 | virtaddr_t page = page_to_virt(pmm_alloc_order(0, GFP_NONE)); 133 | vmm_map_page(self->mmu, virt_to_phys(page), (virtaddr_t)(UTASK_STACK_BOTTOM + off), PAGE_WRITABLE | PAGE_USER_ACCESSIBLE); 134 | } 135 | 136 | // Create a vm_area for the code 137 | struct vm_area *code = kmalloc(sizeof(struct vm_area), KM_NONE); 138 | memset(code, 0, sizeof *code); 139 | code->base = UTASK_ENTRY; 140 | code->len = 2 * PAGE_SIZE; 141 | code->type = VM_AREA_TEXT; 142 | code->flags = VM_AREA_EXECUTABLE; 143 | area_add(self->mmu, code); 144 | 145 | // Create a vm_area for the stack 146 | struct vm_area *stack = kmalloc(sizeof(struct vm_area), KM_NONE); 147 | memset(stack, 0, sizeof *stack); 148 | stack->base = UTASK_STACK_TOP; 149 | stack->len = UTASK_STACK_TOP - UTASK_STACK_BOTTOM; 150 | stack->type = VM_AREA_STACK; 151 | stack->flags = VM_AREA_WRITABLE; 152 | area_add(self->mmu, stack); 153 | 154 | slist_foreach(cur, list, self->mmu->areas) { 155 | klog_verbose("task", "Added area at %p, size %zu\n", (void *)cur->base, cur->len); 156 | } 157 | 158 | // Switch to the new cr3. 159 | // Disable preemption so the cpuset is always correct. 160 | preempt_inc(); 161 | mmu_update_cpuset(self->mmu, percpu_get(id), 1); 162 | write_cr3(virt_to_phys(self->mmu->p4)); 163 | preempt_dec(); 164 | 165 | ret_from_execve((virtaddr_t)entry, UTASK_STACK_TOP); 166 | } 167 | -------------------------------------------------------------------------------- /kernel/smp/cpuset.c: -------------------------------------------------------------------------------- 1 | #include "smp.h" 2 | #include "libk.h" 3 | #include "ds/bitmap.h" 4 | 5 | void cpuset_init(cpuset_t *cpus) 6 | { 7 | *cpus = 0; 8 | } 9 | 10 | void cpuset_clear(cpuset_t *cpus) 11 | { 12 | *cpus = 0; 13 | } 14 | 15 | void cpuset_pin(cpuset_t *cpus) 16 | { 17 | *cpus |= 1; 18 | } 19 | 20 | void cpuset_unpin(cpuset_t *cpus) 21 | { 22 | *cpus &= ~1; 23 | } 24 | 25 | bool cpuset_is_pinned(cpuset_t *cpus) 26 | { 27 | return *cpus | 1; 28 | } 29 | 30 | void cpuset_copy(cpuset_t *dest, cpuset_t *src) 31 | { 32 | *dest = *src; 33 | } 34 | 35 | bool cpuset_query_id(cpuset_t *cpus, cpuid_t id) 36 | { 37 | kassert_dbg(id < MAX_CORES); 38 | return *cpus & (1 << ++id); 39 | } 40 | 41 | void cpuset_set_id(cpuset_t *cpus, cpuid_t id, bool val) 42 | { 43 | kassert_dbg(id < MAX_CORES); 44 | if (val) 45 | *cpus |= (1 << ++id); 46 | else 47 | *cpus &= ~(1 << ++id); 48 | } 49 | 50 | #ifdef VERBOSE 51 | void cpuset_dump(cpuset_t *cpus) 52 | { 53 | for (cpuid_t i = 0; i < sizeof(cpuset_t) * 4; i++) { 54 | if (cpuset_query_id(cpus, i)) 55 | kprintf("%u ", i); 56 | } 57 | } 58 | #endif 59 | -------------------------------------------------------------------------------- /kernel/smp/ipi.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | #include "util.h" 3 | #include "interrupts.h" 4 | #include "percpu.h" 5 | #include "proc.h" 6 | #include "mm.h" 7 | #include "smp.h" 8 | 9 | void ipi_send_fixed(cpuid_t id, uint8_t vec) 10 | { 11 | uint8_t apic_id = percpu_table[id]->apic_id; 12 | lapic_send_ipi(apic_id, IPI_FIXED | vec); 13 | } 14 | 15 | // Begins running the scheduler 16 | void ipi_sched_hint(struct isr_ctx *UNUSED(regs)) 17 | { 18 | lapic_eoi(IRQ_IPI_SCHED_HINT); 19 | sched_run_ap(); 20 | } 21 | 22 | void ipi_abort(struct isr_ctx *UNUSED(regs)) 23 | { 24 | abort_self(); 25 | } 26 | 27 | void ipi_tlb_shootdown(struct isr_ctx *UNUSED(regs)) 28 | { 29 | struct tlb_op *op = tlb_op_location; 30 | for (uintptr_t i = (uintptr_t)op->start; i < (uintptr_t)op->end; i += PAGE_SIZE) { 31 | tlb_flush_single((virtaddr_t)i); 32 | } 33 | atomic_dec_read32(&tlb_remaining_cpus); 34 | lapic_eoi(IRQ_IPI_TLB_SHOOTDOWN); 35 | } 36 | -------------------------------------------------------------------------------- /kernel/smp/smp.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "smp.h" 3 | #include "util.h" 4 | #include "libk.h" 5 | #include "mm.h" 6 | #include "percpu.h" 7 | #include "syscall.h" 8 | #include "drivers/apic.h" 9 | #include "drivers/pit.h" 10 | #include "sync.h" 11 | 12 | extern uintptr_t smp_trampoline_start; 13 | extern uintptr_t smp_trampoline_end; 14 | 15 | volatile bool smp_ap_started_flag; 16 | volatile virtaddr_t smp_ap_stack; 17 | 18 | atomic32_t smp_nr_cpus_ready = { 0 }; 19 | 20 | #define TRAMPOLINE_START 0x1000 21 | // TODO: Add a define for the kernel stack size 22 | 23 | // TODO: This is slow, but we store the value in the per-CPU data structure, so use that instead 24 | cpuid_t smp_cpu_id_full(void) 25 | { 26 | cpuid_t lid = lapic_id(); 27 | for (size_t i = 0; i < lapic_list_size; i++) 28 | if (lapic_list[i].id == lid) 29 | return i; 30 | panic("CPU not found in lapic_list"); 31 | } 32 | 33 | static void smp_boot_ap(size_t index) 34 | { 35 | struct lapic_info *lapic = &lapic_list[index]; 36 | 37 | if (smp_ap_stack == NULL) { 38 | uintptr_t stack_top = (uintptr_t)page_to_virt(pmm_alloc_order(2, GFP_NONE)); 39 | smp_ap_stack = (virtaddr_t)(stack_top + (4 * PAGE_SIZE)); 40 | } 41 | 42 | // Set by the AP when initialisation is complete 43 | smp_ap_started_flag = 0; 44 | 45 | // Adapted from https://nemez.net/osdev/lapic.txt 46 | // Send the INIT IPI 47 | lapic_send_ipi(lapic->id, IPI_INIT); 48 | pit_sleep_ms(10); 49 | 50 | // Send the SIPI (first attempt) 51 | lapic_send_ipi(lapic->id, IPI_START_UP | ((uint32_t)TRAMPOLINE_START / PAGE_SIZE)); 52 | pit_sleep_ms(5); 53 | 54 | if (!smp_ap_started_flag) { 55 | // Send SIPI again (second attempt) 56 | lapic_send_ipi(lapic->id, IPI_START_UP | ((uint32_t)TRAMPOLINE_START / PAGE_SIZE)); 57 | pit_sleep_watch_flag(10, &smp_ap_started_flag, false); 58 | if (!smp_ap_started_flag) { 59 | klog_warn("smp", "CPU %zu failed to boot\n", index); 60 | lapic->present = 0; 61 | return; 62 | } 63 | } 64 | 65 | smp_ap_stack = NULL; 66 | 67 | // Check flag is set 68 | } 69 | 70 | // Boots all the cores 71 | void smp_init(void) 72 | { 73 | klog("smp", "CPU 0 online\n"); 74 | atomic_inc_read32(&smp_nr_cpus_ready); 75 | 76 | uintptr_t vstart = (uintptr_t)&smp_trampoline_start; 77 | uintptr_t vend = (uintptr_t)&smp_trampoline_end; 78 | physaddr_t trampoline_start = TRAMPOLINE_START; 79 | physaddr_t trampoline_end = trampoline_start + (vend - vstart); 80 | for (size_t i = 0; i < (trampoline_end - trampoline_start); i += PAGE_SIZE) { 81 | vmm_map_page(&kernel_mmu, trampoline_start + i, (virtaddr_t)(trampoline_start + i), PAGE_WRITABLE | PAGE_EXECUTABLE); 82 | memcpy((virtaddr_t)(trampoline_start + i), (virtaddr_t)(vstart + i), PAGE_SIZE); 83 | } 84 | 85 | for (size_t i = 1; i < lapic_list_size; i++) 86 | smp_boot_ap(i); 87 | 88 | // Unmap trampoline code from memory. 89 | // We rely on the fact that no CPU is going to access their 90 | // trampoline memory (before doing a cr3 reload to context switch). 91 | for (size_t i = 0; i < (trampoline_end - trampoline_start); i += PAGE_SIZE) { 92 | vmm_map_page(&kernel_mmu, 0, (virtaddr_t)(trampoline_start + i), VMM_UNMAP); 93 | } 94 | 95 | // Free any unused stacks if there were any 96 | if (smp_ap_stack != NULL) { 97 | uintptr_t stack_top = (uintptr_t)smp_ap_stack - (4 * PAGE_SIZE); 98 | pmm_free_order(virt_to_page((virtaddr_t)stack_top), 2); 99 | } 100 | 101 | // Wait for other CPUs to fully finish initialisation 102 | while (smp_nr_cpus() < lapic_list_size) 103 | ; 104 | 105 | klog("smp", "Finished AP boot sequence\n"); 106 | } 107 | 108 | void smp_ap_kmain(void) 109 | { 110 | lapic_enable(); 111 | irq_enable(); 112 | syscall_enable(); 113 | 114 | klog("smp", "CPU %u online\n", smp_cpu_id()); 115 | atomic_inc_read32(&smp_nr_cpus_ready); 116 | 117 | sched_run_ap(); 118 | } 119 | -------------------------------------------------------------------------------- /kernel/smp/spin.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | bits 64 4 | 5 | global spin_lock 6 | spin_lock: 7 | preempt_inc 8 | mov rax, 1 9 | xchg [rdi], rax 10 | test rax, rax 11 | jz .acquired 12 | .retry: 13 | pause 14 | bt qword [rdi], 0 15 | jc .retry 16 | 17 | xchg [rdi], rax 18 | test rax, rax 19 | jnz .retry 20 | .acquired: 21 | ret 22 | 23 | global spin_try_lock 24 | spin_try_lock: 25 | preempt_inc 26 | mov rax, 1 27 | xchg [rdi], rax 28 | test rax, rax 29 | jz .acquired 30 | preempt_dec 31 | mov rax, 0 32 | ret 33 | .acquired: 34 | mov rax, 1 35 | ret 36 | 37 | global spin_unlock 38 | spin_unlock: 39 | mov qword [rdi], 0 40 | preempt_dec 41 | ret 42 | -------------------------------------------------------------------------------- /kernel/smp/trampoline.asm: -------------------------------------------------------------------------------- 1 | ; This all gets copied into low memory before booting an AP 2 | %include "include.asm" 3 | 4 | %define LOAD_ADDR 0x1000 5 | %define TMP_DATA_OFFSET 0x0A00 6 | %define DATA_OFFSET (LOAD_ADDR + TMP_DATA_OFFSET + ($ - tmp_data_virt)) 7 | %define LABEL_OFFSET(label) (LOAD_ADDR + ((label) - smp_trampoline_start)) 8 | 9 | align PAGE_SIZE 10 | global smp_trampoline_start 11 | smp_trampoline_start equ $ 12 | 13 | ; Don't put anything here, since the entry point needs to be page aligned 14 | 15 | bits 16 16 | smp_trampoline_16: 17 | cli 18 | lidt [tmp_idt_ptr_null] 19 | jmp 0x0:LABEL_OFFSET(.fix_cs) 20 | .fix_cs: 21 | xor ax, ax 22 | mov ds, ax 23 | mov es, ax 24 | mov fs, ax 25 | mov gs, ax 26 | mov ss, ax 27 | cld 28 | 29 | lgdt [tmp_gdt_ptr] 30 | 31 | ; Enable PAE, global 32 | mov eax, cr4 33 | or eax, (1 << 5) | (1 << 7) 34 | mov cr4, eax 35 | 36 | ; Jump to long mode 37 | extern p4_table 38 | mov eax, p4_table 39 | mov cr3, eax 40 | 41 | mov ecx, 0xC0000080 ; EFER 42 | rdmsr 43 | or eax, (1 << 8) | (1 << 11) ; Long mode, NX 44 | wrmsr 45 | 46 | ; Enable paging, pmode, write protect 47 | mov eax, (1 << 0) | (1 << 31) | (1 << 16) 48 | mov cr0, eax 49 | 50 | jmp 0x08:LABEL_OFFSET(smp_trampoline_64) 51 | 52 | bits 64 53 | smp_trampoline_64: 54 | lea rax, [tmp_gdt_ptr] 55 | 56 | extern gdt_size 57 | extern gdt64 58 | mov word [tmp_gdt_ptr], gdt_size 59 | mov qword [tmp_gdt_ptr + 2], gdt64 60 | lgdt [tmp_gdt_ptr] 61 | 62 | ; Restore it to its previous state for the next AP 63 | mov word [tmp_gdt_ptr], (tmp_gdt_end - tmp_gdt - 1) 64 | mov qword [tmp_gdt_ptr + 2], tmp_gdt 65 | 66 | jmp far dword [LABEL_OFFSET(.far_ptr)] 67 | .far_ptr: 68 | dd LABEL_OFFSET(.fix_cs) 69 | dw 0x08 70 | .fix_cs: 71 | ; Not entirely sure why 'jmp .fix_rip' does't work 72 | mov rax, .fix_rip 73 | jmp rax 74 | .fix_rip: 75 | 76 | extern smp_ap_stack 77 | mov rsp, [smp_ap_stack] 78 | 79 | extern smp_ap_started_flag 80 | lock inc byte [smp_ap_started_flag] 81 | 82 | extern percpu_init_ap 83 | call percpu_init_ap 84 | 85 | extern idt_load 86 | call idt_load 87 | 88 | extern smp_ap_kmain 89 | call smp_ap_kmain 90 | 91 | .end: 92 | sti 93 | hlt 94 | jmp .end 95 | 96 | times TMP_DATA_OFFSET - ($ - smp_trampoline_start) db 0 97 | tmp_data_virt: 98 | tmp_gdt equ DATA_OFFSET 99 | ; NULL selector 100 | dq 0 101 | ; 64-bit kernel code 102 | dw 0xFFFF 103 | dw 0 104 | db 0 105 | db 10011010b 106 | db 10101111b 107 | db 0 108 | ; 64-bit kernel data 109 | dw 0xFFFF 110 | dw 0 111 | db 0 112 | db 10010010b 113 | db 10101111b 114 | db 0 115 | tmp_gdt_end equ DATA_OFFSET 116 | 117 | align 16 118 | tmp_gdt_ptr equ DATA_OFFSET 119 | dw (tmp_gdt_end - tmp_gdt - 1) 120 | dq tmp_gdt 121 | 122 | ; Any NMI will shutdown, which is what we want 123 | tmp_idt_ptr_null equ DATA_OFFSET 124 | dw 0 125 | dq 0 126 | 127 | global smp_trampoline_end 128 | smp_trampoline_end equ $ 129 | -------------------------------------------------------------------------------- /kernel/syscall/defs.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | #include "syscall.h" 3 | #include "proc.h" 4 | #include "percpu.h" 5 | 6 | #include "gen/syscall_gen.c" 7 | 8 | #define NAME(name) syscall_ ## name 9 | 10 | int64_t syscall_sched_yield(void) 11 | { 12 | sched_yield(); 13 | return 0; 14 | } 15 | 16 | int64_t syscall_write(char c) 17 | { 18 | (void)c; 19 | kprintf("%c", c); 20 | return 0; 21 | } 22 | 23 | int64_t syscall_fork(uint64_t flags, struct callee_regs *regs, virtaddr_t return_addr) 24 | { 25 | if (flags & TASK_KTHREAD) 26 | return -1; 27 | struct task *child = task_fork(current, return_addr, flags, regs); 28 | task_wakeup(child); 29 | sched_yield(); 30 | return child->tid; 31 | } 32 | 33 | int64_t syscall_exit(int code) 34 | { 35 | task_exit(current, code); 36 | panic("exit returned"); 37 | return -1; 38 | } 39 | -------------------------------------------------------------------------------- /kernel/syscall/syscall.asm: -------------------------------------------------------------------------------- 1 | %include "include.asm" 2 | 3 | section .rodata 4 | no_syscall_message: 5 | db 0x1B, '[41mFatal error: syscall/sysret are not supported', 0xA, 0 ; Newline, NUL 6 | 7 | section .text 8 | ; Enables the 'syscall' and 'sysret' instructions 9 | global syscall_enable 10 | syscall_enable: 11 | ; Check that syscall and sysret are actually available 12 | mov eax, 0x80000001 13 | xor ecx, ecx 14 | cpuid 15 | test edx, (1 << 11) 16 | jz .no_syscall 17 | 18 | ; IA32_STAR MSR 19 | mov ecx, 0xC0000081 20 | rdmsr 21 | ; Load user 32-bit cs into STAR[63:48] and load kernel cs into STAR[47:32] 22 | mov edx, 0x00180008 23 | wrmsr 24 | 25 | ; IA32_LSTAR MSR 26 | mov ecx, 0xC0000082 27 | rdmsr 28 | ; Load rip into LSTAR 29 | mov rdi, syscall_entry 30 | mov eax, edi 31 | shr rdi, 32 32 | mov edx, edi 33 | wrmsr 34 | 35 | ; Enable both instructions 36 | mov ecx, 0xC0000080 ; IA32_EFER MSR 37 | rdmsr 38 | or eax, (1 << 0) 39 | wrmsr 40 | 41 | ; Set FMASK MSR for rflags 42 | mov ecx, 0xC0000084 ; IA32_FMASK MSR 43 | rdmsr 44 | or eax, (1 << 9) | (1 << 10) ; Disable interrupts, clear direction flag 45 | wrmsr 46 | 47 | ret 48 | .no_syscall: 49 | mov rdi, no_syscall_message 50 | xor rax, rax ; Needed for varargs 51 | extern kprintf 52 | call kprintf 53 | extern abort 54 | jmp abort 55 | 56 | ; Entry point for the 'syscall' instruction 57 | syscall_entry: 58 | ; rip is in rcx 59 | ; rflags is in r11 60 | ; WARNING: Do not clobber any argument registers like rdi 61 | ; TODO: Save all registers except rcx, r11, rax 62 | 63 | ; Switch to kernel stack 64 | mov [PERCPU_RSP_SCRATCH], rsp 65 | mov rsp, [PERCPU_CURRENT] 66 | mov rsp, [rsp + 0x8] 67 | 68 | ; Setup simulated IRQ frame 69 | push qword (GDT_USER_DATA | 0x3) 70 | push qword [PERCPU_RSP_SCRATCH] ; Old RSP 71 | push r11 72 | push qword (GDT_USER_CODE | 0x3) 73 | push rcx ; rip 74 | 75 | ; Interrupts will be safely handled on the kernel stack 76 | sti 77 | 78 | ; Was the syscall out of range? 79 | cmp rax, NUM_SYSCALLS 80 | jae .bad_syscall 81 | 82 | ; Was it a fork? We need to save the callee-saved registers in that case. 83 | cmp rax, SYSCALL_FORK 84 | je .syscall_fork 85 | 86 | ; Call the syscall function 87 | extern syscall_table 88 | mov rax, [syscall_table + rax * 8] 89 | call rax 90 | 91 | jmp .done 92 | .bad_syscall: 93 | mov rax, ENOSYS 94 | .done: 95 | xor rcx, rcx 96 | xor r11, r11 97 | iretq 98 | .syscall_fork: 99 | test rsi, rsi ; Check if a child stack was passed or not 100 | cmovz rsi, [rsp + 24] ; Read user rsp off stack 101 | push r15 102 | push r14 103 | push r13 104 | push r12 105 | push rbx 106 | push rbp 107 | push rsi ; rsp 108 | mov rsi, rsp ; Callee-saved registers 109 | mov rdx, rcx ; Return address of child 110 | extern syscall_fork 111 | call syscall_fork 112 | add rsp, 8 113 | pop rbp 114 | pop rbx 115 | pop r12 116 | pop r13 117 | pop r14 118 | pop r15 119 | jmp .done 120 | -------------------------------------------------------------------------------- /libk/abort.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | #include "interrupts.h" 3 | #include "drivers/apic.h" 4 | 5 | __attribute__((noreturn)) void abort_self(void) 6 | { 7 | while (1) { 8 | asm volatile ( 9 | "cli\n" 10 | "hlt\n" 11 | : 12 | : 13 | : 14 | ); 15 | } 16 | __builtin_unreachable(); 17 | } 18 | 19 | __attribute__((noreturn)) void abort(void) 20 | { 21 | lapic_send_ipi(0, IPI_BROADCAST | IPI_FIXED | IRQ_IPI_ABORT); 22 | abort_self(); 23 | } 24 | 25 | -------------------------------------------------------------------------------- /libk/ds/bitmap.c: -------------------------------------------------------------------------------- 1 | #include "ds/bitmap.h" 2 | 3 | // TODO: Optimize this (lookup tables, etc) 4 | int bitmap_find_hole(unsigned char *map, size_t size, size_t hole_size) 5 | { 6 | size_t i, temp = 0; 7 | for (i = 0; i < size; i++) { 8 | if (temp == hole_size) 9 | return i - temp; 10 | else if (bitmap_test(map, i) == 0) 11 | temp++; 12 | else 13 | temp = 0; 14 | } 15 | return -1; 16 | } 17 | -------------------------------------------------------------------------------- /libk/ds/rbtree.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "ds/rbtree.h" 3 | 4 | // Credit to https://github.com/pvachon/rbtree, much of this has been adapted from there 5 | 6 | static inline void rotate_right(struct rbtree *tree, struct rb_node *node) 7 | { 8 | struct rb_node *x = node; 9 | struct rb_node *y = x->left; 10 | 11 | x->left = y->right; 12 | 13 | if (y->right != NULL) { 14 | struct rb_node *yright = y->right; 15 | rb_set_parent(yright, x); 16 | } 17 | 18 | rb_set_parent(y, rb_parent(x)); 19 | 20 | if (rb_parent(x) == NULL) { 21 | tree->root = y; 22 | } else { 23 | struct rb_node *xp = rb_parent(x); 24 | if (x == xp->left) { 25 | xp->left = y; 26 | } else { 27 | xp->right = y; 28 | } 29 | } 30 | 31 | y->right = x; 32 | rb_set_parent(x, y); 33 | } 34 | 35 | static inline void rotate_left(struct rbtree *tree, struct rb_node *node) 36 | { 37 | struct rb_node *x = node; 38 | struct rb_node *y = x->right; 39 | 40 | x->right = y->left; 41 | 42 | if (y->left != NULL) { 43 | struct rb_node *yleft = y->left; 44 | rb_set_parent(yleft, x); 45 | } 46 | 47 | rb_set_parent(y, rb_parent(x)); 48 | 49 | if (rb_parent(x) == NULL) { 50 | tree->root = y; 51 | } else { 52 | struct rb_node *xp = rb_parent(x); 53 | if (x == xp->left) { 54 | xp->left = y; 55 | } else { 56 | xp->right = y; 57 | } 58 | } 59 | 60 | y->left = x; 61 | rb_set_parent(x, y); 62 | } 63 | 64 | static inline void insert_case1(struct rbtree *tree, struct rb_node *node) 65 | { 66 | rb_set_color(node, RB_BLACK); 67 | tree->root = node; 68 | } 69 | 70 | static inline void insert_case2(struct rbtree *tree, struct rb_node *node) 71 | { 72 | (void)node; 73 | (void)tree; 74 | } 75 | 76 | static inline struct rb_node *insert_case3(struct rbtree *tree, struct rb_node *node) 77 | { 78 | rb_set_color(rb_parent(node), RB_BLACK); 79 | rb_set_color(rb_uncle(node), RB_BLACK); 80 | rb_set_color(rb_grandparent(node), RB_RED); 81 | (void)tree; 82 | return rb_grandparent(node); 83 | } 84 | 85 | static inline void insert_case4(struct rbtree *tree, struct rb_node *node) 86 | { 87 | struct rb_node *parent = rb_parent(node), *grandparent = rb_grandparent(node); 88 | 89 | if (grandparent->left && node == grandparent->left->right) { 90 | rotate_left(tree, parent); 91 | node = node->left; 92 | } else if (grandparent->right && node == grandparent->right->left) { 93 | rotate_right(tree, parent); 94 | node = node->right; 95 | } 96 | 97 | parent = rb_parent(node); 98 | grandparent = rb_grandparent(node); 99 | 100 | if (node == parent->left) { 101 | rotate_right(tree, grandparent); 102 | } else { 103 | rotate_left(tree, grandparent); 104 | } 105 | 106 | rb_set_color(parent, RB_BLACK); 107 | rb_set_color(grandparent, RB_RED); 108 | } 109 | 110 | void rb_insert(struct rbtree *tree, struct rb_node *node, bool most_left) 111 | { 112 | if (node == NULL) 113 | return; 114 | 115 | struct rb_node *original = node; 116 | 117 | while (1) { 118 | struct rb_node *parent = rb_parent(node); 119 | if (parent == NULL) { 120 | insert_case1(tree, node); 121 | break; 122 | } else if (rb_color(parent) == RB_BLACK) { 123 | insert_case2(tree, node); 124 | break; 125 | } else if (rb_color(rb_uncle(node)) == RB_RED) { 126 | node = insert_case3(tree, node); 127 | } else { 128 | insert_case4(tree, node); 129 | break; 130 | } 131 | } 132 | 133 | if (most_left && original->left == NULL) 134 | tree->most_left = original; 135 | } 136 | 137 | // Swaps two nodes in the tree 138 | static inline void rb_swap(struct rbtree *tree, struct rb_node *x, struct rb_node *y) 139 | { 140 | struct rb_node *left = x->left; 141 | struct rb_node *right = x->right; 142 | struct rb_node *parent = rb_parent(x); 143 | 144 | rb_set_parent(y, parent); 145 | 146 | if (parent != NULL) { 147 | if (parent->left == x) { 148 | parent->left = y; 149 | } else { 150 | parent->right = y; 151 | } 152 | } else { 153 | if (tree->root == x) { 154 | tree->root = y; 155 | } 156 | } 157 | 158 | y->right = right; 159 | if (right != NULL) { 160 | rb_set_parent(right, y); 161 | } 162 | x->right = NULL; 163 | 164 | y->left = left; 165 | if (left != NULL) { 166 | rb_set_parent(left, y); 167 | } 168 | x->left = NULL; 169 | 170 | rb_set_color(y, rb_color(x)); 171 | rb_set_parent(x, NULL); 172 | } 173 | 174 | // Target must not already be in the tree. 175 | void rb_replace(struct rbtree *tree, struct rb_node *victim, struct rb_node *target) 176 | { 177 | *target = *victim; 178 | 179 | if (target->left) 180 | rb_set_parent(target->left, target); 181 | if (target->right) 182 | rb_set_parent(target->right, target); 183 | 184 | if (victim == tree->root) 185 | tree->root = target; 186 | } 187 | 188 | static inline void erase_parent_link(struct rbtree *tree, struct rb_node *node, struct rb_node *replacement) 189 | { 190 | struct rb_node *parent = rb_parent(node); 191 | if (parent == NULL) 192 | tree->root = replacement; 193 | else if (parent->left == node) 194 | parent->left = replacement; 195 | else if (parent->right == node) 196 | parent->right = replacement; 197 | } 198 | 199 | static inline void erase_rebalance(struct rbtree *tree, struct rb_node *node, struct rb_node *parent, bool node_is_left) 200 | { 201 | struct rb_node *x = node; 202 | struct rb_node *xp = parent; 203 | int is_left = node_is_left; 204 | 205 | while (x != tree->root && (x == NULL || rb_is_black(x))) { 206 | // Sibling 207 | struct rb_node *w = is_left ? xp->right : xp->left; 208 | 209 | if (w != NULL && rb_is_red(w)) { 210 | /* Case 1: */ 211 | rb_set_color(w, RB_BLACK); 212 | rb_set_color(xp, RB_RED); 213 | if (is_left) { 214 | rotate_left(tree, xp); 215 | } else { 216 | rotate_right(tree, xp); 217 | } 218 | w = is_left ? xp->right : xp->left; 219 | } 220 | 221 | struct rb_node *wleft = w != NULL ? w->left : NULL; 222 | struct rb_node *wright = w != NULL ? w->right : NULL; 223 | if ( (wleft == NULL || rb_is_black(wleft)) && 224 | (wright == NULL || rb_is_black(wright)) ) 225 | { 226 | /* Case 2: */ 227 | if (w != NULL) { 228 | rb_set_color(w, RB_RED); 229 | } 230 | x = xp; 231 | xp = rb_parent(x); 232 | is_left = xp && (x == xp->left); 233 | } else { 234 | if (is_left && (wright == NULL || rb_is_black(wright))) { 235 | /* Case 3a: */ 236 | rb_set_color(w, RB_RED); 237 | if (wleft) { 238 | rb_set_color(wleft, RB_BLACK); 239 | } 240 | rotate_right(tree, w); 241 | w = xp->right; 242 | } else if (!is_left && (wleft == NULL || rb_is_black(wleft))) { 243 | /* Case 3b: */ 244 | rb_set_color(w, RB_RED); 245 | if (wright) { 246 | rb_set_color(wright, RB_BLACK); 247 | } 248 | rotate_left(tree, w); 249 | w = xp->left; 250 | } 251 | 252 | /* Case 4: */ 253 | wleft = w->left; 254 | wright = w->right; 255 | 256 | rb_set_color(w, rb_color(xp)); 257 | rb_set_color(xp, RB_BLACK); 258 | 259 | if (is_left && wright != NULL) { 260 | rb_set_color(wright, RB_BLACK); 261 | rotate_left(tree, xp); 262 | } else if (!is_left && wleft != NULL) { 263 | rb_set_color(wleft, RB_BLACK); 264 | rotate_right(tree, xp); 265 | } 266 | x = tree->root; 267 | } 268 | } 269 | 270 | if (x != NULL) { 271 | rb_set_color(x, RB_BLACK); 272 | } 273 | } 274 | 275 | #if 0 276 | static void __debug_tree(struct rb_node *node) 277 | { 278 | if (node == NULL) 279 | return; 280 | 281 | fprintf(stderr, "%p: %s - (parent %p, left %p, right %p)\n", node, rb_is_red(node) ? "RED " : "BLACK", 282 | rb_parent(node), node->left, node->right); 283 | __debug_tree(node->left); 284 | __debug_tree(node->right); 285 | } 286 | 287 | static void debug_tree(struct rbtree *tree) 288 | { 289 | __debug_tree(tree->root); 290 | fprintf(stderr, "---\n"); 291 | } 292 | #endif 293 | 294 | void rb_erase(struct rbtree *tree, struct rb_node *node) 295 | { 296 | if (node == NULL) 297 | return; 298 | 299 | if (node == tree->most_left) 300 | tree->most_left = rb_next(node); 301 | 302 | struct rb_node *y, *x, *xp; 303 | 304 | // Node has two children. Do the standard BST removal method so that we only have 305 | // to worry about deleting nodes that have at most one child (e.g one leaf node). 306 | if (node->left && node->right) { 307 | // Find max element in left subtree and swap it 308 | struct rb_node *max; 309 | for (max = node->left; max->right != NULL; max = max->right) 310 | ; 311 | y = max; 312 | } else 313 | y = node; 314 | 315 | if (y->left != NULL) { 316 | x = y->left; 317 | } else { 318 | x = y->right; 319 | } 320 | 321 | if (x != NULL) { 322 | rb_set_parent(x, rb_parent(y)); 323 | xp = rb_parent(x); 324 | } else { 325 | xp = rb_parent(y); 326 | } 327 | 328 | int is_left = 0; 329 | if (rb_parent(y) == NULL) { 330 | tree->root = x; 331 | xp = NULL; 332 | } else { 333 | struct rb_node *yp = rb_parent(y); 334 | if (y == yp->left) { 335 | yp->left = x; 336 | is_left = 1; 337 | } else { 338 | yp->right = x; 339 | is_left = 0; 340 | } 341 | } 342 | 343 | int y_color = rb_color(y); 344 | 345 | // Swap in the node 346 | if (y != node) { 347 | rb_swap(tree, node, y); 348 | if (xp == node) { 349 | xp = y; 350 | } 351 | } 352 | 353 | if (y_color == RB_BLACK) { 354 | erase_rebalance(tree, x, xp, is_left); 355 | } 356 | 357 | node->left = NULL; 358 | node->right = NULL; 359 | rb_set_parent(node, NULL); 360 | } 361 | -------------------------------------------------------------------------------- /libk/kprintf.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "libk.h" 5 | #include "sync.h" 6 | #include "drivers/vga_tmode.h" 7 | #include "drivers/serial.h" 8 | 9 | #define to_hex_char(x) ((x) < 10 ? (x) + '0' : (x) - 10 + 'A') 10 | #define ATOI_BUFLEN 256 11 | 12 | spinlock_t kprintf_lock; 13 | 14 | static inline void kprintf_write_char(char c) 15 | { 16 | serial_write_com(1, c); 17 | vga_tmode_putchar(c); 18 | } 19 | 20 | static inline void kprintf_write_str(char *s) 21 | { 22 | while (*s) 23 | kprintf_write_char(*s++); 24 | } 25 | 26 | static inline char digit_to_char(unsigned int digit) 27 | { 28 | kassert_dbg(digit < 16); 29 | return "0123456789ABCDEF"[digit]; 30 | } 31 | 32 | static int atoi_print(uint64_t num, bool sign, unsigned int base) 33 | { 34 | char buf[ATOI_BUFLEN], *p_buf = buf + sizeof buf - 2; 35 | int nwritten = 0; 36 | buf[ATOI_BUFLEN - 1] = '\0'; 37 | 38 | if (base == 16) { 39 | kprintf_write_char('0'); 40 | kprintf_write_char('x'); 41 | } 42 | 43 | if (num == 0) { 44 | kprintf_write_char('0'); 45 | return 1; 46 | } else if (sign) { 47 | int64_t n = (int64_t)num; 48 | if (n < 0) { 49 | kprintf_write_char('-'); 50 | nwritten++; 51 | n = -n; 52 | } 53 | while (n != 0) { 54 | *p_buf-- = digit_to_char(n % base); 55 | n /= base; 56 | nwritten++; 57 | } 58 | } else { 59 | uint64_t n = num; 60 | while (n != 0) { 61 | *p_buf-- = digit_to_char(n % base); 62 | n /= base; 63 | nwritten++; 64 | } 65 | } 66 | kprintf_write_str(p_buf + 1); 67 | return nwritten; 68 | } 69 | 70 | // Doesn't use locks. Calls must be serialised. 71 | static int __kvprintf(const char *fmt, va_list params) 72 | { 73 | size_t nwritten = 0; 74 | const char *pfmt = fmt; 75 | 76 | while (*pfmt) { 77 | if (*pfmt == '%') { 78 | switch (pfmt[1]) { 79 | case '%': 80 | kprintf_write_char('%'); 81 | nwritten++; 82 | break; 83 | case 'c': 84 | kprintf_write_char((char)va_arg(params, int)); 85 | nwritten++; 86 | break; 87 | case 's': { 88 | const char *s = va_arg(params, const char *); 89 | if (s == NULL) 90 | s = "(null)"; 91 | kprintf_write_str((char *)s); 92 | nwritten += strlen(s); 93 | break; 94 | } 95 | case 'p': { 96 | char buf[17], *p_buf = buf + 15; 97 | buf[16] = '\0'; 98 | uintptr_t ptr = (uintptr_t)va_arg(params, void *); 99 | while (p_buf >= buf) { 100 | *p_buf-- = to_hex_char(ptr & 0xF); 101 | ptr >>= 4; 102 | } 103 | kprintf_write_char('0'); 104 | kprintf_write_char('x'); 105 | kprintf_write_str(buf); 106 | nwritten += 18; 107 | break; 108 | } 109 | case 'l': 110 | if (pfmt[2] == 'l') 111 | pfmt++; 112 | __attribute__((fallthrough)); 113 | case 'z': 114 | if (pfmt[2] == 'd' || pfmt[2] == 'i') 115 | nwritten += atoi_print(va_arg(params, long), true, 10); 116 | else if (pfmt[2] == 'u') 117 | nwritten += atoi_print(va_arg(params, unsigned long), false, 10); 118 | else if (pfmt[2] == 'x') 119 | nwritten += atoi_print(va_arg(params, unsigned long), false, 16); 120 | pfmt++; 121 | break; 122 | case 'i': 123 | case 'd': 124 | nwritten += atoi_print(va_arg(params, int), true, 10); 125 | break; 126 | case 'u': 127 | nwritten += atoi_print(va_arg(params, unsigned int), false, 10); 128 | break; 129 | case 'x': 130 | nwritten += atoi_print(va_arg(params, unsigned int), false, 16); 131 | break; 132 | default: 133 | panic("kprintf: unknown format string character '%c'", pfmt[1]); 134 | break; 135 | } 136 | pfmt++; 137 | nwritten++; 138 | } else 139 | kprintf_write_char(*pfmt); 140 | pfmt++; 141 | } 142 | 143 | return nwritten; 144 | } 145 | 146 | int kprintf_nolock(const char *fmt, ...) 147 | { 148 | va_list params; 149 | va_start(params, fmt); 150 | bool locked = spin_try_lock(&kprintf_lock); 151 | int nwritten = __kvprintf(fmt, params); 152 | if (locked) spin_unlock(&kprintf_lock); 153 | va_end(params); 154 | return nwritten; 155 | } 156 | 157 | // Make sure not to call this from an NMI handler 158 | int kprintf(const char *fmt, ...) 159 | { 160 | va_list params; 161 | uint64_t rflags; 162 | va_start(params, fmt); 163 | spin_lock_irqsave(&kprintf_lock, rflags); 164 | int nwritten = __kvprintf(fmt, params); 165 | spin_unlock_irqsave(&kprintf_lock, rflags); 166 | va_end(params); 167 | return nwritten; 168 | } 169 | -------------------------------------------------------------------------------- /libk/ssp.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "libk.h" 3 | 4 | uintptr_t __stack_chk_guard = 0xc9073a36f93e7732; 5 | 6 | __attribute__((noreturn)) void __stack_chk_fail(void) 7 | { 8 | // TODO: Call a special exception handler that swaps stacks? 9 | panic("Stack smashing detected"); 10 | } 11 | -------------------------------------------------------------------------------- /libk/string.c: -------------------------------------------------------------------------------- 1 | #include "libk.h" 2 | 3 | int LIBK_FN(memcmp)(const void *s1, const void *s2, size_t n) 4 | { 5 | const unsigned char *a = s1; 6 | const unsigned char *b = s2; 7 | for (size_t i = 0; i < n; i++) { 8 | if (a[i] < b[i]) 9 | return -1; 10 | else if (b[i] < a[i]) 11 | return 1; 12 | } 13 | return 0; 14 | } 15 | 16 | void *LIBK_FN(memcpy)(void *restrict s1, const void *restrict s2, size_t n) 17 | { 18 | unsigned char *dst = s1; 19 | const unsigned char *src = s2; 20 | for (size_t i = 0; i < n; i++) 21 | dst[i] = src[i]; 22 | return s1; 23 | } 24 | 25 | void *LIBK_FN(memmove)(void *s1, const void *s2, size_t n) 26 | { 27 | unsigned char *dst = s1; 28 | const unsigned char *src = s2; 29 | if (s1 < s2) { 30 | for (size_t i = 0; i < n; i++) 31 | dst[i] = src[i]; 32 | } else { 33 | for (size_t i = n; i > 0; i--) 34 | dst[i - 1] = src[i - 1]; 35 | } 36 | return s1; 37 | } 38 | 39 | void *LIBK_FN(memset)(void *s, int c, size_t n) 40 | { 41 | unsigned char *p = s, *end = p + n; 42 | for (; p != end; p++) { 43 | *p = (unsigned char)c; 44 | } 45 | return s; 46 | } 47 | 48 | void *LIBK_FN(memchr)(const void *s, int c, size_t n) 49 | { 50 | const unsigned char *p = (const unsigned char *)s; 51 | while (n-- > 0) { 52 | if (*p == (unsigned char)c) 53 | return (void *)p; 54 | else 55 | p++; 56 | } 57 | return NULL; 58 | } 59 | 60 | size_t LIBK_FN(strlen)(const char *s) 61 | { 62 | size_t len = 0; 63 | while (*s++) 64 | len++; 65 | return len; 66 | } 67 | -------------------------------------------------------------------------------- /libk/tests/tmain.c: -------------------------------------------------------------------------------- 1 | #include "snow.h" 2 | 3 | #define LIBK_TEST 4 | #include "libk.h" 5 | 6 | #include "../string.c" 7 | #include "tstring.c" 8 | 9 | snow_main(); 10 | -------------------------------------------------------------------------------- /libk/tests/tstring.c: -------------------------------------------------------------------------------- 1 | #include "snow.h" 2 | 3 | describe(string, { 4 | 5 | subdesc(memcpy, { 6 | it("copies 10 bytes", { 7 | char dst[10] = { 0 }; 8 | char src[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 9 | __libk_memcpy(dst, src, 10); 10 | asserteq_buf(dst, src, 10); 11 | }); 12 | }); 13 | 14 | subdesc(memset, { 15 | it("sets 10 bytes", { 16 | char dst[10] = { 0 }; 17 | char expected[10] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; 18 | __libk_memset(dst, 1, 10); 19 | asserteq_buf(dst, expected, 10); 20 | }); 21 | }); 22 | 23 | subdesc(memmove, { 24 | it("moves 10 non-overlapping bytes", { 25 | char dst[10] = { 0 }; 26 | char src[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 27 | __libk_memcpy(dst, src, 10); 28 | asserteq_buf(dst, src, 10); 29 | }); 30 | 31 | it("moves 10 low-overlapping bytes", { 32 | char buf[15] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; 33 | char expected[10] = { 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; 34 | __libk_memmove(buf, buf + 4, 10); 35 | asserteq_buf(buf, expected, 10); 36 | }); 37 | 38 | it("moves 10 high-overlapping bytes", { 39 | char buf[15] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; 40 | char expected[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 41 | __libk_memmove(buf + 4, buf, 10); 42 | asserteq_buf(buf + 4, expected, 10); 43 | }); 44 | }); 45 | 46 | subdesc(memcmp, { 47 | it("compares 10 equal bytes", { 48 | char a[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 49 | char b[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 50 | asserteq(__libk_memcmp(a, b, 10), 0); 51 | }); 52 | 53 | it("compares 10 non-equal bytes", { 54 | char a[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 55 | char b[10] = { 1, 2, 4, 4, 5, 6, 7, 8, 9, 10 }; 56 | assertneq(__libk_memcmp(a, b, 10), 0); 57 | }); 58 | }); 59 | 60 | subdesc(memchr, { 61 | it("finds a needle", { 62 | char a[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 63 | asserteq(__libk_memchr(a, 3, 10), (void *)(a + 2)); 64 | }); 65 | 66 | it("returns NULL if no needle was found", { 67 | char a[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 68 | asserteq(__libk_memchr(a, 11, 10), NULL); 69 | }); 70 | 71 | it("truncates the integer argument", { 72 | char a[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; 73 | asserteq(__libk_memchr(a, 256 + 3, 10), (void *)(a + 2)); 74 | }); 75 | }); 76 | 77 | subdesc(strlen, { 78 | it("returns 10 for a string of length 10", { 79 | char *tst = "0123456789"; 80 | asserteq(__libk_strlen(tst), 10); 81 | }); 82 | 83 | it("returns 0 on a null byte", { 84 | char null = '\0'; 85 | asserteq(__libk_strlen(&null), 0); 86 | }); 87 | }); 88 | }); 89 | -------------------------------------------------------------------------------- /libk/ubsan.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "libk.h" 3 | #include "util.h" 4 | 5 | #define is_aligned(value, alignment) !(value & (alignment - 1)) 6 | 7 | struct source_location { 8 | const char *filename; 9 | uint32_t line; 10 | uint32_t column; 11 | }; 12 | 13 | struct type_descriptor { 14 | uint16_t kind; 15 | uint16_t info; 16 | char name[]; 17 | }; 18 | 19 | struct type_mismatch_info { 20 | struct source_location location; 21 | struct type_descriptor *type; 22 | uintptr_t alignment; 23 | uint8_t type_check_kind; 24 | }; 25 | 26 | struct overflow_data { 27 | struct source_location location; 28 | struct type_descriptor *type; 29 | }; 30 | 31 | struct out_of_bounds_data { 32 | struct source_location location; 33 | struct type_descriptor *array_type; 34 | struct type_descriptor *index_type; 35 | }; 36 | 37 | struct unreachable_data { 38 | struct source_location location; 39 | }; 40 | 41 | struct invalid_value_data { 42 | struct source_location location; 43 | struct type_descriptor *type; 44 | }; 45 | 46 | struct source_location unknown_location = { 47 | "", 0, 0 48 | }; 49 | 50 | __attribute__((noreturn)) 51 | static void ubsan_abort(struct source_location *location, const char *message) 52 | { 53 | if (!location || !location->filename) 54 | location = &unknown_location; 55 | panic("ubsan fatal error: %s at %s:%d\n", message, location->filename, location->line); 56 | } 57 | 58 | void __ubsan_handle_type_mismatch(struct type_mismatch_info *type_mismatch, uintptr_t pointer); 59 | void __ubsan_handle_add_overflow(struct overflow_data *data, uintptr_t lhs, uintptr_t rhs); 60 | void __ubsan_handle_sub_overflow(struct overflow_data *data, uintptr_t lhs, uintptr_t rhs); 61 | void __ubsan_handle_negate_overflow(struct overflow_data *data, uintptr_t old_value); 62 | void __ubsan_handle_divrem_overflow(struct overflow_data *data, uintptr_t lhs, uintptr_t rhs); 63 | void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, uintptr_t index); 64 | void __ubsan_handle_builtin_unreachable(struct unreachable_data *data); 65 | void __ubsan_handle_mul_overflow(struct overflow_data *data, uintptr_t lhs, uintptr_t rhs); 66 | void __ubsan_handle_shift_out_of_bounds(struct out_of_bounds_data *data, uintptr_t lhs, uintptr_t rhs); 67 | void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, uintptr_t value); 68 | 69 | void __ubsan_handle_type_mismatch(struct type_mismatch_info *type_mismatch, uintptr_t pointer) 70 | { 71 | char *message; 72 | if (pointer == 0) 73 | message = "null pointer access"; 74 | else if (type_mismatch->alignment != 0 && (pointer & (type_mismatch->alignment - 1))) 75 | message = "unaligned memory access"; 76 | else 77 | message = "type mismatch"; 78 | ubsan_abort(&type_mismatch->location, message); 79 | } 80 | 81 | void __ubsan_handle_add_overflow(struct overflow_data *data, uintptr_t UNUSED(lhs), uintptr_t UNUSED(rhs)) 82 | { 83 | ubsan_abort(&data->location, "addition overflow"); 84 | } 85 | 86 | void __ubsan_handle_sub_overflow(struct overflow_data *data, uintptr_t UNUSED(lhs), uintptr_t UNUSED(rhs)) 87 | { 88 | ubsan_abort(&data->location, "subtraction overflow"); 89 | } 90 | 91 | void __ubsan_handle_negate_overflow(struct overflow_data *data, uintptr_t UNUSED(old_value)) 92 | { 93 | ubsan_abort(&data->location, "negation overflow"); 94 | } 95 | 96 | void __ubsan_handle_divrem_overflow(struct overflow_data *data, uintptr_t UNUSED(lhs), uintptr_t UNUSED(rhs)) 97 | { 98 | ubsan_abort(&data->location, "division remainder overflow"); 99 | } 100 | 101 | void __ubsan_handle_out_of_bounds(struct out_of_bounds_data *data, uintptr_t UNUSED(index)) 102 | { 103 | ubsan_abort(&data->location, "out of bounds"); 104 | } 105 | 106 | void __ubsan_handle_builtin_unreachable(struct unreachable_data *data) 107 | { 108 | ubsan_abort(&data->location, "reached builtin_unreachable"); 109 | } 110 | 111 | void __ubsan_handle_mul_overflow(struct overflow_data *data, uintptr_t UNUSED(lhs), uintptr_t UNUSED(rhs)) 112 | { 113 | ubsan_abort(&data->location, "multiplication overflow"); 114 | } 115 | 116 | void __ubsan_handle_shift_out_of_bounds(struct out_of_bounds_data *data, uintptr_t UNUSED(lhs), uintptr_t UNUSED(rhs)) 117 | { 118 | ubsan_abort(&data->location, "shift out of bounds"); 119 | } 120 | 121 | void __ubsan_handle_load_invalid_value(struct invalid_value_data *data, uintptr_t UNUSED(value)) 122 | { 123 | ubsan_abort(&data->location, "invalid value load"); 124 | } 125 | -------------------------------------------------------------------------------- /linker.ld: -------------------------------------------------------------------------------- 1 | ENTRY(_start) 2 | OUTPUT_FORMAT(elf64-x86-64) 3 | 4 | phys = 0x100000; 5 | offset = 0xFFFFFFFF80000000; 6 | virt = offset + phys; 7 | 8 | SECTIONS 9 | { 10 | . = virt; 11 | 12 | .rodata ALIGN(0x1000) : AT(ADDR(.rodata) - offset) 13 | { 14 | _rodata_begin_phys = . - offset; 15 | *(.multiboot_header) 16 | *(.rodata) 17 | *(.rodata*) 18 | *(.eh_frame) 19 | _rodata_end_phys = . - offset; 20 | } 21 | 22 | .text ALIGN(0x1000) : AT(ADDR(.text) - offset) 23 | { 24 | _text_begin_phys = . - offset; 25 | *(.text) 26 | *(.text*) 27 | *(.init) 28 | *(.fini) 29 | *(.ctors) 30 | *(.dtors) 31 | _text_end_phys = . - offset; 32 | } 33 | 34 | .data ALIGN(0x1000) : AT(ADDR(.data) - offset) 35 | { 36 | _data_begin_phys = . - offset; 37 | *(.data) 38 | _data_end_phys = . - offset; 39 | } 40 | 41 | .bss ALIGN(0x1000) : AT(ADDR(.bss) - offset) 42 | { 43 | _bss_begin_phys = . - offset; 44 | *(.bss) 45 | *(COMMON) 46 | _bss_end_phys = . - offset; 47 | } 48 | 49 | /DISCARD/ : 50 | { 51 | *(.comment) 52 | } 53 | 54 | _kernel_end_phys = . + 0x1000 - offset; 55 | 56 | } 57 | -------------------------------------------------------------------------------- /util/err_gen.py: -------------------------------------------------------------------------------- 1 | import sys as _sys 2 | 3 | if len(_sys.argv) != 1: 4 | print("Incorrect number of arguments") 5 | exit(-1) 6 | 7 | print("\t\033[32;1mGenerating\033[0m include/gen/err_gen.py") 8 | 9 | err_list = [] 10 | next_value = 1 11 | 12 | def error(name, desc): 13 | global next_value 14 | err_list.append({ "name": name, "desc": desc, "value": next_value }) 15 | next_value += 1 16 | 17 | error("EINVAL", "invalid") 18 | error("ENOTDIR", "not a valid directory") 19 | error("ENOTSUP", "not supported") 20 | error("EBUSY", "busy") 21 | error("ENOENT", "no directory entry") 22 | 23 | defs = [ 24 | "#pragma once", 25 | "" 26 | ] 27 | 28 | strerror = [ 29 | "", 30 | "static inline const char *strerror(err_t e)", 31 | "{", 32 | "\tconst char *rv;", 33 | "\t", 34 | "\tswitch (e) {", 35 | "\t\tcase 0: rv = \"no error\"; break;", 36 | "\t\tdefault: rv = \"unknown error\"; break;" 37 | ] 38 | 39 | for err in err_list: 40 | defs.append("#define %s (-%dL)" % (err["name"], err["value"])) 41 | strerror.append("\t\tcase %s: rv = \"%s\"; break;" % (err["name"], err["name"] + " (" + err["desc"] + ")")) 42 | 43 | strerror.append("\t}") 44 | strerror.append("") 45 | strerror.append("\treturn rv;") 46 | strerror.append("}") 47 | 48 | out = "\n".join(defs) + "\n".join(strerror) 49 | 50 | path = "include/gen/err_gen.h" 51 | 52 | target_file = open(path, "w") 53 | target_file.write(out) 54 | target_file.close() 55 | -------------------------------------------------------------------------------- /util/syscall_gen.py: -------------------------------------------------------------------------------- 1 | import sys as _sys 2 | 3 | if len(_sys.argv) != 2: 4 | print("Incorrect number of arguments") 5 | exit(-1) 6 | elif ["c", "h", "asm"].index(_sys.argv[1]) == -1: 7 | print("Invalid filetype %s" % _sys.argv[1]) 8 | exit(-1) 9 | 10 | print("\t\033[32;1mGenerating\033[0m include/gen/syscall_gen.%s" % _sys.argv[1]) 11 | 12 | syscall_list = [] 13 | 14 | def syscall(name, args="void"): 15 | syscall_list.append({ "name": name, "args": args }) 16 | 17 | syscall("write", "char c") 18 | syscall("fork", "uint64_t flags, struct callee_regs *regs, virtaddr_t return_addr") 19 | syscall("exit", "int code") 20 | syscall("sched_yield") 21 | 22 | decls = [ 23 | "#include \"mm_types.h\"", 24 | "#include \"proc.h\"", 25 | "", 26 | ] 27 | defs = [ 28 | "#define ENOSYS 0xFFFFFFFFFFFFFFFFLL", 29 | "#define NUM_SYSCALLS %d" % len(syscall_list) 30 | ] 31 | table = [ 32 | "syscall_t syscall_table[NUM_SYSCALLS] = {" 33 | ] 34 | asm_defs = [ 35 | "%define ENOSYS 0xFFFFFFFFFFFFFFFF", 36 | "%%define NUM_SYSCALLS %d" % len(syscall_list) 37 | ] 38 | 39 | for i in range(0, len(syscall_list)): 40 | sys = syscall_list[i] 41 | defs.append("#define SYSCALL_%s %d" % (sys["name"].upper(), i)) 42 | table.append("\t[SYSCALL_%s] = (syscall_t)syscall_%s," % (sys["name"].upper(), sys["name"])) 43 | decls.append("int64_t syscall_%s(%s);" % (sys["name"], sys["args"])) 44 | asm_defs.append("%%define SYSCALL_%s %d" % (sys["name"].upper(), i)) 45 | 46 | table.append("};") 47 | 48 | defs = "\n".join(defs) 49 | table = "\n".join(table) 50 | decls = "\n".join(decls) 51 | asm_defs = "\n".join(asm_defs) 52 | 53 | h_out = """#pragma once 54 | 55 | %s 56 | 57 | %s 58 | 59 | extern syscall_t syscall_table[NUM_SYSCALLS]; 60 | """ % (defs, decls) 61 | 62 | c_out = """%s 63 | """ % table 64 | 65 | asm_out = """%s 66 | """ % asm_defs 67 | 68 | out_data = { "c": c_out, "h": h_out, "asm": asm_out } 69 | 70 | prefix = "include/gen/syscall_gen" 71 | path = prefix + "." + _sys.argv[1] 72 | 73 | target_file = open(path, "w") 74 | target_file.write(out_data[_sys.argv[1]]) 75 | target_file.close() 76 | --------------------------------------------------------------------------------