Use native syscalls instead of interrupt-based ones, move syscall from

driver to arch, and add x86_64 specific native syscalls.

kernel/src/arch/mod.rs

@@ -1,3 +1,4 @@
 pub mod arch;
+pub mod syscall;
 #[cfg(target_arch = "x86_64")]
 pub mod x86_64;

kernel/src/arch/syscall.rs

@@ -5,6 +5,7 @@ use core::{
 
 use x86_64::{
     VirtAddr,
+    instructions::interrupts,
     structures::paging::{FrameAllocator, Mapper, Page, PageTableFlags, Size4KiB},
 };
 
@@ -190,7 +191,11 @@ pub unsafe extern "C" fn syscall_dispatch(
     arg0: isize,
     arg1: isize,
     arg2: isize,
+    arg3: isize,
+    arg4: isize,
+    arg5: isize,
 ) -> isize {
+    interrupts::enable();
     match num {
         BRK => unsafe { sbrk(arg0) },
         WRITE => {

kernel/src/arch/x86_64/gdt.rs

@@ -10,7 +10,8 @@ use x86_64::structures::{
 pub const DOUBLE_FAULT_IST_INDEX: u16 = 0;
 
 lazy_static! {
-    static ref TSS: TaskStateSegment = {
+    #[repr(C, align(4096))]
+    pub static ref TSS: TaskStateSegment = {
         let mut tss = TaskStateSegment::new();
         tss.interrupt_stack_table[DOUBLE_FAULT_IST_INDEX as usize] = {
             const STACK_SIZE: usize = 4096 * 5;
@@ -40,22 +41,33 @@ lazy_static! {
             SegmentSelector,
             SegmentSelector,
             SegmentSelector,
+            SegmentSelector,
             SegmentSelector
         )
     ) = {
         let mut gdt = GlobalDescriptorTable::new();
         let kernel_code_selector = gdt.append(Descriptor::kernel_code_segment());
         let kernel_data_selector = gdt.append(Descriptor::kernel_data_segment());
-        let user_code_selector = gdt.append(Descriptor::user_code_segment());
+
+        let user_code32_selector = gdt.append(Descriptor::UserSegment(
+            (x86_64::structures::gdt::DescriptorFlags::USER_SEGMENT
+                | x86_64::structures::gdt::DescriptorFlags::PRESENT
+                | x86_64::structures::gdt::DescriptorFlags::EXECUTABLE
+                | x86_64::structures::gdt::DescriptorFlags::DPL_RING_3)
+                .bits(),
+        ));
+
         let user_data_selector = gdt.append(Descriptor::user_data_segment());
+        let user_code_selector = gdt.append(Descriptor::user_code_segment());
         let tss_selector = gdt.append(Descriptor::tss_segment(&TSS));
         (
             gdt,
             (
                 kernel_code_selector,
                 kernel_data_selector,
-                user_code_selector,
+                user_code32_selector,
                 user_data_selector,
+                user_code_selector,
                 tss_selector,
             ),
         )
@@ -70,12 +82,12 @@ pub fn load_gdt_x86_64() {
         DS::set_reg(GDT.1.1);
         ES::set_reg(GDT.1.1);
         SS::set_reg(GDT.1.1);
-        load_tss(GDT.1.4);
+        load_tss(GDT.1.5);
     }
 }
 
 pub fn user_code_selector() -> SegmentSelector {
-    GDT.1.2
+    GDT.1.4
 }
 
 pub fn user_data_selector() -> SegmentSelector {

kernel/src/arch/x86_64/init.rs

@@ -3,6 +3,7 @@ use crate::{
         gdt::load_gdt_x86_64,
         interrupts::{PICS, init_idt_x86_64},
         mouse::setup_mouse,
+        syscall::init_syscalls,
     },
     driver::mouse::MOUSE,
 };
@@ -74,6 +75,8 @@ pub fn init_x86_64<'a>(
 
     init_idt_x86_64();
 
+    init_syscalls();
+
     unsafe {
         let mut pics = PICS.lock();
         pics.initialize();

kernel/src/arch/x86_64/interrupts.rs

@@ -7,7 +7,6 @@ use lazy_static::lazy_static;
 use pic8259::ChainedPics;
 use spin::Mutex;
 use x86_64::{
-    VirtAddr,
     instructions::port::Port,
     registers::control::Cr2,
     structures::idt::{InterruptDescriptorTable, InterruptStackFrame, PageFaultErrorCode},
@@ -44,10 +43,6 @@ lazy_static! {
             idt.double_fault
                 .set_handler_fn(double_fault_handler)
                 .set_stack_index(gdt::DOUBLE_FAULT_IST_INDEX);
-            #[allow(function_casts_as_integer)]
-            idt[0x80]
-                .set_handler_addr(VirtAddr::new(syscall_interrupt_handler as u64))
-                .set_privilege_level(x86_64::PrivilegeLevel::Ring3);
         }
         idt[InterruptIndex::Timer.as_u8()].set_handler_fn(timer_interrupt_handler);
         idt.page_fault.set_handler_fn(page_fault_handler);
@@ -136,50 +131,3 @@ pub extern "x86-interrupt" fn keyboard_interrupt_handler(_stack_frame: Interrupt
             .notify_end_of_interrupt(InterruptIndex::Keyboard.as_u8());
     }
 }
-
-#[unsafe(naked)]
-unsafe extern "C" fn syscall_interrupt_handler() {
-    core::arch::naked_asm!(
-        // push all registers
-        "push r15",
-        "push r14",
-        "push r13",
-        "push r12",
-        "push r11",
-        "push r10",
-        "push r9",
-        "push r8",
-        "push rbp",
-        "push rbx",
-        "push rcx",
-        "push rdx",
-        "push rsi",
-        "push rdi",
-        "push rax",
-        "sub rsp, 8",
-        "sti",          // allow IRQ interrupts
-        "mov rcx, rdx", // arg2
-        "mov rdx, rsi", // arg1
-        "mov rsi, rdi", // arg0
-        "mov rdi, rax", // num
-        "call syscall_dispatch",
-        "add rsp, 8",
-        "add rsp, 8",
-        // pop them in reverse orser
-        "pop rdi",
-        "pop rsi",
-        "pop rdx",
-        "pop rcx",
-        "pop rbx",
-        "pop rbp",
-        "pop r8",
-        "pop r9",
-        "pop r10",
-        "pop r11",
-        "pop r12",
-        "pop r13",
-        "pop r14",
-        "pop r15",
-        "iretq",
-    )
-}

kernel/src/arch/x86_64/mod.rs

@@ -5,4 +5,5 @@ pub mod init;
 pub mod interrupts;
 pub mod mouse;
 pub mod paging;
+pub mod syscall;
 pub mod usermode;

kernel/src/arch/x86_64/syscall.rs

@@ -0,0 +1,136 @@
+use core::arch::asm;
+
+use lazy_static::lazy_static;
+use x86_64::instructions::tlb::{self, flush_all};
+
+use crate::arch::x86_64::gdt::{GDT, TSS};
+
+const IA32_EFER: u32 = 0xC0000080;
+const IA32_EFER_SCE: u64 = 1; // system call enable bit
+const IA32_STAR: u32 = 0xC0000081;
+const IA32_LSTAR: u32 = 0xC0000082;
+const IA32_FMASK: u32 = 0xC0000084;
+const IA32_KERNEL_GS_BASE: u32 = 0xC0000102;
+
+#[repr(C)]
+pub struct PerCpuData {
+    pub user_rsp: u64,
+    pub kernel_rsp: u64,
+}
+
+static mut PER_CPU: PerCpuData = PerCpuData {
+    user_rsp: 0,
+    kernel_rsp: 0,
+};
+
+pub unsafe fn wrmsr(msr: u32, value: u64) {
+    let low = value as u32;
+    let high = (value >> 32) as u32;
+    unsafe {
+        asm!(
+            "wrmsr",
+            in("ecx") msr,
+            in("eax") low,
+            in("edx") high,
+        );
+    }
+}
+
+pub unsafe fn rdmsr(msr: u32) -> u64 {
+    let low: u32;
+    let high: u32;
+    unsafe {
+        asm!(
+            "rdmsr",
+            in("ecx") msr,
+            lateout("eax") low,
+            lateout("edx") high,
+        );
+    }
+    ((high as u64) << 32) | (low as u64)
+}
+
+pub fn init_syscalls() {
+    unsafe {
+        flush_all();
+
+        let efer = rdmsr(IA32_EFER);
+        wrmsr(IA32_EFER, efer | IA32_EFER_SCE); // enable syscalls
+
+        let per_cpu_addr = &raw const PER_CPU as u64;
+        wrmsr(IA32_KERNEL_GS_BASE, per_cpu_addr); // per-cpu kernel data
+
+        asm!("mov gs, ax", in("ax") 0u16);
+
+        let kernel_stack_top = TSS.privilege_stack_table[0].as_u64();
+        PER_CPU.kernel_rsp = kernel_stack_top;
+
+        #[allow(function_casts_as_integer)]
+        wrmsr(IA32_LSTAR, syscall_entry as u64); // set syscall entry function
+
+        wrmsr(IA32_FMASK, 1 << 9); // Mask interupts during syscalls
+
+        let kernel_cs = (GDT.1.0.0 & !0b11) as u64;
+        let user_ds = (GDT.1.3.0 & !0b11) as u64;
+
+        let sysret_base = user_ds - 8;
+
+        let star = (kernel_cs << 32) | (sysret_base << 48);
+        wrmsr(IA32_STAR, star);
+    }
+}
+
+#[unsafe(naked)]
+#[unsafe(no_mangle)]
+unsafe extern "C" fn syscall_entry() {
+    core::arch::naked_asm!(
+        r#"
+            .intel_syntax noprefix
+
+            swapgs
+
+            // save user stack
+            mov gs:[0], rsp
+
+            // use kernel stack
+            mov rsp, gs:[8]
+
+            push r11
+            push rcx
+            push rax
+            push rdi
+            push rsi
+            push rdx
+            push r10
+            push r8
+            push r9
+
+            mov rdi, rax
+            mov rsi, [rsp + 40]
+            mov rdx, [rsp + 32]
+            mov rcx, [rsp + 24]
+            mov r8, [rsp + 16]
+            mov r9, [rsp + 8]
+            push qword ptr [rsp]
+
+            call syscall_dispatch
+
+            add rsp, 8 // fix alignment
+
+            pop r9
+            pop r8
+            pop r10
+            pop rdx
+            pop rsi
+            pop rdi
+            add rsp, 8
+
+            pop rcx
+            pop r11
+
+            mov rsp, gs:[0]
+
+            swapgs
+            sysretq"#,
+    );
+}

kernel/src/driver/elf/loader.rs

@@ -2,8 +2,9 @@ use core::ptr::null;
 
 use x86_64::structures::paging::OffsetPageTable;
 
-use crate::driver::{
+use crate::{
-    elf::{
+    arch::syscall::{malloc, memset},
+    driver::elf::{
         header::{
             ET_DYN, ET_EXEC, ET_REL, Elf64Ehdr, Elf64Rel, Elf64Shdr, SHF_ALLOC, SHT_NOBITS, SHT_REL,
         },
@@ -12,7 +13,6 @@ use crate::driver::{
         section::elf_sheader,
         validation::validate_elf,
     },
-    syscall::{malloc, memset},
 };
 
 pub fn load_file(mapper: &mut OffsetPageTable, elf_bytes: &[u8]) -> (*const u8, u64) {

kernel/src/driver/elf/program.rs

@@ -12,19 +12,15 @@ use x86_64::{
 };
 
 use crate::{
-    arch::arch::FRAME_ALLOCATOR,
+    arch::{arch::FRAME_ALLOCATOR, syscall::memset},
-    driver::{
+    driver::elf::{
-        elf::{
+        header::{
-            header::{
+            DT_JMPREL, DT_NEEDED, DT_NULL, DT_PLTREL, DT_PLTRELSZ, DT_RELA, DT_RELAENT, DT_RELASZ,
-                DT_JMPREL, DT_NEEDED, DT_NULL, DT_PLTREL, DT_PLTRELSZ, DT_RELA, DT_RELAENT,
+            DT_STRSZ, DT_STRTAB, DT_SYMENT, DT_SYMTAB, Elf64Dyn, Elf64Ehdr, Elf64Phdr, Elf64Rela,
-                DT_RELASZ, DT_STRSZ, DT_STRTAB, DT_SYMENT, DT_SYMTAB, Elf64Dyn, Elf64Ehdr,
+            Elf64Sym, PF_X, PT_DYNAMIC, PT_LOAD, R_X86_64_64, R_X86_64_GLOB_DAT,
-                Elf64Phdr, Elf64Rela, Elf64Sym, PF_X, PT_DYNAMIC, PT_LOAD, R_X86_64_64,
+            R_X86_64_JUMP_SLOT, R_X86_64_NONE, R_X86_64_RELATIVE, SHN_UNDEF, STB_WEAK,
-                R_X86_64_GLOB_DAT, R_X86_64_JUMP_SLOT, R_X86_64_NONE, R_X86_64_RELATIVE, SHN_UNDEF,
-                STB_WEAK,
-            },
-            reloc::elf_do_reloc,
         },
-        syscall::memset,
+        reloc::elf_do_reloc,
     },
     util::{align_down, align_up},
 };

kernel/src/driver/mod.rs

@@ -3,5 +3,4 @@ pub mod graphics;
 pub mod keyboard;
 pub mod mouse;
 pub mod serial;
-pub mod syscall;
 pub mod timer;