diff --git a/awkernel_lib/src/arch/rv32/delay.rs b/awkernel_lib/src/arch/rv32/delay.rs index 1af9de1e8..5a1b249ad 100644 --- a/awkernel_lib/src/arch/rv32/delay.rs +++ b/awkernel_lib/src/arch/rv32/delay.rs @@ -38,7 +38,7 @@ impl Delay for super::RV32 { while { let test: u32; core::arch::asm!( - "rdcycleh {0}; rdcycle {1}; rdcycleh {2}", + "rdcycleh {0:x}; rdcycle {1:x}; rdcycleh {2:x}", out(reg) cycleh, out(reg) cycle, out(reg) test diff --git a/awkernel_lib/src/arch/rv64.rs b/awkernel_lib/src/arch/rv64.rs index e23965343..0021ad3bb 100644 --- a/awkernel_lib/src/arch/rv64.rs +++ b/awkernel_lib/src/arch/rv64.rs @@ -13,6 +13,22 @@ pub struct RV64; impl super::Arch for RV64 {} +/// # Safety +/// +/// This function must be called at initialization, +/// and called by the primary CPU. +pub unsafe fn init_primary() { + delay::init_primary(); +} + +/// # Safety +/// +/// This function must be called at initialization, +/// and called by non-primary CPUs. +pub unsafe fn init_non_primary() { + delay::init_non_primary(); +} + pub fn init_page_allocator() { frame_allocator::init_page_allocator(); } @@ -39,3 +55,4 @@ pub fn translate_kernel_address(vpn: address::VirtPageNum) -> Option usize { let hartid: usize; unsafe { - core::arch::asm!("csrr {}, mhartid", out(reg) hartid); + // In M-mode, read from tp register (set during boot) + // This avoids repeatedly accessing mhartid CSR + core::arch::asm!("mv {}, tp", out(reg) hartid); } hartid } diff --git a/awkernel_lib/src/arch/rv64/interrupt.rs b/awkernel_lib/src/arch/rv64/interrupt.rs index 72ade36d8..f82576645 100644 --- a/awkernel_lib/src/arch/rv64/interrupt.rs +++ b/awkernel_lib/src/arch/rv64/interrupt.rs @@ -1,5 +1,7 @@ use crate::interrupt::Interrupt; +// RV64 runs in M-mode without OpenSBI, so we use mstatus +// MIE (Machine Interrupt Enable) is bit 3 (0x08) in mstatus impl Interrupt for super::RV64 { fn get_flag() -> usize { let x: usize; diff --git a/awkernel_lib/src/interrupt.rs b/awkernel_lib/src/interrupt.rs index 12aaf798a..ae2fdce4e 100644 --- a/awkernel_lib/src/interrupt.rs +++ b/awkernel_lib/src/interrupt.rs @@ -403,6 +403,57 @@ pub fn handle_preemption() { preemption(); } +/// Handle all pending interrupt requests for RISC-V. +/// This is called from the M-mode interrupt handler. +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] +pub fn handle_irqs(is_task: bool) { + use crate::{heap, unwind::catch_unwind}; + use core::mem::transmute; + + let handlers = IRQ_HANDLERS.read(); + let mut need_preemption = false; + + let controller = INTERRUPT_CONTROLLER.read(); + if let Some(ctrl) = controller.as_ref() { + let iter = ctrl.pending_irqs(); + drop(controller); // unlock + + // Use the primary allocator. + #[cfg(not(feature = "std"))] + let _guard = { + let g = heap::TALLOC.save(); + unsafe { heap::TALLOC.use_primary() }; + g + }; + + for irq in iter { + if irq == PREEMPT_IRQ.load(Ordering::Relaxed) { + need_preemption = true; + continue; + } + + if let Some((_, handler)) = handlers.get(&irq) { + if let Err(err) = catch_unwind(|| { + handler(irq); + }) { + log::warn!("an interrupt handler has been panicked\n{err:?}"); + } + } + } + } + + if need_preemption && is_task { + let ptr = PREEMPT_FN.load(Ordering::Relaxed); + let preemption = unsafe { transmute::<*mut (), fn()>(ptr) }; + preemption(); + } + + // Because IPIs are edge-trigger, + // IPI sent during interrupt handlers will be ignored. + // To notify the interrupt again, we setup a timer. + crate::cpu::reset_wakeup_timer(); +} + /// Disable interrupts and automatically restored the configuration. /// /// ``` diff --git a/kernel/ld/rv64-link.lds b/kernel/ld/rv64-link.lds index c85dd2016..19d184e13 100644 --- a/kernel/ld/rv64-link.lds +++ b/kernel/ld/rv64-link.lds @@ -3,10 +3,11 @@ ENTRY(_start) SECTIONS { - . = 0x80200000; + /* Boot code at DRAM start for -bios none */ + . = 0x80000000; PROVIDE (__executable_start = .); __ram_start = .; - + PROVIDE (stext = .); .init : { KEEP(*(.init)) } .text : { *(.text .text.* .gnu.linkonce.t*) } diff --git a/kernel/src/arch/rv64.rs b/kernel/src/arch/rv64.rs index 58902c929..c532699dc 100644 --- a/kernel/src/arch/rv64.rs +++ b/kernel/src/arch/rv64.rs @@ -1,3 +1,4 @@ pub mod config; mod console; +mod interrupt_controller; mod kernel_main; diff --git a/kernel/src/arch/rv64/boot.S b/kernel/src/arch/rv64/boot.S index 8e3095ac4..3740c2482 100644 --- a/kernel/src/arch/rv64/boot.S +++ b/kernel/src/arch/rv64/boot.S @@ -7,13 +7,28 @@ .attribute arch, "rv64gc" _start: + # Read hart ID from mhartid CSR (M-mode) + csrr a0, mhartid + mv tp, a0 # save hartid to thread pointer for later use + + # Preserve DTB pointer that QEMU/firmware passes in a1 + la t0, dtb_ptr + sd a1, 0(t0) + + # set up a simple trap handler for M-mode + la t0, early_trap_handler + csrw mtvec, t0 + + # Initialize mscratch to 0 (will be set properly later if needed) + csrw mscratch, zero + # set up the 8MB initial stack for each cpu. li sp, 0x80800000 - li a0, 0x00800000 - csrr a1, mhartid - addi a1, a1, 1 - mul a0, a0, a1 - add sp, sp, a0 + li t0, 0x00800000 + mv t1, a0 # use hartid + addi t1, t1, 1 + mul t0, t0, t1 + add sp, sp, t0 # clear the BSS la t0, __bss_start la t1, __bss_end @@ -24,4 +39,203 @@ _start: # jump to kernel_main jal ra, kernel_main 2: - j 2b \ No newline at end of file + j 2b + +# M-mode trap handler - handles interrupts and exceptions +.align 4 +early_trap_handler: + # Check if this is an interrupt or exception + csrr t0, mcause + blt t0, zero, handle_interrupt # If MSB is set, it's an interrupt + + # This is an exception - print debug info and halt + j unhandled_trap + +handle_interrupt: + # Get interrupt code (lower bits of mcause) + li t1, 0x7FFFFFFFFFFFFFFF + and t0, t0, t1 + + # Check interrupt type + li t1, 7 # M-mode timer interrupt + beq t0, t1, handle_timer_interrupt + + li t1, 3 # M-mode software interrupt (IPI) + beq t0, t1, handle_software_interrupt + + # Unknown interrupt - just return + mret + +handle_timer_interrupt: + # Save all registers that might be clobbered by the Rust handler + addi sp, sp, -256 + sd ra, 0(sp) + sd t0, 8(sp) + sd t1, 16(sp) + sd t2, 24(sp) + sd a0, 32(sp) + sd a1, 40(sp) + sd a2, 48(sp) + sd a3, 56(sp) + sd a4, 64(sp) + sd a5, 72(sp) + sd a6, 80(sp) + sd a7, 88(sp) + sd t3, 96(sp) + sd t4, 104(sp) + sd t5, 112(sp) + sd t6, 120(sp) + + # Disable timer interrupt temporarily by setting mtimecmp to max + # MTIMECMP base is 0x02004000, each hart has 8-byte register + csrr t0, mhartid + slli t0, t0, 3 # Multiply by 8 + li t1, 0x02004000 + add t0, t0, t1 + li t1, -1 + sd t1, 0(t0) + + # Call the Rust timer handler + call riscv_handle_timer + + # Restore registers + ld ra, 0(sp) + ld t0, 8(sp) + ld t1, 16(sp) + ld t2, 24(sp) + ld a0, 32(sp) + ld a1, 40(sp) + ld a2, 48(sp) + ld a3, 56(sp) + ld a4, 64(sp) + ld a5, 72(sp) + ld a6, 80(sp) + ld a7, 88(sp) + ld t3, 96(sp) + ld t4, 104(sp) + ld t5, 112(sp) + ld t6, 120(sp) + addi sp, sp, 256 + + # Return from interrupt + mret + +handle_software_interrupt: + # Save all registers that might be clobbered by the Rust handler + addi sp, sp, -256 + sd ra, 0(sp) + sd t0, 8(sp) + sd t1, 16(sp) + sd t2, 24(sp) + sd a0, 32(sp) + sd a1, 40(sp) + sd a2, 48(sp) + sd a3, 56(sp) + sd a4, 64(sp) + sd a5, 72(sp) + sd a6, 80(sp) + sd a7, 88(sp) + sd t3, 96(sp) + sd t4, 104(sp) + sd t5, 112(sp) + sd t6, 120(sp) + + # Clear the software interrupt by writing 0 to MSIP + # MSIP base is 0x02000000, each hart has 4-byte register + csrr t0, mhartid + slli t0, t0, 2 # Multiply by 4 + li t1, 0x02000000 + add t0, t0, t1 + sw zero, 0(t0) # Clear MSIP + + # Call the Rust IPI handler + call riscv_handle_ipi + + # Restore registers + ld ra, 0(sp) + ld t0, 8(sp) + ld t1, 16(sp) + ld t2, 24(sp) + ld a0, 32(sp) + ld a1, 40(sp) + ld a2, 48(sp) + ld a3, 56(sp) + ld a4, 64(sp) + ld a5, 72(sp) + ld a6, 80(sp) + ld a7, 88(sp) + ld t3, 96(sp) + ld t4, 104(sp) + ld t5, 112(sp) + ld t6, 120(sp) + addi sp, sp, 256 + + # Return from interrupt + mret + +unhandled_trap: + # Write 'TRAP!' to UART + li t0, 0x10000000 + li t1, 'T' + sb t1, 0(t0) + li t1, 'R' + sb t1, 0(t0) + li t1, 'A' + sb t1, 0(t0) + li t1, 'P' + sb t1, 0(t0) + li t1, '!' + sb t1, 0(t0) + li t1, '\r' + sb t1, 0(t0) + li t1, '\n' + sb t1, 0(t0) + + # Print mcause + csrr a0, mcause + call print_hex + + # Print mepc + csrr a0, mepc + call print_hex + + # Print mtval + csrr a0, mtval + call print_hex + + # Infinite loop +1: + j 1b + +# Helper function to print a hex value +print_hex: + li t0, 0x10000000 + li t2, 60 # shift amount: 60, 56, 52, ..., 0 + li t3, 16 # counter +1: + srl t1, a0, t2 + andi t1, t1, 0xF + li t4, 10 + blt t1, t4, 2f + addi t1, t1, 'A'-10 + j 3f +2: + addi t1, t1, '0' +3: + sb t1, 0(t0) + addi t2, t2, -4 + addi t3, t3, -1 + bnez t3, 1b + + # Print newline + li t1, '\r' + sb t1, 0(t0) + li t1, '\n' + sb t1, 0(t0) + ret + + .section .data + .align 3 + .global dtb_ptr +dtb_ptr: + .dword 0 diff --git a/kernel/src/arch/rv64/config.rs b/kernel/src/arch/rv64/config.rs index a6333deee..bc9ffe72c 100644 --- a/kernel/src/arch/rv64/config.rs +++ b/kernel/src/arch/rv64/config.rs @@ -1,4 +1,2 @@ -pub const HEAP_START: usize = 0x0008_8000_0000; - pub const PREEMPT_IRQ: u16 = 0; pub const WAKEUP_IRQ: u16 = 1; diff --git a/kernel/src/arch/rv64/interrupt_controller.rs b/kernel/src/arch/rv64/interrupt_controller.rs new file mode 100644 index 000000000..1d7112d2d --- /dev/null +++ b/kernel/src/arch/rv64/interrupt_controller.rs @@ -0,0 +1,197 @@ +use alloc::boxed::Box; +use awkernel_lib::interrupt::InterruptController; +use core::ptr::{read_volatile, write_volatile}; +use core::sync::atomic::Ordering; + +/// RISC-V PLIC (Platform-Level Interrupt Controller) implementation +/// Combined with CLINT/ACLINT for IPI support +pub struct RiscvPlic { + base_address: usize, + max_priority: u32, + num_sources: u16, +} + +// CLINT/ACLINT base address for IPIs +const ACLINT_BASE: usize = 0x0200_0000; +const MSIP_OFFSET: usize = 0x0000; // Machine Software Interrupt Pending + +// Import the detected CPU count from kernel_main +use super::kernel_main::NUM_CPUS; + +impl RiscvPlic { + /// Create a new RISC-V PLIC controller + pub const fn new(base_address: usize, num_sources: u16) -> Self { + Self { + base_address, + max_priority: 7, // Typical PLIC priority levels: 0-7 + num_sources, + } + } + + /// Get the priority register address for a given interrupt source + fn priority_reg(&self, source: u16) -> *mut u32 { + (self.base_address + (source as usize * 4)) as *mut u32 + } + + /// Get the enable register address for a given context and interrupt source + fn enable_reg(&self, context: usize, source: u16) -> *mut u32 { + let reg_index = source as usize / 32; + let base = self.base_address + 0x2000 + context * 0x80; + (base + reg_index * 4) as *mut u32 + } + + /// Get the threshold register address for a given context + fn threshold_reg(&self, context: usize) -> *mut u32 { + (self.base_address + 0x200000 + context * 0x1000) as *mut u32 + } + + /// Get the claim register address for a given context + fn claim_reg(&self, context: usize) -> *mut u32 { + (self.base_address + 0x200004 + context * 0x1000) as *mut u32 + } + + /// Set priority for an interrupt source + fn set_priority(&self, source: u16, priority: u32) { + if source > 0 && source <= self.num_sources && priority <= self.max_priority { + let reg = self.priority_reg(source); + unsafe { write_volatile(reg, priority) }; + } + } + + /// Enable interrupt for a specific context + fn enable_interrupt(&self, context: usize, source: u16) { + if source > 0 && source <= self.num_sources { + let reg = self.enable_reg(context, source); + let bit_pos = source as usize % 32; + unsafe { + let current = read_volatile(reg); + write_volatile(reg, current | (1 << bit_pos)); + } + } + } + + /// Disable interrupt for a specific context + fn disable_interrupt(&self, context: usize, source: u16) { + if source > 0 && source <= self.num_sources { + let reg = self.enable_reg(context, source); + let bit_pos = source as usize % 32; + unsafe { + let current = read_volatile(reg); + write_volatile(reg, current & !(1 << bit_pos)); + } + } + } + + /// Get the current hart ID (CPU ID) + fn get_hart_id(&self) -> usize { + // Use mhartid CSR to get hart ID + let hart_id: usize; + unsafe { + core::arch::asm!("csrr {}, mhartid", out(reg) hart_id); + } + hart_id + } + + /// Get supervisor mode context for current hart + fn get_supervisor_context(&self) -> usize { + // Typically supervisor mode context = hartid * 2 + 1 + self.get_hart_id() * 2 + 1 + } + + /// Send software interrupt (IPI) to a specific hart + fn send_software_interrupt(&self, hart_id: u32) { + // MSIP register for each hart is at ACLINT_BASE + MSIP_OFFSET + (hart_id * 4) + let msip_addr = (ACLINT_BASE + MSIP_OFFSET + (hart_id as usize * 4)) as *mut u32; + unsafe { + write_volatile(msip_addr, 1); + } + } + + /// Enable machine software interrupts + fn enable_software_interrupts(&self) { + unsafe { + // Set MIE.MSIE (Machine Software Interrupt Enable) bit (bit 3) + core::arch::asm!("csrrs t0, mie, {}", in(reg) 1 << 3); + } + } +} + +impl InterruptController for RiscvPlic { + fn enable_irq(&mut self, irq: u16) { + // Set a reasonable priority for the interrupt + self.set_priority(irq, 1); + + // Enable for supervisor mode (we're running in supervisor mode) + let context = self.get_supervisor_context(); + self.enable_interrupt(context, irq); + } + + fn disable_irq(&mut self, irq: u16) { + let context = self.get_supervisor_context(); + self.disable_interrupt(context, irq); + } + + fn pending_irqs(&self) -> Box> { + // Check pending interrupts by claiming them + let context = self.get_supervisor_context(); + let claim_reg = self.claim_reg(context); + + let mut pending = alloc::vec::Vec::new(); + + unsafe { + let claimed = read_volatile(claim_reg); + if claimed != 0 { + pending.push(claimed as u16); + // Complete the interrupt (write back the claim) + write_volatile(claim_reg, claimed); + } + } + + Box::new(pending.into_iter()) + } + + fn send_ipi(&mut self, _irq: u16, cpu_id: u32) { + // Send machine software interrupt to the target hart + self.send_software_interrupt(cpu_id); + } + + fn send_ipi_broadcast(&mut self, _irq: u16) { + // Send IPI to all CPUs + let num_cpus = NUM_CPUS.load(Ordering::Acquire) as u32; + for cpu_id in 0..num_cpus { + self.send_software_interrupt(cpu_id); + } + } + + fn send_ipi_broadcast_without_self(&mut self, _irq: u16) { + // Send IPI to all CPUs except current + let current_hart = self.get_hart_id() as u32; + let num_cpus = NUM_CPUS.load(Ordering::Acquire) as u32; + for cpu_id in 0..num_cpus { + if cpu_id != current_hart { + self.send_software_interrupt(cpu_id); + } + } + } + + fn init_non_primary(&mut self) { + // Set threshold to 0 to accept all interrupts + let context = self.get_supervisor_context(); + let threshold_reg = self.threshold_reg(context); + unsafe { write_volatile(threshold_reg, 0) }; + + // Enable software interrupts for IPIs + self.enable_software_interrupts(); + } + + fn irq_range(&self) -> (u16, u16) { + // PLIC interrupt sources typically range from 1 to num_sources + (1, self.num_sources + 1) + } + + fn irq_range_for_pnp(&self) -> (u16, u16) { + // Reserve lower IRQs for system use, higher for PnP + let start = self.num_sources / 2; + (start, self.num_sources + 1) + } +} diff --git a/kernel/src/arch/rv64/kernel_main.rs b/kernel/src/arch/rv64/kernel_main.rs index 03955b43f..3fbd1b659 100644 --- a/kernel/src/arch/rv64/kernel_main.rs +++ b/kernel/src/arch/rv64/kernel_main.rs @@ -7,7 +7,6 @@ use awkernel_lib::{cpu, heap}; use core::{ arch::global_asm, fmt::Write, - // mem::MaybeUninit, sync::atomic::{AtomicBool, Ordering}, }; use ns16550a::Uart; @@ -16,19 +15,34 @@ const UART_BASE: usize = 0x1000_0000; const HEAP_SIZE: usize = 1024 * 1024 * 512; -// TODO: set initial stack 4MB for each CPU on 0x8040_0000. see boot.S -// const MAX_HARTS: usize = 8; -// const INITIAL_STACK: usize = 0x8040_0000; -// const INITIAL_STACK_SIZE: usize = 0x0040_0000; -// #[repr(align(4096))] -// struct InitialStack([MaybeUninit; INITIAL_STACK_SIZE * MAX_HARTS]); -// #[no_mangle] -// static INITIAL_STACK: InitialStack = unsafe { MaybeUninit::uninit().assume_init() }; - static PRIMARY_INITIALIZED: AtomicBool = AtomicBool::new(false); +pub(super) static NUM_CPUS: core::sync::atomic::AtomicUsize = + core::sync::atomic::AtomicUsize::new(4); global_asm!(include_str!("boot.S")); +#[no_mangle] +pub unsafe extern "C" fn riscv_handle_ipi() { + awkernel_lib::interrupt::handle_irqs(true); +} + +#[no_mangle] +pub unsafe extern "C" fn riscv_handle_timer() { + awkernel_lib::interrupt::handle_irqs(true); +} + +unsafe fn init_interrupt_controller() { + use super::interrupt_controller::RiscvPlic; + use alloc::boxed::Box; + + const PLIC_BASE: usize = 0x0c000000; + const NUM_SOURCES: u16 = 128; + let plic = Box::new(RiscvPlic::new(PLIC_BASE, NUM_SOURCES)); + awkernel_lib::interrupt::register_interrupt_controller(plic); + + core::arch::asm!("csrrs t0, mie, {}", in(reg) 1 << 3); +} + #[no_mangle] pub unsafe extern "C" fn kernel_main() { unsafe { crate::config::init() }; @@ -42,34 +56,21 @@ pub unsafe extern "C" fn kernel_main() { } unsafe fn primary_hart(hartid: usize) { - // setup interrupt; TODO; - super::console::init_port(UART_BASE); - // Initialize memory management (page allocator) awkernel_lib::arch::rv64::init_page_allocator(); - - // Initialize virtual memory system awkernel_lib::arch::rv64::init_kernel_space(); - - // Activate virtual memory (enable MMU and page tables) awkernel_lib::arch::rv64::activate_kernel_space(); - // Verify VM system is working by getting kernel token - let _kernel_token = awkernel_lib::arch::rv64::get_kernel_token(); - - // setup the VM let backup_start = HEAP_START; let backup_size = BACKUP_HEAP_SIZE; let primary_start = HEAP_START + BACKUP_HEAP_SIZE; let primary_size = HEAP_SIZE; - // enable heap allocator heap::init_primary(primary_start, primary_size); heap::init_backup(backup_start, backup_size); - heap::TALLOC.use_primary_then_backup(); // use backup allocator + heap::TALLOC.use_primary_then_backup(); - // initialize serial device and dump booting logo let mut port = Uart::new(UART_BASE); port.init( ns16550a::WordLength::EIGHT, @@ -84,20 +85,19 @@ unsafe fn primary_hart(hartid: usize) { let _ = port.write_str("\r\nAwkernel is booting\r\n\r\n"); - // initialize console driver to which log messages are dumped console::init(UART_BASE); - // switch to S-Mode; TODO; - // * currntly this impl. holds both kernel and userland - // * in M-Mode, which is the highest priority. + init_interrupt_controller(); - // wake up another harts + log::info!("AWkernel RV64 PLIC interrupt controller initialized"); + + NUM_CPUS.store(4, Ordering::Release); PRIMARY_INITIALIZED.store(true, Ordering::SeqCst); let kernel_info = KernelInfo { info: (), cpu_id: hartid, - num_cpu: 4, // TODO: get the number of CPUs + num_cpu: NUM_CPUS.load(Ordering::Acquire), }; crate::main::<()>(kernel_info); @@ -108,12 +108,15 @@ unsafe fn non_primary_hart(hartid: usize) { core::hint::spin_loop(); } - heap::TALLOC.use_primary_then_backup(); // use backup allocator + awkernel_lib::interrupt::init_non_primary(); + heap::TALLOC.use_primary_then_backup(); + + let num_cpu = NUM_CPUS.load(Ordering::Acquire); let kernel_info = KernelInfo { info: (), cpu_id: hartid, - num_cpu: 4, // TODO: get the number of CPUs + num_cpu, }; crate::main::<()>(kernel_info);