// SPDX-License-Identifier: MIT OR Apache-2.0 // // Copyright (c) 2020-2022 Andre Richter //! Memory Management Unit. #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; mod alloc; mod mapping_record; mod translation_table; mod types; use crate::{ bsp, memory::{Address, Physical, Virtual}, synchronization::{self, interface::Mutex}, warn, }; use core::{fmt, num::NonZeroUsize}; pub use types::*; //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- /// MMU enable errors variants. #[allow(missing_docs)] #[derive(Debug)] pub enum MMUEnableError { AlreadyEnabled, Other(&'static str), } /// Memory Management interfaces. pub mod interface { use super::*; /// MMU functions. pub trait MMU { /// Turns on the MMU for the first time and enables data and instruction caching. /// /// # Safety /// /// - Changes the HW's global state. unsafe fn enable_mmu_and_caching( &self, phys_tables_base_addr: Address, ) -> Result<(), MMUEnableError>; /// Returns true if the MMU is enabled, false otherwise. fn is_enabled(&self) -> bool; } } /// Describes the characteristics of a translation granule. pub struct TranslationGranule; /// Describes properties of an address space. pub struct AddressSpace; /// Intended to be implemented for [`AddressSpace`]. pub trait AssociatedTranslationTable { /// A translation table whose address range is: /// /// [u64::MAX, (u64::MAX - AS_SIZE) + 1] type TableStartFromTop; /// A translation table whose address range is: /// /// [AS_SIZE - 1, 0] type TableStartFromBottom; } //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- use interface::MMU; use synchronization::interface::ReadWriteEx; use translation_table::interface::TranslationTable; /// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's /// MMIO VA allocator with it. fn kernel_init_mmio_va_allocator() { let region = bsp::memory::mmu::virt_mmio_remap_region(); alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.initialize(region)); } /// Map a region in the kernel's translation tables. /// /// No input checks done, input is passed through to the architectural implementation. /// /// # Safety /// /// - See `map_at()`. /// - Does not prevent aliasing. unsafe fn kernel_map_at_unchecked( name: &'static str, virt_region: &MemoryRegion, phys_region: &MemoryRegion, attr: &AttributeFields, ) -> Result<(), &'static str> { bsp::memory::mmu::kernel_translation_tables() .write(|tables| tables.map_at(virt_region, phys_region, attr))?; kernel_add_mapping_record(name, virt_region, phys_region, attr); Ok(()) } /// Try to translate a kernel virtual address to a physical address. /// /// Will only succeed if there exists a valid mapping for the input address. fn try_kernel_virt_addr_to_phys_addr( virt_addr: Address, ) -> Result, &'static str> { bsp::memory::mmu::kernel_translation_tables() .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr)) } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- impl fmt::Display for MMUEnableError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { MMUEnableError::AlreadyEnabled => write!(f, "MMU is already enabled"), MMUEnableError::Other(x) => write!(f, "{}", x), } } } impl TranslationGranule { /// The granule's size. pub const SIZE: usize = Self::size_checked(); /// The granule's mask. pub const MASK: usize = Self::SIZE - 1; /// The granule's shift, aka log2(size). pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; const fn size_checked() -> usize { assert!(GRANULE_SIZE.is_power_of_two()); GRANULE_SIZE } } impl AddressSpace { /// The address space size. pub const SIZE: usize = Self::size_checked(); /// The address space shift, aka log2(size). pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize; const fn size_checked() -> usize { assert!(AS_SIZE.is_power_of_two()); // Check for architectural restrictions as well. Self::arch_address_space_size_sanity_checks(); AS_SIZE } } /// Add an entry to the mapping info record. pub fn kernel_add_mapping_record( name: &'static str, virt_region: &MemoryRegion, phys_region: &MemoryRegion, attr: &AttributeFields, ) { if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) { warn!("{}", x); } } /// MMIO remapping in the kernel translation tables. /// /// Typically used by device drivers. /// /// # Safety /// /// - Same as `kernel_map_at_unchecked()`, minus the aliasing part. pub unsafe fn kernel_map_mmio( name: &'static str, mmio_descriptor: &MMIODescriptor, ) -> Result, &'static str> { let phys_region = MemoryRegion::from(*mmio_descriptor); let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page(); // Check if an identical region has been mapped for another driver. If so, reuse it. let virt_addr = if let Some(addr) = mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name) { addr // Otherwise, allocate a new region and map it. } else { let num_pages = match NonZeroUsize::new(phys_region.num_pages()) { None => return Err("Requested 0 pages"), Some(x) => x, }; let virt_region = alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?; kernel_map_at_unchecked( name, &virt_region, &phys_region, &AttributeFields { mem_attributes: MemAttributes::Device, acc_perms: AccessPermissions::ReadWrite, execute_never: true, }, )?; virt_region.start_addr() }; Ok(virt_addr + offset_into_start_page) } /// Try to translate a kernel virtual page address to a physical page address. /// /// Will only succeed if there exists a valid mapping for the input page. pub fn try_kernel_virt_page_addr_to_phys_page_addr( virt_page_addr: PageAddress, ) -> Result, &'static str> { bsp::memory::mmu::kernel_translation_tables() .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr)) } /// Try to get the attributes of a kernel page. /// /// Will only succeed if there exists a valid mapping for the input page. pub fn try_kernel_page_attributes( virt_page_addr: PageAddress, ) -> Result { bsp::memory::mmu::kernel_translation_tables() .read(|tables| tables.try_page_attributes(virt_page_addr)) } /// Enable the MMU and data + instruction caching. /// /// # Safety /// /// - Crucial function during kernel init. Changes the the complete memory view of the processor. #[inline(always)] pub unsafe fn enable_mmu_and_caching( phys_tables_base_addr: Address, ) -> Result<(), MMUEnableError> { arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr) } /// Finish initialization of the MMU subsystem. pub fn post_enable_init() { kernel_init_mmio_va_allocator(); } /// Human-readable print of all recorded kernel mappings. pub fn kernel_print_mappings() { mapping_record::kernel_print() }