From 57c6f72936478c4104574f98adffbb7de80845ee Mon Sep 17 00:00:00 2001 From: Andre Richter Date: Sat, 13 Nov 2021 12:07:47 +0100 Subject: [PATCH] Make address translation a feature of the tables --- 14_virtual_mem_part2_mmio_remap/README.md | 162 +++--- .../aarch64/memory/mmu/translation_table.rs | 50 +- .../src/bsp/raspberrypi/memory/mmu.rs | 27 +- .../src/memory/mmu/types.rs | 19 +- .../README.md | 511 ++++++++++-------- .../src/_arch/aarch64/memory/mmu.rs | 31 +- .../aarch64/memory/mmu/translation_table.rs | 148 ++++- .../src/bsp/raspberrypi/memory/mmu.rs | 66 +-- .../src/memory.rs | 17 +- .../src/memory/mmu.rs | 47 +- .../src/memory/mmu/translation_table.rs | 26 +- .../src/memory/mmu/types.rs | 28 +- .../README.md | 93 ++-- .../src/_arch/aarch64/memory/mmu.rs | 31 +- .../aarch64/memory/mmu/translation_table.rs | 148 ++++- .../src/bsp/raspberrypi/memory/mmu.rs | 66 +-- .../src/memory.rs | 17 +- .../src/memory/mmu.rs | 47 +- .../src/memory/mmu/translation_table.rs | 26 +- .../src/memory/mmu/types.rs | 28 +- 20 files changed, 905 insertions(+), 683 deletions(-) diff --git a/14_virtual_mem_part2_mmio_remap/README.md b/14_virtual_mem_part2_mmio_remap/README.md index dda4cc7f..cda4dcc9 100644 --- a/14_virtual_mem_part2_mmio_remap/README.md +++ b/14_virtual_mem_part2_mmio_remap/README.md @@ -457,19 +457,23 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans val.write( STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64) + STAGE1_TABLE_DESCRIPTOR::TYPE::Table -@@ -230,7 +229,10 @@ +@@ -230,10 +229,13 @@ } /// Create an instance. - pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self { -+ pub fn from_output_addr( -+ phys_output_addr: *const Page, ++ pub fn from_output_page( ++ phys_output_page: *const Page, + attribute_fields: &AttributeFields, + ) -> Self { let val = InMemoryRegister::::new(0); - let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; -@@ -244,50 +246,193 @@ +- let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; ++ let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) + + STAGE1_PAGE_DESCRIPTOR::AF::True +@@ -244,50 +246,201 @@ Self { value: val.get() } } @@ -517,12 +521,6 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans - /// Iterates over all static translation table entries and fills them at once. - /// - /// # Safety -- /// -- /// - Modifies a `static mut`. Ensure it only happens from here. -- pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { -- for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { -- *l2_entry = -- TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize()); + /// The start address of the table's MMIO range. + #[inline(always)] + fn mmio_start_addr(&self) -> Address { @@ -544,11 +542,11 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans + + /// Helper to calculate the lvl2 and lvl3 indices from an address. + #[inline(always)] -+ fn lvl2_lvl3_index_from( ++ fn lvl2_lvl3_index_from_page( + &self, -+ addr: *const Page, ++ virt_page: *const Page, + ) -> Result<(usize, usize), &'static str> { -+ let addr = addr as usize; ++ let addr = virt_page as usize; + let lvl2_index = addr >> Granule512MiB::SHIFT; + let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; + @@ -559,24 +557,42 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans + Ok((lvl2_index, lvl3_index)) + } + -+ /// Returns the PageDescriptor corresponding to the supplied Page. ++ /// Sets the PageDescriptor corresponding to the supplied page address. + /// +- /// - Modifies a `static mut`. Ensure it only happens from here. +- pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { +- for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { +- *l2_entry = +- TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize()); ++ /// Doesn't allow overriding an already valid page. + #[inline(always)] -+ fn page_descriptor_from( ++ fn set_page_descriptor_from_page( + &mut self, -+ addr: *const Page, -+ ) -> Result<&mut PageDescriptor, &'static str> { -+ let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; -+ -+ Ok(&mut self.lvl3[lvl2_index][lvl3_index]) ++ virt_page: *const Page, ++ new_desc: &PageDescriptor, ++ ) -> Result<(), &'static str> { ++ let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; ++ let desc = &mut self.lvl3[lvl2_index][lvl3_index]; + +- for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { +- let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); ++ if desc.is_valid() { ++ return Err("Virtual page is already mapped"); ++ } + +- let (phys_output_addr, attribute_fields) = +- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; ++ *desc = *new_desc; ++ Ok(()) + } +} -+ + +- *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields); +- } +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ - -- for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { -- let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); ++ +impl memory::mmu::translation_table::interface::TranslationTable + for FixedSizeTranslationTable +{ @@ -587,9 +603,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans + + // Populate the l2 entries. + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { -+ let desc = -+ TableDescriptor::from_next_lvl_table_addr(self.lvl3[lvl2_nr].phys_start_addr()); -+ *lvl2_entry = desc; ++ let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr(); ++ ++ let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); ++ *lvl2_entry = new_desc; + } + + self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; @@ -608,33 +625,28 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans + ) -> Result<(), &'static str> { + assert!(self.initialized, "Translation tables not initialized"); + -+ let p = phys_pages.as_slice(); + let v = virt_pages.as_slice(); ++ let p = phys_pages.as_slice(); + + // No work to do for empty slices. + if v.is_empty() { + return Ok(()); + } - -- let (phys_output_addr, attribute_fields) = -- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; ++ + if v.len() != p.len() { + return Err("Tried to map page slices with unequal sizes"); + } - -- *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields); ++ + if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() { + return Err("Tried to map outside of physical address space"); + } + + let iter = p.iter().zip(v.iter()); + for (phys_page, virt_page) in iter { -+ let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; -+ if page_descriptor.is_valid() { -+ return Err("Virtual page is already mapped"); - } ++ let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr); ++ let virt_page = virt_page.as_ptr(); + -+ *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); ++ self.set_page_descriptor_from_page(virt_page, &new_desc)?; } Ok(()) @@ -680,7 +692,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans } } -@@ -296,6 +441,9 @@ +@@ -296,6 +449,9 @@ //-------------------------------------------------------------------------------------------------- #[cfg(test)] @@ -1468,7 +1480,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 14_vir diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs --- 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +++ 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs -@@ -4,70 +4,157 @@ +@@ -4,70 +4,150 @@ //! BSP Memory Management Unit. @@ -1483,7 +1495,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, + MemAttributes, Page, PageSliceDescriptor, TranslationGranule, + }, -+ Physical, Virtual, ++ Address, Physical, Virtual, + }, + synchronization::InitStateLock, +}; @@ -1504,16 +1516,16 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to +/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; -+ + +-const NUM_MEM_RANGES: usize = 2; +/// The kernel's virtual address space defined by this BSP. +pub type KernelVirtAddrSpace = AddressSpace<{ 8 * 1024 * 1024 * 1024 }>; --const NUM_MEM_RANGES: usize = 2; +-/// The virtual memory layout. +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- - --/// The virtual memory layout. ++ +/// The kernel translation tables. /// -/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. @@ -1571,8 +1583,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + let num_pages = size_to_num_pages(super::rx_size()); + + PageSliceDescriptor::from_addr(super::virt_rx_start(), num_pages) -+} -+ + } + +-fn mmio_range_inclusive() -> RangeInclusive { +- RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE) +/// The Read+Write (RW) pages of the kernel binary. +fn virt_rw_page_desc() -> PageSliceDescriptor { + let num_pages = size_to_num_pages(super::rw_size()); @@ -1587,23 +1601,14 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) +} + -+// The binary is still identity mapped, so we don't need to convert in the following. ++// The binary is still identity mapped, so use this trivial conversion function for mapping below. + -+/// The Read+Execute (RX) pages of the kernel binary. -+fn phys_rx_page_desc() -> PageSliceDescriptor { -+ virt_rx_page_desc().into() - } - --fn mmio_range_inclusive() -> RangeInclusive { -- RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE) -+/// The Read+Write (RW) pages of the kernel binary. -+fn phys_rw_page_desc() -> PageSliceDescriptor { -+ virt_rw_page_desc().into() -+} ++fn kernel_virt_to_phys_page_slice( ++ virt_slice: PageSliceDescriptor, ++) -> PageSliceDescriptor { ++ let phys_start_addr = Address::::new(virt_slice.start_addr().into_usize()); + -+/// The boot core's stack. -+fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { -+ virt_boot_core_stack_page_desc().into() ++ PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages()) } //-------------------------------------------------------------------------------------------------- @@ -1635,7 +1640,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + generic_mmu::kernel_map_pages_at( + "Kernel code and RO data", + &virt_rx_page_desc(), -+ &phys_rx_page_desc(), ++ &kernel_virt_to_phys_page_slice(virt_rx_page_desc()), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, @@ -1646,7 +1651,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + generic_mmu::kernel_map_pages_at( + "Kernel data and bss", + &virt_rw_page_desc(), -+ &phys_rw_page_desc(), ++ &kernel_virt_to_phys_page_slice(virt_rw_page_desc()), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, @@ -1657,7 +1662,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + generic_mmu::kernel_map_pages_at( + "Kernel boot-core stack", + &virt_boot_core_stack_page_desc(), -+ &phys_boot_core_stack_page_desc(), ++ &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc()), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, @@ -1669,7 +1674,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs } //-------------------------------------------------------------------------------------------------- -@@ -77,19 +164,24 @@ +@@ -77,19 +157,24 @@ #[cfg(test)] mod tests { use super::*; @@ -1701,7 +1706,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs assert!(end >= start); } } -@@ -97,18 +189,38 @@ +@@ -97,18 +182,38 @@ /// Ensure the kernel's virtual memory layout is free of overlaps. #[kernel_test] fn virt_mem_layout_has_no_overlaps() { @@ -2482,7 +2487,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.r diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs --- 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs +++ 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs -@@ -0,0 +1,210 @@ +@@ -0,0 +1,201 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2021 Andre Richter @@ -2491,7 +2496,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual + +use crate::{ + bsp, common, -+ memory::{Address, AddressType, Physical, Virtual}, ++ memory::{Address, AddressType, Physical}, +}; +use core::{convert::From, marker::PhantomData}; + @@ -2577,11 +2582,11 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual + } + + /// Return a pointer to the first page of the described slice. -+ const fn first_page_ptr(&self) -> *const Page { ++ const fn first_page(&self) -> *const Page { + self.start.into_usize() as *const _ + } + -+ /// Return the number of Pages the slice describes. ++ /// Return the number of pages the slice describes. + pub const fn num_pages(&self) -> usize { + self.num_pages + } @@ -2611,22 +2616,13 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual + (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) + } + -+ /// Return a non-mutable slice of Pages. ++ /// Return a non-mutable slice of pages. + /// + /// # Safety + /// + /// - Same as applies for `core::slice::from_raw_parts`. + pub unsafe fn as_slice(&self) -> &[Page] { -+ core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) -+ } -+} -+ -+impl From> for PageSliceDescriptor { -+ fn from(desc: PageSliceDescriptor) -> Self { -+ Self { -+ start: Address::new(desc.start.into_usize()), -+ num_pages: desc.num_pages, -+ } ++ core::slice::from_raw_parts(self.first_page(), self.num_pages) + } +} + diff --git a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs index 0a9f6d52..0c36cac6 100644 --- a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs +++ b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -229,13 +229,13 @@ impl PageDescriptor { } /// Create an instance. - pub fn from_output_addr( - phys_output_addr: *const Page, + pub fn from_output_page( + phys_output_page: *const Page, attribute_fields: &AttributeFields, ) -> Self { let val = InMemoryRegister::::new(0); - let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; + let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT; val.write( STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) + STAGE1_PAGE_DESCRIPTOR::AF::True @@ -308,11 +308,11 @@ impl FixedSizeTranslationTable { /// Helper to calculate the lvl2 and lvl3 indices from an address. #[inline(always)] - fn lvl2_lvl3_index_from( + fn lvl2_lvl3_index_from_page( &self, - addr: *const Page, + virt_page: *const Page, ) -> Result<(usize, usize), &'static str> { - let addr = addr as usize; + let addr = virt_page as usize; let lvl2_index = addr >> Granule512MiB::SHIFT; let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; @@ -323,15 +323,24 @@ impl FixedSizeTranslationTable { Ok((lvl2_index, lvl3_index)) } - /// Returns the PageDescriptor corresponding to the supplied Page. + /// Sets the PageDescriptor corresponding to the supplied page address. + /// + /// Doesn't allow overriding an already valid page. #[inline(always)] - fn page_descriptor_from( + fn set_page_descriptor_from_page( &mut self, - addr: *const Page, - ) -> Result<&mut PageDescriptor, &'static str> { - let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + virt_page: *const Page, + new_desc: &PageDescriptor, + ) -> Result<(), &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; + let desc = &mut self.lvl3[lvl2_index][lvl3_index]; - Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + if desc.is_valid() { + return Err("Virtual page is already mapped"); + } + + *desc = *new_desc; + Ok(()) } } @@ -349,9 +358,10 @@ impl memory::mmu::translation_table::interface::Transla // Populate the l2 entries. for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { - let desc = - TableDescriptor::from_next_lvl_table_addr(self.lvl3[lvl2_nr].phys_start_addr()); - *lvl2_entry = desc; + let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr(); + + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); + *lvl2_entry = new_desc; } self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; @@ -370,8 +380,8 @@ impl memory::mmu::translation_table::interface::Transla ) -> Result<(), &'static str> { assert!(self.initialized, "Translation tables not initialized"); - let p = phys_pages.as_slice(); let v = virt_pages.as_slice(); + let p = phys_pages.as_slice(); // No work to do for empty slices. if v.is_empty() { @@ -388,12 +398,10 @@ impl memory::mmu::translation_table::interface::Transla let iter = p.iter().zip(v.iter()); for (phys_page, virt_page) in iter { - let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; - if page_descriptor.is_valid() { - return Err("Virtual page is already mapped"); - } + let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr); + let virt_page = virt_page.as_ptr(); - *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); + self.set_page_descriptor_from_page(virt_page, &new_desc)?; } Ok(()) diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs index 5bd2fdbd..b99268b4 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs @@ -12,7 +12,7 @@ use crate::{ AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, MemAttributes, Page, PageSliceDescriptor, TranslationGranule, }, - Physical, Virtual, + Address, Physical, Virtual, }, synchronization::InitStateLock, }; @@ -81,21 +81,14 @@ fn virt_boot_core_stack_page_desc() -> PageSliceDescriptor { PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) } -// The binary is still identity mapped, so we don't need to convert in the following. +// The binary is still identity mapped, so use this trivial conversion function for mapping below. -/// The Read+Execute (RX) pages of the kernel binary. -fn phys_rx_page_desc() -> PageSliceDescriptor { - virt_rx_page_desc().into() -} - -/// The Read+Write (RW) pages of the kernel binary. -fn phys_rw_page_desc() -> PageSliceDescriptor { - virt_rw_page_desc().into() -} +fn kernel_virt_to_phys_page_slice( + virt_slice: PageSliceDescriptor, +) -> PageSliceDescriptor { + let phys_start_addr = Address::::new(virt_slice.start_addr().into_usize()); -/// The boot core's stack. -fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { - virt_boot_core_stack_page_desc().into() + PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages()) } //-------------------------------------------------------------------------------------------------- @@ -124,7 +117,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { generic_mmu::kernel_map_pages_at( "Kernel code and RO data", &virt_rx_page_desc(), - &phys_rx_page_desc(), + &kernel_virt_to_phys_page_slice(virt_rx_page_desc()), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadOnly, @@ -135,7 +128,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { generic_mmu::kernel_map_pages_at( "Kernel data and bss", &virt_rw_page_desc(), - &phys_rw_page_desc(), + &kernel_virt_to_phys_page_slice(virt_rw_page_desc()), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadWrite, @@ -146,7 +139,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { generic_mmu::kernel_map_pages_at( "Kernel boot-core stack", &virt_boot_core_stack_page_desc(), - &phys_boot_core_stack_page_desc(), + &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc()), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadWrite, diff --git a/14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs b/14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs index 2e52e8f3..bbd4e9e8 100644 --- a/14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs +++ b/14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs @@ -6,7 +6,7 @@ use crate::{ bsp, common, - memory::{Address, AddressType, Physical, Virtual}, + memory::{Address, AddressType, Physical}, }; use core::{convert::From, marker::PhantomData}; @@ -92,11 +92,11 @@ impl PageSliceDescriptor { } /// Return a pointer to the first page of the described slice. - const fn first_page_ptr(&self) -> *const Page { + const fn first_page(&self) -> *const Page { self.start.into_usize() as *const _ } - /// Return the number of Pages the slice describes. + /// Return the number of pages the slice describes. pub const fn num_pages(&self) -> usize { self.num_pages } @@ -126,22 +126,13 @@ impl PageSliceDescriptor { (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) } - /// Return a non-mutable slice of Pages. + /// Return a non-mutable slice of pages. /// /// # Safety /// /// - Same as applies for `core::slice::from_raw_parts`. pub unsafe fn as_slice(&self) -> &[Page] { - core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) - } -} - -impl From> for PageSliceDescriptor { - fn from(desc: PageSliceDescriptor) -> Self { - Self { - start: Address::new(desc.start.into_usize()), - num_pages: desc.num_pages, - } + core::slice::from_raw_parts(self.first_page(), self.num_pages) } } diff --git a/15_virtual_mem_part3_precomputed_tables/README.md b/15_virtual_mem_part3_precomputed_tables/README.md index c564a1b5..efc28eff 100644 --- a/15_virtual_mem_part3_precomputed_tables/README.md +++ b/15_virtual_mem_part3_precomputed_tables/README.md @@ -257,7 +257,7 @@ ffff000000010008: 44 33 22 11 ffff00000001000c: 00 00 00 00 ``` -Both times, the `movabsq` instruction gets emitted. It means that the address is put into the target +Both times, the `movabs` instruction gets emitted. It means that the address is put into the target register using hardcoded `immediate values`. PC-relative address calculation is not used here. Hence, this code would return the `absolute` address in both cases. Which means in the second case, even when the binary would be loaded at `0x8_0000`, the return value would be @@ -350,11 +350,11 @@ ffff000000010008: 44 33 22 11 ffff00000001000c: 00 00 00 00 ``` -What changed compared to earlier is that `rx_start_address()` now indirects through the `Global +What changed compared to earlier is that `get_address_of_global()` now indirects through the `Global Offset Table`, as has been promised by the compiler's documentation. Specifically, -`rx_start_address()` addresses the `GOT` using PC-relative addressing (distance from code to `GOT` -must always be fixed), and loads the first 64 bit word from the start of the `GOT`, which happens to -be `0xffff_0000_0001_0008`. +`get_address_of_global()` addresses the `GOT` using PC-relative addressing (distance from code to +`GOT` must always be fixed), and loads the first 64 bit word from the start of the `GOT`, which +happens to be `0xffff_0000_0001_0008`. Okay okay okay... So when we use `-fpic`, we get the **absolute** address of `global_data_word` even on `AArch64` now. How does this help when the code executes from `0x8_0000`? @@ -674,7 +674,7 @@ Precomputing kernel translation tables and patching kernel ELF Generating Boot-core stack | 0x0000_0000_0010_0000 | 512 KiB -------------------------------------------------- Patching Kernel table struct at physical 0x9_0000 - Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0040 + Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0060 Finished in 0.03s ``` @@ -685,11 +685,11 @@ tutorial, anything else, like `MMIO-remapping`, can and will happen lazily durin Two more things that changed in this tutorial, but won't be explained in detail: -- Since virtual memory in `EL1` is now active from the start, any attempt to convert from - `Address` to `Address` is now done through the `TryFrom` trait, which - internally uses HW-supported address translation of the CPU. So either the translation succeeds - because there is a valid virtual-to-physical mapping in the currently used translation tables, or - an `Err()` is returned. +- Since virtual memory in `EL1` is now active from the start, any attempt to convert from a kernel + `Address` to `Address` is now done using the function + `mmu::try_kernel_virt_addr_to_phys_addr()`, which internally uses a new function that has been + added to the `TranslationTable` interface. If there is no valid virtual-to-physical mapping + present in the tables, an `Err()` is returned. - The precomputed translation table mappings won't automatically have entries in the kernel's `mapping info record`, which is used to print mapping info during boot. Mapping record entries are not computed offline in order to reduce complexity. For this reason, the `BSP` code, which in @@ -724,7 +724,7 @@ Precomputing kernel translation tables and patching kernel ELF Generating Boot-core stack | 0x0000_0000_0010_0000 | 512 KiB -------------------------------------------------- Patching Kernel table struct at physical 0x9_0000 - Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0040 + Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0060 Finished in 0.03s Minipush 1.0 @@ -732,6 +732,7 @@ Minipush 1.0 [MP] ⏳ Waiting for /dev/ttyUSB0 [MP] ✅ Serial connected [MP] 🔌 Please power the target now + __ __ _ _ _ _ | \/ (_)_ _ (_) | ___ __ _ __| | | |\/| | | ' \| | |__/ _ \/ _` / _` | @@ -859,15 +860,6 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s 15_virtua diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs --- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs +++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs -@@ -23,7 +23,7 @@ - Address, Physical, Virtual, - }, - }; --use core::convert; -+use core::convert::{self, TryInto}; - use tock_registers::{ - interfaces::{Readable, Writeable}, - register_bitfields, @@ -124,7 +124,7 @@ } @@ -888,7 +880,64 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati Address::new(self as *const _ as usize) } } -@@ -273,7 +272,7 @@ +@@ -220,6 +219,35 @@ + } + } + ++/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes. ++impl convert::TryFrom> for AttributeFields { ++ type Error = &'static str; ++ ++ fn try_from( ++ desc: InMemoryRegister, ++ ) -> Result { ++ let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) { ++ memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM, ++ memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device, ++ _ => return Err("Unexpected memory attribute"), ++ }; ++ ++ let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) { ++ Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly, ++ Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite, ++ _ => return Err("Unexpected access permission"), ++ }; ++ ++ let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0; ++ ++ Ok(AttributeFields { ++ mem_attributes, ++ acc_perms, ++ execute_never, ++ }) ++ } ++} ++ + impl PageDescriptor { + /// Create an instance. + /// +@@ -252,6 +280,20 @@ + InMemoryRegister::::new(self.value) + .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) + } ++ ++ /// Returns the output page. ++ fn output_page(&self) -> *const Page { ++ let shifted = InMemoryRegister::::new(self.value) ++ .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB); ++ let addr = shifted << Granule64KiB::SHIFT; ++ ++ addr as *const Page ++ } ++ ++ /// Returns the attributes. ++ fn try_attributes(&self) -> Result { ++ InMemoryRegister::::new(self.value).try_into() ++ } + } + + //-------------------------------------------------------------------------------------------------- +@@ -273,7 +315,7 @@ /// Create an instance. #[allow(clippy::assertions_on_constants)] @@ -897,7 +946,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); // Can't have a zero-sized address space. -@@ -282,11 +281,20 @@ +@@ -282,11 +324,20 @@ Self { lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], @@ -920,7 +969,26 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati /// The start address of the table's MMIO range. #[inline(always)] fn mmio_start_addr(&self) -> Address { -@@ -342,24 +350,26 @@ +@@ -323,6 +374,18 @@ + Ok((lvl2_index, lvl3_index)) + } + ++ /// Returns the PageDescriptor corresponding to the supplied page address. ++ #[inline(always)] ++ fn page_descriptor_from_page( ++ &self, ++ virt_page: *const Page, ++ ) -> Result<&PageDescriptor, &'static str> { ++ let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; ++ let desc = &self.lvl3[lvl2_index][lvl3_index]; ++ ++ Ok(desc) ++ } ++ + /// Sets the PageDescriptor corresponding to the supplied page address. + /// + /// Doesn't allow overriding an already valid page. +@@ -351,14 +414,15 @@ impl memory::mmu::translation_table::interface::TranslationTable for FixedSizeTranslationTable { @@ -933,16 +1001,13 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati // Populate the l2 entries. for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { -- let desc = -- TableDescriptor::from_next_lvl_table_addr(self.lvl3[lvl2_nr].phys_start_addr()); -+ let addr = self.lvl3[lvl2_nr] -+ .virt_start_addr() -+ .try_into() -+ .map_err(|_| "Translation error")?; -+ -+ let desc = TableDescriptor::from_next_lvl_table_addr(addr); - *lvl2_entry = desc; - } +- let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr(); ++ let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr(); ++ let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?; + + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); + *lvl2_entry = new_desc; +@@ -366,10 +430,8 @@ self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; self.initialized = true; @@ -954,61 +1019,52 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati } unsafe fn map_pages_at( +@@ -442,6 +504,44 @@ -diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs ---- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs -+++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs -@@ -15,7 +15,7 @@ - - use crate::{ - bsp, memory, -- memory::{mmu::TranslationGranule, Address, Physical}, -+ memory::{mmu::TranslationGranule, Address, Physical, Virtual}, - }; - use core::intrinsics::unlikely; - use cortex_a::{asm::barrier, registers::*}; -@@ -109,7 +109,7 @@ - //------------------------------------------------------------------------------ - // OS Interface Code - //------------------------------------------------------------------------------ --use memory::mmu::MMUEnableError; -+use memory::mmu::{MMUEnableError, TranslationError}; - - impl memory::mmu::interface::MMU for MemoryManagementUnit { - unsafe fn enable_mmu_and_caching( -@@ -153,4 +153,31 @@ - fn is_enabled(&self) -> bool { - SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) + false } + -+ fn try_virt_to_phys( ++ fn try_virt_page_to_phys_page( + &self, -+ virt: Address, -+ ) -> Result, TranslationError> { -+ if !self.is_enabled() { -+ return Err(TranslationError::MMUDisabled); -+ } ++ virt_page: *const Page, ++ ) -> Result<*const Page, &'static str> { ++ let page_desc = self.page_descriptor_from_page(virt_page)?; + -+ let addr = virt.into_usize() as u64; -+ unsafe { -+ asm!( -+ "AT S1E1R, {0}", -+ in(reg) addr, -+ options(readonly, nostack, preserves_flags) -+ ); ++ if !page_desc.is_valid() { ++ return Err("Page marked invalid"); + } + -+ let par_el1 = PAR_EL1.extract(); -+ if par_el1.matches_all(PAR_EL1::F::TranslationAborted) { -+ return Err(TranslationError::Aborted); ++ Ok(page_desc.output_page()) ++ } ++ ++ fn try_page_attributes( ++ &self, ++ virt_page: *const Page, ++ ) -> Result { ++ let page_desc = self.page_descriptor_from_page(virt_page)?; ++ ++ if !page_desc.is_valid() { ++ return Err("Page marked invalid"); + } + -+ let phys_addr = (par_el1.read(PAR_EL1::PA) << 12) | (addr & 0xFFF); ++ page_desc.try_attributes() ++ } ++ ++ /// Try to translate a virtual address to a physical address. ++ /// ++ /// Will only succeed if there exists a valid mapping for the input address. ++ fn try_virt_addr_to_phys_addr( ++ &self, ++ virt_addr: Address, ++ ) -> Result, &'static str> { ++ let page = self.try_virt_page_to_phys_page(virt_addr.as_page_ptr())?; + -+ Ok(Address::new(phys_addr as usize)) ++ Ok(Address::new(page as usize + virt_addr.offset_into_page())) + } } + //-------------------------------------------------------------------------------------------------- + diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/console.rs 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/console.rs --- 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/console.rs +++ 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/console.rs @@ -1087,15 +1143,18 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld 15_virtual diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs --- 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +++ 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs -@@ -16,6 +16,7 @@ +@@ -9,8 +9,8 @@ + memory::{ + mmu as generic_mmu, + mmu::{ +- AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, +- MemAttributes, Page, PageSliceDescriptor, TranslationGranule, ++ AddressSpace, AssociatedTranslationTable, AttributeFields, Page, PageSliceDescriptor, ++ TranslationGranule, + }, + Address, Physical, Virtual, }, - synchronization::InitStateLock, - }; -+use core::convert::TryInto; - - //-------------------------------------------------------------------------------------------------- - // Private Definitions -@@ -33,7 +34,7 @@ +@@ -33,7 +33,7 @@ pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; /// The kernel's virtual address space defined by this BSP. @@ -1104,7 +1163,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v //-------------------------------------------------------------------------------------------------- // Global instances -@@ -45,13 +46,33 @@ +@@ -45,13 +45,33 @@ /// /// That is, `size_of(InitStateLock) == size_of(KernelTranslationTable)`. /// There is a unit tests that checks this porperty. @@ -1139,82 +1198,92 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v /// Helper function for calculating the number of pages the given parameter spans. const fn size_to_num_pages(size: usize) -> usize { assert!(size > 0); -@@ -81,21 +102,22 @@ +@@ -81,16 +101,22 @@ PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) } --// The binary is still identity mapped, so we don't need to convert in the following. +-// The binary is still identity mapped, so use this trivial conversion function for mapping below. +- +// There is no reason to expect the following conversions to fail, since they were generated offline -+// by the `translation table tool`. If it doesn't work, a panic due to the unwrap is justified. - - /// The Read+Execute (RX) pages of the kernel binary. - fn phys_rx_page_desc() -> PageSliceDescriptor { -- virt_rx_page_desc().into() -+ virt_rx_page_desc().try_into().unwrap() - } - - /// The Read+Write (RW) pages of the kernel binary. - fn phys_rw_page_desc() -> PageSliceDescriptor { -- virt_rw_page_desc().into() -+ virt_rw_page_desc().try_into().unwrap() - } - - /// The boot core's stack. - fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { -- virt_boot_core_stack_page_desc().into() -+ virt_boot_core_stack_page_desc().try_into().unwrap() ++// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified. + fn kernel_virt_to_phys_page_slice( + virt_slice: PageSliceDescriptor, + ) -> PageSliceDescriptor { +- let phys_start_addr = Address::::new(virt_slice.start_addr().into_usize()); ++ let phys_first_page = ++ generic_mmu::try_kernel_virt_page_to_phys_page(virt_slice.first_page()).unwrap(); ++ let phys_start_addr = Address::new(phys_first_page as usize); + + PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages()) } ++fn kernel_page_attributes(virt_page: *const Page) -> AttributeFields { ++ generic_mmu::try_kernel_page_attributes(virt_page).unwrap() ++} ++ //-------------------------------------------------------------------------------------------------- -@@ -115,13 +137,15 @@ + // Public Code + //-------------------------------------------------------------------------------------------------- +@@ -108,112 +134,33 @@ ) as *const Page<_> } -/// Map the kernel binary. -+/// Add mapping records for the kernel binary. - /// +-/// -/// # Safety -+/// The actual translation table entries for the kernel binary are generated using the offline -+/// `translation table tool` and patched into the kernel binary. This function just adds the mapping -+/// record entries. ++/// Add mapping records for the kernel binary. /// -/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. -pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { - generic_mmu::kernel_map_pages_at( -+/// It must be ensured that these entries are in sync with the offline tool. ++/// The actual translation table entries for the kernel binary are generated using the offline ++/// `translation table tool` and patched into the kernel binary. This function just adds the mapping ++/// record entries. +pub fn kernel_add_mapping_records_for_precomputed() { ++ let virt_rx_page_desc = virt_rx_page_desc(); + generic_mmu::kernel_add_mapping_record( "Kernel code and RO data", - &virt_rx_page_desc(), - &phys_rx_page_desc(), -@@ -130,9 +154,9 @@ - acc_perms: AccessPermissions::ReadOnly, - execute_never: false, - }, +- &virt_rx_page_desc(), +- &kernel_virt_to_phys_page_slice(virt_rx_page_desc()), +- &AttributeFields { +- mem_attributes: MemAttributes::CacheableDRAM, +- acc_perms: AccessPermissions::ReadOnly, +- execute_never: false, +- }, - )?; ++ &virt_rx_page_desc, ++ &kernel_virt_to_phys_page_slice(virt_rx_page_desc), ++ &kernel_page_attributes(virt_rx_page_desc.first_page()), + ); - generic_mmu::kernel_map_pages_at( ++ let virt_rw_page_desc = virt_rw_page_desc(); + generic_mmu::kernel_add_mapping_record( "Kernel data and bss", - &virt_rw_page_desc(), - &phys_rw_page_desc(), -@@ -141,9 +165,9 @@ - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, +- &virt_rw_page_desc(), +- &kernel_virt_to_phys_page_slice(virt_rw_page_desc()), +- &AttributeFields { +- mem_attributes: MemAttributes::CacheableDRAM, +- acc_perms: AccessPermissions::ReadWrite, +- execute_never: true, +- }, - )?; ++ &virt_rw_page_desc, ++ &kernel_virt_to_phys_page_slice(virt_rw_page_desc), ++ &kernel_page_attributes(virt_rw_page_desc.first_page()), + ); - generic_mmu::kernel_map_pages_at( ++ let virt_boot_core_stack_page_desc = virt_boot_core_stack_page_desc(); + generic_mmu::kernel_add_mapping_record( "Kernel boot-core stack", - &virt_boot_core_stack_page_desc(), - &phys_boot_core_stack_page_desc(), -@@ -152,75 +176,5 @@ - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, +- &virt_boot_core_stack_page_desc(), +- &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc()), +- &AttributeFields { +- mem_attributes: MemAttributes::CacheableDRAM, +- acc_perms: AccessPermissions::ReadWrite, +- execute_never: true, +- }, - )?; - - Ok(()) @@ -1286,6 +1355,9 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v - - assert!(bss_range.contains(&kernel_tables_addr)); - } ++ &virt_boot_core_stack_page_desc, ++ &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc), ++ &kernel_page_attributes(virt_boot_core_stack_page_desc.first_page()), + ); } @@ -1345,7 +1417,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15 use crate::memory::{ mmu::{AttributeFields, PageSliceDescriptor}, - Address, Physical, Virtual, -+ Physical, Virtual, ++ Address, Page, Physical, Virtual, }; //-------------------------------------------------------------------------------------------------- @@ -1361,7 +1433,38 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15 /// Map the given virtual pages to the given physical pages. /// -@@ -80,7 +77,7 @@ +@@ -70,6 +67,30 @@ + + /// Check if a virtual page splice is in the "MMIO region". + fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; ++ ++ /// Try to translate a virtual page pointer to a physical page pointer. ++ /// ++ /// Will only succeed if there exists a valid mapping for the input page. ++ fn try_virt_page_to_phys_page( ++ &self, ++ virt_page: *const Page, ++ ) -> Result<*const Page, &'static str>; ++ ++ /// Try to get the attributes of a page. ++ /// ++ /// Will only succeed if there exists a valid mapping for the input page. ++ fn try_page_attributes( ++ &self, ++ virt_page: *const Page, ++ ) -> Result; ++ ++ /// Try to translate a virtual address to a physical address. ++ /// ++ /// Will only succeed if there exists a valid mapping for the input address. ++ fn try_virt_addr_to_phys_addr( ++ &self, ++ virt_addr: Address, ++ ) -> Result, &'static str>; + } + } + +@@ -80,7 +101,7 @@ #[cfg(test)] mod tests { use super::*; @@ -1370,7 +1473,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15 use arch_translation_table::MinSizeTranslationTable; use interface::TranslationTable; use test_macros::kernel_test; -@@ -89,9 +86,9 @@ +@@ -89,9 +110,9 @@ #[kernel_test] fn translationtable_implementation_sanity() { // This will occupy a lot of space on the stack. @@ -1386,75 +1489,20 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15 diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs 15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs --- 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs +++ 15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs -@@ -8,7 +8,10 @@ - bsp, common, - memory::{Address, AddressType, Physical, Virtual}, - }; --use core::{convert::From, marker::PhantomData}; -+use core::{ -+ convert::{From, TryFrom}, -+ marker::PhantomData, -+}; - - //-------------------------------------------------------------------------------------------------- - // Public Definitions -@@ -136,12 +139,16 @@ +@@ -92,7 +92,7 @@ } - } --impl From> for PageSliceDescriptor { -- fn from(desc: PageSliceDescriptor) -> Self { -- Self { -- start: Address::new(desc.start.into_usize()), -+impl TryFrom> for PageSliceDescriptor { -+ type Error = super::TranslationError; -+ -+ fn try_from(desc: PageSliceDescriptor) -> Result { -+ let phys_start = super::try_virt_to_phys(desc.start)?; -+ -+ Ok(Self { -+ start: phys_start, - num_pages: desc.num_pages, -- } -+ }) + /// Return a pointer to the first page of the described slice. +- const fn first_page(&self) -> *const Page { ++ pub const fn first_page(&self) -> *const Page { + self.start.into_usize() as *const _ } - } diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs --- 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs +++ 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs -@@ -33,6 +33,14 @@ - Other(&'static str), - } - -+/// Translation error variants. -+#[allow(missing_docs)] -+#[derive(Debug)] -+pub enum TranslationError { -+ MMUDisabled, -+ Aborted, -+} -+ - /// Memory Management interfaces. - pub mod interface { - use super::*; -@@ -51,6 +59,14 @@ - - /// Returns true if the MMU is enabled, false otherwise. - fn is_enabled(&self) -> bool; -+ -+ /// Try to translate a virtual address to a physical address. -+ /// -+ /// Will only succeed if there exists a valid mapping for the input VA. -+ fn try_virt_to_phys( -+ &self, -+ virt: Address, -+ ) -> Result, TranslationError>; - } - } - -@@ -92,9 +108,7 @@ +@@ -92,9 +92,7 @@ bsp::memory::mmu::kernel_translation_tables() .write(|tables| tables.map_pages_at(virt_pages, phys_pages, attr))?; @@ -1465,7 +1513,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3 Ok(()) } -@@ -146,6 +160,18 @@ +@@ -146,6 +144,18 @@ } } @@ -1484,14 +1532,14 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3 /// Raw mapping of virtual to physical pages in the kernel translation tables. /// /// Prevents mapping into the MMIO range of the tables. -@@ -214,21 +240,11 @@ +@@ -214,21 +224,34 @@ Ok(virt_addr + offset_into_start_page) } -/// Map the kernel's binary. Returns the translation table's base address. -/// -/// # Safety -+/// Try to translate a virtual address to a physical address. ++/// Try to translate a kernel virtual page pointer to a physical page pointer. /// -/// - See [`bsp::memory::mmu::kernel_map_binary()`]. -pub unsafe fn kernel_map_binary() -> Result, &'static str> { @@ -1500,17 +1548,38 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3 - tables.init(); - tables.phys_base_address() - }); -- ++/// Will only succeed if there exists a valid mapping for the input page. ++pub fn try_kernel_virt_page_to_phys_page( ++ virt_page: *const Page, ++) -> Result<*const Page, &'static str> { ++ bsp::memory::mmu::kernel_translation_tables() ++ .read(|tables| tables.try_virt_page_to_phys_page(virt_page)) ++} + - bsp::memory::mmu::kernel_map_binary()?; -- ++/// Try to get the attributes of a kernel page. ++/// ++/// Will only succeed if there exists a valid mapping for the input page. ++pub fn try_kernel_page_attributes( ++ virt_page: *const Page, ++) -> Result { ++ bsp::memory::mmu::kernel_translation_tables() ++ .read(|tables| tables.try_page_attributes(virt_page)) ++} + - Ok(phys_kernel_tables_base_addr) -+/// Will only succeed if there exists a valid mapping for the input VA. -+pub fn try_virt_to_phys(virt: Address) -> Result, TranslationError> { -+ arch_mmu::mmu().try_virt_to_phys(virt) ++/// Try to translate a kernel virtual address to a physical address. ++/// ++/// Will only succeed if there exists a valid mapping for the input address. ++fn try_kernel_virt_addr_to_phys_addr( ++ virt_addr: Address, ++) -> Result, &'static str> { ++ bsp::memory::mmu::kernel_translation_tables() ++ .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr)) } /// Enable the MMU and data + instruction caching. -@@ -236,6 +252,7 @@ +@@ -236,6 +259,7 @@ /// # Safety /// /// - Crucial function during kernel init. Changes the the complete memory view of the processor. @@ -1522,29 +1591,39 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3 diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory.rs 15_virtual_mem_part3_precomputed_tables/src/memory.rs --- 14_virtual_mem_part2_mmio_remap/src/memory.rs +++ 15_virtual_mem_part3_precomputed_tables/src/memory.rs -@@ -8,6 +8,7 @@ +@@ -6,12 +6,13 @@ + + pub mod mmu; - use crate::common; +-use crate::common; ++use crate::{bsp, common}; use core::{ -+ convert::TryFrom, fmt, marker::PhantomData, ops::{AddAssign, SubAssign}, -@@ -67,6 +68,14 @@ - } - } + }; ++use mmu::Page; -+impl TryFrom> for Address { -+ type Error = mmu::TranslationError; + //-------------------------------------------------------------------------------------------------- + // Public Definitions +@@ -65,6 +66,17 @@ + pub const fn into_usize(self) -> usize { + self.value + } + -+ fn try_from(virt: Address) -> Result { -+ mmu::try_virt_to_phys(virt) ++ /// Return a pointer to the page that contains this address. ++ pub const fn as_page_ptr(&self) -> *const Page { ++ self.align_down(bsp::memory::mmu::KernelGranule::SIZE) ++ .into_usize() as *const _ + } -+} + - impl core::ops::Add for Address { - type Output = Self; ++ /// Return the address' offset into the underlying page. ++ pub const fn offset_into_page(&self) -> usize { ++ self.value & bsp::memory::mmu::KernelGranule::MASK ++ } + } + impl core::ops::Add for Address { diff -uNr 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs --- 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs diff --git a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs index 86b5337c..2cad10b7 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs @@ -15,7 +15,7 @@ use crate::{ bsp, memory, - memory::{mmu::TranslationGranule, Address, Physical, Virtual}, + memory::{mmu::TranslationGranule, Address, Physical}, }; use core::intrinsics::unlikely; use cortex_a::{asm::barrier, registers::*}; @@ -109,7 +109,7 @@ pub fn mmu() -> &'static impl memory::mmu::interface::MMU { //------------------------------------------------------------------------------ // OS Interface Code //------------------------------------------------------------------------------ -use memory::mmu::{MMUEnableError, TranslationError}; +use memory::mmu::MMUEnableError; impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn enable_mmu_and_caching( @@ -153,31 +153,4 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { fn is_enabled(&self) -> bool { SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } - - fn try_virt_to_phys( - &self, - virt: Address, - ) -> Result, TranslationError> { - if !self.is_enabled() { - return Err(TranslationError::MMUDisabled); - } - - let addr = virt.into_usize() as u64; - unsafe { - asm!( - "AT S1E1R, {0}", - in(reg) addr, - options(readonly, nostack, preserves_flags) - ); - } - - let par_el1 = PAR_EL1.extract(); - if par_el1.matches_all(PAR_EL1::F::TranslationAborted) { - return Err(TranslationError::Aborted); - } - - let phys_addr = (par_el1.read(PAR_EL1::PA) << 12) | (addr & 0xFFF); - - Ok(Address::new(phys_addr as usize)) - } } diff --git a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs index d4fa1f39..222f15ce 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -23,7 +23,7 @@ use crate::{ Address, Physical, Virtual, }, }; -use core::convert::{self, TryInto}; +use core::convert; use tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, @@ -219,6 +219,35 @@ impl convert::From } } +/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes. +impl convert::TryFrom> for AttributeFields { + type Error = &'static str; + + fn try_from( + desc: InMemoryRegister, + ) -> Result { + let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) { + memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM, + memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device, + _ => return Err("Unexpected memory attribute"), + }; + + let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) { + Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly, + Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite, + _ => return Err("Unexpected access permission"), + }; + + let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0; + + Ok(AttributeFields { + mem_attributes, + acc_perms, + execute_never, + }) + } +} + impl PageDescriptor { /// Create an instance. /// @@ -228,13 +257,13 @@ impl PageDescriptor { } /// Create an instance. - pub fn from_output_addr( - phys_output_addr: *const Page, + pub fn from_output_page( + phys_output_page: *const Page, attribute_fields: &AttributeFields, ) -> Self { let val = InMemoryRegister::::new(0); - let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; + let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT; val.write( STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) + STAGE1_PAGE_DESCRIPTOR::AF::True @@ -251,6 +280,20 @@ impl PageDescriptor { InMemoryRegister::::new(self.value) .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) } + + /// Returns the output page. + fn output_page(&self) -> *const Page { + let shifted = InMemoryRegister::::new(self.value) + .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB); + let addr = shifted << Granule64KiB::SHIFT; + + addr as *const Page + } + + /// Returns the attributes. + fn try_attributes(&self) -> Result { + InMemoryRegister::::new(self.value).try_into() + } } //-------------------------------------------------------------------------------------------------- @@ -316,11 +359,11 @@ impl FixedSizeTranslationTable { /// Helper to calculate the lvl2 and lvl3 indices from an address. #[inline(always)] - fn lvl2_lvl3_index_from( + fn lvl2_lvl3_index_from_page( &self, - addr: *const Page, + virt_page: *const Page, ) -> Result<(usize, usize), &'static str> { - let addr = addr as usize; + let addr = virt_page as usize; let lvl2_index = addr >> Granule512MiB::SHIFT; let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; @@ -331,15 +374,36 @@ impl FixedSizeTranslationTable { Ok((lvl2_index, lvl3_index)) } - /// Returns the PageDescriptor corresponding to the supplied Page. + /// Returns the PageDescriptor corresponding to the supplied page address. #[inline(always)] - fn page_descriptor_from( + fn page_descriptor_from_page( + &self, + virt_page: *const Page, + ) -> Result<&PageDescriptor, &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; + let desc = &self.lvl3[lvl2_index][lvl3_index]; + + Ok(desc) + } + + /// Sets the PageDescriptor corresponding to the supplied page address. + /// + /// Doesn't allow overriding an already valid page. + #[inline(always)] + fn set_page_descriptor_from_page( &mut self, - addr: *const Page, - ) -> Result<&mut PageDescriptor, &'static str> { - let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + virt_page: *const Page, + new_desc: &PageDescriptor, + ) -> Result<(), &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; + let desc = &mut self.lvl3[lvl2_index][lvl3_index]; + + if desc.is_valid() { + return Err("Virtual page is already mapped"); + } - Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + *desc = *new_desc; + Ok(()) } } @@ -357,13 +421,11 @@ impl memory::mmu::translation_table::interface::Transla // Populate the l2 entries. for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { - let addr = self.lvl3[lvl2_nr] - .virt_start_addr() - .try_into() - .map_err(|_| "Translation error")?; + let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr(); + let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?; - let desc = TableDescriptor::from_next_lvl_table_addr(addr); - *lvl2_entry = desc; + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); + *lvl2_entry = new_desc; } self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; @@ -380,8 +442,8 @@ impl memory::mmu::translation_table::interface::Transla ) -> Result<(), &'static str> { assert!(self.initialized, "Translation tables not initialized"); - let p = phys_pages.as_slice(); let v = virt_pages.as_slice(); + let p = phys_pages.as_slice(); // No work to do for empty slices. if v.is_empty() { @@ -398,12 +460,10 @@ impl memory::mmu::translation_table::interface::Transla let iter = p.iter().zip(v.iter()); for (phys_page, virt_page) in iter { - let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; - if page_descriptor.is_valid() { - return Err("Virtual page is already mapped"); - } + let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr); + let virt_page = virt_page.as_ptr(); - *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); + self.set_page_descriptor_from_page(virt_page, &new_desc)?; } Ok(()) @@ -444,6 +504,44 @@ impl memory::mmu::translation_table::interface::Transla false } + + fn try_virt_page_to_phys_page( + &self, + virt_page: *const Page, + ) -> Result<*const Page, &'static str> { + let page_desc = self.page_descriptor_from_page(virt_page)?; + + if !page_desc.is_valid() { + return Err("Page marked invalid"); + } + + Ok(page_desc.output_page()) + } + + fn try_page_attributes( + &self, + virt_page: *const Page, + ) -> Result { + let page_desc = self.page_descriptor_from_page(virt_page)?; + + if !page_desc.is_valid() { + return Err("Page marked invalid"); + } + + page_desc.try_attributes() + } + + /// Try to translate a virtual address to a physical address. + /// + /// Will only succeed if there exists a valid mapping for the input address. + fn try_virt_addr_to_phys_addr( + &self, + virt_addr: Address, + ) -> Result, &'static str> { + let page = self.try_virt_page_to_phys_page(virt_addr.as_page_ptr())?; + + Ok(Address::new(page as usize + virt_addr.offset_into_page())) + } } //-------------------------------------------------------------------------------------------------- diff --git a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs index ab2f60ac..588a5728 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs @@ -9,14 +9,13 @@ use crate::{ memory::{ mmu as generic_mmu, mmu::{ - AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, - MemAttributes, Page, PageSliceDescriptor, TranslationGranule, + AddressSpace, AssociatedTranslationTable, AttributeFields, Page, PageSliceDescriptor, + TranslationGranule, }, - Physical, Virtual, + Address, Physical, Virtual, }, synchronization::InitStateLock, }; -use core::convert::TryInto; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -103,21 +102,19 @@ fn virt_boot_core_stack_page_desc() -> PageSliceDescriptor { } // There is no reason to expect the following conversions to fail, since they were generated offline -// by the `translation table tool`. If it doesn't work, a panic due to the unwrap is justified. - -/// The Read+Execute (RX) pages of the kernel binary. -fn phys_rx_page_desc() -> PageSliceDescriptor { - virt_rx_page_desc().try_into().unwrap() -} - -/// The Read+Write (RW) pages of the kernel binary. -fn phys_rw_page_desc() -> PageSliceDescriptor { - virt_rw_page_desc().try_into().unwrap() +// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified. +fn kernel_virt_to_phys_page_slice( + virt_slice: PageSliceDescriptor, +) -> PageSliceDescriptor { + let phys_first_page = + generic_mmu::try_kernel_virt_page_to_phys_page(virt_slice.first_page()).unwrap(); + let phys_start_addr = Address::new(phys_first_page as usize); + + PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages()) } -/// The boot core's stack. -fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { - virt_boot_core_stack_page_desc().try_into().unwrap() +fn kernel_page_attributes(virt_page: *const Page) -> AttributeFields { + generic_mmu::try_kernel_page_attributes(virt_page).unwrap() } //-------------------------------------------------------------------------------------------------- @@ -142,39 +139,28 @@ pub fn phys_addr_space_end_page() -> *const Page { /// The actual translation table entries for the kernel binary are generated using the offline /// `translation table tool` and patched into the kernel binary. This function just adds the mapping /// record entries. -/// -/// It must be ensured that these entries are in sync with the offline tool. pub fn kernel_add_mapping_records_for_precomputed() { + let virt_rx_page_desc = virt_rx_page_desc(); generic_mmu::kernel_add_mapping_record( "Kernel code and RO data", - &virt_rx_page_desc(), - &phys_rx_page_desc(), - &AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadOnly, - execute_never: false, - }, + &virt_rx_page_desc, + &kernel_virt_to_phys_page_slice(virt_rx_page_desc), + &kernel_page_attributes(virt_rx_page_desc.first_page()), ); + let virt_rw_page_desc = virt_rw_page_desc(); generic_mmu::kernel_add_mapping_record( "Kernel data and bss", - &virt_rw_page_desc(), - &phys_rw_page_desc(), - &AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, + &virt_rw_page_desc, + &kernel_virt_to_phys_page_slice(virt_rw_page_desc), + &kernel_page_attributes(virt_rw_page_desc.first_page()), ); + let virt_boot_core_stack_page_desc = virt_boot_core_stack_page_desc(); generic_mmu::kernel_add_mapping_record( "Kernel boot-core stack", - &virt_boot_core_stack_page_desc(), - &phys_boot_core_stack_page_desc(), - &AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, + &virt_boot_core_stack_page_desc, + &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc), + &kernel_page_attributes(virt_boot_core_stack_page_desc.first_page()), ); } diff --git a/15_virtual_mem_part3_precomputed_tables/src/memory.rs b/15_virtual_mem_part3_precomputed_tables/src/memory.rs index 4ac96904..1780e7d4 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/memory.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/memory.rs @@ -6,13 +6,13 @@ pub mod mmu; -use crate::common; +use crate::{bsp, common}; use core::{ - convert::TryFrom, fmt, marker::PhantomData, ops::{AddAssign, SubAssign}, }; +use mmu::Page; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -66,13 +66,16 @@ impl Address { pub const fn into_usize(self) -> usize { self.value } -} -impl TryFrom> for Address { - type Error = mmu::TranslationError; + /// Return a pointer to the page that contains this address. + pub const fn as_page_ptr(&self) -> *const Page { + self.align_down(bsp::memory::mmu::KernelGranule::SIZE) + .into_usize() as *const _ + } - fn try_from(virt: Address) -> Result { - mmu::try_virt_to_phys(virt) + /// Return the address' offset into the underlying page. + pub const fn offset_into_page(&self) -> usize { + self.value & bsp::memory::mmu::KernelGranule::MASK } } diff --git a/15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs b/15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs index 106d3a90..3c3e5176 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs @@ -33,14 +33,6 @@ pub enum MMUEnableError { Other(&'static str), } -/// Translation error variants. -#[allow(missing_docs)] -#[derive(Debug)] -pub enum TranslationError { - MMUDisabled, - Aborted, -} - /// Memory Management interfaces. pub mod interface { use super::*; @@ -59,14 +51,6 @@ pub mod interface { /// Returns true if the MMU is enabled, false otherwise. fn is_enabled(&self) -> bool; - - /// Try to translate a virtual address to a physical address. - /// - /// Will only succeed if there exists a valid mapping for the input VA. - fn try_virt_to_phys( - &self, - virt: Address, - ) -> Result, TranslationError>; } } @@ -240,11 +224,34 @@ pub unsafe fn kernel_map_mmio( Ok(virt_addr + offset_into_start_page) } -/// Try to translate a virtual address to a physical address. +/// Try to translate a kernel virtual page pointer to a physical page pointer. +/// +/// Will only succeed if there exists a valid mapping for the input page. +pub fn try_kernel_virt_page_to_phys_page( + virt_page: *const Page, +) -> Result<*const Page, &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_virt_page_to_phys_page(virt_page)) +} + +/// Try to get the attributes of a kernel page. +/// +/// Will only succeed if there exists a valid mapping for the input page. +pub fn try_kernel_page_attributes( + virt_page: *const Page, +) -> Result { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_page_attributes(virt_page)) +} + +/// Try to translate a kernel virtual address to a physical address. /// -/// Will only succeed if there exists a valid mapping for the input VA. -pub fn try_virt_to_phys(virt: Address) -> Result, TranslationError> { - arch_mmu::mmu().try_virt_to_phys(virt) +/// Will only succeed if there exists a valid mapping for the input address. +fn try_kernel_virt_addr_to_phys_addr( + virt_addr: Address, +) -> Result, &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr)) } /// Enable the MMU and data + instruction caching. diff --git a/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/translation_table.rs b/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/translation_table.rs index bf9bed8c..bda05c7d 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/translation_table.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/translation_table.rs @@ -10,7 +10,7 @@ mod arch_translation_table; use crate::memory::{ mmu::{AttributeFields, PageSliceDescriptor}, - Physical, Virtual, + Address, Page, Physical, Virtual, }; //-------------------------------------------------------------------------------------------------- @@ -67,6 +67,30 @@ pub mod interface { /// Check if a virtual page splice is in the "MMIO region". fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; + + /// Try to translate a virtual page pointer to a physical page pointer. + /// + /// Will only succeed if there exists a valid mapping for the input page. + fn try_virt_page_to_phys_page( + &self, + virt_page: *const Page, + ) -> Result<*const Page, &'static str>; + + /// Try to get the attributes of a page. + /// + /// Will only succeed if there exists a valid mapping for the input page. + fn try_page_attributes( + &self, + virt_page: *const Page, + ) -> Result; + + /// Try to translate a virtual address to a physical address. + /// + /// Will only succeed if there exists a valid mapping for the input address. + fn try_virt_addr_to_phys_addr( + &self, + virt_addr: Address, + ) -> Result, &'static str>; } } diff --git a/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs b/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs index dc05fd70..b14089ed 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs @@ -6,12 +6,9 @@ use crate::{ bsp, common, - memory::{Address, AddressType, Physical, Virtual}, -}; -use core::{ - convert::{From, TryFrom}, - marker::PhantomData, + memory::{Address, AddressType, Physical}, }; +use core::{convert::From, marker::PhantomData}; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -95,11 +92,11 @@ impl PageSliceDescriptor { } /// Return a pointer to the first page of the described slice. - const fn first_page_ptr(&self) -> *const Page { + pub const fn first_page(&self) -> *const Page { self.start.into_usize() as *const _ } - /// Return the number of Pages the slice describes. + /// Return the number of pages the slice describes. pub const fn num_pages(&self) -> usize { self.num_pages } @@ -129,26 +126,13 @@ impl PageSliceDescriptor { (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) } - /// Return a non-mutable slice of Pages. + /// Return a non-mutable slice of pages. /// /// # Safety /// /// - Same as applies for `core::slice::from_raw_parts`. pub unsafe fn as_slice(&self) -> &[Page] { - core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) - } -} - -impl TryFrom> for PageSliceDescriptor { - type Error = super::TranslationError; - - fn try_from(desc: PageSliceDescriptor) -> Result { - let phys_start = super::try_virt_to_phys(desc.start)?; - - Ok(Self { - start: phys_start, - num_pages: desc.num_pages, - }) + core::slice::from_raw_parts(self.first_page(), self.num_pages) } } diff --git a/16_virtual_mem_part4_higher_half_kernel/README.md b/16_virtual_mem_part4_higher_half_kernel/README.md index 28005223..8eeee7c3 100644 --- a/16_virtual_mem_part4_higher_half_kernel/README.md +++ b/16_virtual_mem_part4_higher_half_kernel/README.md @@ -144,7 +144,7 @@ Precomputing kernel translation tables and patching kernel ELF Generating Boot-core stack | 0xffff_ffff_8010_0000 | 512 KiB -------------------------------------------------- Patching Kernel table struct at physical 0x9_0000 - Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0060 + Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0080 Finished in 0.03s Minipush 1.0 @@ -152,6 +152,7 @@ Minipush 1.0 [MP] ⏳ Waiting for /dev/ttyUSB0 [MP] ✅ Serial connected [MP] 🔌 Please power the target now + __ __ _ _ _ _ | \/ (_)_ _ (_) | ___ __ _ __| | | |\/| | | ' \| | |__/ _ \/ _` / _` | @@ -163,19 +164,20 @@ Minipush 1.0 [MP] ⏩ Pushing 387 KiB =======================================🦀 100% 96 KiB/s Time: 00:00:04 [ML] Loaded! Executing the payload now -[ 4.316420] mingo version 0.16.0 -[ 4.316627] Booting on: Raspberry Pi 3 -[ 4.317082] MMU online: -[ 4.317375] ------------------------------------------------------------------------------------------------------------------------------------------- -[ 4.319119] Virtual Physical Size Attr Entity -[ 4.320863] ------------------------------------------------------------------------------------------------------------------------------------------- -[ 4.322610] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data -[ 4.324223] 0xffff_ffff_8009_0000..0xffff_ffff_800e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C RW XN | Kernel data and bss -[ 4.325793] 0xffff_ffff_8010_0000..0xffff_ffff_8017_ffff --> 0x00_0010_0000..0x00_0017_ffff | 512 KiB | C RW XN | Kernel boot-core stack -[ 4.327397] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_3f20_0000..0x00_3f20_ffff | 64 KiB | Dev RW XN | BCM GPIO -[ 4.328847] | BCM PL011 UART -[ 4.330365] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_3f00_0000..0x00_3f00_ffff | 64 KiB | Dev RW XN | BCM Peripheral Interrupt Controller -[ 4.332108] ------------------------------------------------------------------------------------------------------------------------------------------- +[ 4.318584] mingo version 0.16.0 +[ 4.318792] Booting on: Raspberry Pi 3 +[ 4.319247] MMU online: +[ 4.319540] ------------------------------------------------------------------------------------------------------------------------------------------- +[ 4.321284] Virtual Physical Size Attr Entity +[ 4.323028] ------------------------------------------------------------------------------------------------------------------------------------------- +[ 4.324773] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data +[ 4.326387] 0xffff_ffff_8009_0000..0xffff_ffff_800e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C RW XN | Kernel data and bss +[ 4.327957] 0xffff_ffff_8010_0000..0xffff_ffff_8017_ffff --> 0x00_0010_0000..0x00_0017_ffff | 512 KiB | C RW XN | Kernel boot-core stack +[ 4.329560] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_3f20_0000..0x00_3f20_ffff | 64 KiB | Dev RW XN | BCM GPIO +[ 4.331012] | BCM PL011 UART +[ 4.332529] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_3f00_0000..0x00_3f00_ffff | 64 KiB | Dev RW XN | BCM Peripheral Interrupt Controller +[ 4.334273] ------------------------------------------------------------------------------------------------------------------------------------------- + ``` Raspberry Pi 4: @@ -189,18 +191,20 @@ Precomputing kernel translation tables and patching kernel ELF Section Start Virt Addr Size -------------------------------------------------- Generating Code and RO data | 0xffff_ffff_8008_0000 | 64 KiB - Generating Data and bss | 0xffff_ffff_8009_0000 | 448 KiB - Generating Boot-core stack | 0xffff_ffff_8011_0000 | 512 KiB + Generating Data and bss | 0xffff_ffff_8009_0000 | 384 KiB + Generating Boot-core stack | 0xffff_ffff_8010_0000 | 512 KiB -------------------------------------------------- - Patching Kernel table struct at physical 0xa_0000 - Patching Value of kernel table physical base address (0xe_0000) at physical 0x8_0068 + Patching Kernel table struct at physical 0x9_0000 + Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0080 Finished in 0.03s + Minipush 1.0 [MP] ⏳ Waiting for /dev/ttyUSB0 [MP] ✅ Serial connected [MP] 🔌 Please power the target now + __ __ _ _ _ _ | \/ (_)_ _ (_) | ___ __ _ __| | | |\/| | | ' \| | |__/ _ \/ _` / _` | @@ -209,23 +213,24 @@ Minipush 1.0 Raspberry Pi 4 [ML] Requesting binary -[MP] ⏩ Pushing 449 KiB ======================================🦀 100% 112 KiB/s Time: 00:00:04 +[MP] ⏩ Pushing 394 KiB =======================================🦀 100% 98 KiB/s Time: 00:00:04 [ML] Loaded! Executing the payload now -[ 5.009551] mingo version 0.16.0 -[ 5.009585] Booting on: Raspberry Pi 4 -[ 5.010040] MMU online: -[ 5.010332] ------------------------------------------------------------------------------------------------------------------------------------------- -[ 5.012077] Virtual Physical Size Attr Entity -[ 5.013821] ------------------------------------------------------------------------------------------------------------------------------------------- -[ 5.015566] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data -[ 5.017181] 0xffff_ffff_8009_0000..0xffff_ffff_800f_ffff --> 0x00_0009_0000..0x00_000f_ffff | 448 KiB | C RW XN | Kernel data and bss -[ 5.018751] 0xffff_ffff_8011_0000..0xffff_ffff_8018_ffff --> 0x00_0011_0000..0x00_0018_ffff | 512 KiB | C RW XN | Kernel boot-core stack -[ 5.020354] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_fe20_0000..0x00_fe20_ffff | 64 KiB | Dev RW XN | BCM GPIO -[ 5.021805] | BCM PL011 UART -[ 5.023322] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_ff84_0000..0x00_ff84_ffff | 64 KiB | Dev RW XN | GICD -[ 5.024730] | GICC -[ 5.026138] ------------------------------------------------------------------------------------------------------------------------------------------- +[ 4.401227] mingo version 0.16.0 +[ 4.401260] Booting on: Raspberry Pi 4 +[ 4.401715] MMU online: +[ 4.402008] ------------------------------------------------------------------------------------------------------------------------------------------- +[ 4.403752] Virtual Physical Size Attr Entity +[ 4.405496] ------------------------------------------------------------------------------------------------------------------------------------------- +[ 4.407241] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data +[ 4.408855] 0xffff_ffff_8009_0000..0xffff_ffff_800e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C RW XN | Kernel data and bss +[ 4.410425] 0xffff_ffff_8010_0000..0xffff_ffff_8017_ffff --> 0x00_0010_0000..0x00_0017_ffff | 512 KiB | C RW XN | Kernel boot-core stack +[ 4.412028] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_fe20_0000..0x00_fe20_ffff | 64 KiB | Dev RW XN | BCM GPIO +[ 4.413480] | BCM PL011 UART +[ 4.414997] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_ff84_0000..0x00_ff84_ffff | 64 KiB | Dev RW XN | GICD +[ 4.416405] | GICC +[ 4.417814] ------------------------------------------------------------------------------------------------------------------------------------------- + ``` ## Diff to previous @@ -361,7 +366,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t /// Page descriptors, covering 64 KiB windows per entry. lvl3: [[PageDescriptor; 8192]; NUM_TABLES], -@@ -262,14 +262,23 @@ +@@ -305,14 +305,23 @@ where [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized, { @@ -387,7 +392,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t /// Create an instance. #[allow(clippy::assertions_on_constants)] const fn _new(for_precompute: bool) -> Self { -@@ -298,20 +307,32 @@ +@@ -341,20 +350,32 @@ /// The start address of the table's MMIO range. #[inline(always)] fn mmio_start_addr(&self) -> Address { @@ -424,12 +429,12 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t } /// Helper to calculate the lvl2 and lvl3 indices from an address. -@@ -320,7 +341,12 @@ +@@ -363,7 +384,12 @@ &self, - addr: *const Page, + virt_page: *const Page, ) -> Result<(usize, usize), &'static str> { -- let addr = addr as usize; -+ let mut addr = addr as usize; +- let addr = virt_page as usize; ++ let mut addr = virt_page as usize; + + if START_FROM_TOP { + addr -= Self::START_FROM_TOP_OFFSET.into_usize() @@ -438,7 +443,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t let lvl2_index = addr >> Granule512MiB::SHIFT; let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; -@@ -347,8 +373,9 @@ +@@ -411,8 +437,9 @@ // OS Interface Code //------------------------------------------------------------------------------ @@ -450,7 +455,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t { fn init(&mut self) -> Result<(), &'static str> { if self.initialized { -@@ -423,12 +450,16 @@ +@@ -483,12 +510,16 @@ return Err("Not enough MMIO space left"); } @@ -468,7 +473,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t Ok(PageSliceDescriptor::from_addr(addr, num_pages)) } -@@ -451,7 +482,7 @@ +@@ -549,7 +580,7 @@ //-------------------------------------------------------------------------------------------------- #[cfg(test)] @@ -571,7 +576,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld 16 diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs --- 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs -@@ -23,7 +23,7 @@ +@@ -22,7 +22,7 @@ //-------------------------------------------------------------------------------------------------- type KernelTranslationTable = @@ -600,7 +605,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/lib.rs 16_virtual_mem_part diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs --- 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs -@@ -80,6 +80,11 @@ +@@ -64,6 +64,11 @@ pub trait AssociatedTranslationTable { /// A translation table whose address range is: /// diff --git a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu.rs b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu.rs index 830fb9da..84191af9 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu.rs @@ -15,7 +15,7 @@ use crate::{ bsp, memory, - memory::{mmu::TranslationGranule, Address, Physical, Virtual}, + memory::{mmu::TranslationGranule, Address, Physical}, }; use core::intrinsics::unlikely; use cortex_a::{asm::barrier, registers::*}; @@ -111,7 +111,7 @@ pub fn mmu() -> &'static impl memory::mmu::interface::MMU { //------------------------------------------------------------------------------ // OS Interface Code //------------------------------------------------------------------------------ -use memory::mmu::{MMUEnableError, TranslationError}; +use memory::mmu::MMUEnableError; impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn enable_mmu_and_caching( @@ -155,31 +155,4 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { fn is_enabled(&self) -> bool { SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } - - fn try_virt_to_phys( - &self, - virt: Address, - ) -> Result, TranslationError> { - if !self.is_enabled() { - return Err(TranslationError::MMUDisabled); - } - - let addr = virt.into_usize() as u64; - unsafe { - asm!( - "AT S1E1R, {0}", - in(reg) addr, - options(readonly, nostack, preserves_flags) - ); - } - - let par_el1 = PAR_EL1.extract(); - if par_el1.matches_all(PAR_EL1::F::TranslationAborted) { - return Err(TranslationError::Aborted); - } - - let phys_addr = (par_el1.read(PAR_EL1::PA) << 12) | (addr & 0xFFF); - - Ok(Address::new(phys_addr as usize)) - } } diff --git a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu/translation_table.rs b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu/translation_table.rs index 662ee3ee..ead8a990 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu/translation_table.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -23,7 +23,7 @@ use crate::{ Address, Physical, Virtual, }, }; -use core::convert::{self, TryInto}; +use core::convert; use tock_registers::{ interfaces::{Readable, Writeable}, register_bitfields, @@ -219,6 +219,35 @@ impl convert::From } } +/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes. +impl convert::TryFrom> for AttributeFields { + type Error = &'static str; + + fn try_from( + desc: InMemoryRegister, + ) -> Result { + let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) { + memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM, + memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device, + _ => return Err("Unexpected memory attribute"), + }; + + let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) { + Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly, + Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite, + _ => return Err("Unexpected access permission"), + }; + + let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0; + + Ok(AttributeFields { + mem_attributes, + acc_perms, + execute_never, + }) + } +} + impl PageDescriptor { /// Create an instance. /// @@ -228,13 +257,13 @@ impl PageDescriptor { } /// Create an instance. - pub fn from_output_addr( - phys_output_addr: *const Page, + pub fn from_output_page( + phys_output_page: *const Page, attribute_fields: &AttributeFields, ) -> Self { let val = InMemoryRegister::::new(0); - let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; + let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT; val.write( STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) + STAGE1_PAGE_DESCRIPTOR::AF::True @@ -251,6 +280,20 @@ impl PageDescriptor { InMemoryRegister::::new(self.value) .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) } + + /// Returns the output page. + fn output_page(&self) -> *const Page { + let shifted = InMemoryRegister::::new(self.value) + .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB); + let addr = shifted << Granule64KiB::SHIFT; + + addr as *const Page + } + + /// Returns the attributes. + fn try_attributes(&self) -> Result { + InMemoryRegister::::new(self.value).try_into() + } } //-------------------------------------------------------------------------------------------------- @@ -337,11 +380,11 @@ impl /// Helper to calculate the lvl2 and lvl3 indices from an address. #[inline(always)] - fn lvl2_lvl3_index_from( + fn lvl2_lvl3_index_from_page( &self, - addr: *const Page, + virt_page: *const Page, ) -> Result<(usize, usize), &'static str> { - let mut addr = addr as usize; + let mut addr = virt_page as usize; if START_FROM_TOP { addr -= Self::START_FROM_TOP_OFFSET.into_usize() @@ -357,15 +400,36 @@ impl Ok((lvl2_index, lvl3_index)) } - /// Returns the PageDescriptor corresponding to the supplied Page. + /// Returns the PageDescriptor corresponding to the supplied page address. #[inline(always)] - fn page_descriptor_from( + fn page_descriptor_from_page( + &self, + virt_page: *const Page, + ) -> Result<&PageDescriptor, &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; + let desc = &self.lvl3[lvl2_index][lvl3_index]; + + Ok(desc) + } + + /// Sets the PageDescriptor corresponding to the supplied page address. + /// + /// Doesn't allow overriding an already valid page. + #[inline(always)] + fn set_page_descriptor_from_page( &mut self, - addr: *const Page, - ) -> Result<&mut PageDescriptor, &'static str> { - let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + virt_page: *const Page, + new_desc: &PageDescriptor, + ) -> Result<(), &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?; + let desc = &mut self.lvl3[lvl2_index][lvl3_index]; + + if desc.is_valid() { + return Err("Virtual page is already mapped"); + } - Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + *desc = *new_desc; + Ok(()) } } @@ -384,13 +448,11 @@ impl // Populate the l2 entries. for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { - let addr = self.lvl3[lvl2_nr] - .virt_start_addr() - .try_into() - .map_err(|_| "Translation error")?; + let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr(); + let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?; - let desc = TableDescriptor::from_next_lvl_table_addr(addr); - *lvl2_entry = desc; + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); + *lvl2_entry = new_desc; } self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; @@ -407,8 +469,8 @@ impl ) -> Result<(), &'static str> { assert!(self.initialized, "Translation tables not initialized"); - let p = phys_pages.as_slice(); let v = virt_pages.as_slice(); + let p = phys_pages.as_slice(); // No work to do for empty slices. if v.is_empty() { @@ -425,12 +487,10 @@ impl let iter = p.iter().zip(v.iter()); for (phys_page, virt_page) in iter { - let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; - if page_descriptor.is_valid() { - return Err("Virtual page is already mapped"); - } + let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr); + let virt_page = virt_page.as_ptr(); - *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); + self.set_page_descriptor_from_page(virt_page, &new_desc)?; } Ok(()) @@ -475,6 +535,44 @@ impl false } + + fn try_virt_page_to_phys_page( + &self, + virt_page: *const Page, + ) -> Result<*const Page, &'static str> { + let page_desc = self.page_descriptor_from_page(virt_page)?; + + if !page_desc.is_valid() { + return Err("Page marked invalid"); + } + + Ok(page_desc.output_page()) + } + + fn try_page_attributes( + &self, + virt_page: *const Page, + ) -> Result { + let page_desc = self.page_descriptor_from_page(virt_page)?; + + if !page_desc.is_valid() { + return Err("Page marked invalid"); + } + + page_desc.try_attributes() + } + + /// Try to translate a virtual address to a physical address. + /// + /// Will only succeed if there exists a valid mapping for the input address. + fn try_virt_addr_to_phys_addr( + &self, + virt_addr: Address, + ) -> Result, &'static str> { + let page = self.try_virt_page_to_phys_page(virt_addr.as_page_ptr())?; + + Ok(Address::new(page as usize + virt_addr.offset_into_page())) + } } //-------------------------------------------------------------------------------------------------- diff --git a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs index 34d220f0..8e851a8d 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs @@ -9,14 +9,13 @@ use crate::{ memory::{ mmu as generic_mmu, mmu::{ - AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, - MemAttributes, Page, PageSliceDescriptor, TranslationGranule, + AddressSpace, AssociatedTranslationTable, AttributeFields, Page, PageSliceDescriptor, + TranslationGranule, }, - Physical, Virtual, + Address, Physical, Virtual, }, synchronization::InitStateLock, }; -use core::convert::TryInto; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -103,21 +102,19 @@ fn virt_boot_core_stack_page_desc() -> PageSliceDescriptor { } // There is no reason to expect the following conversions to fail, since they were generated offline -// by the `translation table tool`. If it doesn't work, a panic due to the unwrap is justified. - -/// The Read+Execute (RX) pages of the kernel binary. -fn phys_rx_page_desc() -> PageSliceDescriptor { - virt_rx_page_desc().try_into().unwrap() -} - -/// The Read+Write (RW) pages of the kernel binary. -fn phys_rw_page_desc() -> PageSliceDescriptor { - virt_rw_page_desc().try_into().unwrap() +// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified. +fn kernel_virt_to_phys_page_slice( + virt_slice: PageSliceDescriptor, +) -> PageSliceDescriptor { + let phys_first_page = + generic_mmu::try_kernel_virt_page_to_phys_page(virt_slice.first_page()).unwrap(); + let phys_start_addr = Address::new(phys_first_page as usize); + + PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages()) } -/// The boot core's stack. -fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { - virt_boot_core_stack_page_desc().try_into().unwrap() +fn kernel_page_attributes(virt_page: *const Page) -> AttributeFields { + generic_mmu::try_kernel_page_attributes(virt_page).unwrap() } //-------------------------------------------------------------------------------------------------- @@ -142,39 +139,28 @@ pub fn phys_addr_space_end_page() -> *const Page { /// The actual translation table entries for the kernel binary are generated using the offline /// `translation table tool` and patched into the kernel binary. This function just adds the mapping /// record entries. -/// -/// It must be ensured that these entries are in sync with the offline tool. pub fn kernel_add_mapping_records_for_precomputed() { + let virt_rx_page_desc = virt_rx_page_desc(); generic_mmu::kernel_add_mapping_record( "Kernel code and RO data", - &virt_rx_page_desc(), - &phys_rx_page_desc(), - &AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadOnly, - execute_never: false, - }, + &virt_rx_page_desc, + &kernel_virt_to_phys_page_slice(virt_rx_page_desc), + &kernel_page_attributes(virt_rx_page_desc.first_page()), ); + let virt_rw_page_desc = virt_rw_page_desc(); generic_mmu::kernel_add_mapping_record( "Kernel data and bss", - &virt_rw_page_desc(), - &phys_rw_page_desc(), - &AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, + &virt_rw_page_desc, + &kernel_virt_to_phys_page_slice(virt_rw_page_desc), + &kernel_page_attributes(virt_rw_page_desc.first_page()), ); + let virt_boot_core_stack_page_desc = virt_boot_core_stack_page_desc(); generic_mmu::kernel_add_mapping_record( "Kernel boot-core stack", - &virt_boot_core_stack_page_desc(), - &phys_boot_core_stack_page_desc(), - &AttributeFields { - mem_attributes: MemAttributes::CacheableDRAM, - acc_perms: AccessPermissions::ReadWrite, - execute_never: true, - }, + &virt_boot_core_stack_page_desc, + &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc), + &kernel_page_attributes(virt_boot_core_stack_page_desc.first_page()), ); } diff --git a/16_virtual_mem_part4_higher_half_kernel/src/memory.rs b/16_virtual_mem_part4_higher_half_kernel/src/memory.rs index 4ac96904..1780e7d4 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/memory.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/memory.rs @@ -6,13 +6,13 @@ pub mod mmu; -use crate::common; +use crate::{bsp, common}; use core::{ - convert::TryFrom, fmt, marker::PhantomData, ops::{AddAssign, SubAssign}, }; +use mmu::Page; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -66,13 +66,16 @@ impl Address { pub const fn into_usize(self) -> usize { self.value } -} -impl TryFrom> for Address { - type Error = mmu::TranslationError; + /// Return a pointer to the page that contains this address. + pub const fn as_page_ptr(&self) -> *const Page { + self.align_down(bsp::memory::mmu::KernelGranule::SIZE) + .into_usize() as *const _ + } - fn try_from(virt: Address) -> Result { - mmu::try_virt_to_phys(virt) + /// Return the address' offset into the underlying page. + pub const fn offset_into_page(&self) -> usize { + self.value & bsp::memory::mmu::KernelGranule::MASK } } diff --git a/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs b/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs index f9c40d35..d4def284 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs @@ -33,14 +33,6 @@ pub enum MMUEnableError { Other(&'static str), } -/// Translation error variants. -#[allow(missing_docs)] -#[derive(Debug)] -pub enum TranslationError { - MMUDisabled, - Aborted, -} - /// Memory Management interfaces. pub mod interface { use super::*; @@ -59,14 +51,6 @@ pub mod interface { /// Returns true if the MMU is enabled, false otherwise. fn is_enabled(&self) -> bool; - - /// Try to translate a virtual address to a physical address. - /// - /// Will only succeed if there exists a valid mapping for the input VA. - fn try_virt_to_phys( - &self, - virt: Address, - ) -> Result, TranslationError>; } } @@ -245,11 +229,34 @@ pub unsafe fn kernel_map_mmio( Ok(virt_addr + offset_into_start_page) } -/// Try to translate a virtual address to a physical address. +/// Try to translate a kernel virtual page pointer to a physical page pointer. +/// +/// Will only succeed if there exists a valid mapping for the input page. +pub fn try_kernel_virt_page_to_phys_page( + virt_page: *const Page, +) -> Result<*const Page, &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_virt_page_to_phys_page(virt_page)) +} + +/// Try to get the attributes of a kernel page. +/// +/// Will only succeed if there exists a valid mapping for the input page. +pub fn try_kernel_page_attributes( + virt_page: *const Page, +) -> Result { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_page_attributes(virt_page)) +} + +/// Try to translate a kernel virtual address to a physical address. /// -/// Will only succeed if there exists a valid mapping for the input VA. -pub fn try_virt_to_phys(virt: Address) -> Result, TranslationError> { - arch_mmu::mmu().try_virt_to_phys(virt) +/// Will only succeed if there exists a valid mapping for the input address. +fn try_kernel_virt_addr_to_phys_addr( + virt_addr: Address, +) -> Result, &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr)) } /// Enable the MMU and data + instruction caching. diff --git a/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/translation_table.rs b/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/translation_table.rs index bf9bed8c..bda05c7d 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/translation_table.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/translation_table.rs @@ -10,7 +10,7 @@ mod arch_translation_table; use crate::memory::{ mmu::{AttributeFields, PageSliceDescriptor}, - Physical, Virtual, + Address, Page, Physical, Virtual, }; //-------------------------------------------------------------------------------------------------- @@ -67,6 +67,30 @@ pub mod interface { /// Check if a virtual page splice is in the "MMIO region". fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; + + /// Try to translate a virtual page pointer to a physical page pointer. + /// + /// Will only succeed if there exists a valid mapping for the input page. + fn try_virt_page_to_phys_page( + &self, + virt_page: *const Page, + ) -> Result<*const Page, &'static str>; + + /// Try to get the attributes of a page. + /// + /// Will only succeed if there exists a valid mapping for the input page. + fn try_page_attributes( + &self, + virt_page: *const Page, + ) -> Result; + + /// Try to translate a virtual address to a physical address. + /// + /// Will only succeed if there exists a valid mapping for the input address. + fn try_virt_addr_to_phys_addr( + &self, + virt_addr: Address, + ) -> Result, &'static str>; } } diff --git a/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/types.rs b/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/types.rs index dc05fd70..b14089ed 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/types.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/types.rs @@ -6,12 +6,9 @@ use crate::{ bsp, common, - memory::{Address, AddressType, Physical, Virtual}, -}; -use core::{ - convert::{From, TryFrom}, - marker::PhantomData, + memory::{Address, AddressType, Physical}, }; +use core::{convert::From, marker::PhantomData}; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -95,11 +92,11 @@ impl PageSliceDescriptor { } /// Return a pointer to the first page of the described slice. - const fn first_page_ptr(&self) -> *const Page { + pub const fn first_page(&self) -> *const Page { self.start.into_usize() as *const _ } - /// Return the number of Pages the slice describes. + /// Return the number of pages the slice describes. pub const fn num_pages(&self) -> usize { self.num_pages } @@ -129,26 +126,13 @@ impl PageSliceDescriptor { (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) } - /// Return a non-mutable slice of Pages. + /// Return a non-mutable slice of pages. /// /// # Safety /// /// - Same as applies for `core::slice::from_raw_parts`. pub unsafe fn as_slice(&self) -> &[Page] { - core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) - } -} - -impl TryFrom> for PageSliceDescriptor { - type Error = super::TranslationError; - - fn try_from(desc: PageSliceDescriptor) -> Result { - let phys_start = super::try_virt_to_phys(desc.start)?; - - Ok(Self { - start: phys_start, - num_pages: desc.num_pages, - }) + core::slice::from_raw_parts(self.first_page(), self.num_pages) } }