Archives
/
rust-raspberrypi-OS-tutorials
mirror of https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make address translation a feature of the tables

Browse Source
pull/143/head
Andre Richter 3 years ago
parent baa634f390
commit 57c6f72936
No known key found for this signature in database
GPG Key ID: 2116C1AB102F615E
20 changed files with 905 additions and 683 deletions
Show Stats Download Patch File Download Diff File

162
14_virtual_mem_part2_mmio_remap/README.md
Unescape Escape View File

@ -457,19 +457,23 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
val.write( val.write(
STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64) STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)
+ STAGE1_TABLE_DESCRIPTOR::TYPE::Table + STAGE1_TABLE_DESCRIPTOR::TYPE::Table
@@ -230,7 +229,10 @@ @@ -230,10 +229,13 @@
} }
/// Create an instance. /// Create an instance.
- pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self { - pub fn from_output_addr(phys_output_addr: usize, attribute_fields: &AttributeFields) -> Self {
+ pub fn from_output_addr( + pub fn from_output_page(
+ phys_output_addr: *const Page<Physical>, + phys_output_page: *const Page<Physical>,
+ attribute_fields: &AttributeFields, + attribute_fields: &AttributeFields,
+ ) -> Self { + ) -> Self {
let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0); let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);
let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; - let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT;
@@ -244,50 +246,193 @@ + let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT;
val.write(
STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)
+ STAGE1_PAGE_DESCRIPTOR::AF::True
@@ -244,50 +246,201 @@
Self { value: val.get() } Self { value: val.get() }
} }
@ -517,12 +521,6 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
- /// Iterates over all static translation table entries and fills them at once. - /// Iterates over all static translation table entries and fills them at once.
- /// - ///
- /// # Safety - /// # Safety
- ///
- /// - Modifies a `static mut`. Ensure it only happens from here.
- pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {
- for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {
- *l2_entry =
- TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());
+ /// The start address of the table's MMIO range. + /// The start address of the table's MMIO range.
+ #[inline(always)] + #[inline(always)]
+ fn mmio_start_addr(&self) -> Address<Virtual> { + fn mmio_start_addr(&self) -> Address<Virtual> {
@ -544,11 +542,11 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
+ +
+ /// Helper to calculate the lvl2 and lvl3 indices from an address. + /// Helper to calculate the lvl2 and lvl3 indices from an address.
+ #[inline(always)] + #[inline(always)]
+ fn lvl2_lvl3_index_from( + fn lvl2_lvl3_index_from_page(
+ &self, + &self,
+ addr: *const Page<Virtual>, + virt_page: *const Page<Virtual>,
+ ) -> Result<(usize, usize), &'static str> { + ) -> Result<(usize, usize), &'static str> {
+ let addr = addr as usize; + let addr = virt_page as usize;
+ let lvl2_index = addr >> Granule512MiB::SHIFT; + let lvl2_index = addr >> Granule512MiB::SHIFT;
+ let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; + let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;
+ +
@ -559,24 +557,42 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
+ Ok((lvl2_index, lvl3_index)) + Ok((lvl2_index, lvl3_index))
+ } + }
+ +
+ /// Returns the PageDescriptor corresponding to the supplied Page. + /// Sets the PageDescriptor corresponding to the supplied page address.
///
- /// - Modifies a `static mut`. Ensure it only happens from here.
- pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> {
- for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() {
- *l2_entry =
- TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].phys_start_addr_usize());
+ /// Doesn't allow overriding an already valid page.
+ #[inline(always)] + #[inline(always)]
+ fn page_descriptor_from( + fn set_page_descriptor_from_page(
+ &mut self, + &mut self,
+ addr: *const Page<Virtual>, + virt_page: *const Page<Virtual>,
+ ) -> Result<&mut PageDescriptor, &'static str> { + new_desc: &PageDescriptor,
+ let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + ) -> Result<(), &'static str> {
+ + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
+ Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + let desc = &mut self.lvl3[lvl2_index][lvl3_index];
- for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {
- let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);
+ if desc.is_valid() {
+ return Err("Virtual page is already mapped");
+ }
- let (phys_output_addr, attribute_fields) =
- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;
+ *desc = *new_desc;
+ Ok(())
+ } + }
+} +}
+
- *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);
- }
+//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------
+// OS Interface Code +// OS Interface Code
+//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------
+
- for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() {
- let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT);
+impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable +impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable
+ for FixedSizeTranslationTable<NUM_TABLES> + for FixedSizeTranslationTable<NUM_TABLES>
+{ +{
@ -587,9 +603,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
+ +
+ // Populate the l2 entries. + // Populate the l2 entries.
+ for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {
+ let desc = + let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr();
+ TableDescriptor::from_next_lvl_table_addr(self.lvl3[lvl2_nr].phys_start_addr()); +
+ *lvl2_entry = desc; + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);
+ *lvl2_entry = new_desc;
+ } + }
+ +
+ self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; + self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX;
@ -608,33 +625,28 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
+ ) -> Result<(), &'static str> { + ) -> Result<(), &'static str> {
+ assert!(self.initialized, "Translation tables not initialized"); + assert!(self.initialized, "Translation tables not initialized");
+ +
+ let p = phys_pages.as_slice();
+ let v = virt_pages.as_slice(); + let v = virt_pages.as_slice();
+ let p = phys_pages.as_slice();
+ +
+ // No work to do for empty slices. + // No work to do for empty slices.
+ if v.is_empty() { + if v.is_empty() {
+ return Ok(()); + return Ok(());
+ } + }
+
- let (phys_output_addr, attribute_fields) =
- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?;
+ if v.len() != p.len() { + if v.len() != p.len() {
+ return Err("Tried to map page slices with unequal sizes"); + return Err("Tried to map page slices with unequal sizes");
+ } + }
+
- *l3_entry = PageDescriptor::from_output_addr(phys_output_addr, &attribute_fields);
+ if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() { + if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() {
+ return Err("Tried to map outside of physical address space"); + return Err("Tried to map outside of physical address space");
+ } + }
+ +
+ let iter = p.iter().zip(v.iter()); + let iter = p.iter().zip(v.iter());
+ for (phys_page, virt_page) in iter { + for (phys_page, virt_page) in iter {
+ let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; + let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr);
+ if page_descriptor.is_valid() { + let virt_page = virt_page.as_ptr();
+ return Err("Virtual page is already mapped");
}
+ +
+ *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); + self.set_page_descriptor_from_page(virt_page, &new_desc)?;
} }
Ok(()) Ok(())
@ -680,7 +692,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/trans
} }
} }
@@ -296,6 +441,9 @@ @@ -296,6 +449,9 @@
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
#[cfg(test)] #[cfg(test)]
@ -1468,7 +1480,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 14_vir
diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs
--- 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs --- 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+++ 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +++ 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs
@@ -4,70 +4,157 @@ @@ -4,70 +4,150 @@
//! BSP Memory Management Unit. //! BSP Memory Management Unit.
@ -1483,7 +1495,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+ AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, + AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields,
+ MemAttributes, Page, PageSliceDescriptor, TranslationGranule, + MemAttributes, Page, PageSliceDescriptor, TranslationGranule,
+ }, + },
+ Physical, Virtual, + Address, Physical, Virtual,
+ }, + },
+ synchronization::InitStateLock, + synchronization::InitStateLock,
+}; +};
@ -1504,16 +1516,16 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to +/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to
+/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. +/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`.
+pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;
+
-const NUM_MEM_RANGES: usize = 2;
+/// The kernel's virtual address space defined by this BSP. +/// The kernel's virtual address space defined by this BSP.
+pub type KernelVirtAddrSpace = AddressSpace<{ 8 * 1024 * 1024 * 1024 }>; +pub type KernelVirtAddrSpace = AddressSpace<{ 8 * 1024 * 1024 * 1024 }>;
-const NUM_MEM_RANGES: usize = 2; -/// The virtual memory layout.
+//-------------------------------------------------------------------------------------------------- +//--------------------------------------------------------------------------------------------------
+// Global instances +// Global instances
+//-------------------------------------------------------------------------------------------------- +//--------------------------------------------------------------------------------------------------
+
-/// The virtual memory layout.
+/// The kernel translation tables. +/// The kernel translation tables.
/// ///
-/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. -/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM.
@ -1571,8 +1583,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+ let num_pages = size_to_num_pages(super::rx_size()); + let num_pages = size_to_num_pages(super::rx_size());
+ +
+ PageSliceDescriptor::from_addr(super::virt_rx_start(), num_pages) + PageSliceDescriptor::from_addr(super::virt_rx_start(), num_pages)
+} }
+
-fn mmio_range_inclusive() -> RangeInclusive<usize> {
- RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)
+/// The Read+Write (RW) pages of the kernel binary. +/// The Read+Write (RW) pages of the kernel binary.
+fn virt_rw_page_desc() -> PageSliceDescriptor<Virtual> { +fn virt_rw_page_desc() -> PageSliceDescriptor<Virtual> {
+ let num_pages = size_to_num_pages(super::rw_size()); + let num_pages = size_to_num_pages(super::rw_size());
@ -1587,23 +1601,14 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+ PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) + PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages)
+} +}
+ +
+// The binary is still identity mapped, so we don't need to convert in the following. +// The binary is still identity mapped, so use this trivial conversion function for mapping below.
+ +
+/// The Read+Execute (RX) pages of the kernel binary. +fn kernel_virt_to_phys_page_slice(
+fn phys_rx_page_desc() -> PageSliceDescriptor<Physical> { + virt_slice: PageSliceDescriptor<Virtual>,
+ virt_rx_page_desc().into() +) -> PageSliceDescriptor<Physical> {
} + let phys_start_addr = Address::<Physical>::new(virt_slice.start_addr().into_usize());
-fn mmio_range_inclusive() -> RangeInclusive<usize> {
- RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE)
+/// The Read+Write (RW) pages of the kernel binary.
+fn phys_rw_page_desc() -> PageSliceDescriptor<Physical> {
+ virt_rw_page_desc().into()
+}
+ +
+/// The boot core's stack. + PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages())
+fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor<Physical> {
+ virt_boot_core_stack_page_desc().into()
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -1635,7 +1640,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+ generic_mmu::kernel_map_pages_at( + generic_mmu::kernel_map_pages_at(
+ "Kernel code and RO data", + "Kernel code and RO data",
+ &virt_rx_page_desc(), + &virt_rx_page_desc(),
+ &phys_rx_page_desc(), + &kernel_virt_to_phys_page_slice(virt_rx_page_desc()),
+ &AttributeFields { + &AttributeFields {
+ mem_attributes: MemAttributes::CacheableDRAM, + mem_attributes: MemAttributes::CacheableDRAM,
+ acc_perms: AccessPermissions::ReadOnly, + acc_perms: AccessPermissions::ReadOnly,
@ -1646,7 +1651,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+ generic_mmu::kernel_map_pages_at( + generic_mmu::kernel_map_pages_at(
+ "Kernel data and bss", + "Kernel data and bss",
+ &virt_rw_page_desc(), + &virt_rw_page_desc(),
+ &phys_rw_page_desc(), + &kernel_virt_to_phys_page_slice(virt_rw_page_desc()),
+ &AttributeFields { + &AttributeFields {
+ mem_attributes: MemAttributes::CacheableDRAM, + mem_attributes: MemAttributes::CacheableDRAM,
+ acc_perms: AccessPermissions::ReadWrite, + acc_perms: AccessPermissions::ReadWrite,
@ -1657,7 +1662,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
+ generic_mmu::kernel_map_pages_at( + generic_mmu::kernel_map_pages_at(
+ "Kernel boot-core stack", + "Kernel boot-core stack",
+ &virt_boot_core_stack_page_desc(), + &virt_boot_core_stack_page_desc(),
+ &phys_boot_core_stack_page_desc(), + &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc()),
+ &AttributeFields { + &AttributeFields {
+ mem_attributes: MemAttributes::CacheableDRAM, + mem_attributes: MemAttributes::CacheableDRAM,
+ acc_perms: AccessPermissions::ReadWrite, + acc_perms: AccessPermissions::ReadWrite,
@ -1669,7 +1674,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@@ -77,19 +164,24 @@ @@ -77,19 +157,24 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@ -1701,7 +1706,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs
assert!(end >= start); assert!(end >= start);
} }
} }
@@ -97,18 +189,38 @@ @@ -97,18 +182,38 @@
/// Ensure the kernel's virtual memory layout is free of overlaps. /// Ensure the kernel's virtual memory layout is free of overlaps.
#[kernel_test] #[kernel_test]
fn virt_mem_layout_has_no_overlaps() { fn virt_mem_layout_has_no_overlaps() {
@ -2482,7 +2487,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.r
diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs
--- 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs --- 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs
+++ 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs +++ 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs
@@ -0,0 +1,210 @@ @@ -0,0 +1,201 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0 +// SPDX-License-Identifier: MIT OR Apache-2.0
+// +//
+// Copyright (c) 2020-2021 Andre Richter <andre.o.richter@gmail.com> +// Copyright (c) 2020-2021 Andre Richter <andre.o.richter@gmail.com>
@ -2491,7 +2496,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual
+ +
+use crate::{ +use crate::{
+ bsp, common, + bsp, common,
+ memory::{Address, AddressType, Physical, Virtual}, + memory::{Address, AddressType, Physical},
+}; +};
+use core::{convert::From, marker::PhantomData}; +use core::{convert::From, marker::PhantomData};
+ +
@ -2577,11 +2582,11 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual
+ } + }
+ +
+ /// Return a pointer to the first page of the described slice. + /// Return a pointer to the first page of the described slice.
+ const fn first_page_ptr(&self) -> *const Page<ATYPE> { + const fn first_page(&self) -> *const Page<ATYPE> {
+ self.start.into_usize() as *const _ + self.start.into_usize() as *const _
+ } + }
+ +
+ /// Return the number of Pages the slice describes. + /// Return the number of pages the slice describes.
+ pub const fn num_pages(&self) -> usize { + pub const fn num_pages(&self) -> usize {
+ self.num_pages + self.num_pages
+ } + }
@ -2611,22 +2616,13 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 14_virtual
+ (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) + (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive())
+ } + }
+ +
+ /// Return a non-mutable slice of Pages. + /// Return a non-mutable slice of pages.
+ /// + ///
+ /// # Safety + /// # Safety
+ /// + ///
+ /// - Same as applies for `core::slice::from_raw_parts`. + /// - Same as applies for `core::slice::from_raw_parts`.
+ pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] { + pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] {
+ core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) + core::slice::from_raw_parts(self.first_page(), self.num_pages)
+ }
+}
+
+impl From<PageSliceDescriptor<Virtual>> for PageSliceDescriptor<Physical> {
+ fn from(desc: PageSliceDescriptor<Virtual>) -> Self {
+ Self {
+ start: Address::new(desc.start.into_usize()),
+ num_pages: desc.num_pages,
+ }
+ } + }
+} +}
+ +

50
14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs
Unescape Escape View File

@ -229,13 +229,13 @@ impl PageDescriptor {
} }
/// Create an instance. /// Create an instance.
pub fn from_output_addr( pub fn from_output_page(
phys_output_addr: *const Page<Physical>, phys_output_page: *const Page<Physical>,
attribute_fields: &AttributeFields, attribute_fields: &AttributeFields,
) -> Self { ) -> Self {
let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0); let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);
let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT;
val.write( val.write(
STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)
+ STAGE1_PAGE_DESCRIPTOR::AF::True + STAGE1_PAGE_DESCRIPTOR::AF::True
@ -308,11 +308,11 @@ impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {
/// Helper to calculate the lvl2 and lvl3 indices from an address. /// Helper to calculate the lvl2 and lvl3 indices from an address.
#[inline(always)] #[inline(always)]
fn lvl2_lvl3_index_from( fn lvl2_lvl3_index_from_page(
&self, &self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<(usize, usize), &'static str> { ) -> Result<(usize, usize), &'static str> {
let addr = addr as usize; let addr = virt_page as usize;
let lvl2_index = addr >> Granule512MiB::SHIFT; let lvl2_index = addr >> Granule512MiB::SHIFT;
let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;
@ -323,15 +323,24 @@ impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {
Ok((lvl2_index, lvl3_index)) Ok((lvl2_index, lvl3_index))
} }
/// Returns the PageDescriptor corresponding to the supplied Page. /// Sets the PageDescriptor corresponding to the supplied page address.
///
/// Doesn't allow overriding an already valid page.
#[inline(always)] #[inline(always)]
fn page_descriptor_from( fn set_page_descriptor_from_page(
&mut self, &mut self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<&mut PageDescriptor, &'static str> { new_desc: &PageDescriptor,
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; ) -> Result<(), &'static str> {
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
let desc = &mut self.lvl3[lvl2_index][lvl3_index];
Ok(&mut self.lvl3[lvl2_index][lvl3_index]) if desc.is_valid() {
return Err("Virtual page is already mapped");
}
*desc = *new_desc;
Ok(())
} }
} }
@ -349,9 +358,10 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
// Populate the l2 entries. // Populate the l2 entries.
for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {
let desc = let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr();
TableDescriptor::from_next_lvl_table_addr(self.lvl3[lvl2_nr].phys_start_addr());
*lvl2_entry = desc; let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);
*lvl2_entry = new_desc;
} }
self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX;
@ -370,8 +380,8 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
) -> Result<(), &'static str> { ) -> Result<(), &'static str> {
assert!(self.initialized, "Translation tables not initialized"); assert!(self.initialized, "Translation tables not initialized");
let p = phys_pages.as_slice();
let v = virt_pages.as_slice(); let v = virt_pages.as_slice();
let p = phys_pages.as_slice();
// No work to do for empty slices. // No work to do for empty slices.
if v.is_empty() { if v.is_empty() {
@ -388,12 +398,10 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
let iter = p.iter().zip(v.iter()); let iter = p.iter().zip(v.iter());
for (phys_page, virt_page) in iter { for (phys_page, virt_page) in iter {
let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr);
if page_descriptor.is_valid() { let virt_page = virt_page.as_ptr();
return Err("Virtual page is already mapped");
}
*page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); self.set_page_descriptor_from_page(virt_page, &new_desc)?;
} }
Ok(()) Ok(())

27
14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs
Unescape Escape View File

@ -12,7 +12,7 @@ use crate::{
AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields,
MemAttributes, Page, PageSliceDescriptor, TranslationGranule, MemAttributes, Page, PageSliceDescriptor, TranslationGranule,
}, },
Physical, Virtual, Address, Physical, Virtual,
}, },
synchronization::InitStateLock, synchronization::InitStateLock,
}; };
@ -81,21 +81,14 @@ fn virt_boot_core_stack_page_desc() -> PageSliceDescriptor<Virtual> {
PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages)
} }
// The binary is still identity mapped, so we don't need to convert in the following. // The binary is still identity mapped, so use this trivial conversion function for mapping below.
/// The Read+Execute (RX) pages of the kernel binary. fn kernel_virt_to_phys_page_slice(
fn phys_rx_page_desc() -> PageSliceDescriptor<Physical> { virt_slice: PageSliceDescriptor<Virtual>,
virt_rx_page_desc().into() ) -> PageSliceDescriptor<Physical> {
} let phys_start_addr = Address::<Physical>::new(virt_slice.start_addr().into_usize());
/// The Read+Write (RW) pages of the kernel binary.
fn phys_rw_page_desc() -> PageSliceDescriptor<Physical> {
virt_rw_page_desc().into()
}
/// The boot core's stack. PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages())
fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor<Physical> {
virt_boot_core_stack_page_desc().into()
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -124,7 +117,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> {
generic_mmu::kernel_map_pages_at( generic_mmu::kernel_map_pages_at(
"Kernel code and RO data", "Kernel code and RO data",
&virt_rx_page_desc(), &virt_rx_page_desc(),
&phys_rx_page_desc(), &kernel_virt_to_phys_page_slice(virt_rx_page_desc()),
&AttributeFields { &AttributeFields {
mem_attributes: MemAttributes::CacheableDRAM, mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadOnly, acc_perms: AccessPermissions::ReadOnly,
@ -135,7 +128,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> {
generic_mmu::kernel_map_pages_at( generic_mmu::kernel_map_pages_at(
"Kernel data and bss", "Kernel data and bss",
&virt_rw_page_desc(), &virt_rw_page_desc(),
&phys_rw_page_desc(), &kernel_virt_to_phys_page_slice(virt_rw_page_desc()),
&AttributeFields { &AttributeFields {
mem_attributes: MemAttributes::CacheableDRAM, mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadWrite, acc_perms: AccessPermissions::ReadWrite,
@ -146,7 +139,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> {
generic_mmu::kernel_map_pages_at( generic_mmu::kernel_map_pages_at(
"Kernel boot-core stack", "Kernel boot-core stack",
&virt_boot_core_stack_page_desc(), &virt_boot_core_stack_page_desc(),
&phys_boot_core_stack_page_desc(), &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc()),
&AttributeFields { &AttributeFields {
mem_attributes: MemAttributes::CacheableDRAM, mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadWrite, acc_perms: AccessPermissions::ReadWrite,

19
14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs
Unescape Escape View File

@ -6,7 +6,7 @@
use crate::{ use crate::{
bsp, common, bsp, common,
memory::{Address, AddressType, Physical, Virtual}, memory::{Address, AddressType, Physical},
}; };
use core::{convert::From, marker::PhantomData}; use core::{convert::From, marker::PhantomData};
@ -92,11 +92,11 @@ impl<ATYPE: AddressType> PageSliceDescriptor<ATYPE> {
} }
/// Return a pointer to the first page of the described slice. /// Return a pointer to the first page of the described slice.
const fn first_page_ptr(&self) -> *const Page<ATYPE> { const fn first_page(&self) -> *const Page<ATYPE> {
self.start.into_usize() as *const _ self.start.into_usize() as *const _
} }
/// Return the number of Pages the slice describes. /// Return the number of pages the slice describes.
pub const fn num_pages(&self) -> usize { pub const fn num_pages(&self) -> usize {
self.num_pages self.num_pages
} }
@ -126,22 +126,13 @@ impl<ATYPE: AddressType> PageSliceDescriptor<ATYPE> {
(addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive())
} }
/// Return a non-mutable slice of Pages. /// Return a non-mutable slice of pages.
/// ///
/// # Safety /// # Safety
/// ///
/// - Same as applies for `core::slice::from_raw_parts`. /// - Same as applies for `core::slice::from_raw_parts`.
pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] { pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] {
core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) core::slice::from_raw_parts(self.first_page(), self.num_pages)
}
}
impl From<PageSliceDescriptor<Virtual>> for PageSliceDescriptor<Physical> {
fn from(desc: PageSliceDescriptor<Virtual>) -> Self {
Self {
start: Address::new(desc.start.into_usize()),
num_pages: desc.num_pages,
}
} }
} }

511
15_virtual_mem_part3_precomputed_tables/README.md
Unescape Escape View File

@ -257,7 +257,7 @@ ffff000000010008: 44 33 22 11
ffff00000001000c: 00 00 00 00 ffff00000001000c: 00 00 00 00
``` ```
Both times, the `movabsq` instruction gets emitted. It means that the address is put into the target Both times, the `movabs` instruction gets emitted. It means that the address is put into the target
register using hardcoded `immediate values`. PC-relative address calculation is not used here. register using hardcoded `immediate values`. PC-relative address calculation is not used here.
Hence, this code would return the `absolute` address in both cases. Which means in the second case, Hence, this code would return the `absolute` address in both cases. Which means in the second case,
even when the binary would be loaded at `0x8_0000`, the return value would be even when the binary would be loaded at `0x8_0000`, the return value would be
@ -350,11 +350,11 @@ ffff000000010008: 44 33 22 11
ffff00000001000c: 00 00 00 00 ffff00000001000c: 00 00 00 00
``` ```
What changed compared to earlier is that `rx_start_address()` now indirects through the `Global What changed compared to earlier is that `get_address_of_global()` now indirects through the `Global
Offset Table`, as has been promised by the compiler's documentation. Specifically, Offset Table`, as has been promised by the compiler's documentation. Specifically,
`rx_start_address()` addresses the `GOT` using PC-relative addressing (distance from code to `GOT` `get_address_of_global()` addresses the `GOT` using PC-relative addressing (distance from code to
must always be fixed), and loads the first 64 bit word from the start of the `GOT`, which happens to `GOT` must always be fixed), and loads the first 64 bit word from the start of the `GOT`, which
be `0xffff_0000_0001_0008`. happens to be `0xffff_0000_0001_0008`.
Okay okay okay... So when we use `-fpic`, we get the **absolute** address of `global_data_word` even Okay okay okay... So when we use `-fpic`, we get the **absolute** address of `global_data_word` even
on `AArch64` now. How does this help when the code executes from `0x8_0000`? on `AArch64` now. How does this help when the code executes from `0x8_0000`?
@ -674,7 +674,7 @@ Precomputing kernel translation tables and patching kernel ELF
Generating Boot-core stack | 0x0000_0000_0010_0000 | 512 KiB Generating Boot-core stack | 0x0000_0000_0010_0000 | 512 KiB
-------------------------------------------------- --------------------------------------------------
Patching Kernel table struct at physical 0x9_0000 Patching Kernel table struct at physical 0x9_0000
Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0040 Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0060
Finished in 0.03s Finished in 0.03s
``` ```
@ -685,11 +685,11 @@ tutorial, anything else, like `MMIO-remapping`, can and will happen lazily durin
Two more things that changed in this tutorial, but won't be explained in detail: Two more things that changed in this tutorial, but won't be explained in detail:
- Since virtual memory in `EL1` is now active from the start, any attempt to convert from - Since virtual memory in `EL1` is now active from the start, any attempt to convert from a kernel
`Address<Virtual>` to `Address<Physical>` is now done through the `TryFrom` trait, which `Address<Virtual>` to `Address<Physical>` is now done using the function
internally uses HW-supported address translation of the CPU. So either the translation succeeds `mmu::try_kernel_virt_addr_to_phys_addr()`, which internally uses a new function that has been
because there is a valid virtual-to-physical mapping in the currently used translation tables, or added to the `TranslationTable` interface. If there is no valid virtual-to-physical mapping
an `Err()` is returned. present in the tables, an `Err()` is returned.
- The precomputed translation table mappings won't automatically have entries in the kernel's - The precomputed translation table mappings won't automatically have entries in the kernel's
`mapping info record`, which is used to print mapping info during boot. Mapping record entries are `mapping info record`, which is used to print mapping info during boot. Mapping record entries are
not computed offline in order to reduce complexity. For this reason, the `BSP` code, which in not computed offline in order to reduce complexity. For this reason, the `BSP` code, which in
@ -724,7 +724,7 @@ Precomputing kernel translation tables and patching kernel ELF
Generating Boot-core stack | 0x0000_0000_0010_0000 | 512 KiB Generating Boot-core stack | 0x0000_0000_0010_0000 | 512 KiB
-------------------------------------------------- --------------------------------------------------
Patching Kernel table struct at physical 0x9_0000 Patching Kernel table struct at physical 0x9_0000
Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0040 Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0060
Finished in 0.03s Finished in 0.03s
Minipush 1.0 Minipush 1.0
@ -732,6 +732,7 @@ Minipush 1.0
[MP] ⏳ Waiting for /dev/ttyUSB0 [MP] ⏳ Waiting for /dev/ttyUSB0
[MP] ✅ Serial connected [MP] ✅ Serial connected
[MP] 🔌 Please power the target now [MP] 🔌 Please power the target now
__ __ _ _ _ _ __ __ _ _ _ _
| \/ (_)_ _ (_) | ___ __ _ __| | | \/ (_)_ _ (_) | ___ __ _ __| |
| |\/| | | ' \| | |__/ _ \/ _` / _` | | |\/| | | ' \| | |__/ _ \/ _` / _` |
@ -859,15 +860,6 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s 15_virtua
diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs
--- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs --- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs +++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs
@@ -23,7 +23,7 @@
Address, Physical, Virtual,
},
};
-use core::convert;
+use core::convert::{self, TryInto};
use tock_registers::{
interfaces::{Readable, Writeable},
register_bitfields,
@@ -124,7 +124,7 @@ @@ -124,7 +124,7 @@
} }
@ -888,7 +880,64 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati
Address::new(self as *const _ as usize) Address::new(self as *const _ as usize)
} }
} }
@@ -273,7 +272,7 @@ @@ -220,6 +219,35 @@
}
}
+/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.
+impl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {
+ type Error = &'static str;
+
+ fn try_from(
+ desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,
+ ) -> Result<AttributeFields, Self::Error> {
+ let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {
+ memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,
+ memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,
+ _ => return Err("Unexpected memory attribute"),
+ };
+
+ let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {
+ Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,
+ Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,
+ _ => return Err("Unexpected access permission"),
+ };
+
+ let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;
+
+ Ok(AttributeFields {
+ mem_attributes,
+ acc_perms,
+ execute_never,
+ })
+ }
+}
+
impl PageDescriptor {
/// Create an instance.
///
@@ -252,6 +280,20 @@
InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)
.is_set(STAGE1_PAGE_DESCRIPTOR::VALID)
}
+
+ /// Returns the output page.
+ fn output_page(&self) -> *const Page<Physical> {
+ let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)
+ .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB);
+ let addr = shifted << Granule64KiB::SHIFT;
+
+ addr as *const Page<Physical>
+ }
+
+ /// Returns the attributes.
+ fn try_attributes(&self) -> Result<AttributeFields, &'static str> {
+ InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()
+ }
}
//--------------------------------------------------------------------------------------------------
@@ -273,7 +315,7 @@
/// Create an instance. /// Create an instance.
#[allow(clippy::assertions_on_constants)] #[allow(clippy::assertions_on_constants)]
@ -897,7 +946,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati
assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE);
// Can't have a zero-sized address space. // Can't have a zero-sized address space.
@@ -282,11 +281,20 @@ @@ -282,11 +324,20 @@
Self { Self {
lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES],
lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES],
@ -920,7 +969,26 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati
/// The start address of the table's MMIO range. /// The start address of the table's MMIO range.
#[inline(always)] #[inline(always)]
fn mmio_start_addr(&self) -> Address<Virtual> { fn mmio_start_addr(&self) -> Address<Virtual> {
@@ -342,24 +350,26 @@ @@ -323,6 +374,18 @@
Ok((lvl2_index, lvl3_index))
}
+ /// Returns the PageDescriptor corresponding to the supplied page address.
+ #[inline(always)]
+ fn page_descriptor_from_page(
+ &self,
+ virt_page: *const Page<Virtual>,
+ ) -> Result<&PageDescriptor, &'static str> {
+ let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
+ let desc = &self.lvl3[lvl2_index][lvl3_index];
+
+ Ok(desc)
+ }
+
/// Sets the PageDescriptor corresponding to the supplied page address.
///
/// Doesn't allow overriding an already valid page.
@@ -351,14 +414,15 @@
impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::TranslationTable
for FixedSizeTranslationTable<NUM_TABLES> for FixedSizeTranslationTable<NUM_TABLES>
{ {
@ -933,16 +1001,13 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati
// Populate the l2 entries. // Populate the l2 entries.
for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {
- let desc = - let phys_table_addr = self.lvl3[lvl2_nr].phys_start_addr();
- TableDescriptor::from_next_lvl_table_addr(self.lvl3[lvl2_nr].phys_start_addr()); + let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();
+ let addr = self.lvl3[lvl2_nr] + let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;
+ .virt_start_addr()
+ .try_into() let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);
+ .map_err(|_| "Translation error")?; *lvl2_entry = new_desc;
+ @@ -366,10 +430,8 @@
+ let desc = TableDescriptor::from_next_lvl_table_addr(addr);
*lvl2_entry = desc;
}
self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX;
self.initialized = true; self.initialized = true;
@ -954,61 +1019,52 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translati
} }
unsafe fn map_pages_at( unsafe fn map_pages_at(
@@ -442,6 +504,44 @@
diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs false
--- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs
@@ -15,7 +15,7 @@
use crate::{
bsp, memory,
- memory::{mmu::TranslationGranule, Address, Physical},
+ memory::{mmu::TranslationGranule, Address, Physical, Virtual},
};
use core::intrinsics::unlikely;
use cortex_a::{asm::barrier, registers::*};
@@ -109,7 +109,7 @@
//------------------------------------------------------------------------------
// OS Interface Code
//------------------------------------------------------------------------------
-use memory::mmu::MMUEnableError;
+use memory::mmu::{MMUEnableError, TranslationError};
impl memory::mmu::interface::MMU for MemoryManagementUnit {
unsafe fn enable_mmu_and_caching(
@@ -153,4 +153,31 @@
fn is_enabled(&self) -> bool {
SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)
} }
+ +
+ fn try_virt_to_phys( + fn try_virt_page_to_phys_page(
+ &self, + &self,
+ virt: Address<Virtual>, + virt_page: *const Page<Virtual>,
+ ) -> Result<Address<Physical>, TranslationError> { + ) -> Result<*const Page<Physical>, &'static str> {
+ if !self.is_enabled() { + let page_desc = self.page_descriptor_from_page(virt_page)?;
+ return Err(TranslationError::MMUDisabled);
+ }
+ +
+ let addr = virt.into_usize() as u64; + if !page_desc.is_valid() {
+ unsafe { + return Err("Page marked invalid");
+ asm!(
+ "AT S1E1R, {0}",
+ in(reg) addr,
+ options(readonly, nostack, preserves_flags)
+ );
+ } + }
+ +
+ let par_el1 = PAR_EL1.extract(); + Ok(page_desc.output_page())
+ if par_el1.matches_all(PAR_EL1::F::TranslationAborted) { + }
+ return Err(TranslationError::Aborted); +
+ fn try_page_attributes(
+ &self,
+ virt_page: *const Page<Virtual>,
+ ) -> Result<AttributeFields, &'static str> {
+ let page_desc = self.page_descriptor_from_page(virt_page)?;
+
+ if !page_desc.is_valid() {
+ return Err("Page marked invalid");
+ } + }
+ +
+ let phys_addr = (par_el1.read(PAR_EL1::PA) << 12) | (addr & 0xFFF); + page_desc.try_attributes()
+ }
+
+ /// Try to translate a virtual address to a physical address.
+ ///
+ /// Will only succeed if there exists a valid mapping for the input address.
+ fn try_virt_addr_to_phys_addr(
+ &self,
+ virt_addr: Address<Virtual>,
+ ) -> Result<Address<Physical>, &'static str> {
+ let page = self.try_virt_page_to_phys_page(virt_addr.as_page_ptr())?;
+ +
+ Ok(Address::new(phys_addr as usize)) + Ok(Address::new(page as usize + virt_addr.offset_into_page()))
+ } + }
} }
//--------------------------------------------------------------------------------------------------
diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/console.rs 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/console.rs diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/console.rs 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/console.rs
--- 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/console.rs --- 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/console.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/console.rs +++ 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/console.rs
@ -1087,15 +1143,18 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld 15_virtual
diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs
--- 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs --- 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs +++ 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs
@@ -16,6 +16,7 @@ @@ -9,8 +9,8 @@
memory::{
mmu as generic_mmu,
mmu::{
- AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields,
- MemAttributes, Page, PageSliceDescriptor, TranslationGranule,
+ AddressSpace, AssociatedTranslationTable, AttributeFields, Page, PageSliceDescriptor,
+ TranslationGranule,
},
Address, Physical, Virtual,
}, },
synchronization::InitStateLock, @@ -33,7 +33,7 @@
};
+use core::convert::TryInto;
//--------------------------------------------------------------------------------------------------
// Private Definitions
@@ -33,7 +34,7 @@
pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>;
/// The kernel's virtual address space defined by this BSP. /// The kernel's virtual address space defined by this BSP.
@ -1104,7 +1163,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Global instances // Global instances
@@ -45,13 +46,33 @@ @@ -45,13 +45,33 @@
/// ///
/// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`. /// That is, `size_of(InitStateLock<KernelTranslationTable>) == size_of(KernelTranslationTable)`.
/// There is a unit tests that checks this porperty. /// There is a unit tests that checks this porperty.
@ -1139,82 +1198,92 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v
/// Helper function for calculating the number of pages the given parameter spans. /// Helper function for calculating the number of pages the given parameter spans.
const fn size_to_num_pages(size: usize) -> usize { const fn size_to_num_pages(size: usize) -> usize {
assert!(size > 0); assert!(size > 0);
@@ -81,21 +102,22 @@ @@ -81,16 +101,22 @@
PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages)
} }
-// The binary is still identity mapped, so we don't need to convert in the following. -// The binary is still identity mapped, so use this trivial conversion function for mapping below.
-
+// There is no reason to expect the following conversions to fail, since they were generated offline +// There is no reason to expect the following conversions to fail, since they were generated offline
+// by the `translation table tool`. If it doesn't work, a panic due to the unwrap is justified. +// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.
fn kernel_virt_to_phys_page_slice(
/// The Read+Execute (RX) pages of the kernel binary. virt_slice: PageSliceDescriptor<Virtual>,
fn phys_rx_page_desc() -> PageSliceDescriptor<Physical> { ) -> PageSliceDescriptor<Physical> {
- virt_rx_page_desc().into() - let phys_start_addr = Address::<Physical>::new(virt_slice.start_addr().into_usize());
+ virt_rx_page_desc().try_into().unwrap() + let phys_first_page =
} + generic_mmu::try_kernel_virt_page_to_phys_page(virt_slice.first_page()).unwrap();
+ let phys_start_addr = Address::new(phys_first_page as usize);
/// The Read+Write (RW) pages of the kernel binary.
fn phys_rw_page_desc() -> PageSliceDescriptor<Physical> { PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages())
- virt_rw_page_desc().into()
+ virt_rw_page_desc().try_into().unwrap()
}
/// The boot core's stack.
fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor<Physical> {
- virt_boot_core_stack_page_desc().into()
+ virt_boot_core_stack_page_desc().try_into().unwrap()
} }
+fn kernel_page_attributes(virt_page: *const Page<Virtual>) -> AttributeFields {
+ generic_mmu::try_kernel_page_attributes(virt_page).unwrap()
+}
+
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@@ -115,13 +137,15 @@ // Public Code
//--------------------------------------------------------------------------------------------------
@@ -108,112 +134,33 @@
) as *const Page<_> ) as *const Page<_>
} }
-/// Map the kernel binary. -/// Map the kernel binary.
+/// Add mapping records for the kernel binary. -///
///
-/// # Safety -/// # Safety
+/// The actual translation table entries for the kernel binary are generated using the offline +/// Add mapping records for the kernel binary.
+/// `translation table tool` and patched into the kernel binary. This function just adds the mapping
+/// record entries.
/// ///
-/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. -/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking.
-pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { -pub unsafe fn kernel_map_binary() -> Result<(), &'static str> {
- generic_mmu::kernel_map_pages_at( - generic_mmu::kernel_map_pages_at(
+/// It must be ensured that these entries are in sync with the offline tool. +/// The actual translation table entries for the kernel binary are generated using the offline
+/// `translation table tool` and patched into the kernel binary. This function just adds the mapping
+/// record entries.
+pub fn kernel_add_mapping_records_for_precomputed() { +pub fn kernel_add_mapping_records_for_precomputed() {
+ let virt_rx_page_desc = virt_rx_page_desc();
+ generic_mmu::kernel_add_mapping_record( + generic_mmu::kernel_add_mapping_record(
"Kernel code and RO data", "Kernel code and RO data",
&virt_rx_page_desc(), - &virt_rx_page_desc(),
&phys_rx_page_desc(), - &kernel_virt_to_phys_page_slice(virt_rx_page_desc()),
@@ -130,9 +154,9 @@ - &AttributeFields {
acc_perms: AccessPermissions::ReadOnly, - mem_attributes: MemAttributes::CacheableDRAM,
execute_never: false, - acc_perms: AccessPermissions::ReadOnly,
}, - execute_never: false,
- },
- )?; - )?;
+ &virt_rx_page_desc,
+ &kernel_virt_to_phys_page_slice(virt_rx_page_desc),
+ &kernel_page_attributes(virt_rx_page_desc.first_page()),
+ ); + );
- generic_mmu::kernel_map_pages_at( - generic_mmu::kernel_map_pages_at(
+ let virt_rw_page_desc = virt_rw_page_desc();
+ generic_mmu::kernel_add_mapping_record( + generic_mmu::kernel_add_mapping_record(
"Kernel data and bss", "Kernel data and bss",
&virt_rw_page_desc(), - &virt_rw_page_desc(),
&phys_rw_page_desc(), - &kernel_virt_to_phys_page_slice(virt_rw_page_desc()),
@@ -141,9 +165,9 @@ - &AttributeFields {
acc_perms: AccessPermissions::ReadWrite, - mem_attributes: MemAttributes::CacheableDRAM,
execute_never: true, - acc_perms: AccessPermissions::ReadWrite,
}, - execute_never: true,
- },
- )?; - )?;
+ &virt_rw_page_desc,
+ &kernel_virt_to_phys_page_slice(virt_rw_page_desc),
+ &kernel_page_attributes(virt_rw_page_desc.first_page()),
+ ); + );
- generic_mmu::kernel_map_pages_at( - generic_mmu::kernel_map_pages_at(
+ let virt_boot_core_stack_page_desc = virt_boot_core_stack_page_desc();
+ generic_mmu::kernel_add_mapping_record( + generic_mmu::kernel_add_mapping_record(
"Kernel boot-core stack", "Kernel boot-core stack",
&virt_boot_core_stack_page_desc(), - &virt_boot_core_stack_page_desc(),
&phys_boot_core_stack_page_desc(), - &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc()),
@@ -152,75 +176,5 @@ - &AttributeFields {
acc_perms: AccessPermissions::ReadWrite, - mem_attributes: MemAttributes::CacheableDRAM,
execute_never: true, - acc_perms: AccessPermissions::ReadWrite,
}, - execute_never: true,
- },
- )?; - )?;
- -
- Ok(()) - Ok(())
@ -1286,6 +1355,9 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v
- -
- assert!(bss_range.contains(&kernel_tables_addr)); - assert!(bss_range.contains(&kernel_tables_addr));
- } - }
+ &virt_boot_core_stack_page_desc,
+ &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc),
+ &kernel_page_attributes(virt_boot_core_stack_page_desc.first_page()),
+ ); + );
} }
@ -1345,7 +1417,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15
use crate::memory::{ use crate::memory::{
mmu::{AttributeFields, PageSliceDescriptor}, mmu::{AttributeFields, PageSliceDescriptor},
- Address, Physical, Virtual, - Address, Physical, Virtual,
+ Physical, Virtual, + Address, Page, Physical, Virtual,
}; };
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -1361,7 +1433,38 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15
/// Map the given virtual pages to the given physical pages. /// Map the given virtual pages to the given physical pages.
/// ///
@@ -80,7 +77,7 @@ @@ -70,6 +67,30 @@
/// Check if a virtual page splice is in the "MMIO region".
fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor<Virtual>) -> bool;
+
+ /// Try to translate a virtual page pointer to a physical page pointer.
+ ///
+ /// Will only succeed if there exists a valid mapping for the input page.
+ fn try_virt_page_to_phys_page(
+ &self,
+ virt_page: *const Page<Virtual>,
+ ) -> Result<*const Page<Physical>, &'static str>;
+
+ /// Try to get the attributes of a page.
+ ///
+ /// Will only succeed if there exists a valid mapping for the input page.
+ fn try_page_attributes(
+ &self,
+ virt_page: *const Page<Virtual>,
+ ) -> Result<AttributeFields, &'static str>;
+
+ /// Try to translate a virtual address to a physical address.
+ ///
+ /// Will only succeed if there exists a valid mapping for the input address.
+ fn try_virt_addr_to_phys_addr(
+ &self,
+ virt_addr: Address<Virtual>,
+ ) -> Result<Address<Physical>, &'static str>;
}
}
@@ -80,7 +101,7 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@ -1370,7 +1473,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15
use arch_translation_table::MinSizeTranslationTable; use arch_translation_table::MinSizeTranslationTable;
use interface::TranslationTable; use interface::TranslationTable;
use test_macros::kernel_test; use test_macros::kernel_test;
@@ -89,9 +86,9 @@ @@ -89,9 +110,9 @@
#[kernel_test] #[kernel_test]
fn translationtable_implementation_sanity() { fn translationtable_implementation_sanity() {
// This will occupy a lot of space on the stack. // This will occupy a lot of space on the stack.
@ -1386,75 +1489,20 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs 15
diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs 15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs 15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs
--- 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs --- 14_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs +++ 15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs
@@ -8,7 +8,10 @@ @@ -92,7 +92,7 @@
bsp, common,
memory::{Address, AddressType, Physical, Virtual},
};
-use core::{convert::From, marker::PhantomData};
+use core::{
+ convert::{From, TryFrom},
+ marker::PhantomData,
+};
//--------------------------------------------------------------------------------------------------
// Public Definitions
@@ -136,12 +139,16 @@
} }
}
-impl From<PageSliceDescriptor<Virtual>> for PageSliceDescriptor<Physical> { /// Return a pointer to the first page of the described slice.
- fn from(desc: PageSliceDescriptor<Virtual>) -> Self { - const fn first_page(&self) -> *const Page<ATYPE> {
- Self { + pub const fn first_page(&self) -> *const Page<ATYPE> {
- start: Address::new(desc.start.into_usize()), self.start.into_usize() as *const _
+impl TryFrom<PageSliceDescriptor<Virtual>> for PageSliceDescriptor<Physical> {
+ type Error = super::TranslationError;
+
+ fn try_from(desc: PageSliceDescriptor<Virtual>) -> Result<Self, Self::Error> {
+ let phys_start = super::try_virt_to_phys(desc.start)?;
+
+ Ok(Self {
+ start: phys_start,
num_pages: desc.num_pages,
- }
+ })
} }
}
diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs
--- 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs --- 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs +++ 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs
@@ -33,6 +33,14 @@ @@ -92,9 +92,7 @@
Other(&'static str),
}
+/// Translation error variants.
+#[allow(missing_docs)]
+#[derive(Debug)]
+pub enum TranslationError {
+ MMUDisabled,
+ Aborted,
+}
+
/// Memory Management interfaces.
pub mod interface {
use super::*;
@@ -51,6 +59,14 @@
/// Returns true if the MMU is enabled, false otherwise.
fn is_enabled(&self) -> bool;
+
+ /// Try to translate a virtual address to a physical address.
+ ///
+ /// Will only succeed if there exists a valid mapping for the input VA.
+ fn try_virt_to_phys(
+ &self,
+ virt: Address<Virtual>,
+ ) -> Result<Address<Physical>, TranslationError>;
}
}
@@ -92,9 +108,7 @@
bsp::memory::mmu::kernel_translation_tables() bsp::memory::mmu::kernel_translation_tables()
.write(|tables| tables.map_pages_at(virt_pages, phys_pages, attr))?; .write(|tables| tables.map_pages_at(virt_pages, phys_pages, attr))?;
@ -1465,7 +1513,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3
Ok(()) Ok(())
} }
@@ -146,6 +160,18 @@ @@ -146,6 +144,18 @@
} }
} }
@ -1484,14 +1532,14 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3
/// Raw mapping of virtual to physical pages in the kernel translation tables. /// Raw mapping of virtual to physical pages in the kernel translation tables.
/// ///
/// Prevents mapping into the MMIO range of the tables. /// Prevents mapping into the MMIO range of the tables.
@@ -214,21 +240,11 @@ @@ -214,21 +224,34 @@
Ok(virt_addr + offset_into_start_page) Ok(virt_addr + offset_into_start_page)
} }
-/// Map the kernel's binary. Returns the translation table's base address. -/// Map the kernel's binary. Returns the translation table's base address.
-/// -///
-/// # Safety -/// # Safety
+/// Try to translate a virtual address to a physical address. +/// Try to translate a kernel virtual page pointer to a physical page pointer.
/// ///
-/// - See [`bsp::memory::mmu::kernel_map_binary()`]. -/// - See [`bsp::memory::mmu::kernel_map_binary()`].
-pub unsafe fn kernel_map_binary() -> Result<Address<Physical>, &'static str> { -pub unsafe fn kernel_map_binary() -> Result<Address<Physical>, &'static str> {
@ -1500,17 +1548,38 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3
- tables.init(); - tables.init();
- tables.phys_base_address() - tables.phys_base_address()
- }); - });
- +/// Will only succeed if there exists a valid mapping for the input page.
+pub fn try_kernel_virt_page_to_phys_page(
+ virt_page: *const Page<Virtual>,
+) -> Result<*const Page<Physical>, &'static str> {
+ bsp::memory::mmu::kernel_translation_tables()
+ .read(|tables| tables.try_virt_page_to_phys_page(virt_page))
+}
- bsp::memory::mmu::kernel_map_binary()?; - bsp::memory::mmu::kernel_map_binary()?;
- +/// Try to get the attributes of a kernel page.
+///
+/// Will only succeed if there exists a valid mapping for the input page.
+pub fn try_kernel_page_attributes(
+ virt_page: *const Page<Virtual>,
+) -> Result<AttributeFields, &'static str> {
+ bsp::memory::mmu::kernel_translation_tables()
+ .read(|tables| tables.try_page_attributes(virt_page))
+}
- Ok(phys_kernel_tables_base_addr) - Ok(phys_kernel_tables_base_addr)
+/// Will only succeed if there exists a valid mapping for the input VA. +/// Try to translate a kernel virtual address to a physical address.
+pub fn try_virt_to_phys(virt: Address<Virtual>) -> Result<Address<Physical>, TranslationError> { +///
+ arch_mmu::mmu().try_virt_to_phys(virt) +/// Will only succeed if there exists a valid mapping for the input address.
+fn try_kernel_virt_addr_to_phys_addr(
+ virt_addr: Address<Virtual>,
+) -> Result<Address<Physical>, &'static str> {
+ bsp::memory::mmu::kernel_translation_tables()
+ .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))
} }
/// Enable the MMU and data + instruction caching. /// Enable the MMU and data + instruction caching.
@@ -236,6 +252,7 @@ @@ -236,6 +259,7 @@
/// # Safety /// # Safety
/// ///
/// - Crucial function during kernel init. Changes the the complete memory view of the processor. /// - Crucial function during kernel init. Changes the the complete memory view of the processor.
@ -1522,29 +1591,39 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory/mmu.rs 15_virtual_mem_part3
diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory.rs 15_virtual_mem_part3_precomputed_tables/src/memory.rs diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory.rs 15_virtual_mem_part3_precomputed_tables/src/memory.rs
--- 14_virtual_mem_part2_mmio_remap/src/memory.rs --- 14_virtual_mem_part2_mmio_remap/src/memory.rs
+++ 15_virtual_mem_part3_precomputed_tables/src/memory.rs +++ 15_virtual_mem_part3_precomputed_tables/src/memory.rs
@@ -8,6 +8,7 @@ @@ -6,12 +6,13 @@
pub mod mmu;
use crate::common; -use crate::common;
+use crate::{bsp, common};
use core::{ use core::{
+ convert::TryFrom,
fmt, fmt,
marker::PhantomData, marker::PhantomData,
ops::{AddAssign, SubAssign}, ops::{AddAssign, SubAssign},
@@ -67,6 +68,14 @@ };
} +use mmu::Page;
}
+impl TryFrom<Address<Virtual>> for Address<Physical> { //--------------------------------------------------------------------------------------------------
+ type Error = mmu::TranslationError; // Public Definitions
@@ -65,6 +66,17 @@
pub const fn into_usize(self) -> usize {
self.value
}
+ +
+ fn try_from(virt: Address<Virtual>) -> Result<Self, Self::Error> { + /// Return a pointer to the page that contains this address.
+ mmu::try_virt_to_phys(virt) + pub const fn as_page_ptr(&self) -> *const Page<ATYPE> {
+ self.align_down(bsp::memory::mmu::KernelGranule::SIZE)
+ .into_usize() as *const _
+ } + }
+}
+ +
impl<ATYPE: AddressType> core::ops::Add<usize> for Address<ATYPE> { + /// Return the address' offset into the underlying page.
type Output = Self; + pub const fn offset_into_page(&self) -> usize {
+ self.value & bsp::memory::mmu::KernelGranule::MASK
+ }
}
impl<ATYPE: AddressType> core::ops::Add<usize> for Address<ATYPE> {
diff -uNr 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs diff -uNr 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs
--- 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs --- 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs

31
15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu.rs
Unescape Escape View File

@ -15,7 +15,7 @@
use crate::{ use crate::{
bsp, memory, bsp, memory,
memory::{mmu::TranslationGranule, Address, Physical, Virtual}, memory::{mmu::TranslationGranule, Address, Physical},
}; };
use core::intrinsics::unlikely; use core::intrinsics::unlikely;
use cortex_a::{asm::barrier, registers::*}; use cortex_a::{asm::barrier, registers::*};
@ -109,7 +109,7 @@ pub fn mmu() -> &'static impl memory::mmu::interface::MMU {
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// OS Interface Code // OS Interface Code
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
use memory::mmu::{MMUEnableError, TranslationError}; use memory::mmu::MMUEnableError;
impl memory::mmu::interface::MMU for MemoryManagementUnit { impl memory::mmu::interface::MMU for MemoryManagementUnit {
unsafe fn enable_mmu_and_caching( unsafe fn enable_mmu_and_caching(
@ -153,31 +153,4 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit {
fn is_enabled(&self) -> bool { fn is_enabled(&self) -> bool {
SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)
} }
fn try_virt_to_phys(
&self,
virt: Address<Virtual>,
) -> Result<Address<Physical>, TranslationError> {
if !self.is_enabled() {
return Err(TranslationError::MMUDisabled);
}
let addr = virt.into_usize() as u64;
unsafe {
asm!(
"AT S1E1R, {0}",
in(reg) addr,
options(readonly, nostack, preserves_flags)
);
}
let par_el1 = PAR_EL1.extract();
if par_el1.matches_all(PAR_EL1::F::TranslationAborted) {
return Err(TranslationError::Aborted);
}
let phys_addr = (par_el1.read(PAR_EL1::PA) << 12) | (addr & 0xFFF);
Ok(Address::new(phys_addr as usize))
}
} }

148
15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/translation_table.rs
Unescape Escape View File

@ -23,7 +23,7 @@ use crate::{
Address, Physical, Virtual, Address, Physical, Virtual,
}, },
}; };
use core::convert::{self, TryInto}; use core::convert;
use tock_registers::{ use tock_registers::{
interfaces::{Readable, Writeable}, interfaces::{Readable, Writeable},
register_bitfields, register_bitfields,
@ -219,6 +219,35 @@ impl convert::From<AttributeFields>
} }
} }
/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.
impl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {
type Error = &'static str;
fn try_from(
desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,
) -> Result<AttributeFields, Self::Error> {
let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {
memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,
memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,
_ => return Err("Unexpected memory attribute"),
};
let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {
Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,
Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,
_ => return Err("Unexpected access permission"),
};
let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;
Ok(AttributeFields {
mem_attributes,
acc_perms,
execute_never,
})
}
}
impl PageDescriptor { impl PageDescriptor {
/// Create an instance. /// Create an instance.
/// ///
@ -228,13 +257,13 @@ impl PageDescriptor {
} }
/// Create an instance. /// Create an instance.
pub fn from_output_addr( pub fn from_output_page(
phys_output_addr: *const Page<Physical>, phys_output_page: *const Page<Physical>,
attribute_fields: &AttributeFields, attribute_fields: &AttributeFields,
) -> Self { ) -> Self {
let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0); let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);
let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT;
val.write( val.write(
STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)
+ STAGE1_PAGE_DESCRIPTOR::AF::True + STAGE1_PAGE_DESCRIPTOR::AF::True
@ -251,6 +280,20 @@ impl PageDescriptor {
InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value) InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)
.is_set(STAGE1_PAGE_DESCRIPTOR::VALID) .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)
} }
/// Returns the output page.
fn output_page(&self) -> *const Page<Physical> {
let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)
.read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB);
let addr = shifted << Granule64KiB::SHIFT;
addr as *const Page<Physical>
}
/// Returns the attributes.
fn try_attributes(&self) -> Result<AttributeFields, &'static str> {
InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()
}
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -316,11 +359,11 @@ impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {
/// Helper to calculate the lvl2 and lvl3 indices from an address. /// Helper to calculate the lvl2 and lvl3 indices from an address.
#[inline(always)] #[inline(always)]
fn lvl2_lvl3_index_from( fn lvl2_lvl3_index_from_page(
&self, &self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<(usize, usize), &'static str> { ) -> Result<(usize, usize), &'static str> {
let addr = addr as usize; let addr = virt_page as usize;
let lvl2_index = addr >> Granule512MiB::SHIFT; let lvl2_index = addr >> Granule512MiB::SHIFT;
let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;
@ -331,15 +374,36 @@ impl<const NUM_TABLES: usize> FixedSizeTranslationTable<NUM_TABLES> {
Ok((lvl2_index, lvl3_index)) Ok((lvl2_index, lvl3_index))
} }
/// Returns the PageDescriptor corresponding to the supplied Page. /// Returns the PageDescriptor corresponding to the supplied page address.
#[inline(always)] #[inline(always)]
fn page_descriptor_from( fn page_descriptor_from_page(
&self,
virt_page: *const Page<Virtual>,
) -> Result<&PageDescriptor, &'static str> {
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
let desc = &self.lvl3[lvl2_index][lvl3_index];
Ok(desc)
}
/// Sets the PageDescriptor corresponding to the supplied page address.
///
/// Doesn't allow overriding an already valid page.
#[inline(always)]
fn set_page_descriptor_from_page(
&mut self, &mut self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<&mut PageDescriptor, &'static str> { new_desc: &PageDescriptor,
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; ) -> Result<(), &'static str> {
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
let desc = &mut self.lvl3[lvl2_index][lvl3_index];
if desc.is_valid() {
return Err("Virtual page is already mapped");
}
Ok(&mut self.lvl3[lvl2_index][lvl3_index]) *desc = *new_desc;
Ok(())
} }
} }
@ -357,13 +421,11 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
// Populate the l2 entries. // Populate the l2 entries.
for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {
let addr = self.lvl3[lvl2_nr] let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();
.virt_start_addr() let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;
.try_into()
.map_err(|_| "Translation error")?;
let desc = TableDescriptor::from_next_lvl_table_addr(addr); let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);
*lvl2_entry = desc; *lvl2_entry = new_desc;
} }
self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX;
@ -380,8 +442,8 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
) -> Result<(), &'static str> { ) -> Result<(), &'static str> {
assert!(self.initialized, "Translation tables not initialized"); assert!(self.initialized, "Translation tables not initialized");
let p = phys_pages.as_slice();
let v = virt_pages.as_slice(); let v = virt_pages.as_slice();
let p = phys_pages.as_slice();
// No work to do for empty slices. // No work to do for empty slices.
if v.is_empty() { if v.is_empty() {
@ -398,12 +460,10 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
let iter = p.iter().zip(v.iter()); let iter = p.iter().zip(v.iter());
for (phys_page, virt_page) in iter { for (phys_page, virt_page) in iter {
let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr);
if page_descriptor.is_valid() { let virt_page = virt_page.as_ptr();
return Err("Virtual page is already mapped");
}
*page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); self.set_page_descriptor_from_page(virt_page, &new_desc)?;
} }
Ok(()) Ok(())
@ -444,6 +504,44 @@ impl<const NUM_TABLES: usize> memory::mmu::translation_table::interface::Transla
false false
} }
fn try_virt_page_to_phys_page(
&self,
virt_page: *const Page<Virtual>,
) -> Result<*const Page<Physical>, &'static str> {
let page_desc = self.page_descriptor_from_page(virt_page)?;
if !page_desc.is_valid() {
return Err("Page marked invalid");
}
Ok(page_desc.output_page())
}
fn try_page_attributes(
&self,
virt_page: *const Page<Virtual>,
) -> Result<AttributeFields, &'static str> {
let page_desc = self.page_descriptor_from_page(virt_page)?;
if !page_desc.is_valid() {
return Err("Page marked invalid");
}
page_desc.try_attributes()
}
/// Try to translate a virtual address to a physical address.
///
/// Will only succeed if there exists a valid mapping for the input address.
fn try_virt_addr_to_phys_addr(
&self,
virt_addr: Address<Virtual>,
) -> Result<Address<Physical>, &'static str> {
let page = self.try_virt_page_to_phys_page(virt_addr.as_page_ptr())?;
Ok(Address::new(page as usize + virt_addr.offset_into_page()))
}
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------

66
15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs
Unescape Escape View File

@ -9,14 +9,13 @@ use crate::{
memory::{ memory::{
mmu as generic_mmu, mmu as generic_mmu,
mmu::{ mmu::{
AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, AddressSpace, AssociatedTranslationTable, AttributeFields, Page, PageSliceDescriptor,
MemAttributes, Page, PageSliceDescriptor, TranslationGranule, TranslationGranule,
}, },
Physical, Virtual, Address, Physical, Virtual,
}, },
synchronization::InitStateLock, synchronization::InitStateLock,
}; };
use core::convert::TryInto;
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Private Definitions // Private Definitions
@ -103,21 +102,19 @@ fn virt_boot_core_stack_page_desc() -> PageSliceDescriptor<Virtual> {
} }
// There is no reason to expect the following conversions to fail, since they were generated offline // There is no reason to expect the following conversions to fail, since they were generated offline
// by the `translation table tool`. If it doesn't work, a panic due to the unwrap is justified. // by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.
fn kernel_virt_to_phys_page_slice(
/// The Read+Execute (RX) pages of the kernel binary. virt_slice: PageSliceDescriptor<Virtual>,
fn phys_rx_page_desc() -> PageSliceDescriptor<Physical> { ) -> PageSliceDescriptor<Physical> {
virt_rx_page_desc().try_into().unwrap() let phys_first_page =
} generic_mmu::try_kernel_virt_page_to_phys_page(virt_slice.first_page()).unwrap();
let phys_start_addr = Address::new(phys_first_page as usize);
/// The Read+Write (RW) pages of the kernel binary.
fn phys_rw_page_desc() -> PageSliceDescriptor<Physical> { PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages())
virt_rw_page_desc().try_into().unwrap()
} }
/// The boot core's stack. fn kernel_page_attributes(virt_page: *const Page<Virtual>) -> AttributeFields {
fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor<Physical> { generic_mmu::try_kernel_page_attributes(virt_page).unwrap()
virt_boot_core_stack_page_desc().try_into().unwrap()
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -142,39 +139,28 @@ pub fn phys_addr_space_end_page() -> *const Page<Physical> {
/// The actual translation table entries for the kernel binary are generated using the offline /// The actual translation table entries for the kernel binary are generated using the offline
/// `translation table tool` and patched into the kernel binary. This function just adds the mapping /// `translation table tool` and patched into the kernel binary. This function just adds the mapping
/// record entries. /// record entries.
///
/// It must be ensured that these entries are in sync with the offline tool.
pub fn kernel_add_mapping_records_for_precomputed() { pub fn kernel_add_mapping_records_for_precomputed() {
let virt_rx_page_desc = virt_rx_page_desc();
generic_mmu::kernel_add_mapping_record( generic_mmu::kernel_add_mapping_record(
"Kernel code and RO data", "Kernel code and RO data",
&virt_rx_page_desc(), &virt_rx_page_desc,
&phys_rx_page_desc(), &kernel_virt_to_phys_page_slice(virt_rx_page_desc),
&AttributeFields { &kernel_page_attributes(virt_rx_page_desc.first_page()),
mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadOnly,
execute_never: false,
},
); );
let virt_rw_page_desc = virt_rw_page_desc();
generic_mmu::kernel_add_mapping_record( generic_mmu::kernel_add_mapping_record(
"Kernel data and bss", "Kernel data and bss",
&virt_rw_page_desc(), &virt_rw_page_desc,
&phys_rw_page_desc(), &kernel_virt_to_phys_page_slice(virt_rw_page_desc),
&AttributeFields { &kernel_page_attributes(virt_rw_page_desc.first_page()),
mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadWrite,
execute_never: true,
},
); );
let virt_boot_core_stack_page_desc = virt_boot_core_stack_page_desc();
generic_mmu::kernel_add_mapping_record( generic_mmu::kernel_add_mapping_record(
"Kernel boot-core stack", "Kernel boot-core stack",
&virt_boot_core_stack_page_desc(), &virt_boot_core_stack_page_desc,
&phys_boot_core_stack_page_desc(), &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc),
&AttributeFields { &kernel_page_attributes(virt_boot_core_stack_page_desc.first_page()),
mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadWrite,
execute_never: true,
},
); );
} }

17
15_virtual_mem_part3_precomputed_tables/src/memory.rs
Unescape Escape View File

@ -6,13 +6,13 @@
pub mod mmu; pub mod mmu;
use crate::common; use crate::{bsp, common};
use core::{ use core::{
convert::TryFrom,
fmt, fmt,
marker::PhantomData, marker::PhantomData,
ops::{AddAssign, SubAssign}, ops::{AddAssign, SubAssign},
}; };
use mmu::Page;
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Public Definitions // Public Definitions
@ -66,13 +66,16 @@ impl<ATYPE: AddressType> Address<ATYPE> {
pub const fn into_usize(self) -> usize { pub const fn into_usize(self) -> usize {
self.value self.value
} }
}
impl TryFrom<Address<Virtual>> for Address<Physical> { /// Return a pointer to the page that contains this address.
type Error = mmu::TranslationError; pub const fn as_page_ptr(&self) -> *const Page<ATYPE> {
self.align_down(bsp::memory::mmu::KernelGranule::SIZE)
.into_usize() as *const _
}
fn try_from(virt: Address<Virtual>) -> Result<Self, Self::Error> { /// Return the address' offset into the underlying page.
mmu::try_virt_to_phys(virt) pub const fn offset_into_page(&self) -> usize {
self.value & bsp::memory::mmu::KernelGranule::MASK
} }
} }

47
15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs
Unescape Escape View File

@ -33,14 +33,6 @@ pub enum MMUEnableError {
Other(&'static str), Other(&'static str),
} }
/// Translation error variants.
#[allow(missing_docs)]
#[derive(Debug)]
pub enum TranslationError {
MMUDisabled,
Aborted,
}
/// Memory Management interfaces. /// Memory Management interfaces.
pub mod interface { pub mod interface {
use super::*; use super::*;
@ -59,14 +51,6 @@ pub mod interface {
/// Returns true if the MMU is enabled, false otherwise. /// Returns true if the MMU is enabled, false otherwise.
fn is_enabled(&self) -> bool; fn is_enabled(&self) -> bool;
/// Try to translate a virtual address to a physical address.
///
/// Will only succeed if there exists a valid mapping for the input VA.
fn try_virt_to_phys(
&self,
virt: Address<Virtual>,
) -> Result<Address<Physical>, TranslationError>;
} }
} }
@ -240,11 +224,34 @@ pub unsafe fn kernel_map_mmio(
Ok(virt_addr + offset_into_start_page) Ok(virt_addr + offset_into_start_page)
} }
/// Try to translate a virtual address to a physical address. /// Try to translate a kernel virtual page pointer to a physical page pointer.
///
/// Will only succeed if there exists a valid mapping for the input page.
pub fn try_kernel_virt_page_to_phys_page(
virt_page: *const Page<Virtual>,
) -> Result<*const Page<Physical>, &'static str> {
bsp::memory::mmu::kernel_translation_tables()
.read(|tables| tables.try_virt_page_to_phys_page(virt_page))
}
/// Try to get the attributes of a kernel page.
///
/// Will only succeed if there exists a valid mapping for the input page.
pub fn try_kernel_page_attributes(
virt_page: *const Page<Virtual>,
) -> Result<AttributeFields, &'static str> {
bsp::memory::mmu::kernel_translation_tables()
.read(|tables| tables.try_page_attributes(virt_page))
}
/// Try to translate a kernel virtual address to a physical address.
/// ///
/// Will only succeed if there exists a valid mapping for the input VA. /// Will only succeed if there exists a valid mapping for the input address.
pub fn try_virt_to_phys(virt: Address<Virtual>) -> Result<Address<Physical>, TranslationError> { fn try_kernel_virt_addr_to_phys_addr(
arch_mmu::mmu().try_virt_to_phys(virt) virt_addr: Address<Virtual>,
) -> Result<Address<Physical>, &'static str> {
bsp::memory::mmu::kernel_translation_tables()
.read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))
} }
/// Enable the MMU and data + instruction caching. /// Enable the MMU and data + instruction caching.

26
15_virtual_mem_part3_precomputed_tables/src/memory/mmu/translation_table.rs
Unescape Escape View File

@ -10,7 +10,7 @@ mod arch_translation_table;
use crate::memory::{ use crate::memory::{
mmu::{AttributeFields, PageSliceDescriptor}, mmu::{AttributeFields, PageSliceDescriptor},
Physical, Virtual, Address, Page, Physical, Virtual,
}; };
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -67,6 +67,30 @@ pub mod interface {
/// Check if a virtual page splice is in the "MMIO region". /// Check if a virtual page splice is in the "MMIO region".
fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor<Virtual>) -> bool; fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor<Virtual>) -> bool;
/// Try to translate a virtual page pointer to a physical page pointer.
///
/// Will only succeed if there exists a valid mapping for the input page.
fn try_virt_page_to_phys_page(
&self,
virt_page: *const Page<Virtual>,
) -> Result<*const Page<Physical>, &'static str>;
/// Try to get the attributes of a page.
///
/// Will only succeed if there exists a valid mapping for the input page.
fn try_page_attributes(
&self,
virt_page: *const Page<Virtual>,
) -> Result<AttributeFields, &'static str>;
/// Try to translate a virtual address to a physical address.
///
/// Will only succeed if there exists a valid mapping for the input address.
fn try_virt_addr_to_phys_addr(
&self,
virt_addr: Address<Virtual>,
) -> Result<Address<Physical>, &'static str>;
} }
} }

28
15_virtual_mem_part3_precomputed_tables/src/memory/mmu/types.rs
Unescape Escape View File

@ -6,12 +6,9 @@
use crate::{ use crate::{
bsp, common, bsp, common,
memory::{Address, AddressType, Physical, Virtual}, memory::{Address, AddressType, Physical},
};
use core::{
convert::{From, TryFrom},
marker::PhantomData,
}; };
use core::{convert::From, marker::PhantomData};
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Public Definitions // Public Definitions
@ -95,11 +92,11 @@ impl<ATYPE: AddressType> PageSliceDescriptor<ATYPE> {
} }
/// Return a pointer to the first page of the described slice. /// Return a pointer to the first page of the described slice.
const fn first_page_ptr(&self) -> *const Page<ATYPE> { pub const fn first_page(&self) -> *const Page<ATYPE> {
self.start.into_usize() as *const _ self.start.into_usize() as *const _
} }
/// Return the number of Pages the slice describes. /// Return the number of pages the slice describes.
pub const fn num_pages(&self) -> usize { pub const fn num_pages(&self) -> usize {
self.num_pages self.num_pages
} }
@ -129,26 +126,13 @@ impl<ATYPE: AddressType> PageSliceDescriptor<ATYPE> {
(addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive())
} }
/// Return a non-mutable slice of Pages. /// Return a non-mutable slice of pages.
/// ///
/// # Safety /// # Safety
/// ///
/// - Same as applies for `core::slice::from_raw_parts`. /// - Same as applies for `core::slice::from_raw_parts`.
pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] { pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] {
core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) core::slice::from_raw_parts(self.first_page(), self.num_pages)
}
}
impl TryFrom<PageSliceDescriptor<Virtual>> for PageSliceDescriptor<Physical> {
type Error = super::TranslationError;
fn try_from(desc: PageSliceDescriptor<Virtual>) -> Result<Self, Self::Error> {
let phys_start = super::try_virt_to_phys(desc.start)?;
Ok(Self {
start: phys_start,
num_pages: desc.num_pages,
})
} }
} }

93
16_virtual_mem_part4_higher_half_kernel/README.md
Unescape Escape View File

@ -144,7 +144,7 @@ Precomputing kernel translation tables and patching kernel ELF
Generating Boot-core stack | 0xffff_ffff_8010_0000 | 512 KiB Generating Boot-core stack | 0xffff_ffff_8010_0000 | 512 KiB
-------------------------------------------------- --------------------------------------------------
Patching Kernel table struct at physical 0x9_0000 Patching Kernel table struct at physical 0x9_0000
Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0060 Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0080
Finished in 0.03s Finished in 0.03s
Minipush 1.0 Minipush 1.0
@ -152,6 +152,7 @@ Minipush 1.0
[MP] ⏳ Waiting for /dev/ttyUSB0 [MP] ⏳ Waiting for /dev/ttyUSB0
[MP] ✅ Serial connected [MP] ✅ Serial connected
[MP] 🔌 Please power the target now [MP] 🔌 Please power the target now
__ __ _ _ _ _ __ __ _ _ _ _
| \/ (_)_ _ (_) | ___ __ _ __| | | \/ (_)_ _ (_) | ___ __ _ __| |
| |\/| | | ' \| | |__/ _ \/ _` / _` | | |\/| | | ' \| | |__/ _ \/ _` / _` |
@ -163,19 +164,20 @@ Minipush 1.0
[MP] ⏩ Pushing 387 KiB =======================================🦀 100% 96 KiB/s Time: 00:00:04 [MP] ⏩ Pushing 387 KiB =======================================🦀 100% 96 KiB/s Time: 00:00:04
[ML] Loaded! Executing the payload now [ML] Loaded! Executing the payload now
[ 4.316420] mingo version 0.16.0 [ 4.318584] mingo version 0.16.0
[ 4.316627] Booting on: Raspberry Pi 3 [ 4.318792] Booting on: Raspberry Pi 3
[ 4.317082] MMU online: [ 4.319247] MMU online:
[ 4.317375] ------------------------------------------------------------------------------------------------------------------------------------------- [ 4.319540] -------------------------------------------------------------------------------------------------------------------------------------------
[ 4.319119] Virtual Physical Size Attr Entity [ 4.321284] Virtual Physical Size Attr Entity
[ 4.320863] ------------------------------------------------------------------------------------------------------------------------------------------- [ 4.323028] -------------------------------------------------------------------------------------------------------------------------------------------
[ 4.322610] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data [ 4.324773] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data
[ 4.324223] 0xffff_ffff_8009_0000..0xffff_ffff_800e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C RW XN | Kernel data and bss [ 4.326387] 0xffff_ffff_8009_0000..0xffff_ffff_800e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C RW XN | Kernel data and bss
[ 4.325793] 0xffff_ffff_8010_0000..0xffff_ffff_8017_ffff --> 0x00_0010_0000..0x00_0017_ffff | 512 KiB | C RW XN | Kernel boot-core stack [ 4.327957] 0xffff_ffff_8010_0000..0xffff_ffff_8017_ffff --> 0x00_0010_0000..0x00_0017_ffff | 512 KiB | C RW XN | Kernel boot-core stack
[ 4.327397] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_3f20_0000..0x00_3f20_ffff | 64 KiB | Dev RW XN | BCM GPIO [ 4.329560] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_3f20_0000..0x00_3f20_ffff | 64 KiB | Dev RW XN | BCM GPIO
[ 4.328847] | BCM PL011 UART [ 4.331012] | BCM PL011 UART
[ 4.330365] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_3f00_0000..0x00_3f00_ffff | 64 KiB | Dev RW XN | BCM Peripheral Interrupt Controller [ 4.332529] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_3f00_0000..0x00_3f00_ffff | 64 KiB | Dev RW XN | BCM Peripheral Interrupt Controller
[ 4.332108] ------------------------------------------------------------------------------------------------------------------------------------------- [ 4.334273] -------------------------------------------------------------------------------------------------------------------------------------------
``` ```
Raspberry Pi 4: Raspberry Pi 4:
@ -189,18 +191,20 @@ Precomputing kernel translation tables and patching kernel ELF
Section Start Virt Addr Size Section Start Virt Addr Size
-------------------------------------------------- --------------------------------------------------
Generating Code and RO data | 0xffff_ffff_8008_0000 | 64 KiB Generating Code and RO data | 0xffff_ffff_8008_0000 | 64 KiB
Generating Data and bss | 0xffff_ffff_8009_0000 | 448 KiB Generating Data and bss | 0xffff_ffff_8009_0000 | 384 KiB
Generating Boot-core stack | 0xffff_ffff_8011_0000 | 512 KiB Generating Boot-core stack | 0xffff_ffff_8010_0000 | 512 KiB
-------------------------------------------------- --------------------------------------------------
Patching Kernel table struct at physical 0xa_0000 Patching Kernel table struct at physical 0x9_0000
Patching Value of kernel table physical base address (0xe_0000) at physical 0x8_0068 Patching Value of kernel table physical base address (0xd_0000) at physical 0x8_0080
Finished in 0.03s Finished in 0.03s
Minipush 1.0 Minipush 1.0
[MP] ⏳ Waiting for /dev/ttyUSB0 [MP] ⏳ Waiting for /dev/ttyUSB0
[MP] ✅ Serial connected [MP] ✅ Serial connected
[MP] 🔌 Please power the target now [MP] 🔌 Please power the target now
__ __ _ _ _ _ __ __ _ _ _ _
| \/ (_)_ _ (_) | ___ __ _ __| | | \/ (_)_ _ (_) | ___ __ _ __| |
| |\/| | | ' \| | |__/ _ \/ _` / _` | | |\/| | | ' \| | |__/ _ \/ _` / _` |
@ -209,23 +213,24 @@ Minipush 1.0
Raspberry Pi 4 Raspberry Pi 4
[ML] Requesting binary [ML] Requesting binary
[MP] ⏩ Pushing 449 KiB ======================================🦀 100% 112 KiB/s Time: 00:00:04 [MP] ⏩ Pushing 394 KiB =======================================🦀 100% 98 KiB/s Time: 00:00:04
[ML] Loaded! Executing the payload now [ML] Loaded! Executing the payload now
[ 5.009551] mingo version 0.16.0 [ 4.401227] mingo version 0.16.0
[ 5.009585] Booting on: Raspberry Pi 4 [ 4.401260] Booting on: Raspberry Pi 4
[ 5.010040] MMU online: [ 4.401715] MMU online:
[ 5.010332] ------------------------------------------------------------------------------------------------------------------------------------------- [ 4.402008] -------------------------------------------------------------------------------------------------------------------------------------------
[ 5.012077] Virtual Physical Size Attr Entity [ 4.403752] Virtual Physical Size Attr Entity
[ 5.013821] ------------------------------------------------------------------------------------------------------------------------------------------- [ 4.405496] -------------------------------------------------------------------------------------------------------------------------------------------
[ 5.015566] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data [ 4.407241] 0xffff_ffff_8008_0000..0xffff_ffff_8008_ffff --> 0x00_0008_0000..0x00_0008_ffff | 64 KiB | C RO X | Kernel code and RO data
[ 5.017181] 0xffff_ffff_8009_0000..0xffff_ffff_800f_ffff --> 0x00_0009_0000..0x00_000f_ffff | 448 KiB | C RW XN | Kernel data and bss [ 4.408855] 0xffff_ffff_8009_0000..0xffff_ffff_800e_ffff --> 0x00_0009_0000..0x00_000e_ffff | 384 KiB | C RW XN | Kernel data and bss
[ 5.018751] 0xffff_ffff_8011_0000..0xffff_ffff_8018_ffff --> 0x00_0011_0000..0x00_0018_ffff | 512 KiB | C RW XN | Kernel boot-core stack [ 4.410425] 0xffff_ffff_8010_0000..0xffff_ffff_8017_ffff --> 0x00_0010_0000..0x00_0017_ffff | 512 KiB | C RW XN | Kernel boot-core stack
[ 5.020354] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_fe20_0000..0x00_fe20_ffff | 64 KiB | Dev RW XN | BCM GPIO [ 4.412028] 0xffff_ffff_f000_0000..0xffff_ffff_f000_ffff --> 0x00_fe20_0000..0x00_fe20_ffff | 64 KiB | Dev RW XN | BCM GPIO
[ 5.021805] | BCM PL011 UART [ 4.413480] | BCM PL011 UART
[ 5.023322] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_ff84_0000..0x00_ff84_ffff | 64 KiB | Dev RW XN | GICD [ 4.414997] 0xffff_ffff_f001_0000..0xffff_ffff_f001_ffff --> 0x00_ff84_0000..0x00_ff84_ffff | 64 KiB | Dev RW XN | GICD
[ 5.024730] | GICC [ 4.416405] | GICC
[ 5.026138] ------------------------------------------------------------------------------------------------------------------------------------------- [ 4.417814] -------------------------------------------------------------------------------------------------------------------------------------------
``` ```
## Diff to previous ## Diff to previous
@ -361,7 +366,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t
/// Page descriptors, covering 64 KiB windows per entry. /// Page descriptors, covering 64 KiB windows per entry.
lvl3: [[PageDescriptor; 8192]; NUM_TABLES], lvl3: [[PageDescriptor; 8192]; NUM_TABLES],
@@ -262,14 +262,23 @@ @@ -305,14 +305,23 @@
where where
[u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized, [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized,
{ {
@ -387,7 +392,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t
/// Create an instance. /// Create an instance.
#[allow(clippy::assertions_on_constants)] #[allow(clippy::assertions_on_constants)]
const fn _new(for_precompute: bool) -> Self { const fn _new(for_precompute: bool) -> Self {
@@ -298,20 +307,32 @@ @@ -341,20 +350,32 @@
/// The start address of the table's MMIO range. /// The start address of the table's MMIO range.
#[inline(always)] #[inline(always)]
fn mmio_start_addr(&self) -> Address<Virtual> { fn mmio_start_addr(&self) -> Address<Virtual> {
@ -424,12 +429,12 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t
} }
/// Helper to calculate the lvl2 and lvl3 indices from an address. /// Helper to calculate the lvl2 and lvl3 indices from an address.
@@ -320,7 +341,12 @@ @@ -363,7 +384,12 @@
&self, &self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<(usize, usize), &'static str> { ) -> Result<(usize, usize), &'static str> {
- let addr = addr as usize; - let addr = virt_page as usize;
+ let mut addr = addr as usize; + let mut addr = virt_page as usize;
+ +
+ if START_FROM_TOP { + if START_FROM_TOP {
+ addr -= Self::START_FROM_TOP_OFFSET.into_usize() + addr -= Self::START_FROM_TOP_OFFSET.into_usize()
@ -438,7 +443,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t
let lvl2_index = addr >> Granule512MiB::SHIFT; let lvl2_index = addr >> Granule512MiB::SHIFT;
let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT; let lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT;
@@ -347,8 +373,9 @@ @@ -411,8 +437,9 @@
// OS Interface Code // OS Interface Code
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -450,7 +455,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t
{ {
fn init(&mut self) -> Result<(), &'static str> { fn init(&mut self) -> Result<(), &'static str> {
if self.initialized { if self.initialized {
@@ -423,12 +450,16 @@ @@ -483,12 +510,16 @@
return Err("Not enough MMIO space left"); return Err("Not enough MMIO space left");
} }
@ -468,7 +473,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/memory/mmu/t
Ok(PageSliceDescriptor::from_addr(addr, num_pages)) Ok(PageSliceDescriptor::from_addr(addr, num_pages))
} }
@@ -451,7 +482,7 @@ @@ -549,7 +580,7 @@
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
#[cfg(test)] #[cfg(test)]
@ -571,7 +576,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld 16
diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs
--- 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs --- 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu.rs
+++ 16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs
@@ -23,7 +23,7 @@ @@ -22,7 +22,7 @@
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
type KernelTranslationTable = type KernelTranslationTable =
@ -600,7 +605,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/lib.rs 16_virtual_mem_part
diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs
--- 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs --- 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs
+++ 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs
@@ -80,6 +80,11 @@ @@ -64,6 +64,11 @@
pub trait AssociatedTranslationTable { pub trait AssociatedTranslationTable {
/// A translation table whose address range is: /// A translation table whose address range is:
/// ///

31
16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu.rs
Unescape Escape View File

@ -15,7 +15,7 @@
use crate::{ use crate::{
bsp, memory, bsp, memory,
memory::{mmu::TranslationGranule, Address, Physical, Virtual}, memory::{mmu::TranslationGranule, Address, Physical},
}; };
use core::intrinsics::unlikely; use core::intrinsics::unlikely;
use cortex_a::{asm::barrier, registers::*}; use cortex_a::{asm::barrier, registers::*};
@ -111,7 +111,7 @@ pub fn mmu() -> &'static impl memory::mmu::interface::MMU {
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// OS Interface Code // OS Interface Code
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
use memory::mmu::{MMUEnableError, TranslationError}; use memory::mmu::MMUEnableError;
impl memory::mmu::interface::MMU for MemoryManagementUnit { impl memory::mmu::interface::MMU for MemoryManagementUnit {
unsafe fn enable_mmu_and_caching( unsafe fn enable_mmu_and_caching(
@ -155,31 +155,4 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit {
fn is_enabled(&self) -> bool { fn is_enabled(&self) -> bool {
SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)
} }
fn try_virt_to_phys(
&self,
virt: Address<Virtual>,
) -> Result<Address<Physical>, TranslationError> {
if !self.is_enabled() {
return Err(TranslationError::MMUDisabled);
}
let addr = virt.into_usize() as u64;
unsafe {
asm!(
"AT S1E1R, {0}",
in(reg) addr,
options(readonly, nostack, preserves_flags)
);
}
let par_el1 = PAR_EL1.extract();
if par_el1.matches_all(PAR_EL1::F::TranslationAborted) {
return Err(TranslationError::Aborted);
}
let phys_addr = (par_el1.read(PAR_EL1::PA) << 12) | (addr & 0xFFF);
Ok(Address::new(phys_addr as usize))
}
} }

148
16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/memory/mmu/translation_table.rs
Unescape Escape View File

@ -23,7 +23,7 @@ use crate::{
Address, Physical, Virtual, Address, Physical, Virtual,
}, },
}; };
use core::convert::{self, TryInto}; use core::convert;
use tock_registers::{ use tock_registers::{
interfaces::{Readable, Writeable}, interfaces::{Readable, Writeable},
register_bitfields, register_bitfields,
@ -219,6 +219,35 @@ impl convert::From<AttributeFields>
} }
} }
/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes.
impl convert::TryFrom<InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>> for AttributeFields {
type Error = &'static str;
fn try_from(
desc: InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>,
) -> Result<AttributeFields, Self::Error> {
let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) {
memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM,
memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device,
_ => return Err("Unexpected memory attribute"),
};
let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) {
Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly,
Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite,
_ => return Err("Unexpected access permission"),
};
let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0;
Ok(AttributeFields {
mem_attributes,
acc_perms,
execute_never,
})
}
}
impl PageDescriptor { impl PageDescriptor {
/// Create an instance. /// Create an instance.
/// ///
@ -228,13 +257,13 @@ impl PageDescriptor {
} }
/// Create an instance. /// Create an instance.
pub fn from_output_addr( pub fn from_output_page(
phys_output_addr: *const Page<Physical>, phys_output_page: *const Page<Physical>,
attribute_fields: &AttributeFields, attribute_fields: &AttributeFields,
) -> Self { ) -> Self {
let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0); let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);
let shifted = phys_output_addr as u64 >> Granule64KiB::SHIFT; let shifted = phys_output_page as u64 >> Granule64KiB::SHIFT;
val.write( val.write(
STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted) STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted)
+ STAGE1_PAGE_DESCRIPTOR::AF::True + STAGE1_PAGE_DESCRIPTOR::AF::True
@ -251,6 +280,20 @@ impl PageDescriptor {
InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value) InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)
.is_set(STAGE1_PAGE_DESCRIPTOR::VALID) .is_set(STAGE1_PAGE_DESCRIPTOR::VALID)
} }
/// Returns the output page.
fn output_page(&self) -> *const Page<Physical> {
let shifted = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value)
.read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB);
let addr = shifted << Granule64KiB::SHIFT;
addr as *const Page<Physical>
}
/// Returns the attributes.
fn try_attributes(&self) -> Result<AttributeFields, &'static str> {
InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(self.value).try_into()
}
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -337,11 +380,11 @@ impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>
/// Helper to calculate the lvl2 and lvl3 indices from an address. /// Helper to calculate the lvl2 and lvl3 indices from an address.
#[inline(always)] #[inline(always)]
fn lvl2_lvl3_index_from( fn lvl2_lvl3_index_from_page(
&self, &self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<(usize, usize), &'static str> { ) -> Result<(usize, usize), &'static str> {
let mut addr = addr as usize; let mut addr = virt_page as usize;
if START_FROM_TOP { if START_FROM_TOP {
addr -= Self::START_FROM_TOP_OFFSET.into_usize() addr -= Self::START_FROM_TOP_OFFSET.into_usize()
@ -357,15 +400,36 @@ impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>
Ok((lvl2_index, lvl3_index)) Ok((lvl2_index, lvl3_index))
} }
/// Returns the PageDescriptor corresponding to the supplied Page. /// Returns the PageDescriptor corresponding to the supplied page address.
#[inline(always)] #[inline(always)]
fn page_descriptor_from( fn page_descriptor_from_page(
&self,
virt_page: *const Page<Virtual>,
) -> Result<&PageDescriptor, &'static str> {
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
let desc = &self.lvl3[lvl2_index][lvl3_index];
Ok(desc)
}
/// Sets the PageDescriptor corresponding to the supplied page address.
///
/// Doesn't allow overriding an already valid page.
#[inline(always)]
fn set_page_descriptor_from_page(
&mut self, &mut self,
addr: *const Page<Virtual>, virt_page: *const Page<Virtual>,
) -> Result<&mut PageDescriptor, &'static str> { new_desc: &PageDescriptor,
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; ) -> Result<(), &'static str> {
let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page(virt_page)?;
let desc = &mut self.lvl3[lvl2_index][lvl3_index];
if desc.is_valid() {
return Err("Virtual page is already mapped");
}
Ok(&mut self.lvl3[lvl2_index][lvl3_index]) *desc = *new_desc;
Ok(())
} }
} }
@ -384,13 +448,11 @@ impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>
// Populate the l2 entries. // Populate the l2 entries.
for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() {
let addr = self.lvl3[lvl2_nr] let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr();
.virt_start_addr() let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?;
.try_into()
.map_err(|_| "Translation error")?;
let desc = TableDescriptor::from_next_lvl_table_addr(addr); let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr);
*lvl2_entry = desc; *lvl2_entry = new_desc;
} }
self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX;
@ -407,8 +469,8 @@ impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>
) -> Result<(), &'static str> { ) -> Result<(), &'static str> {
assert!(self.initialized, "Translation tables not initialized"); assert!(self.initialized, "Translation tables not initialized");
let p = phys_pages.as_slice();
let v = virt_pages.as_slice(); let v = virt_pages.as_slice();
let p = phys_pages.as_slice();
// No work to do for empty slices. // No work to do for empty slices.
if v.is_empty() { if v.is_empty() {
@ -425,12 +487,10 @@ impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>
let iter = p.iter().zip(v.iter()); let iter = p.iter().zip(v.iter());
for (phys_page, virt_page) in iter { for (phys_page, virt_page) in iter {
let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; let new_desc = PageDescriptor::from_output_page(phys_page.as_ptr(), attr);
if page_descriptor.is_valid() { let virt_page = virt_page.as_ptr();
return Err("Virtual page is already mapped");
}
*page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), attr); self.set_page_descriptor_from_page(virt_page, &new_desc)?;
} }
Ok(()) Ok(())
@ -475,6 +535,44 @@ impl<const NUM_TABLES: usize, const START_FROM_TOP: bool>
false false
} }
fn try_virt_page_to_phys_page(
&self,
virt_page: *const Page<Virtual>,
) -> Result<*const Page<Physical>, &'static str> {
let page_desc = self.page_descriptor_from_page(virt_page)?;
if !page_desc.is_valid() {
return Err("Page marked invalid");
}
Ok(page_desc.output_page())
}
fn try_page_attributes(
&self,
virt_page: *const Page<Virtual>,
) -> Result<AttributeFields, &'static str> {
let page_desc = self.page_descriptor_from_page(virt_page)?;
if !page_desc.is_valid() {
return Err("Page marked invalid");
}
page_desc.try_attributes()
}
/// Try to translate a virtual address to a physical address.
///
/// Will only succeed if there exists a valid mapping for the input address.
fn try_virt_addr_to_phys_addr(
&self,
virt_addr: Address<Virtual>,
) -> Result<Address<Physical>, &'static str> {
let page = self.try_virt_page_to_phys_page(virt_addr.as_page_ptr())?;
Ok(Address::new(page as usize + virt_addr.offset_into_page()))
}
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------

66
16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory/mmu.rs
Unescape Escape View File

@ -9,14 +9,13 @@ use crate::{
memory::{ memory::{
mmu as generic_mmu, mmu as generic_mmu,
mmu::{ mmu::{
AccessPermissions, AddressSpace, AssociatedTranslationTable, AttributeFields, AddressSpace, AssociatedTranslationTable, AttributeFields, Page, PageSliceDescriptor,
MemAttributes, Page, PageSliceDescriptor, TranslationGranule, TranslationGranule,
}, },
Physical, Virtual, Address, Physical, Virtual,
}, },
synchronization::InitStateLock, synchronization::InitStateLock,
}; };
use core::convert::TryInto;
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Private Definitions // Private Definitions
@ -103,21 +102,19 @@ fn virt_boot_core_stack_page_desc() -> PageSliceDescriptor<Virtual> {
} }
// There is no reason to expect the following conversions to fail, since they were generated offline // There is no reason to expect the following conversions to fail, since they were generated offline
// by the `translation table tool`. If it doesn't work, a panic due to the unwrap is justified. // by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified.
fn kernel_virt_to_phys_page_slice(
/// The Read+Execute (RX) pages of the kernel binary. virt_slice: PageSliceDescriptor<Virtual>,
fn phys_rx_page_desc() -> PageSliceDescriptor<Physical> { ) -> PageSliceDescriptor<Physical> {
virt_rx_page_desc().try_into().unwrap() let phys_first_page =
} generic_mmu::try_kernel_virt_page_to_phys_page(virt_slice.first_page()).unwrap();
let phys_start_addr = Address::new(phys_first_page as usize);
/// The Read+Write (RW) pages of the kernel binary.
fn phys_rw_page_desc() -> PageSliceDescriptor<Physical> { PageSliceDescriptor::from_addr(phys_start_addr, virt_slice.num_pages())
virt_rw_page_desc().try_into().unwrap()
} }
/// The boot core's stack. fn kernel_page_attributes(virt_page: *const Page<Virtual>) -> AttributeFields {
fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor<Physical> { generic_mmu::try_kernel_page_attributes(virt_page).unwrap()
virt_boot_core_stack_page_desc().try_into().unwrap()
} }
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -142,39 +139,28 @@ pub fn phys_addr_space_end_page() -> *const Page<Physical> {
/// The actual translation table entries for the kernel binary are generated using the offline /// The actual translation table entries for the kernel binary are generated using the offline
/// `translation table tool` and patched into the kernel binary. This function just adds the mapping /// `translation table tool` and patched into the kernel binary. This function just adds the mapping
/// record entries. /// record entries.
///
/// It must be ensured that these entries are in sync with the offline tool.
pub fn kernel_add_mapping_records_for_precomputed() { pub fn kernel_add_mapping_records_for_precomputed() {
let virt_rx_page_desc = virt_rx_page_desc();
generic_mmu::kernel_add_mapping_record( generic_mmu::kernel_add_mapping_record(
"Kernel code and RO data", "Kernel code and RO data",
&virt_rx_page_desc(), &virt_rx_page_desc,
&phys_rx_page_desc(), &kernel_virt_to_phys_page_slice(virt_rx_page_desc),
&AttributeFields { &kernel_page_attributes(virt_rx_page_desc.first_page()),
mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadOnly,
execute_never: false,
},
); );
let virt_rw_page_desc = virt_rw_page_desc();
generic_mmu::kernel_add_mapping_record( generic_mmu::kernel_add_mapping_record(
"Kernel data and bss", "Kernel data and bss",
&virt_rw_page_desc(), &virt_rw_page_desc,
&phys_rw_page_desc(), &kernel_virt_to_phys_page_slice(virt_rw_page_desc),
&AttributeFields { &kernel_page_attributes(virt_rw_page_desc.first_page()),
mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadWrite,
execute_never: true,
},
); );
let virt_boot_core_stack_page_desc = virt_boot_core_stack_page_desc();
generic_mmu::kernel_add_mapping_record( generic_mmu::kernel_add_mapping_record(
"Kernel boot-core stack", "Kernel boot-core stack",
&virt_boot_core_stack_page_desc(), &virt_boot_core_stack_page_desc,
&phys_boot_core_stack_page_desc(), &kernel_virt_to_phys_page_slice(virt_boot_core_stack_page_desc),
&AttributeFields { &kernel_page_attributes(virt_boot_core_stack_page_desc.first_page()),
mem_attributes: MemAttributes::CacheableDRAM,
acc_perms: AccessPermissions::ReadWrite,
execute_never: true,
},
); );
} }

17
16_virtual_mem_part4_higher_half_kernel/src/memory.rs
Unescape Escape View File

@ -6,13 +6,13 @@
pub mod mmu; pub mod mmu;
use crate::common; use crate::{bsp, common};
use core::{ use core::{
convert::TryFrom,
fmt, fmt,
marker::PhantomData, marker::PhantomData,
ops::{AddAssign, SubAssign}, ops::{AddAssign, SubAssign},
}; };
use mmu::Page;
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Public Definitions // Public Definitions
@ -66,13 +66,16 @@ impl<ATYPE: AddressType> Address<ATYPE> {
pub const fn into_usize(self) -> usize { pub const fn into_usize(self) -> usize {
self.value self.value
} }
}
impl TryFrom<Address<Virtual>> for Address<Physical> { /// Return a pointer to the page that contains this address.
type Error = mmu::TranslationError; pub const fn as_page_ptr(&self) -> *const Page<ATYPE> {
self.align_down(bsp::memory::mmu::KernelGranule::SIZE)
.into_usize() as *const _
}
fn try_from(virt: Address<Virtual>) -> Result<Self, Self::Error> { /// Return the address' offset into the underlying page.
mmu::try_virt_to_phys(virt) pub const fn offset_into_page(&self) -> usize {
self.value & bsp::memory::mmu::KernelGranule::MASK
} }
} }

47
16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs
Unescape Escape View File

@ -33,14 +33,6 @@ pub enum MMUEnableError {
Other(&'static str), Other(&'static str),
} }
/// Translation error variants.
#[allow(missing_docs)]
#[derive(Debug)]
pub enum TranslationError {
MMUDisabled,
Aborted,
}
/// Memory Management interfaces. /// Memory Management interfaces.
pub mod interface { pub mod interface {
use super::*; use super::*;
@ -59,14 +51,6 @@ pub mod interface {
/// Returns true if the MMU is enabled, false otherwise. /// Returns true if the MMU is enabled, false otherwise.
fn is_enabled(&self) -> bool; fn is_enabled(&self) -> bool;
/// Try to translate a virtual address to a physical address.
///
/// Will only succeed if there exists a valid mapping for the input VA.
fn try_virt_to_phys(
&self,
virt: Address<Virtual>,
) -> Result<Address<Physical>, TranslationError>;
} }
} }
@ -245,11 +229,34 @@ pub unsafe fn kernel_map_mmio(
Ok(virt_addr + offset_into_start_page) Ok(virt_addr + offset_into_start_page)
} }
/// Try to translate a virtual address to a physical address. /// Try to translate a kernel virtual page pointer to a physical page pointer.
///
/// Will only succeed if there exists a valid mapping for the input page.
pub fn try_kernel_virt_page_to_phys_page(
virt_page: *const Page<Virtual>,
) -> Result<*const Page<Physical>, &'static str> {
bsp::memory::mmu::kernel_translation_tables()
.read(|tables| tables.try_virt_page_to_phys_page(virt_page))
}
/// Try to get the attributes of a kernel page.
///
/// Will only succeed if there exists a valid mapping for the input page.
pub fn try_kernel_page_attributes(
virt_page: *const Page<Virtual>,
) -> Result<AttributeFields, &'static str> {
bsp::memory::mmu::kernel_translation_tables()
.read(|tables| tables.try_page_attributes(virt_page))
}
/// Try to translate a kernel virtual address to a physical address.
/// ///
/// Will only succeed if there exists a valid mapping for the input VA. /// Will only succeed if there exists a valid mapping for the input address.
pub fn try_virt_to_phys(virt: Address<Virtual>) -> Result<Address<Physical>, TranslationError> { fn try_kernel_virt_addr_to_phys_addr(
arch_mmu::mmu().try_virt_to_phys(virt) virt_addr: Address<Virtual>,
) -> Result<Address<Physical>, &'static str> {
bsp::memory::mmu::kernel_translation_tables()
.read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr))
} }
/// Enable the MMU and data + instruction caching. /// Enable the MMU and data + instruction caching.

26
16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/translation_table.rs
Unescape Escape View File

@ -10,7 +10,7 @@ mod arch_translation_table;
use crate::memory::{ use crate::memory::{
mmu::{AttributeFields, PageSliceDescriptor}, mmu::{AttributeFields, PageSliceDescriptor},
Physical, Virtual, Address, Page, Physical, Virtual,
}; };
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
@ -67,6 +67,30 @@ pub mod interface {
/// Check if a virtual page splice is in the "MMIO region". /// Check if a virtual page splice is in the "MMIO region".
fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor<Virtual>) -> bool; fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor<Virtual>) -> bool;
/// Try to translate a virtual page pointer to a physical page pointer.
///
/// Will only succeed if there exists a valid mapping for the input page.
fn try_virt_page_to_phys_page(
&self,
virt_page: *const Page<Virtual>,
) -> Result<*const Page<Physical>, &'static str>;
/// Try to get the attributes of a page.
///
/// Will only succeed if there exists a valid mapping for the input page.
fn try_page_attributes(
&self,
virt_page: *const Page<Virtual>,
) -> Result<AttributeFields, &'static str>;
/// Try to translate a virtual address to a physical address.
///
/// Will only succeed if there exists a valid mapping for the input address.
fn try_virt_addr_to_phys_addr(
&self,
virt_addr: Address<Virtual>,
) -> Result<Address<Physical>, &'static str>;
} }
} }

28
16_virtual_mem_part4_higher_half_kernel/src/memory/mmu/types.rs
Unescape Escape View File

@ -6,12 +6,9 @@
use crate::{ use crate::{
bsp, common, bsp, common,
memory::{Address, AddressType, Physical, Virtual}, memory::{Address, AddressType, Physical},
};
use core::{
convert::{From, TryFrom},
marker::PhantomData,
}; };
use core::{convert::From, marker::PhantomData};
//-------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------
// Public Definitions // Public Definitions
@ -95,11 +92,11 @@ impl<ATYPE: AddressType> PageSliceDescriptor<ATYPE> {
} }
/// Return a pointer to the first page of the described slice. /// Return a pointer to the first page of the described slice.
const fn first_page_ptr(&self) -> *const Page<ATYPE> { pub const fn first_page(&self) -> *const Page<ATYPE> {
self.start.into_usize() as *const _ self.start.into_usize() as *const _
} }
/// Return the number of Pages the slice describes. /// Return the number of pages the slice describes.
pub const fn num_pages(&self) -> usize { pub const fn num_pages(&self) -> usize {
self.num_pages self.num_pages
} }
@ -129,26 +126,13 @@ impl<ATYPE: AddressType> PageSliceDescriptor<ATYPE> {
(addr >= self.start_addr()) && (addr <= self.end_addr_inclusive()) (addr >= self.start_addr()) && (addr <= self.end_addr_inclusive())
} }
/// Return a non-mutable slice of Pages. /// Return a non-mutable slice of pages.
/// ///
/// # Safety /// # Safety
/// ///
/// - Same as applies for `core::slice::from_raw_parts`. /// - Same as applies for `core::slice::from_raw_parts`.
pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] { pub unsafe fn as_slice(&self) -> &[Page<ATYPE>] {
core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) core::slice::from_raw_parts(self.first_page(), self.num_pages)
}
}
impl TryFrom<PageSliceDescriptor<Virtual>> for PageSliceDescriptor<Physical> {
type Error = super::TranslationError;
fn try_from(desc: PageSliceDescriptor<Virtual>) -> Result<Self, Self::Error> {
let phys_start = super::try_virt_to_phys(desc.start)?;
Ok(Self {
start: phys_start,
num_pages: desc.num_pages,
})
} }
} }

Write Preview
Loading…
Cancel
Save