|
|
|
@ -109,7 +109,7 @@ Due to this default return, it is technicall not needed to define normal cacheab
|
|
|
|
|
This file contains the `AArch64` `MMU` driver. The granule is hardcoded here (`64 KiB` page
|
|
|
|
|
descriptors).
|
|
|
|
|
|
|
|
|
|
The actual translation tables are stored in a global instance of the `TranslationTables` struct:
|
|
|
|
|
The actual translation tables are stored in a global instance of the `ArchTranslationTable` struct:
|
|
|
|
|
|
|
|
|
|
```rust
|
|
|
|
|
/// A table descriptor for 64 KiB aperture.
|
|
|
|
@ -117,39 +117,41 @@ The actual translation tables are stored in a global instance of the `Translatio
|
|
|
|
|
/// The output points to the next table.
|
|
|
|
|
#[derive(Copy, Clone)]
|
|
|
|
|
#[repr(transparent)]
|
|
|
|
|
struct TableDescriptor(u64);
|
|
|
|
|
struct TableDescriptor(InMemoryRegister<u64, STAGE1_TABLE_DESCRIPTOR::Register>);
|
|
|
|
|
|
|
|
|
|
/// A page descriptor with 64 KiB aperture.
|
|
|
|
|
///
|
|
|
|
|
/// The output points to physical memory.
|
|
|
|
|
#[derive(Copy, Clone)]
|
|
|
|
|
#[repr(transparent)]
|
|
|
|
|
struct PageDescriptor(u64);
|
|
|
|
|
struct PageDescriptor(InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>);
|
|
|
|
|
|
|
|
|
|
/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB
|
|
|
|
|
/// aligned, hence the "reverse" order of appearance.
|
|
|
|
|
#[repr(C)]
|
|
|
|
|
#[repr(align(65536))]
|
|
|
|
|
struct TranslationTables<const N: usize> {
|
|
|
|
|
struct FixedSizeTranslationTable<const NUM_TABLES: usize> {
|
|
|
|
|
/// Page descriptors, covering 64 KiB windows per entry.
|
|
|
|
|
lvl3: [[PageDescriptor; 8192]; N],
|
|
|
|
|
lvl3: [[PageDescriptor; 8192]; NUM_TABLES],
|
|
|
|
|
|
|
|
|
|
/// Table descriptors, covering 512 MiB windows.
|
|
|
|
|
lvl2: [TableDescriptor; N],
|
|
|
|
|
lvl2: [TableDescriptor; NUM_TABLES],
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4.
|
|
|
|
|
const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT;
|
|
|
|
|
const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT;
|
|
|
|
|
type ArchTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;
|
|
|
|
|
|
|
|
|
|
//--------------------------------------------------------------------------------------------------
|
|
|
|
|
// Global instances
|
|
|
|
|
//--------------------------------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
/// The translation tables.
|
|
|
|
|
///
|
|
|
|
|
/// # Safety
|
|
|
|
|
///
|
|
|
|
|
/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries.
|
|
|
|
|
static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables {
|
|
|
|
|
lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB],
|
|
|
|
|
lvl2: [TableDescriptor(0); ENTRIES_512_MIB],
|
|
|
|
|
};
|
|
|
|
|
static mut TABLES: ArchTranslationTable = ArchTranslationTable::new();
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
They are populated using `bsp::memory::mmu::virt_mem_layout().virt_addr_properties()` and a bunch of
|
|
|
|
@ -298,7 +300,7 @@ Minipush 1.0
|
|
|
|
|
diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs
|
|
|
|
|
--- 10_privilege_level/src/_arch/aarch64/memory/mmu.rs
|
|
|
|
|
+++ 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs
|
|
|
|
|
@@ -0,0 +1,319 @@
|
|
|
|
|
@@ -0,0 +1,333 @@
|
|
|
|
|
+// SPDX-License-Identifier: MIT OR Apache-2.0
|
|
|
|
|
+//
|
|
|
|
|
+// Copyright (c) 2018-2020 Andre Richter <andre.o.richter@gmail.com>
|
|
|
|
@ -311,7 +313,7 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+use crate::{bsp, memory};
|
|
|
|
|
+use core::convert;
|
|
|
|
|
+use cortex_a::{barrier, regs::*};
|
|
|
|
|
+use register::register_bitfields;
|
|
|
|
|
+use register::{register_bitfields, InMemoryRegister};
|
|
|
|
|
+
|
|
|
|
|
+//--------------------------------------------------------------------------------------------------
|
|
|
|
|
+// Private Definitions
|
|
|
|
@ -390,39 +392,30 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+/// The output points to the next table.
|
|
|
|
|
+#[derive(Copy, Clone)]
|
|
|
|
|
+#[repr(transparent)]
|
|
|
|
|
+struct TableDescriptor(u64);
|
|
|
|
|
+struct TableDescriptor(InMemoryRegister<u64, STAGE1_TABLE_DESCRIPTOR::Register>);
|
|
|
|
|
+
|
|
|
|
|
+/// A page descriptor with 64 KiB aperture.
|
|
|
|
|
+///
|
|
|
|
|
+/// The output points to physical memory.
|
|
|
|
|
+#[derive(Copy, Clone)]
|
|
|
|
|
+#[repr(transparent)]
|
|
|
|
|
+struct PageDescriptor(u64);
|
|
|
|
|
+struct PageDescriptor(InMemoryRegister<u64, STAGE1_PAGE_DESCRIPTOR::Register>);
|
|
|
|
|
+
|
|
|
|
|
+/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB
|
|
|
|
|
+/// aligned, hence the "reverse" order of appearance.
|
|
|
|
|
+#[repr(C)]
|
|
|
|
|
+#[repr(align(65536))]
|
|
|
|
|
+struct TranslationTables<const N: usize> {
|
|
|
|
|
+struct FixedSizeTranslationTable<const NUM_TABLES: usize> {
|
|
|
|
|
+ /// Page descriptors, covering 64 KiB windows per entry.
|
|
|
|
|
+ lvl3: [[PageDescriptor; 8192]; N],
|
|
|
|
|
+ lvl3: [[PageDescriptor; 8192]; NUM_TABLES],
|
|
|
|
|
+
|
|
|
|
|
+ /// Table descriptors, covering 512 MiB windows.
|
|
|
|
|
+ lvl2: [TableDescriptor; N],
|
|
|
|
|
+ lvl2: [TableDescriptor; NUM_TABLES],
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4.
|
|
|
|
|
+const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT;
|
|
|
|
|
+
|
|
|
|
|
+/// The translation tables.
|
|
|
|
|
+///
|
|
|
|
|
+/// # Safety
|
|
|
|
|
+///
|
|
|
|
|
+/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries.
|
|
|
|
|
+static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables {
|
|
|
|
|
+ lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB],
|
|
|
|
|
+ lvl2: [TableDescriptor(0); ENTRIES_512_MIB],
|
|
|
|
|
+};
|
|
|
|
|
+const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT;
|
|
|
|
|
+type ArchTranslationTable = FixedSizeTranslationTable<NUM_LVL2_TABLES>;
|
|
|
|
|
+
|
|
|
|
|
+trait BaseAddr {
|
|
|
|
|
+ fn base_addr_u64(&self) -> u64;
|
|
|
|
@ -436,17 +429,20 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+ pub const NORMAL: u64 = 1;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+//--------------------------------------------------------------------------------------------------
|
|
|
|
|
+// Public Definitions
|
|
|
|
|
+//--------------------------------------------------------------------------------------------------
|
|
|
|
|
+
|
|
|
|
|
+/// Memory Management Unit type.
|
|
|
|
|
+pub struct MemoryManagementUnit;
|
|
|
|
|
+struct MemoryManagementUnit;
|
|
|
|
|
+
|
|
|
|
|
+//--------------------------------------------------------------------------------------------------
|
|
|
|
|
+// Global instances
|
|
|
|
|
+//--------------------------------------------------------------------------------------------------
|
|
|
|
|
+
|
|
|
|
|
+/// The translation tables.
|
|
|
|
|
+///
|
|
|
|
|
+/// # Safety
|
|
|
|
|
+///
|
|
|
|
|
+/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries.
|
|
|
|
|
+static mut TABLES: ArchTranslationTable = ArchTranslationTable::new();
|
|
|
|
|
+
|
|
|
|
|
+static MMU: MemoryManagementUnit = MemoryManagementUnit;
|
|
|
|
|
+
|
|
|
|
|
+//--------------------------------------------------------------------------------------------------
|
|
|
|
@ -459,23 +455,26 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fn base_addr_usize(&self) -> usize {
|
|
|
|
|
+ self as *const T as usize
|
|
|
|
|
+ self as *const _ as usize
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+impl convert::From<usize> for TableDescriptor {
|
|
|
|
|
+ fn from(next_lvl_table_addr: usize) -> Self {
|
|
|
|
|
+ let val = InMemoryRegister::<u64, STAGE1_TABLE_DESCRIPTOR::Register>::new(0);
|
|
|
|
|
+
|
|
|
|
|
+ let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT;
|
|
|
|
|
+ let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True
|
|
|
|
|
+ + STAGE1_TABLE_DESCRIPTOR::TYPE::Table
|
|
|
|
|
+ + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64))
|
|
|
|
|
+ .value;
|
|
|
|
|
+ val.write(
|
|
|
|
|
+ STAGE1_TABLE_DESCRIPTOR::VALID::True
|
|
|
|
|
+ + STAGE1_TABLE_DESCRIPTOR::TYPE::Table
|
|
|
|
|
+ + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64),
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
|
|
+ TableDescriptor(val)
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU.
|
|
|
|
|
+/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU.
|
|
|
|
|
+impl convert::From<AttributeFields>
|
|
|
|
|
+ for register::FieldValue<u64, STAGE1_PAGE_DESCRIPTOR::Register>
|
|
|
|
|
+{
|
|
|
|
@ -510,19 +509,35 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+impl PageDescriptor {
|
|
|
|
|
+ /// Create an instance.
|
|
|
|
|
+ fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self {
|
|
|
|
|
+ let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT;
|
|
|
|
|
+ let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True
|
|
|
|
|
+ + STAGE1_PAGE_DESCRIPTOR::AF::True
|
|
|
|
|
+ + attribute_fields.into()
|
|
|
|
|
+ + STAGE1_PAGE_DESCRIPTOR::TYPE::Table
|
|
|
|
|
+ + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64))
|
|
|
|
|
+ .value;
|
|
|
|
|
+ let val = InMemoryRegister::<u64, STAGE1_PAGE_DESCRIPTOR::Register>::new(0);
|
|
|
|
|
+
|
|
|
|
|
+ let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT;
|
|
|
|
|
+ val.write(
|
|
|
|
|
+ STAGE1_PAGE_DESCRIPTOR::VALID::True
|
|
|
|
|
+ + STAGE1_PAGE_DESCRIPTOR::AF::True
|
|
|
|
|
+ + attribute_fields.into()
|
|
|
|
|
+ + STAGE1_PAGE_DESCRIPTOR::TYPE::Table
|
|
|
|
|
+ + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted),
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
|
|
+ Self(val)
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+impl<const NUM_TABLES: usize> FixedSizeTranslationTable<{ NUM_TABLES }> {
|
|
|
|
|
+ /// Create an instance.
|
|
|
|
|
+ pub const fn new() -> Self {
|
|
|
|
|
+ assert!(NUM_TABLES > 0);
|
|
|
|
|
+
|
|
|
|
|
+ Self {
|
|
|
|
|
+ lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES],
|
|
|
|
|
+ lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES],
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/// Setup function for the MAIR_EL1 register.
|
|
|
|
|
+fn set_up_mair() {
|
|
|
|
|
+ // Define the memory types being mapped.
|
|
|
|
@ -561,6 +576,7 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+/// Configure various settings of stage 1 of the EL1 translation regime.
|
|
|
|
|
+fn configure_translation_control() {
|
|
|
|
|
+ let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange);
|
|
|
|
|
+
|
|
|
|
|
+ TCR_EL1.write(
|
|
|
|
|
+ TCR_EL1::TBI0::Ignored
|
|
|
|
|
+ + TCR_EL1::IPS.val(ips)
|
|
|
|
@ -590,7 +606,7 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s
|
|
|
|
|
+ unsafe fn init(&self) -> Result<(), &'static str> {
|
|
|
|
|
+ // Fail early if translation granule is not supported. Both RPis support it, though.
|
|
|
|
|
+ if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) {
|
|
|
|
|
+ return Err("64 KiB translation granule not supported");
|
|
|
|
|
+ return Err("Translation granule not supported in HW");
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // Prepare the memory attribute indirection register.
|
|
|
|
@ -837,16 +853,17 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs
|
|
|
|
|
//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html
|
|
|
|
|
//!
|
|
|
|
|
//! # Code organization and architecture
|
|
|
|
|
@@ -102,6 +104,8 @@
|
|
|
|
|
@@ -102,6 +104,9 @@
|
|
|
|
|
//! - `crate::memory::*`
|
|
|
|
|
//! - `crate::bsp::memory::*`
|
|
|
|
|
|
|
|
|
|
+#![allow(incomplete_features)]
|
|
|
|
|
+#![feature(const_generics)]
|
|
|
|
|
+#![feature(const_panic)]
|
|
|
|
|
#![feature(format_args_nl)]
|
|
|
|
|
#![feature(naked_functions)]
|
|
|
|
|
#![feature(panic_info_message)]
|
|
|
|
|
@@ -129,9 +133,18 @@
|
|
|
|
|
@@ -129,9 +134,18 @@
|
|
|
|
|
/// # Safety
|
|
|
|
|
///
|
|
|
|
|
/// - Only a single core must be active and running this function.
|
|
|
|
@ -866,7 +883,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs
|
|
|
|
|
|
|
|
|
|
for i in bsp::driver::driver_manager().all_device_drivers().iter() {
|
|
|
|
|
if i.init().is_err() {
|
|
|
|
|
@@ -154,6 +167,9 @@
|
|
|
|
|
@@ -154,6 +168,9 @@
|
|
|
|
|
|
|
|
|
|
info!("Booting on: {}", bsp::board_name());
|
|
|
|
|
|
|
|
|
@ -876,7 +893,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs
|
|
|
|
|
let (_, privilege_level) = exception::current_privilege_level();
|
|
|
|
|
info!("Current privilege level: {}", privilege_level);
|
|
|
|
|
|
|
|
|
|
@@ -177,6 +193,13 @@
|
|
|
|
|
@@ -177,6 +194,13 @@
|
|
|
|
|
info!("Timer test, spinning for 1 second");
|
|
|
|
|
time::time_manager().spin_for(Duration::from_secs(1));
|
|
|
|
|
|
|
|
|
|