From e7df5b298298e36a6c928cb2f17ce11580b662c3 Mon Sep 17 00:00:00 2001 From: Andre Richter Date: Mon, 28 Sep 2020 22:31:42 +0200 Subject: [PATCH] Use InMemoryRegister in MMU driver --- 11_virtual_memory/README.md | 121 ++++++++++-------- .../src/_arch/aarch64/memory/mmu.rs | 86 +++++++------ 11_virtual_memory/src/main.rs | 1 + 12_exceptions_part1_groundwork/README.md | 8 +- .../src/_arch/aarch64/memory/mmu.rs | 86 +++++++------ 12_exceptions_part1_groundwork/src/main.rs | 1 + 13_integrated_testing/README.md | 15 ++- .../src/_arch/aarch64/memory/mmu.rs | 86 +++++++------ 13_integrated_testing/src/lib.rs | 1 + 14_exceptions_part2_peripheral_IRQs/README.md | 6 +- .../src/_arch/aarch64/memory/mmu.rs | 86 +++++++------ 11 files changed, 288 insertions(+), 209 deletions(-) diff --git a/11_virtual_memory/README.md b/11_virtual_memory/README.md index 599108e1..fed7cba0 100644 --- a/11_virtual_memory/README.md +++ b/11_virtual_memory/README.md @@ -109,7 +109,7 @@ Due to this default return, it is technicall not needed to define normal cacheab This file contains the `AArch64` `MMU` driver. The granule is hardcoded here (`64 KiB` page descriptors). -The actual translation tables are stored in a global instance of the `TranslationTables` struct: +The actual translation tables are stored in a global instance of the `ArchTranslationTable` struct: ```rust /// A table descriptor for 64 KiB aperture. @@ -117,39 +117,41 @@ The actual translation tables are stored in a global instance of the `Translatio /// The output points to the next table. #[derive(Copy, Clone)] #[repr(transparent)] -struct TableDescriptor(u64); +struct TableDescriptor(InMemoryRegister); /// A page descriptor with 64 KiB aperture. /// /// The output points to physical memory. #[derive(Copy, Clone)] #[repr(transparent)] -struct PageDescriptor(u64); +struct PageDescriptor(InMemoryRegister); /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB /// aligned, hence the "reverse" order of appearance. #[repr(C)] #[repr(align(65536))] -struct TranslationTables { +struct FixedSizeTranslationTable { /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; N], + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; N], + lvl2: [TableDescriptor; NUM_TABLES], } /// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. -const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +type ArchTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- /// The translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. -static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables { - lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], - lvl2: [TableDescriptor(0); ENTRIES_512_MIB], -}; +static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); ``` They are populated using `bsp::memory::mmu::virt_mem_layout().virt_addr_properties()` and a bunch of @@ -298,7 +300,7 @@ Minipush 1.0 diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs --- 10_privilege_level/src/_arch/aarch64/memory/mmu.rs +++ 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs -@@ -0,0 +1,319 @@ +@@ -0,0 +1,333 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter @@ -311,7 +313,7 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s +use crate::{bsp, memory}; +use core::convert; +use cortex_a::{barrier, regs::*}; -+use register::register_bitfields; ++use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions @@ -390,39 +392,30 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(transparent)] -+struct TableDescriptor(u64); ++struct TableDescriptor(InMemoryRegister); + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(transparent)] -+struct PageDescriptor(u64); ++struct PageDescriptor(InMemoryRegister); + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] -+struct TranslationTables { ++struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. -+ lvl3: [[PageDescriptor; 8192]; N], ++ lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. -+ lvl2: [TableDescriptor; N], ++ lvl2: [TableDescriptor; NUM_TABLES], +} + +/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. -+const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -+ -+/// The translation tables. -+/// -+/// # Safety -+/// -+/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. -+static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables { -+ lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], -+ lvl2: [TableDescriptor(0); ENTRIES_512_MIB], -+}; ++const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; ++type ArchTranslationTable = FixedSizeTranslationTable; + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; @@ -436,17 +429,20 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s + pub const NORMAL: u64 = 1; +} + -+//-------------------------------------------------------------------------------------------------- -+// Public Definitions -+//-------------------------------------------------------------------------------------------------- -+ +/// Memory Management Unit type. -+pub struct MemoryManagementUnit; ++struct MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + ++/// The translation tables. ++/// ++/// # Safety ++/// ++/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. ++static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); ++ +static MMU: MemoryManagementUnit = MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- @@ -459,23 +455,26 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s + } + + fn base_addr_usize(&self) -> usize { -+ self as *const T as usize ++ self as *const _ as usize + } +} + +impl convert::From for TableDescriptor { + fn from(next_lvl_table_addr: usize) -> Self { ++ let val = InMemoryRegister::::new(0); ++ + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; -+ let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True -+ + STAGE1_TABLE_DESCRIPTOR::TYPE::Table -+ + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)) -+ .value; ++ val.write( ++ STAGE1_TABLE_DESCRIPTOR::VALID::True ++ + STAGE1_TABLE_DESCRIPTOR::TYPE::Table ++ + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), ++ ); + + TableDescriptor(val) + } +} + -+/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. ++/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ @@ -510,19 +509,35 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s +} + +impl PageDescriptor { ++ /// Create an instance. + fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { -+ let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; -+ let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True -+ + STAGE1_PAGE_DESCRIPTOR::AF::True -+ + attribute_fields.into() -+ + STAGE1_PAGE_DESCRIPTOR::TYPE::Table -+ + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)) -+ .value; ++ let val = InMemoryRegister::::new(0); ++ ++ let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; ++ val.write( ++ STAGE1_PAGE_DESCRIPTOR::VALID::True ++ + STAGE1_PAGE_DESCRIPTOR::AF::True ++ + attribute_fields.into() ++ + STAGE1_PAGE_DESCRIPTOR::TYPE::Table ++ + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), ++ ); + + Self(val) + } +} + ++impl FixedSizeTranslationTable<{ NUM_TABLES }> { ++ /// Create an instance. ++ pub const fn new() -> Self { ++ assert!(NUM_TABLES > 0); ++ ++ Self { ++ lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], ++ lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], ++ } ++ } ++} ++ +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the memory types being mapped. @@ -561,6 +576,7 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s +/// Configure various settings of stage 1 of the EL1 translation regime. +fn configure_translation_control() { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); ++ + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) @@ -590,7 +606,7 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s + unsafe fn init(&self) -> Result<(), &'static str> { + // Fail early if translation granule is not supported. Both RPis support it, though. + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { -+ return Err("64 KiB translation granule not supported"); ++ return Err("Translation granule not supported in HW"); + } + + // Prepare the memory attribute indirection register. @@ -837,16 +853,17 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs //! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html //! //! # Code organization and architecture -@@ -102,6 +104,8 @@ +@@ -102,6 +104,9 @@ //! - `crate::memory::*` //! - `crate::bsp::memory::*` +#![allow(incomplete_features)] +#![feature(const_generics)] ++#![feature(const_panic)] #![feature(format_args_nl)] #![feature(naked_functions)] #![feature(panic_info_message)] -@@ -129,9 +133,18 @@ +@@ -129,9 +134,18 @@ /// # Safety /// /// - Only a single core must be active and running this function. @@ -866,7 +883,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs for i in bsp::driver::driver_manager().all_device_drivers().iter() { if i.init().is_err() { -@@ -154,6 +167,9 @@ +@@ -154,6 +168,9 @@ info!("Booting on: {}", bsp::board_name()); @@ -876,7 +893,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs let (_, privilege_level) = exception::current_privilege_level(); info!("Current privilege level: {}", privilege_level); -@@ -177,6 +193,13 @@ +@@ -177,6 +194,13 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); diff --git a/11_virtual_memory/src/_arch/aarch64/memory/mmu.rs b/11_virtual_memory/src/_arch/aarch64/memory/mmu.rs index e104edbc..dd94f714 100644 --- a/11_virtual_memory/src/_arch/aarch64/memory/mmu.rs +++ b/11_virtual_memory/src/_arch/aarch64/memory/mmu.rs @@ -10,7 +10,7 @@ use super::{AccessPermissions, AttributeFields, MemAttributes}; use crate::{bsp, memory}; use core::convert; use cortex_a::{barrier, regs::*}; -use register::register_bitfields; +use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -89,39 +89,30 @@ const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) /// The output points to the next table. #[derive(Copy, Clone)] #[repr(transparent)] -struct TableDescriptor(u64); +struct TableDescriptor(InMemoryRegister); /// A page descriptor with 64 KiB aperture. /// /// The output points to physical memory. #[derive(Copy, Clone)] #[repr(transparent)] -struct PageDescriptor(u64); +struct PageDescriptor(InMemoryRegister); /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB /// aligned, hence the "reverse" order of appearance. #[repr(C)] #[repr(align(65536))] -struct TranslationTables { +struct FixedSizeTranslationTable { /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; N], + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; N], + lvl2: [TableDescriptor; NUM_TABLES], } /// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. -const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; - -/// The translation tables. -/// -/// # Safety -/// -/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. -static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables { - lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], - lvl2: [TableDescriptor(0); ENTRIES_512_MIB], -}; +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +type ArchTranslationTable = FixedSizeTranslationTable; trait BaseAddr { fn base_addr_u64(&self) -> u64; @@ -135,17 +126,20 @@ mod mair { pub const NORMAL: u64 = 1; } -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - /// Memory Management Unit type. -pub struct MemoryManagementUnit; +struct MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- +/// The translation tables. +/// +/// # Safety +/// +/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); + static MMU: MemoryManagementUnit = MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- @@ -158,23 +152,26 @@ impl BaseAddr for [T; N] { } fn base_addr_usize(&self) -> usize { - self as *const T as usize + self as *const _ as usize } } impl convert::From for TableDescriptor { fn from(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)) - .value; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); TableDescriptor(val) } } -/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. impl convert::From for register::FieldValue { @@ -209,19 +206,35 @@ impl convert::From } impl PageDescriptor { + /// Create an instance. fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)) - .value; + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); Self(val) } } +impl FixedSizeTranslationTable<{ NUM_TABLES }> { + /// Create an instance. + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + + Self { + lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], + lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], + } + } +} + /// Setup function for the MAIR_EL1 register. fn set_up_mair() { // Define the memory types being mapped. @@ -260,6 +273,7 @@ unsafe fn populate_tt_entries() -> Result<(), &'static str> { /// Configure various settings of stage 1 of the EL1 translation regime. fn configure_translation_control() { let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( TCR_EL1::TBI0::Ignored + TCR_EL1::IPS.val(ips) @@ -289,7 +303,7 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn init(&self) -> Result<(), &'static str> { // Fail early if translation granule is not supported. Both RPis support it, though. if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { - return Err("64 KiB translation granule not supported"); + return Err("Translation granule not supported in HW"); } // Prepare the memory attribute indirection register. diff --git a/11_virtual_memory/src/main.rs b/11_virtual_memory/src/main.rs index 6ff3292f..6d28aa69 100644 --- a/11_virtual_memory/src/main.rs +++ b/11_virtual_memory/src/main.rs @@ -106,6 +106,7 @@ #![allow(incomplete_features)] #![feature(const_generics)] +#![feature(const_panic)] #![feature(format_args_nl)] #![feature(naked_functions)] #![feature(panic_info_message)] diff --git a/12_exceptions_part1_groundwork/README.md b/12_exceptions_part1_groundwork/README.md index 13a1ba66..b4f997a3 100644 --- a/12_exceptions_part1_groundwork/README.md +++ b/12_exceptions_part1_groundwork/README.md @@ -957,15 +957,15 @@ diff -uNr 11_virtual_memory/src/bsp.rs 12_exceptions_part1_groundwork/src/bsp.rs diff -uNr 11_virtual_memory/src/main.rs 12_exceptions_part1_groundwork/src/main.rs --- 11_virtual_memory/src/main.rs +++ 12_exceptions_part1_groundwork/src/main.rs -@@ -107,6 +107,7 @@ - #![allow(incomplete_features)] +@@ -108,6 +108,7 @@ #![feature(const_generics)] + #![feature(const_panic)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(naked_functions)] #![feature(panic_info_message)] #![feature(trait_alias)] -@@ -142,6 +143,8 @@ +@@ -143,6 +144,8 @@ use driver::interface::DriverManager; use memory::mmu::interface::MMU; @@ -974,7 +974,7 @@ diff -uNr 11_virtual_memory/src/main.rs 12_exceptions_part1_groundwork/src/main. if let Err(string) = memory::mmu::mmu().init() { panic!("MMU: {}", string); } -@@ -193,13 +196,28 @@ +@@ -194,13 +197,28 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs index e104edbc..dd94f714 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs @@ -10,7 +10,7 @@ use super::{AccessPermissions, AttributeFields, MemAttributes}; use crate::{bsp, memory}; use core::convert; use cortex_a::{barrier, regs::*}; -use register::register_bitfields; +use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -89,39 +89,30 @@ const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) /// The output points to the next table. #[derive(Copy, Clone)] #[repr(transparent)] -struct TableDescriptor(u64); +struct TableDescriptor(InMemoryRegister); /// A page descriptor with 64 KiB aperture. /// /// The output points to physical memory. #[derive(Copy, Clone)] #[repr(transparent)] -struct PageDescriptor(u64); +struct PageDescriptor(InMemoryRegister); /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB /// aligned, hence the "reverse" order of appearance. #[repr(C)] #[repr(align(65536))] -struct TranslationTables { +struct FixedSizeTranslationTable { /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; N], + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; N], + lvl2: [TableDescriptor; NUM_TABLES], } /// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. -const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; - -/// The translation tables. -/// -/// # Safety -/// -/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. -static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables { - lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], - lvl2: [TableDescriptor(0); ENTRIES_512_MIB], -}; +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +type ArchTranslationTable = FixedSizeTranslationTable; trait BaseAddr { fn base_addr_u64(&self) -> u64; @@ -135,17 +126,20 @@ mod mair { pub const NORMAL: u64 = 1; } -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - /// Memory Management Unit type. -pub struct MemoryManagementUnit; +struct MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- +/// The translation tables. +/// +/// # Safety +/// +/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); + static MMU: MemoryManagementUnit = MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- @@ -158,23 +152,26 @@ impl BaseAddr for [T; N] { } fn base_addr_usize(&self) -> usize { - self as *const T as usize + self as *const _ as usize } } impl convert::From for TableDescriptor { fn from(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)) - .value; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); TableDescriptor(val) } } -/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. impl convert::From for register::FieldValue { @@ -209,19 +206,35 @@ impl convert::From } impl PageDescriptor { + /// Create an instance. fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)) - .value; + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); Self(val) } } +impl FixedSizeTranslationTable<{ NUM_TABLES }> { + /// Create an instance. + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + + Self { + lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], + lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], + } + } +} + /// Setup function for the MAIR_EL1 register. fn set_up_mair() { // Define the memory types being mapped. @@ -260,6 +273,7 @@ unsafe fn populate_tt_entries() -> Result<(), &'static str> { /// Configure various settings of stage 1 of the EL1 translation regime. fn configure_translation_control() { let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( TCR_EL1::TBI0::Ignored + TCR_EL1::IPS.val(ips) @@ -289,7 +303,7 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn init(&self) -> Result<(), &'static str> { // Fail early if translation granule is not supported. Both RPis support it, though. if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { - return Err("64 KiB translation granule not supported"); + return Err("Translation granule not supported in HW"); } // Prepare the memory attribute indirection register. diff --git a/12_exceptions_part1_groundwork/src/main.rs b/12_exceptions_part1_groundwork/src/main.rs index 25640f78..d9473f18 100644 --- a/12_exceptions_part1_groundwork/src/main.rs +++ b/12_exceptions_part1_groundwork/src/main.rs @@ -106,6 +106,7 @@ #![allow(incomplete_features)] #![feature(const_generics)] +#![feature(const_panic)] #![feature(format_args_nl)] #![feature(global_asm)] #![feature(naked_functions)] diff --git a/13_integrated_testing/README.md b/13_integrated_testing/README.md index cf1f699a..f87b210b 100644 --- a/13_integrated_testing/README.md +++ b/13_integrated_testing/README.md @@ -1095,7 +1095,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/exception.rs 13_integrated_testing/ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/lib.rs --- 12_exceptions_part1_groundwork/src/lib.rs +++ 13_integrated_testing/src/lib.rs -@@ -0,0 +1,170 @@ +@@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter @@ -1206,6 +1206,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li + +#![allow(incomplete_features)] +#![feature(const_generics)] ++#![feature(const_panic)] +#![feature(custom_inner_attributes)] +#![feature(format_args_nl)] +#![feature(global_asm)] @@ -1270,7 +1271,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/main.rs --- 12_exceptions_part1_groundwork/src/main.rs +++ 13_integrated_testing/src/main.rs -@@ -6,128 +6,12 @@ +@@ -6,129 +6,12 @@ #![doc(html_logo_url = "https://git.io/JeGIp")] //! The `kernel` binary. @@ -1371,9 +1372,11 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m -//! -//! - `crate::memory::*` -//! - `crate::bsp::memory::*` - +- -#![allow(incomplete_features)] -#![feature(const_generics)] +-#![feature(const_panic)] ++ #![feature(format_args_nl)] -#![feature(global_asm)] -#![feature(naked_functions)] @@ -1400,7 +1403,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m /// Early init code. /// -@@ -139,6 +23,7 @@ +@@ -140,6 +23,7 @@ /// - Without it, any atomic operations, e.g. the yet-to-be-introduced spinlocks in the device /// drivers (which currently employ NullLocks instead of spinlocks), will fail to work on /// the RPi SoCs. @@ -1408,7 +1411,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m unsafe fn kernel_init() -> ! { use driver::interface::DriverManager; use memory::mmu::interface::MMU; -@@ -164,9 +49,7 @@ +@@ -165,9 +49,7 @@ /// The main function running after the early init. fn kernel_main() -> ! { use console::interface::All; @@ -1418,7 +1421,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m info!("Booting on: {}", bsp::board_name()); -@@ -193,31 +76,6 @@ +@@ -194,31 +76,6 @@ info!(" {}. {}", i + 1, driver.compatible()); } diff --git a/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs b/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs index e104edbc..dd94f714 100644 --- a/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs +++ b/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs @@ -10,7 +10,7 @@ use super::{AccessPermissions, AttributeFields, MemAttributes}; use crate::{bsp, memory}; use core::convert; use cortex_a::{barrier, regs::*}; -use register::register_bitfields; +use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -89,39 +89,30 @@ const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) /// The output points to the next table. #[derive(Copy, Clone)] #[repr(transparent)] -struct TableDescriptor(u64); +struct TableDescriptor(InMemoryRegister); /// A page descriptor with 64 KiB aperture. /// /// The output points to physical memory. #[derive(Copy, Clone)] #[repr(transparent)] -struct PageDescriptor(u64); +struct PageDescriptor(InMemoryRegister); /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB /// aligned, hence the "reverse" order of appearance. #[repr(C)] #[repr(align(65536))] -struct TranslationTables { +struct FixedSizeTranslationTable { /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; N], + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; N], + lvl2: [TableDescriptor; NUM_TABLES], } /// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. -const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; - -/// The translation tables. -/// -/// # Safety -/// -/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. -static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables { - lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], - lvl2: [TableDescriptor(0); ENTRIES_512_MIB], -}; +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +type ArchTranslationTable = FixedSizeTranslationTable; trait BaseAddr { fn base_addr_u64(&self) -> u64; @@ -135,17 +126,20 @@ mod mair { pub const NORMAL: u64 = 1; } -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - /// Memory Management Unit type. -pub struct MemoryManagementUnit; +struct MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- +/// The translation tables. +/// +/// # Safety +/// +/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); + static MMU: MemoryManagementUnit = MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- @@ -158,23 +152,26 @@ impl BaseAddr for [T; N] { } fn base_addr_usize(&self) -> usize { - self as *const T as usize + self as *const _ as usize } } impl convert::From for TableDescriptor { fn from(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)) - .value; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); TableDescriptor(val) } } -/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. impl convert::From for register::FieldValue { @@ -209,19 +206,35 @@ impl convert::From } impl PageDescriptor { + /// Create an instance. fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)) - .value; + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); Self(val) } } +impl FixedSizeTranslationTable<{ NUM_TABLES }> { + /// Create an instance. + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + + Self { + lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], + lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], + } + } +} + /// Setup function for the MAIR_EL1 register. fn set_up_mair() { // Define the memory types being mapped. @@ -260,6 +273,7 @@ unsafe fn populate_tt_entries() -> Result<(), &'static str> { /// Configure various settings of stage 1 of the EL1 translation regime. fn configure_translation_control() { let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( TCR_EL1::TBI0::Ignored + TCR_EL1::IPS.val(ips) @@ -289,7 +303,7 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn init(&self) -> Result<(), &'static str> { // Fail early if translation granule is not supported. Both RPis support it, though. if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { - return Err("64 KiB translation granule not supported"); + return Err("Translation granule not supported in HW"); } // Prepare the memory attribute indirection register. diff --git a/13_integrated_testing/src/lib.rs b/13_integrated_testing/src/lib.rs index 6c6ed7b7..8c9603e4 100644 --- a/13_integrated_testing/src/lib.rs +++ b/13_integrated_testing/src/lib.rs @@ -108,6 +108,7 @@ #![allow(incomplete_features)] #![feature(const_generics)] +#![feature(const_panic)] #![feature(custom_inner_attributes)] #![feature(format_args_nl)] #![feature(global_asm)] diff --git a/14_exceptions_part2_peripheral_IRQs/README.md b/14_exceptions_part2_peripheral_IRQs/README.md index 0c152be2..b1356ebb 100644 --- a/14_exceptions_part2_peripheral_IRQs/README.md +++ b/14_exceptions_part2_peripheral_IRQs/README.md @@ -2416,20 +2416,20 @@ diff -uNr 13_integrated_testing/src/lib.rs 14_exceptions_part2_peripheral_IRQs/s //! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html //! //! # Code organization and architecture -@@ -107,8 +112,11 @@ +@@ -107,9 +112,11 @@ //! - `crate::bsp::memory::*` #![allow(incomplete_features)] +#![feature(asm)] +#![feature(const_fn)] #![feature(const_generics)] + #![feature(const_panic)] -#![feature(custom_inner_attributes)] -+#![feature(const_panic)] +#![feature(core_intrinsics)] #![feature(format_args_nl)] #![feature(global_asm)] #![feature(linkage)] -@@ -137,6 +145,7 @@ +@@ -138,6 +145,7 @@ pub mod exception; pub mod memory; pub mod print; diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs index e104edbc..dd94f714 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs @@ -10,7 +10,7 @@ use super::{AccessPermissions, AttributeFields, MemAttributes}; use crate::{bsp, memory}; use core::convert; use cortex_a::{barrier, regs::*}; -use register::register_bitfields; +use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -89,39 +89,30 @@ const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) /// The output points to the next table. #[derive(Copy, Clone)] #[repr(transparent)] -struct TableDescriptor(u64); +struct TableDescriptor(InMemoryRegister); /// A page descriptor with 64 KiB aperture. /// /// The output points to physical memory. #[derive(Copy, Clone)] #[repr(transparent)] -struct PageDescriptor(u64); +struct PageDescriptor(InMemoryRegister); /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB /// aligned, hence the "reverse" order of appearance. #[repr(C)] #[repr(align(65536))] -struct TranslationTables { +struct FixedSizeTranslationTable { /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; N], + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; N], + lvl2: [TableDescriptor; NUM_TABLES], } /// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. -const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; - -/// The translation tables. -/// -/// # Safety -/// -/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. -static mut TABLES: TranslationTables<{ ENTRIES_512_MIB }> = TranslationTables { - lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], - lvl2: [TableDescriptor(0); ENTRIES_512_MIB], -}; +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +type ArchTranslationTable = FixedSizeTranslationTable; trait BaseAddr { fn base_addr_u64(&self) -> u64; @@ -135,17 +126,20 @@ mod mair { pub const NORMAL: u64 = 1; } -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - /// Memory Management Unit type. -pub struct MemoryManagementUnit; +struct MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- +/// The translation tables. +/// +/// # Safety +/// +/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); + static MMU: MemoryManagementUnit = MemoryManagementUnit; //-------------------------------------------------------------------------------------------------- @@ -158,23 +152,26 @@ impl BaseAddr for [T; N] { } fn base_addr_usize(&self) -> usize { - self as *const T as usize + self as *const _ as usize } } impl convert::From for TableDescriptor { fn from(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)) - .value; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); TableDescriptor(val) } } -/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. impl convert::From for register::FieldValue { @@ -209,19 +206,35 @@ impl convert::From } impl PageDescriptor { + /// Create an instance. fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; - let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)) - .value; + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); Self(val) } } +impl FixedSizeTranslationTable<{ NUM_TABLES }> { + /// Create an instance. + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + + Self { + lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], + lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], + } + } +} + /// Setup function for the MAIR_EL1 register. fn set_up_mair() { // Define the memory types being mapped. @@ -260,6 +273,7 @@ unsafe fn populate_tt_entries() -> Result<(), &'static str> { /// Configure various settings of stage 1 of the EL1 translation regime. fn configure_translation_control() { let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( TCR_EL1::TBI0::Ignored + TCR_EL1::IPS.val(ips) @@ -289,7 +303,7 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn init(&self) -> Result<(), &'static str> { // Fail early if translation granule is not supported. Both RPis support it, though. if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { - return Err("64 KiB translation granule not supported"); + return Err("Translation granule not supported in HW"); } // Prepare the memory attribute indirection register.