diff --git a/00_before_we_start/README.md b/00_before_we_start/README.md index 02c25118..60e060bc 100644 --- a/00_before_we_start/README.md +++ b/00_before_we_start/README.md @@ -12,7 +12,7 @@ the tutorials advance. Have fun! -## Code organization and architecture +# Code organization and architecture The code is divided into different *modules*, each representing a typical **subsystem** of the `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example, @@ -25,15 +25,22 @@ architecture. For each supported processor architecture, there exists a subfolde for example, `src/_arch/aarch64`. The architecture folders mirror the subsystem modules laid out in `src`. For example, architectural -code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go into -`src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -`src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -module organization. That means a public function `foo()` defined in `src/_arch/aarch64/memory.rs` -would be reachable as `crate::memory::foo()` only. +code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go into +`src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in `src/memory/mmu.rs` +using the `path attribute`. Usually, the chosen module name is the generic module's name prefixed +with `arch_`. -The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. Rather, -it's contents are conditionally pulled into respective files using the `#[path = -"_arch/xxx/yyy.rs"]` attribute. +For example, this is the top of `src/memory/mmu.rs`: + +``` +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/memory/mmu.rs"] +mod arch_mmu; +``` + +Often times, items from the `arch_ module` will be publicly reexported by the parent module. This +way, each architecture specific module can provide its implementation of an item, while the caller +must not be concerned which architecture has been conditionally compiled. ## BSP code @@ -42,9 +49,8 @@ target board specific definitions and functions. These are things such as the bo instances of drivers for devices that are featured on the respective board. Just like processor architecture code, the `BSP` code's module structure tries to mirror the -`kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -whatever is provided must be called starting from the `bsp` namespace, e.g. -`bsp::driver::driver_manager()`. +`kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is provided +must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. ## Kernel interfaces @@ -96,3 +102,10 @@ From a namespace perspective, **memory** subsystem code lives in: - `crate::memory::*` - `crate::bsp::memory::*` + +# Boot flow + +1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. + - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. + + diff --git a/01_wait_forever/Makefile b/01_wait_forever/Makefile index ef6395f1..abecc406 100644 --- a/01_wait_forever/Makefile +++ b/01_wait_forever/Makefile @@ -34,8 +34,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/01_wait_forever/README.md b/01_wait_forever/README.md index 0bd3e742..785e9817 100644 --- a/01_wait_forever/README.md +++ b/01_wait_forever/README.md @@ -23,8 +23,8 @@ - Only `.text` section. - `main.rs`: Important [inner attributes]: - `#![no_std]`, `#![no_main]` -- `cpu.S`: Assembly `_start()` function that executes `wfe` (Wait For Event), halting all cores that - are executing `_start()`. +- `boot.S`: Assembly `_start()` function that executes `wfe` (Wait For Event), halting all cores + that are executing `_start()`. - We (have to) define a `#[panic_handler]` function to make the compiler happy. - Make it `unimplemented!()` because it will be stripped out since it is not used. diff --git a/01_wait_forever/src/_arch/aarch64/cpu.rs b/01_wait_forever/src/_arch/aarch64/cpu.rs deleted file mode 100644 index 6d24fe28..00000000 --- a/01_wait_forever/src/_arch/aarch64/cpu.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural processor code. - -// Assembly counterpart to this file. -global_asm!(include_str!("cpu.S")); diff --git a/01_wait_forever/src/_arch/aarch64/cpu.S b/01_wait_forever/src/_arch/aarch64/cpu/boot.S similarity index 100% rename from 01_wait_forever/src/_arch/aarch64/cpu.S rename to 01_wait_forever/src/_arch/aarch64/cpu/boot.S diff --git a/01_wait_forever/src/_arch/aarch64/cpu/boot.rs b/01_wait_forever/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c3c325d3 --- /dev/null +++ b/01_wait_forever/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +// Assembly counterpart to this file. Includes function _start(). +global_asm!(include_str!("boot.S")); diff --git a/01_wait_forever/src/bsp.rs b/01_wait_forever/src/bsp.rs index 8f0d27c8..2b92251f 100644 --- a/01_wait_forever/src/bsp.rs +++ b/01_wait_forever/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; diff --git a/01_wait_forever/src/cpu.rs b/01_wait_forever/src/cpu.rs index 27aea204..c3962ff8 100644 --- a/01_wait_forever/src/cpu.rs +++ b/01_wait_forever/src/cpu.rs @@ -4,7 +4,4 @@ //! Processor code. -#[cfg(target_arch = "aarch64")] -#[path = "_arch/aarch64/cpu.rs"] -mod arch_cpu; -pub use arch_cpu::*; +mod boot; diff --git a/01_wait_forever/src/cpu/boot.rs b/01_wait_forever/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/01_wait_forever/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/01_wait_forever/src/main.rs b/01_wait_forever/src/main.rs index 88345353..10cf6bb4 100644 --- a/01_wait_forever/src/main.rs +++ b/01_wait_forever/src/main.rs @@ -20,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -37,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -91,14 +97,17 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. #![feature(asm)] #![feature(global_asm)] #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. - mod bsp; mod cpu; mod panic_wait; diff --git a/02_runtime_init/Makefile b/02_runtime_init/Makefile index ef6395f1..abecc406 100644 --- a/02_runtime_init/Makefile +++ b/02_runtime_init/Makefile @@ -34,8 +34,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/02_runtime_init/README.CN.md b/02_runtime_init/README.CN.md index 1369031d..0ffecb49 100644 --- a/02_runtime_init/README.CN.md +++ b/02_runtime_init/README.CN.md @@ -2,7 +2,7 @@ ## tl;dr -我们拓展了`cpu.S`，在第一次启动的时候调用Rust代码。在Rust的代码中先清零了[bss] section，然后通过调用`panic()`挂起CPU。再次运行`make qemu`看看新增加的代码是怎么运行的。 +我们拓展了`boot.S`，在第一次启动的时候调用Rust代码。在Rust的代码中先清零了[bss] section，然后通过调用`panic()`挂起CPU。再次运行`make qemu`看看新增加的代码是怎么运行的。 ## 值得注意的变化 @@ -21,9 +21,23 @@ ## 相比之前的变化（diff） ```diff -diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.S 02_runtime_init/src/_arch/aarch64/cpu.S ---- 01_wait_forever/src/_arch/aarch64/cpu.S -+++ 02_runtime_init/src/_arch/aarch64/cpu.S +diff -uNr 01_wait_forever/Cargo.toml 02_runtime_init/Cargo.toml +--- 01_wait_forever/Cargo.toml ++++ 02_runtime_init/Cargo.toml +@@ -4,6 +4,9 @@ + authors = ["Andre Richter "] + edition = "2018" + ++[profile.release] ++lto = true ++ + # The features section is used to select the target board. + [features] + default = [] + +diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.S 02_runtime_init/src/_arch/aarch64/cpu/boot.S +--- 01_wait_forever/src/_arch/aarch64/cpu/boot.S ++++ 02_runtime_init/src/_arch/aarch64/cpu/boot.S @@ -7,5 +7,15 @@ .global _start @@ -43,10 +57,45 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.S 02_runtime_init/src/_arch/aarc + b 1b // We should never reach here. But just in case, + // park this core aswell +diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs +--- 01_wait_forever/src/_arch/aarch64/cpu.rs ++++ 02_runtime_init/src/_arch/aarch64/cpu.rs +@@ -0,0 +1,30 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! Architectural processor code. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::cpu::arch_cpu ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Pause execution on the core. ++#[inline(always)] ++pub fn wait_forever() -> ! { ++ unsafe { ++ loop { ++ #[rustfmt::skip] ++ asm!( ++ "wfe", ++ options(nomem, nostack, preserves_flags) ++ ); ++ } ++ } ++} + diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld --- 01_wait_forever/src/bsp/raspberrypi/link.ld +++ 02_runtime_init/src/bsp/raspberrypi/link.ld -@@ -13,5 +13,24 @@ +@@ -13,5 +13,27 @@ *(.text._start) *(.text*) } @@ -66,22 +115,97 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra + __bss_start = .; + *(.bss*); + . = ALIGN(8); -+ __bss_end = .; ++ ++ /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ ++ . += 8; ++ __bss_end_inclusive = . - 8; + } + /DISCARD/ : { *(.comment*) } } +diff -uNr 01_wait_forever/src/bsp/raspberrypi/memory.rs 02_runtime_init/src/bsp/raspberrypi/memory.rs +--- 01_wait_forever/src/bsp/raspberrypi/memory.rs ++++ 02_runtime_init/src/bsp/raspberrypi/memory.rs +@@ -0,0 +1,37 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! BSP Memory Management. ++ ++use core::{cell::UnsafeCell, ops::RangeInclusive}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++// Symbols from the linker script. ++extern "Rust" { ++ static __bss_start: UnsafeCell; ++ static __bss_end_inclusive: UnsafeCell; ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Return the inclusive range spanning the .bss section. ++/// ++/// # Safety ++/// ++/// - Values are provided by the linker script and must be trusted as-is. ++/// - The linker-provided addresses must be u64 aligned. ++pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { ++ let range; ++ unsafe { ++ range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); ++ } ++ assert!(!range.is_empty()); ++ ++ range ++} + +diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs +--- 01_wait_forever/src/bsp/raspberrypi.rs ++++ 02_runtime_init/src/bsp/raspberrypi.rs +@@ -4,4 +4,4 @@ + + //! Top-level BSP file for the Raspberry Pi 3 and 4. + +-// Coming soon. ++pub mod memory; + +diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs +--- 01_wait_forever/src/cpu.rs ++++ 02_runtime_init/src/cpu.rs +@@ -4,4 +4,13 @@ + + //! Processor code. + ++#[cfg(target_arch = "aarch64")] ++#[path = "_arch/aarch64/cpu.rs"] ++mod arch_cpu; ++ + mod boot; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_cpu::wait_forever; + diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs --- 01_wait_forever/src/main.rs +++ 02_runtime_init/src/main.rs -@@ -97,10 +97,20 @@ - #![no_main] - #![no_std] +@@ -102,6 +102,7 @@ + //! + //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. ++//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. + //! + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html --// `mod cpu` provides the `_start()` function, the first function to run. -+// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -+// `runtime_init()`, which jumps to `kernel_init()`. +@@ -112,6 +113,15 @@ mod bsp; mod cpu; @@ -102,73 +226,70 @@ diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs diff -uNr 01_wait_forever/src/memory.rs 02_runtime_init/src/memory.rs --- 01_wait_forever/src/memory.rs +++ 02_runtime_init/src/memory.rs -@@ -0,0 +1,29 @@ +@@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//! Memory Management. + -+use core::ops::Range; ++use core::ops::RangeInclusive; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + -+/// Zero out a memory region. ++/// Zero out an inclusive memory range. +/// +/// # Safety +/// +/// - `range.start` and `range.end` must be valid. +/// - `range.start` and `range.end` must be `T` aligned. -+pub unsafe fn zero_volatile(range: Range<*mut T>) ++pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) +where + T: From, +{ -+ let mut ptr = range.start; ++ let mut ptr = *range.start(); ++ let end_inclusive = *range.end(); + -+ while ptr < range.end { ++ while ptr <= end_inclusive { + core::ptr::write_volatile(ptr, T::from(0)); + ptr = ptr.offset(1); + } +} +diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs +--- 01_wait_forever/src/panic_wait.rs ++++ 02_runtime_init/src/panic_wait.rs +@@ -4,9 +4,10 @@ + + //! A panic handler that infinitely waits. + ++use crate::cpu; + use core::panic::PanicInfo; + + #[panic_handler] + fn panic(_info: &PanicInfo) -> ! { +- unimplemented!() ++ cpu::wait_forever() + } + diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.rs --- 01_wait_forever/src/runtime_init.rs +++ 02_runtime_init/src/runtime_init.rs -@@ -0,0 +1,58 @@ +@@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//! Rust runtime initialization code. + -+use crate::memory; -+use core::ops::Range; ++use crate::{bsp, memory}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + -+/// Return the range spanning the .bss section. -+/// -+/// # Safety -+/// -+/// - The symbol-provided addresses must be valid. -+/// - The symbol-provided addresses must be usize aligned. -+unsafe fn bss_range() -> Range<*mut usize> { -+ extern "C" { -+ // Boundaries of the .bss section, provided by linker script symbols. -+ static mut __bss_start: usize; -+ static mut __bss_end: usize; -+ } -+ -+ Range { -+ start: &mut __bss_start, -+ end: &mut __bss_end, -+ } -+} -+ +/// Zero out the .bss section. +/// +/// # Safety @@ -176,7 +297,7 @@ diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.r +/// - Must only be called pre `kernel_init()`. +#[inline(always)] +unsafe fn zero_bss() { -+ memory::zero_volatile(bss_range()); ++ memory::zero_volatile(bsp::memory::bss_range_inclusive()); +} + +//-------------------------------------------------------------------------------------------------- @@ -190,7 +311,7 @@ diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.r +/// +/// - Only a single core must be active and running this function. +#[no_mangle] -+pub unsafe extern "C" fn runtime_init() -> ! { ++pub unsafe fn runtime_init() -> ! { + zero_bss(); + + crate::kernel_init() diff --git a/02_runtime_init/README.md b/02_runtime_init/README.md index 206424f9..7948aae4 100644 --- a/02_runtime_init/README.md +++ b/02_runtime_init/README.md @@ -2,7 +2,7 @@ ## tl;dr -- We extend `cpu.S` to call into Rust code for the first time. There, we zero the [bss] section +- We extend `boot.S` to call into Rust code for the first time. There, we zero the [bss] section before execution is halted with a call to `panic()`. - Check out `make qemu` again to see the additional code run. @@ -37,13 +37,44 @@ diff -uNr 01_wait_forever/Cargo.toml 02_runtime_init/Cargo.toml [features] default = [] +diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.S 02_runtime_init/src/_arch/aarch64/cpu/boot.S +--- 01_wait_forever/src/_arch/aarch64/cpu/boot.S ++++ 02_runtime_init/src/_arch/aarch64/cpu/boot.S +@@ -7,5 +7,15 @@ + .global _start + + _start: +-1: wfe // Wait for event +- b 1b // In case an event happened, jump back to 1 ++ mrs x1, mpidr_el1 // Read Multiprocessor Affinity Register ++ and x1, x1, #3 // Clear all bits except [1:0], which hold core id ++ cbz x1, 2f // Jump to label 2 if we are core 0 ++1: wfe // Wait for event ++ b 1b // In case an event happened, jump back to 1 ++2: // If we are here, we are core0 ++ ldr x1, =_start // Load address of function "_start()" ++ mov sp, x1 // Set start of stack to before our code, aka first ++ // address before "_start()" ++ bl runtime_init // Jump to the "runtime_init()" kernel function ++ b 1b // We should never reach here. But just in case, ++ // park this core aswell + diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs --- 01_wait_forever/src/_arch/aarch64/cpu.rs +++ 02_runtime_init/src/_arch/aarch64/cpu.rs -@@ -6,3 +6,21 @@ - - // Assembly counterpart to this file. - global_asm!(include_str!("cpu.S")); +@@ -0,0 +1,30 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! Architectural processor code. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::cpu::arch_cpu + +//-------------------------------------------------------------------------------------------------- +// Public Code @@ -63,28 +94,6 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aar + } +} -diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.S 02_runtime_init/src/_arch/aarch64/cpu.S ---- 01_wait_forever/src/_arch/aarch64/cpu.S -+++ 02_runtime_init/src/_arch/aarch64/cpu.S -@@ -7,5 +7,15 @@ - .global _start - - _start: --1: wfe // Wait for event -- b 1b // In case an event happened, jump back to 1 -+ mrs x1, mpidr_el1 // Read Multiprocessor Affinity Register -+ and x1, x1, #3 // Clear all bits except [1:0], which hold core id -+ cbz x1, 2f // Jump to label 2 if we are core 0 -+1: wfe // Wait for event -+ b 1b // In case an event happened, jump back to 1 -+2: // If we are here, we are core0 -+ ldr x1, =_start // Load address of function "_start()" -+ mov sp, x1 // Set start of stack to before our code, aka first -+ // address before "_start()" -+ bl runtime_init // Jump to the "runtime_init()" kernel function -+ b 1b // We should never reach here. But just in case, -+ // park this core aswell - diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld --- 01_wait_forever/src/bsp/raspberrypi/link.ld +++ 02_runtime_init/src/bsp/raspberrypi/link.ld @@ -169,16 +178,36 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspber -// Coming soon. +pub mod memory; +diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs +--- 01_wait_forever/src/cpu.rs ++++ 02_runtime_init/src/cpu.rs +@@ -4,4 +4,13 @@ + + //! Processor code. + ++#[cfg(target_arch = "aarch64")] ++#[path = "_arch/aarch64/cpu.rs"] ++mod arch_cpu; ++ + mod boot; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_cpu::wait_forever; + diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs --- 01_wait_forever/src/main.rs +++ 02_runtime_init/src/main.rs -@@ -97,10 +97,20 @@ - #![no_main] - #![no_std] +@@ -102,6 +102,7 @@ + //! + //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. ++//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. --// `mod cpu` provides the `_start()` function, the first function to run. -+// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -+// `runtime_init()`, which jumps to `kernel_init()`. + #![feature(asm)] + #![feature(global_asm)] +@@ -110,6 +111,15 @@ mod bsp; mod cpu; diff --git a/02_runtime_init/src/_arch/aarch64/cpu.rs b/02_runtime_init/src/_arch/aarch64/cpu.rs index 6ca5986a..4fd7e313 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu.rs +++ b/02_runtime_init/src/_arch/aarch64/cpu.rs @@ -3,9 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -// Assembly counterpart to this file. -global_asm!(include_str!("cpu.S")); +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/02_runtime_init/src/_arch/aarch64/cpu.S b/02_runtime_init/src/_arch/aarch64/cpu/boot.S similarity index 100% rename from 02_runtime_init/src/_arch/aarch64/cpu.S rename to 02_runtime_init/src/_arch/aarch64/cpu/boot.S diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c3c325d3 --- /dev/null +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +// Assembly counterpart to this file. Includes function _start(). +global_asm!(include_str!("boot.S")); diff --git a/02_runtime_init/src/bsp.rs b/02_runtime_init/src/bsp.rs index 8f0d27c8..2b92251f 100644 --- a/02_runtime_init/src/bsp.rs +++ b/02_runtime_init/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; diff --git a/02_runtime_init/src/cpu.rs b/02_runtime_init/src/cpu.rs index 27aea204..d8f78082 100644 --- a/02_runtime_init/src/cpu.rs +++ b/02_runtime_init/src/cpu.rs @@ -7,4 +7,10 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::wait_forever; diff --git a/02_runtime_init/src/cpu/boot.rs b/02_runtime_init/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/02_runtime_init/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/02_runtime_init/src/main.rs b/02_runtime_init/src/main.rs index 76e7a772..ba5a877c 100644 --- a/02_runtime_init/src/main.rs +++ b/02_runtime_init/src/main.rs @@ -20,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -37,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -91,15 +97,18 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. #![feature(asm)] #![feature(global_asm)] #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod cpu; mod memory; diff --git a/03_hacky_hello_world/Makefile b/03_hacky_hello_world/Makefile index 90238d09..9d5a8f78 100644 --- a/03_hacky_hello_world/Makefile +++ b/03_hacky_hello_world/Makefile @@ -34,8 +34,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/03_hacky_hello_world/README.md b/03_hacky_hello_world/README.md index 95760f02..11c804bf 100644 --- a/03_hacky_hello_world/README.md +++ b/03_hacky_hello_world/README.md @@ -139,8 +139,17 @@ diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs --- 02_runtime_init/src/main.rs +++ 03_hacky_hello_world/src/main.rs -@@ -93,7 +93,9 @@ - //! - `crate::bsp::memory::*` +@@ -100,19 +100,25 @@ + //! + //! # Boot flow + //! +-//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +-//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. ++//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. ++//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. + //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. ++//! ++//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(asm)] +#![feature(format_args_nl)] @@ -149,9 +158,6 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs #![no_main] #![no_std] -@@ -101,9 +103,11 @@ - // `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; +mod console; mod cpu; @@ -161,7 +167,7 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs mod runtime_init; /// Early init code. -@@ -112,5 +116,7 @@ +@@ -121,5 +127,7 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { @@ -202,7 +208,7 @@ diff -uNr 02_runtime_init/src/print.rs 03_hacky_hello_world/src/print.rs +// +// Copyright (c) 2018-2021 Andre Richter + -+//! Printing facilities. ++//! Printing. + +use crate::{bsp, console}; +use core::fmt; diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu.rs b/03_hacky_hello_world/src/_arch/aarch64/cpu.rs index 6ca5986a..4fd7e313 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu.rs +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu.rs @@ -3,9 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -// Assembly counterpart to this file. -global_asm!(include_str!("cpu.S")); +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu.S b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S similarity index 100% rename from 03_hacky_hello_world/src/_arch/aarch64/cpu.S rename to 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c3c325d3 --- /dev/null +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +// Assembly counterpart to this file. Includes function _start(). +global_asm!(include_str!("boot.S")); diff --git a/03_hacky_hello_world/src/bsp.rs b/03_hacky_hello_world/src/bsp.rs index 8f0d27c8..2b92251f 100644 --- a/03_hacky_hello_world/src/bsp.rs +++ b/03_hacky_hello_world/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; diff --git a/03_hacky_hello_world/src/cpu.rs b/03_hacky_hello_world/src/cpu.rs index 27aea204..d8f78082 100644 --- a/03_hacky_hello_world/src/cpu.rs +++ b/03_hacky_hello_world/src/cpu.rs @@ -7,4 +7,10 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::wait_forever; diff --git a/03_hacky_hello_world/src/cpu/boot.rs b/03_hacky_hello_world/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/03_hacky_hello_world/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/03_hacky_hello_world/src/main.rs b/03_hacky_hello_world/src/main.rs index 3a4cf64c..378e0baf 100644 --- a/03_hacky_hello_world/src/main.rs +++ b/03_hacky_hello_world/src/main.rs @@ -20,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -37,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -91,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(asm)] #![feature(format_args_nl)] @@ -99,9 +113,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/03_hacky_hello_world/src/print.rs b/03_hacky_hello_world/src/print.rs index 59ec9e8e..fa6451c7 100644 --- a/03_hacky_hello_world/src/print.rs +++ b/03_hacky_hello_world/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/04_zero_overhead_abstraction/Makefile b/04_zero_overhead_abstraction/Makefile index 90238d09..9d5a8f78 100644 --- a/04_zero_overhead_abstraction/Makefile +++ b/04_zero_overhead_abstraction/Makefile @@ -34,8 +34,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/04_zero_overhead_abstraction/README.md b/04_zero_overhead_abstraction/README.md index ce452fb7..6c9ad70c 100644 --- a/04_zero_overhead_abstraction/README.md +++ b/04_zero_overhead_abstraction/README.md @@ -23,47 +23,20 @@ diff -uNr 03_hacky_hello_world/Cargo.toml 04_zero_overhead_abstraction/Cargo.tom +cortex-a = { version = "5.x.x" } + -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/smp.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs ---- 03_hacky_hello_world/src/_arch/aarch64/cpu/smp.rs -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs -@@ -0,0 +1,22 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Architectural symmetric multiprocessing. -+ -+use cortex_a::regs::*; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Return the executing core's id. -+#[inline(always)] -+pub fn core_id() -> T -+where -+ T: From, -+{ -+ const CORE_MASK: u64 = 0b11; -+ -+ T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -+} +diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs +--- 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs ++++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs +@@ -11,5 +11,31 @@ + //! + //! crate::cpu::boot::arch_boot -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs ---- 03_hacky_hello_world/src/_arch/aarch64/cpu.rs -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs -@@ -4,8 +4,34 @@ - - //! Architectural processor code. - --// Assembly counterpart to this file. --global_asm!(include_str!("cpu.S")); +-// Assembly counterpart to this file. Includes function _start(). +-global_asm!(include_str!("boot.S")); +use crate::{bsp, cpu}; -+use cortex_a::{asm, regs::*}; ++use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- -+// Boot Code ++// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. @@ -84,32 +57,13 @@ diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu.rs 04_zero_overhead_abstrac + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. -+ wait_forever() ++ cpu::wait_forever() + } +} - //-------------------------------------------------------------------------------------------------- - // Public Code -@@ -14,13 +40,7 @@ - /// Pause execution on the core. - #[inline(always)] - pub fn wait_forever() -> ! { -- unsafe { -- loop { -- #[rustfmt::skip] -- asm!( -- "wfe", -- options(nomem, nostack, preserves_flags) -- ); -- } -+ loop { -+ asm::wfe() - } - } - -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu.S 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.S ---- 03_hacky_hello_world/src/_arch/aarch64/cpu.S -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.S +diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.S +--- 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S ++++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.S @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// @@ -133,6 +87,69 @@ diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu.S 04_zero_overhead_abstract - b 1b // We should never reach here. But just in case, - // park this core aswell +diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/smp.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs +--- 03_hacky_hello_world/src/_arch/aarch64/cpu/smp.rs ++++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs +@@ -0,0 +1,29 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! Architectural symmetric multiprocessing. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::cpu::smp::arch_smp ++ ++use cortex_a::regs::*; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Return the executing core's id. ++#[inline(always)] ++pub fn core_id() -> T ++where ++ T: From, ++{ ++ const CORE_MASK: u64 = 0b11; ++ ++ T::from((MPIDR_EL1.get() & CORE_MASK) as u8) ++} + +diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs +--- 03_hacky_hello_world/src/_arch/aarch64/cpu.rs ++++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs +@@ -11,6 +11,8 @@ + //! + //! crate::cpu::arch_cpu + ++use cortex_a::asm; ++ + //-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- +@@ -18,13 +20,7 @@ + /// Pause execution on the core. + #[inline(always)] + pub fn wait_forever() -> ! { +- unsafe { +- loop { +- #[rustfmt::skip] +- asm!( +- "wfe", +- options(nomem, nostack, preserves_flags) +- ); +- } ++ loop { ++ asm::wfe() + } + } + diff -uNr 03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs 04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs --- 03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs +++ 04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs @@ -193,7 +210,7 @@ diff -uNr 03_hacky_hello_world/src/bsp/raspberrypi.rs 04_zero_overhead_abstracti diff -uNr 03_hacky_hello_world/src/cpu/smp.rs 04_zero_overhead_abstraction/src/cpu/smp.rs --- 03_hacky_hello_world/src/cpu/smp.rs +++ 04_zero_overhead_abstraction/src/cpu/smp.rs -@@ -0,0 +1,10 @@ +@@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -202,25 +219,32 @@ diff -uNr 03_hacky_hello_world/src/cpu/smp.rs 04_zero_overhead_abstraction/src/c + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/smp.rs"] -+mod arch_cpu_smp; -+pub use arch_cpu_smp::*; ++mod arch_smp; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_smp::core_id; diff -uNr 03_hacky_hello_world/src/cpu.rs 04_zero_overhead_abstraction/src/cpu.rs --- 03_hacky_hello_world/src/cpu.rs +++ 04_zero_overhead_abstraction/src/cpu.rs -@@ -8,3 +8,5 @@ - #[path = "_arch/aarch64/cpu.rs"] - mod arch_cpu; - pub use arch_cpu::*; -+ +@@ -10,6 +10,8 @@ + + mod boot; + +pub mod smp; ++ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- diff -uNr 03_hacky_hello_world/src/main.rs 04_zero_overhead_abstraction/src/main.rs --- 03_hacky_hello_world/src/main.rs +++ 04_zero_overhead_abstraction/src/main.rs -@@ -92,9 +92,7 @@ - //! - `crate::memory::*` - //! - `crate::bsp::memory::*` +@@ -106,9 +106,7 @@ + //! + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html -#![feature(asm)] #![feature(format_args_nl)] @@ -228,7 +252,7 @@ diff -uNr 03_hacky_hello_world/src/main.rs 04_zero_overhead_abstraction/src/main #![feature(panic_info_message)] #![no_main] #![no_std] -@@ -116,7 +114,8 @@ +@@ -127,7 +125,8 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs index b02903c9..5f97ea41 100644 --- a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs +++ b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs @@ -3,35 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..549b5927 --- /dev/null +++ b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::runtime_init; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs +++ b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/04_zero_overhead_abstraction/src/bsp.rs b/04_zero_overhead_abstraction/src/bsp.rs index 8f0d27c8..2b92251f 100644 --- a/04_zero_overhead_abstraction/src/bsp.rs +++ b/04_zero_overhead_abstraction/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; diff --git a/04_zero_overhead_abstraction/src/cpu.rs b/04_zero_overhead_abstraction/src/cpu.rs index c9e5af72..6f326b32 100644 --- a/04_zero_overhead_abstraction/src/cpu.rs +++ b/04_zero_overhead_abstraction/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::wait_forever; diff --git a/04_zero_overhead_abstraction/src/cpu/boot.rs b/04_zero_overhead_abstraction/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/04_zero_overhead_abstraction/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/04_zero_overhead_abstraction/src/cpu/smp.rs b/04_zero_overhead_abstraction/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/04_zero_overhead_abstraction/src/cpu/smp.rs +++ b/04_zero_overhead_abstraction/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/04_zero_overhead_abstraction/src/main.rs b/04_zero_overhead_abstraction/src/main.rs index cd4c99e5..7de836fe 100644 --- a/04_zero_overhead_abstraction/src/main.rs +++ b/04_zero_overhead_abstraction/src/main.rs @@ -20,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -37,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -91,15 +97,20 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(format_args_nl)] #![feature(panic_info_message)] #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/04_zero_overhead_abstraction/src/print.rs b/04_zero_overhead_abstraction/src/print.rs index 59ec9e8e..fa6451c7 100644 --- a/04_zero_overhead_abstraction/src/print.rs +++ b/04_zero_overhead_abstraction/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/05_safe_globals/Makefile b/05_safe_globals/Makefile index 90238d09..9d5a8f78 100644 --- a/05_safe_globals/Makefile +++ b/05_safe_globals/Makefile @@ -34,8 +34,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/05_safe_globals/README.md b/05_safe_globals/README.md index 537b0ba3..3db54d6b 100644 --- a/05_safe_globals/README.md +++ b/05_safe_globals/README.md @@ -3,8 +3,8 @@ ## tl;dr - A pseudo-lock is introduced. -- It is a first showcase of OS synchronization primitives and enables safe access to a global data - structure. +- It is a first showcase of OS synchronization primitives and enables safe access to a global data + structure. ## Mutable globals in Rust @@ -26,9 +26,7 @@ variant of a *MUTual EXclusion* primitive. `Mutex` is introduced as a trait in ` and implemented by the `NullLock` in the same file. In order to make the code lean for teaching purposes, it leaves out the actual architecture-specific logic for protection against concurrent access, since we don't need it as long as the kernel only executes on a single core with interrupts -disabled. That is also why it is implemented in the same file as the interface itself. In later -tutorials, an implementation might move to the `_arch` once it pulls in arch-specific code that can -not be further abstracted. +disabled. The `NullLock` focuses on showcasing the Rust core concept of [interior mutability]. Make sure to read up on it. I also recommend to read this article about an [accurate mental model for Rust's @@ -213,7 +211,7 @@ diff -uNr 04_zero_overhead_abstraction/src/console.rs 05_safe_globals/src/consol diff -uNr 04_zero_overhead_abstraction/src/main.rs 05_safe_globals/src/main.rs --- 04_zero_overhead_abstraction/src/main.rs +++ 05_safe_globals/src/main.rs -@@ -94,6 +94,7 @@ +@@ -108,6 +108,7 @@ #![feature(format_args_nl)] #![feature(panic_info_message)] @@ -221,7 +219,7 @@ diff -uNr 04_zero_overhead_abstraction/src/main.rs 05_safe_globals/src/main.rs #![no_main] #![no_std] -@@ -107,6 +108,7 @@ +@@ -118,6 +119,7 @@ mod panic_wait; mod print; mod runtime_init; @@ -229,7 +227,7 @@ diff -uNr 04_zero_overhead_abstraction/src/main.rs 05_safe_globals/src/main.rs /// Early init code. /// -@@ -114,8 +116,15 @@ +@@ -125,8 +127,15 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/05_safe_globals/src/_arch/aarch64/cpu.rs b/05_safe_globals/src/_arch/aarch64/cpu.rs index b02903c9..5f97ea41 100644 --- a/05_safe_globals/src/_arch/aarch64/cpu.rs +++ b/05_safe_globals/src/_arch/aarch64/cpu.rs @@ -3,35 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/05_safe_globals/src/_arch/aarch64/cpu/boot.rs b/05_safe_globals/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..549b5927 --- /dev/null +++ b/05_safe_globals/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::runtime_init; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/05_safe_globals/src/_arch/aarch64/cpu/smp.rs b/05_safe_globals/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/05_safe_globals/src/_arch/aarch64/cpu/smp.rs +++ b/05_safe_globals/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/05_safe_globals/src/bsp.rs b/05_safe_globals/src/bsp.rs index 8f0d27c8..2b92251f 100644 --- a/05_safe_globals/src/bsp.rs +++ b/05_safe_globals/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; diff --git a/05_safe_globals/src/cpu.rs b/05_safe_globals/src/cpu.rs index c9e5af72..6f326b32 100644 --- a/05_safe_globals/src/cpu.rs +++ b/05_safe_globals/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::wait_forever; diff --git a/05_safe_globals/src/cpu/boot.rs b/05_safe_globals/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/05_safe_globals/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/05_safe_globals/src/cpu/smp.rs b/05_safe_globals/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/05_safe_globals/src/cpu/smp.rs +++ b/05_safe_globals/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/05_safe_globals/src/main.rs b/05_safe_globals/src/main.rs index 64d4c5fd..bb6abcb8 100644 --- a/05_safe_globals/src/main.rs +++ b/05_safe_globals/src/main.rs @@ -20,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -37,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -91,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(format_args_nl)] #![feature(panic_info_message)] @@ -98,9 +112,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/05_safe_globals/src/print.rs b/05_safe_globals/src/print.rs index 59ec9e8e..fa6451c7 100644 --- a/05_safe_globals/src/print.rs +++ b/05_safe_globals/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/06_drivers_gpio_uart/Makefile b/06_drivers_gpio_uart/Makefile index 0fde2eb8..ee7ebe38 100644 --- a/06_drivers_gpio_uart/Makefile +++ b/06_drivers_gpio_uart/Makefile @@ -40,8 +40,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/06_drivers_gpio_uart/README.md b/06_drivers_gpio_uart/README.md index 6ca14cfe..2f23d1b5 100644 --- a/06_drivers_gpio_uart/README.md +++ b/06_drivers_gpio_uart/README.md @@ -19,13 +19,16 @@ from kernel code. - Drivers are stored in `src/bsp/device_driver`, and can be reused between `BSP`s. - We introduce the `GPIO` driver, which pinmuxes the RPi's PL011 UART. + - Note how this driver differentiates between **RPi 3** and **RPi4**. Their HW is different, + so we have to account for it in SW. - Most importantly, the `PL011Uart` driver: It implements the `console::interface::*` traits and is from now on used as the main system console output. -- `BSP`s now contain a memory map in `src/bsp/raspberrypi/memory.rs`. In the specific case, they contain the - Raspberry's `MMIO` addresses which are used to instantiate the respectivedevice drivers. +- `BSP`s now contain a memory map in `src/bsp/raspberrypi/memory.rs`. In the specific case, they + contain the Raspberry's `MMIO` addresses which are used to instantiate the respective device + drivers. - We also modify the `panic!` handler, so that it does not anymore rely on `println!`, which uses the globally-shared instance of the `UART` that might be locked when an error is encountered (for - now this can't happen due to the `NullLock`, but with a real lock it becomes an issue). + now, this can't happen due to the `NullLock`, but with a real lock it becomes an issue). - Instead, it creates a new UART driver instance, re-initializes the device and uses that one to print. This increases the chances that the system is able to print a final important message before it suspends itself. @@ -145,7 +148,7 @@ diff -uNr 05_safe_globals/Makefile 06_drivers_gpio_uart/Makefile # BSP-specific arguments ifeq ($(BSP),rpi3) TARGET = aarch64-unknown-none-softfloat -@@ -51,13 +57,23 @@ +@@ -52,13 +58,23 @@ DOCKER_IMAGE = rustembedded/osdev-utils DOCKER_CMD = docker run --rm -v $(shell pwd):/work/tutorial -w /work/tutorial DOCKER_CMD_INTERACT = $(DOCKER_CMD) -i -t @@ -171,7 +174,7 @@ diff -uNr 05_safe_globals/Makefile 06_drivers_gpio_uart/Makefile all: $(KERNEL_BIN) -@@ -78,6 +94,9 @@ +@@ -79,6 +95,9 @@ @$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) endif @@ -185,7 +188,7 @@ diff -uNr 05_safe_globals/Makefile 06_drivers_gpio_uart/Makefile diff -uNr 05_safe_globals/src/_arch/aarch64/cpu.rs 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs --- 05_safe_globals/src/_arch/aarch64/cpu.rs +++ 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs -@@ -37,6 +37,16 @@ +@@ -17,6 +17,16 @@ // Public Code //-------------------------------------------------------------------------------------------------- @@ -1158,7 +1161,7 @@ diff -uNr 05_safe_globals/src/bsp.rs 06_drivers_gpio_uart/src/bsp.rs +++ 06_drivers_gpio_uart/src/bsp.rs @@ -4,6 +4,8 @@ - //! Conditional re-exporting of Board Support Packages. + //! Conditional reexporting of Board Support Packages. +mod device_driver; + @@ -1169,7 +1172,7 @@ diff -uNr 05_safe_globals/src/bsp.rs 06_drivers_gpio_uart/src/bsp.rs diff -uNr 05_safe_globals/src/console.rs 06_drivers_gpio_uart/src/console.rs --- 05_safe_globals/src/console.rs +++ 06_drivers_gpio_uart/src/console.rs -@@ -14,8 +14,26 @@ +@@ -14,8 +14,25 @@ /// Console write functions. pub trait Write { @@ -1179,8 +1182,7 @@ diff -uNr 05_safe_globals/src/console.rs 06_drivers_gpio_uart/src/console.rs /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; + -+ /// Block execution until the last buffered character has been physically put on the TX -+ /// wire. ++ /// Block until the last buffered character has been physically put on the TX wire. + fn flush(&self); + } + @@ -1196,7 +1198,7 @@ diff -uNr 05_safe_globals/src/console.rs 06_drivers_gpio_uart/src/console.rs } /// Console statistics. -@@ -24,8 +42,13 @@ +@@ -24,8 +41,13 @@ fn chars_written(&self) -> usize { 0 } @@ -1212,6 +1214,16 @@ diff -uNr 05_safe_globals/src/console.rs 06_drivers_gpio_uart/src/console.rs + pub trait All = Write + Read + Statistics; } +diff -uNr 05_safe_globals/src/cpu.rs 06_drivers_gpio_uart/src/cpu.rs +--- 05_safe_globals/src/cpu.rs ++++ 06_drivers_gpio_uart/src/cpu.rs +@@ -15,4 +15,4 @@ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- +-pub use arch_cpu::wait_forever; ++pub use arch_cpu::{nop, spin_for_cycles, wait_forever}; + diff -uNr 05_safe_globals/src/driver.rs 06_drivers_gpio_uart/src/driver.rs --- 05_safe_globals/src/driver.rs +++ 06_drivers_gpio_uart/src/driver.rs @@ -1264,30 +1276,15 @@ diff -uNr 05_safe_globals/src/driver.rs 06_drivers_gpio_uart/src/driver.rs diff -uNr 05_safe_globals/src/main.rs 06_drivers_gpio_uart/src/main.rs --- 05_safe_globals/src/main.rs +++ 06_drivers_gpio_uart/src/main.rs -@@ -7,6 +7,14 @@ - - //! The `kernel` binary. - //! -+//! # TL;DR - Overview of important Kernel entities -+//! -+//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -+//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -+//! -+//! [console interface]: ../libkernel/console/interface/index.html -+//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -+//! - //! # Code organization and architecture +@@ -106,6 +106,7 @@ //! - //! The code is divided into different *modules*, each representing a typical **subsystem** of the -@@ -92,6 +100,7 @@ - //! - `crate::memory::*` - //! - `crate::bsp::memory::*` + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html +#![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(panic_info_message)] #![feature(trait_alias)] -@@ -104,6 +113,7 @@ +@@ -115,6 +116,7 @@ mod bsp; mod console; mod cpu; @@ -1295,7 +1292,7 @@ diff -uNr 05_safe_globals/src/main.rs 06_drivers_gpio_uart/src/main.rs mod memory; mod panic_wait; mod print; -@@ -115,16 +125,49 @@ +@@ -126,16 +128,49 @@ /// # Safety /// /// - Only a single core must be active and running this function. @@ -1303,7 +1300,8 @@ diff -uNr 05_safe_globals/src/main.rs 06_drivers_gpio_uart/src/main.rs unsafe fn kernel_init() -> ! { - use console::interface::Statistics; + use driver::interface::DriverManager; -+ + +- println!("[0] Hello from pure Rust!"); + for i in bsp::driver::driver_manager().all_device_drivers().iter() { + if let Err(x) = i.init() { + panic!("Error loading driver: {}: {}", i.compatible(), x); @@ -1315,8 +1313,7 @@ diff -uNr 05_safe_globals/src/main.rs 06_drivers_gpio_uart/src/main.rs + // Transition from unsafe to safe. + kernel_main() +} - -- println!("[0] Hello from pure Rust!"); ++ +/// The main function running after the early init. +fn kernel_main() -> ! { + use bsp::console::console; diff --git a/06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs b/06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs index 2d5d0b04..e16fdecc 100644 --- a/06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs +++ b/06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs @@ -3,35 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..549b5927 --- /dev/null +++ b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::runtime_init; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs +++ b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/06_drivers_gpio_uart/src/bsp.rs b/06_drivers_gpio_uart/src/bsp.rs index 25750249..c558922f 100644 --- a/06_drivers_gpio_uart/src/bsp.rs +++ b/06_drivers_gpio_uart/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/06_drivers_gpio_uart/src/console.rs b/06_drivers_gpio_uart/src/console.rs index 3552823c..c3154ba2 100644 --- a/06_drivers_gpio_uart/src/console.rs +++ b/06_drivers_gpio_uart/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/06_drivers_gpio_uart/src/cpu.rs b/06_drivers_gpio_uart/src/cpu.rs index c9e5af72..103f00db 100644 --- a/06_drivers_gpio_uart/src/cpu.rs +++ b/06_drivers_gpio_uart/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, spin_for_cycles, wait_forever}; diff --git a/06_drivers_gpio_uart/src/cpu/boot.rs b/06_drivers_gpio_uart/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/06_drivers_gpio_uart/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/06_drivers_gpio_uart/src/cpu/smp.rs b/06_drivers_gpio_uart/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/06_drivers_gpio_uart/src/cpu/smp.rs +++ b/06_drivers_gpio_uart/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/06_drivers_gpio_uart/src/main.rs b/06_drivers_gpio_uart/src/main.rs index 1674b9e4..a16c0de7 100644 --- a/06_drivers_gpio_uart/src/main.rs +++ b/06_drivers_gpio_uart/src/main.rs @@ -7,14 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -28,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -45,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -99,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] @@ -107,9 +113,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/06_drivers_gpio_uart/src/print.rs b/06_drivers_gpio_uart/src/print.rs index 59ec9e8e..fa6451c7 100644 --- a/06_drivers_gpio_uart/src/print.rs +++ b/06_drivers_gpio_uart/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/07_uart_chainloader/Makefile b/07_uart_chainloader/Makefile index 1fc61b6b..d32147d6 100644 --- a/07_uart_chainloader/Makefile +++ b/07_uart_chainloader/Makefile @@ -42,8 +42,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/07_uart_chainloader/README.md b/07_uart_chainloader/README.md index fe625110..0bd4bce2 100644 --- a/07_uart_chainloader/README.md +++ b/07_uart_chainloader/README.md @@ -24,20 +24,23 @@ tutorials in a quick manner. Our chainloader is called `MiniLoad` and is inspired by [raspbootin]. You can try it with this tutorial already: -1. Depending on your target hardware:`make` or `BSP=rpi4 make`. +1. Depending on your target hardware, run:`make` or `BSP=rpi4 make`. 1. Copy `kernel8.img` to the SD card. -1. Execute `make chainboot` or `BSP=rpi4 make chainboot`. +1. Run `make chainboot` or `BSP=rpi4 make chainboot`. 1. Connect the USB serial to your host PC. - Wiring diagram at [top-level README](../README.md#-usb-serial-output). - Make sure that you **DID NOT** connect the power pin of the USB serial. Only RX/TX and GND. 1. Connect the RPi to the (USB) power cable. 1. Observe the loader fetching a kernel over `UART`: -> ❗ **NOTE**: By default, `make chainboot` tries to connect to `/dev/ttyUSB0`. -> Should the USB serial on your system have a different name, you have to provide it explicitly. For -> example: -> -> `DEV_SERIAL=/dev/tty.usbserial-0001 make chainboot` +> ❗ **NOTE**: `make chainboot` assumes a default serial device name of `/dev/ttyUSB0`. Depending on +> your host operating system, the device name might differ. For example, on `macOS`, it might be +> something like `/dev/tty.usbserial-0001`. In this case, please give the name explicitly: + + +```console +$ DEV_SERIAL=/dev/tty.usbserial-0001 make chainboot +``` [raspbootin]: https://github.com/mrvn/raspbootin @@ -127,7 +130,7 @@ diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile endif # Export for build.rs -@@ -67,13 +69,14 @@ +@@ -68,13 +70,14 @@ ifeq ($(UNAME_S),Linux) DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV) @@ -145,7 +148,7 @@ diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile all: $(KERNEL_BIN) -@@ -87,15 +90,18 @@ +@@ -88,15 +91,18 @@ $(DOC_CMD) --document-private-items --open ifeq ($(QEMU_MACHINE_TYPE),) @@ -167,7 +170,7 @@ diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile clippy: RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) -@@ -107,7 +113,10 @@ +@@ -108,7 +114,10 @@ readelf --headers $(KERNEL_ELF) objdump: $(KERNEL_ELF) @@ -180,10 +183,10 @@ diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile nm: $(KERNEL_ELF) @$(DOCKER_ELFTOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt -diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs 07_uart_chainloader/src/_arch/aarch64/cpu.rs ---- 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs -+++ 07_uart_chainloader/src/_arch/aarch64/cpu.rs -@@ -22,11 +22,11 @@ +diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs +--- 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs ++++ 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs +@@ -29,11 +29,11 @@ /// actually set (`SP.set()`). #[no_mangle] pub unsafe fn _start() -> ! { @@ -196,8 +199,12 @@ diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs 07_uart_chainloader/src/ + relocate::relocate_self() } else { // If not core0, infinitely wait for events. - wait_forever() -@@ -54,3 +54,19 @@ + cpu::wait_forever() + +diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs 07_uart_chainloader/src/_arch/aarch64/cpu.rs +--- 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs ++++ 07_uart_chainloader/src/_arch/aarch64/cpu.rs +@@ -34,3 +34,19 @@ asm::wfe() } } @@ -383,12 +390,28 @@ diff -uNr 06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 07_uart_chainloader unsafe { range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); +diff -uNr 06_drivers_gpio_uart/src/cpu.rs 07_uart_chainloader/src/cpu.rs +--- 06_drivers_gpio_uart/src/cpu.rs ++++ 07_uart_chainloader/src/cpu.rs +@@ -15,4 +15,4 @@ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- +-pub use arch_cpu::{nop, spin_for_cycles, wait_forever}; ++pub use arch_cpu::{branch_to_raw_addr, nop, spin_for_cycles, wait_forever}; + diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs --- 06_drivers_gpio_uart/src/main.rs +++ 07_uart_chainloader/src/main.rs -@@ -100,7 +100,9 @@ - //! - `crate::memory::*` - //! - `crate::bsp::memory::*` +@@ -102,11 +102,14 @@ + //! + //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. ++//! 2. Once finished with architectural setup, the arch code calls [`relocate::relocate_self()`]. ++//! 3. Finally, [`runtime_init::runtime_init()`] is called. + //! + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html +#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] @@ -396,17 +419,7 @@ diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs #![feature(format_args_nl)] #![feature(panic_info_message)] #![feature(trait_alias)] -@@ -108,7 +110,8 @@ - #![no_std] - - // `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls --// `runtime_init()`, which jumps to `kernel_init()`. -+// `relocate::relocate_self()`. `relocate::relocate_self()` calls `runtime_init()`, which jumps to -+// `kernel_init()`. - - mod bsp; - mod console; -@@ -117,6 +120,7 @@ +@@ -120,6 +123,7 @@ mod memory; mod panic_wait; mod print; @@ -414,7 +427,7 @@ diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs mod runtime_init; mod synchronization; -@@ -145,29 +149,49 @@ +@@ -148,29 +152,49 @@ fn kernel_main() -> ! { use bsp::console::console; use console::interface::All; @@ -478,7 +491,7 @@ diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs + println!("[ML] Loaded! Executing the payload now\n"); + console().flush(); + -+ // Use black magic to get a function pointer. ++ // Use black magic to create a function pointer. + let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) }; + + // Jump to loaded kernel! diff --git a/07_uart_chainloader/demo_payload_rpi3.img b/07_uart_chainloader/demo_payload_rpi3.img index 47a0ae10..0719de41 100755 Binary files a/07_uart_chainloader/demo_payload_rpi3.img and b/07_uart_chainloader/demo_payload_rpi3.img differ diff --git a/07_uart_chainloader/demo_payload_rpi4.img b/07_uart_chainloader/demo_payload_rpi4.img index 14e578c1..c8d8f968 100755 Binary files a/07_uart_chainloader/demo_payload_rpi4.img and b/07_uart_chainloader/demo_payload_rpi4.img differ diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu.rs b/07_uart_chainloader/src/_arch/aarch64/cpu.rs index 184347d7..b53dc269 100644 --- a/07_uart_chainloader/src/_arch/aarch64/cpu.rs +++ b/07_uart_chainloader/src/_arch/aarch64/cpu.rs @@ -3,35 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::relocate; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - relocate::relocate_self() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs b/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..b343792f --- /dev/null +++ b/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::relocate; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + relocate::relocate_self() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs b/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs +++ b/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/07_uart_chainloader/src/bsp.rs b/07_uart_chainloader/src/bsp.rs index 25750249..c558922f 100644 --- a/07_uart_chainloader/src/bsp.rs +++ b/07_uart_chainloader/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/07_uart_chainloader/src/console.rs b/07_uart_chainloader/src/console.rs index 3552823c..c3154ba2 100644 --- a/07_uart_chainloader/src/console.rs +++ b/07_uart_chainloader/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/07_uart_chainloader/src/cpu.rs b/07_uart_chainloader/src/cpu.rs index c9e5af72..c543bd98 100644 --- a/07_uart_chainloader/src/cpu.rs +++ b/07_uart_chainloader/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{branch_to_raw_addr, nop, spin_for_cycles, wait_forever}; diff --git a/07_uart_chainloader/src/cpu/boot.rs b/07_uart_chainloader/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/07_uart_chainloader/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/07_uart_chainloader/src/cpu/smp.rs b/07_uart_chainloader/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/07_uart_chainloader/src/cpu/smp.rs +++ b/07_uart_chainloader/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/07_uart_chainloader/src/main.rs b/07_uart_chainloader/src/main.rs index c8f2a91f..fdb787b5 100644 --- a/07_uart_chainloader/src/main.rs +++ b/07_uart_chainloader/src/main.rs @@ -7,14 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -28,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -45,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -99,6 +97,15 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`relocate::relocate_self()`]. +//! 3. Finally, [`runtime_init::runtime_init()`] is called. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(asm)] #![feature(const_fn_fn_ptr_basics)] @@ -109,10 +116,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `relocate::relocate_self()`. `relocate::relocate_self()` calls `runtime_init()`, which jumps to -// `kernel_init()`. - mod bsp; mod console; mod cpu; @@ -189,7 +192,7 @@ fn kernel_main() -> ! { println!("[ML] Loaded! Executing the payload now\n"); console().flush(); - // Use black magic to get a function pointer. + // Use black magic to create a function pointer. let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) }; // Jump to loaded kernel! diff --git a/07_uart_chainloader/src/print.rs b/07_uart_chainloader/src/print.rs index 59ec9e8e..fa6451c7 100644 --- a/07_uart_chainloader/src/print.rs +++ b/07_uart_chainloader/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/08_timestamps/Makefile b/08_timestamps/Makefile index 071167b9..27944c93 100644 --- a/08_timestamps/Makefile +++ b/08_timestamps/Makefile @@ -40,8 +40,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/08_timestamps/README.md b/08_timestamps/README.md index b79121c0..432ac091 100644 --- a/08_timestamps/README.md +++ b/08_timestamps/README.md @@ -2,7 +2,8 @@ ## tl;dr -- We add abstractions for the architectural timer and implement them for `_arch/aarch64`. +- We add abstractions for timer hardware, and implement them for the ARM architectural timer in + `_arch/aarch64`. - The new timer functions are used to annotate UART prints with timestamps, and to get rid of the cycle-based delays in the `GPIO` and `UART` device drivers, which boosts accuracy. - A `warn!()` macro is added. @@ -68,7 +69,7 @@ diff -uNr 07_uart_chainloader/Makefile 08_timestamps/Makefile endif # Export for build.rs -@@ -75,8 +73,7 @@ +@@ -76,8 +74,7 @@ EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) EXEC_MINIPUSH = ruby ../utils/minipush.rb @@ -78,7 +79,7 @@ diff -uNr 07_uart_chainloader/Makefile 08_timestamps/Makefile all: $(KERNEL_BIN) -@@ -90,18 +87,15 @@ +@@ -91,18 +88,15 @@ $(DOC_CMD) --document-private-items --open ifeq ($(QEMU_MACHINE_TYPE),) @@ -100,7 +101,7 @@ diff -uNr 07_uart_chainloader/Makefile 08_timestamps/Makefile clippy: RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) -@@ -113,10 +107,7 @@ +@@ -114,10 +108,7 @@ readelf --headers $(KERNEL_ELF) objdump: $(KERNEL_ELF) @@ -113,10 +114,10 @@ diff -uNr 07_uart_chainloader/Makefile 08_timestamps/Makefile nm: $(KERNEL_ELF) @$(DOCKER_ELFTOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt -diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/aarch64/cpu.rs ---- 07_uart_chainloader/src/_arch/aarch64/cpu.rs -+++ 08_timestamps/src/_arch/aarch64/cpu.rs -@@ -22,11 +22,11 @@ +diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs 08_timestamps/src/_arch/aarch64/cpu/boot.rs +--- 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs ++++ 08_timestamps/src/_arch/aarch64/cpu/boot.rs +@@ -29,11 +29,11 @@ /// actually set (`SP.set()`). #[no_mangle] pub unsafe fn _start() -> ! { @@ -129,8 +130,12 @@ diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/a + runtime_init::runtime_init() } else { // If not core0, infinitely wait for events. - wait_forever() -@@ -39,14 +39,6 @@ + cpu::wait_forever() + +diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/aarch64/cpu.rs +--- 07_uart_chainloader/src/_arch/aarch64/cpu.rs ++++ 08_timestamps/src/_arch/aarch64/cpu.rs +@@ -19,14 +19,6 @@ pub use asm::nop; @@ -145,7 +150,7 @@ diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/a /// Pause execution on the core. #[inline(always)] pub fn wait_forever() -> ! { -@@ -54,19 +46,3 @@ +@@ -34,19 +26,3 @@ asm::wfe() } } @@ -169,12 +174,19 @@ diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/a diff -uNr 07_uart_chainloader/src/_arch/aarch64/time.rs 08_timestamps/src/_arch/aarch64/time.rs --- 07_uart_chainloader/src/_arch/aarch64/time.rs +++ 08_timestamps/src/_arch/aarch64/time.rs -@@ -0,0 +1,98 @@ +@@ -0,0 +1,105 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//! Architectural timer primitives. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::time::arch_time + +use crate::{time, warn}; +use core::time::Duration; @@ -227,7 +239,7 @@ diff -uNr 07_uart_chainloader/src/_arch/aarch64/time.rs 08_timestamps/src/_arch/ + } + + // Calculate the register compare value. -+ let frq = CNTFRQ_EL0.get() as u64; ++ let frq = CNTFRQ_EL0.get(); + let x = match frq.checked_mul(duration.as_nanos() as u64) { + None => { + warn!("Spin duration too long, skipping"); @@ -483,24 +495,28 @@ diff -uNr 07_uart_chainloader/src/bsp/raspberrypi/memory.rs 08_timestamps/src/bs unsafe { range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); +diff -uNr 07_uart_chainloader/src/cpu.rs 08_timestamps/src/cpu.rs +--- 07_uart_chainloader/src/cpu.rs ++++ 08_timestamps/src/cpu.rs +@@ -15,4 +15,4 @@ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- +-pub use arch_cpu::{branch_to_raw_addr, nop, spin_for_cycles, wait_forever}; ++pub use arch_cpu::{nop, wait_forever}; + diff -uNr 07_uart_chainloader/src/main.rs 08_timestamps/src/main.rs --- 07_uart_chainloader/src/main.rs +++ 08_timestamps/src/main.rs -@@ -11,9 +11,11 @@ - //! - //! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. - //! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -+//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. +@@ -102,14 +102,11 @@ //! - //! [console interface]: ../libkernel/console/interface/index.html - //! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -+//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html + //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +-//! 2. Once finished with architectural setup, the arch code calls [`relocate::relocate_self()`]. +-//! 3. Finally, [`runtime_init::runtime_init()`] is called. ++//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! - //! # Code organization and architecture - //! -@@ -100,9 +102,7 @@ - //! - `crate::memory::*` - //! - `crate::bsp::memory::*` + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html -#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] @@ -508,17 +524,7 @@ diff -uNr 07_uart_chainloader/src/main.rs 08_timestamps/src/main.rs #![feature(format_args_nl)] #![feature(panic_info_message)] #![feature(trait_alias)] -@@ -110,8 +110,7 @@ - #![no_std] - - // `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls --// `relocate::relocate_self()`. `relocate::relocate_self()` calls `runtime_init()`, which jumps to --// `kernel_init()`. -+// `runtime_init()`, which jumps to `kernel_init()`. - - mod bsp; - mod console; -@@ -120,9 +119,9 @@ +@@ -123,9 +120,9 @@ mod memory; mod panic_wait; mod print; @@ -529,7 +535,7 @@ diff -uNr 07_uart_chainloader/src/main.rs 08_timestamps/src/main.rs /// Early init code. /// -@@ -147,51 +146,31 @@ +@@ -150,51 +147,31 @@ /// The main function running after the early init. fn kernel_main() -> ! { @@ -593,7 +599,7 @@ diff -uNr 07_uart_chainloader/src/main.rs 08_timestamps/src/main.rs + // Test a failing timer case. + time::time_manager().spin_for(Duration::from_nanos(1)); -- // Use black magic to get a function pointer. +- // Use black magic to create a function pointer. - let kernel: fn() -> ! = unsafe { core::mem::transmute(kernel_addr) }; - - // Jump to loaded kernel! @@ -760,7 +766,7 @@ diff -uNr 07_uart_chainloader/src/runtime_init.rs 08_timestamps/src/runtime_init diff -uNr 07_uart_chainloader/src/time.rs 08_timestamps/src/time.rs --- 07_uart_chainloader/src/time.rs +++ 08_timestamps/src/time.rs -@@ -0,0 +1,35 @@ +@@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2021 Andre Richter @@ -770,7 +776,11 @@ diff -uNr 07_uart_chainloader/src/time.rs 08_timestamps/src/time.rs +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/time.rs"] +mod arch_time; -+pub use arch_time::*; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_time::time_manager; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions @@ -781,8 +791,6 @@ diff -uNr 07_uart_chainloader/src/time.rs 08_timestamps/src/time.rs + use core::time::Duration; + + /// Time management functions. -+ /// -+ /// The `BSP` is supposed to supply one global instance. + pub trait TimeManager { + /// The timer's resolution. + fn resolution(&self) -> Duration; diff --git a/08_timestamps/src/_arch/aarch64/cpu.rs b/08_timestamps/src/_arch/aarch64/cpu.rs index a65a0682..d3b07461 100644 --- a/08_timestamps/src/_arch/aarch64/cpu.rs +++ b/08_timestamps/src/_arch/aarch64/cpu.rs @@ -3,35 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/08_timestamps/src/_arch/aarch64/cpu/boot.rs b/08_timestamps/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..549b5927 --- /dev/null +++ b/08_timestamps/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::runtime_init; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/08_timestamps/src/_arch/aarch64/cpu/smp.rs b/08_timestamps/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/08_timestamps/src/_arch/aarch64/cpu/smp.rs +++ b/08_timestamps/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/08_timestamps/src/_arch/aarch64/time.rs b/08_timestamps/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/08_timestamps/src/_arch/aarch64/time.rs +++ b/08_timestamps/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/08_timestamps/src/bsp.rs b/08_timestamps/src/bsp.rs index 25750249..c558922f 100644 --- a/08_timestamps/src/bsp.rs +++ b/08_timestamps/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/08_timestamps/src/console.rs b/08_timestamps/src/console.rs index 3552823c..c3154ba2 100644 --- a/08_timestamps/src/console.rs +++ b/08_timestamps/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/08_timestamps/src/cpu.rs b/08_timestamps/src/cpu.rs index c9e5af72..3834f183 100644 --- a/08_timestamps/src/cpu.rs +++ b/08_timestamps/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; diff --git a/08_timestamps/src/cpu/boot.rs b/08_timestamps/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/08_timestamps/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/08_timestamps/src/cpu/smp.rs b/08_timestamps/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/08_timestamps/src/cpu/smp.rs +++ b/08_timestamps/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/08_timestamps/src/main.rs b/08_timestamps/src/main.rs index b9db5fef..5a7b9f55 100644 --- a/08_timestamps/src/main.rs +++ b/08_timestamps/src/main.rs @@ -7,16 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -30,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -47,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -101,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] @@ -109,9 +113,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/08_timestamps/src/print.rs b/08_timestamps/src/print.rs index 1ea96b6a..5a563811 100644 --- a/08_timestamps/src/print.rs +++ b/08_timestamps/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/08_timestamps/src/time.rs b/08_timestamps/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/08_timestamps/src/time.rs +++ b/08_timestamps/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/09_hw_debug_JTAG/Makefile b/09_hw_debug_JTAG/Makefile index 9254018c..9105d297 100644 --- a/09_hw_debug_JTAG/Makefile +++ b/09_hw_debug_JTAG/Makefile @@ -44,8 +44,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/09_hw_debug_JTAG/README.md b/09_hw_debug_JTAG/README.md index 12cf0cc1..918f7258 100644 --- a/09_hw_debug_JTAG/README.md +++ b/09_hw_debug_JTAG/README.md @@ -287,8 +287,8 @@ Hence, please ensure the following order of connecting the devices to your box: 1. Connect the USB serial. 2. Afterwards, the Olimex debugger. -This way, Linux enumerates the devices accordingly. This has to be done only once. It is fine to -disconnect and connect the serial multiple times, e.g. for kicking off different `make jtagboot` +This way, the host OS enumerates the devices accordingly. This has to be done only once. It is fine +to disconnect and connect the serial multiple times, e.g. for kicking off different `make jtagboot` runs, while keeping the debugger connected. ## Additional resources @@ -324,7 +324,7 @@ diff -uNr 08_timestamps/Makefile 09_hw_debug_JTAG/Makefile LINKER_FILE = src/bsp/raspberrypi/link.ld RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 endif -@@ -58,9 +62,12 @@ +@@ -59,9 +63,12 @@ DOCKER_CMD = docker run --rm -v $(shell pwd):/work/tutorial -w /work/tutorial DOCKER_CMD_INTERACT = $(DOCKER_CMD) -i -t DOCKER_ARG_DIR_UTILS = -v $(shell pwd)/../utils:/work/utils @@ -337,7 +337,7 @@ diff -uNr 08_timestamps/Makefile 09_hw_debug_JTAG/Makefile DOCKER_ELFTOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE) # Dockerize commands that require USB device passthrough only on Linux -@@ -68,12 +75,17 @@ +@@ -69,12 +76,17 @@ DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV) DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_UTILS) $(DOCKER_IMAGE) @@ -356,7 +356,7 @@ diff -uNr 08_timestamps/Makefile 09_hw_debug_JTAG/Makefile all: $(KERNEL_BIN) -@@ -97,6 +109,23 @@ +@@ -98,6 +110,23 @@ chainboot: $(KERNEL_BIN) @$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN) diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs b/09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs index a65a0682..d3b07461 100644 --- a/09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs +++ b/09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs @@ -3,35 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..549b5927 --- /dev/null +++ b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::runtime_init; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs +++ b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/time.rs b/09_hw_debug_JTAG/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/09_hw_debug_JTAG/src/_arch/aarch64/time.rs +++ b/09_hw_debug_JTAG/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/09_hw_debug_JTAG/src/bsp.rs b/09_hw_debug_JTAG/src/bsp.rs index 25750249..c558922f 100644 --- a/09_hw_debug_JTAG/src/bsp.rs +++ b/09_hw_debug_JTAG/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/09_hw_debug_JTAG/src/console.rs b/09_hw_debug_JTAG/src/console.rs index 3552823c..c3154ba2 100644 --- a/09_hw_debug_JTAG/src/console.rs +++ b/09_hw_debug_JTAG/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/09_hw_debug_JTAG/src/cpu.rs b/09_hw_debug_JTAG/src/cpu.rs index c9e5af72..3834f183 100644 --- a/09_hw_debug_JTAG/src/cpu.rs +++ b/09_hw_debug_JTAG/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; diff --git a/09_hw_debug_JTAG/src/cpu/boot.rs b/09_hw_debug_JTAG/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/09_hw_debug_JTAG/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/09_hw_debug_JTAG/src/cpu/smp.rs b/09_hw_debug_JTAG/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/09_hw_debug_JTAG/src/cpu/smp.rs +++ b/09_hw_debug_JTAG/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/09_hw_debug_JTAG/src/main.rs b/09_hw_debug_JTAG/src/main.rs index b9db5fef..5a7b9f55 100644 --- a/09_hw_debug_JTAG/src/main.rs +++ b/09_hw_debug_JTAG/src/main.rs @@ -7,16 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -30,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -47,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -101,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] @@ -109,9 +113,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/09_hw_debug_JTAG/src/print.rs b/09_hw_debug_JTAG/src/print.rs index 1ea96b6a..5a563811 100644 --- a/09_hw_debug_JTAG/src/print.rs +++ b/09_hw_debug_JTAG/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/09_hw_debug_JTAG/src/time.rs b/09_hw_debug_JTAG/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/09_hw_debug_JTAG/src/time.rs +++ b/09_hw_debug_JTAG/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/10_privilege_level/Makefile b/10_privilege_level/Makefile index 9254018c..9105d297 100644 --- a/10_privilege_level/Makefile +++ b/10_privilege_level/Makefile @@ -44,8 +44,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/10_privilege_level/README.md b/10_privilege_level/README.md index 8cd31381..2556ec99 100644 --- a/10_privilege_level/README.md +++ b/10_privilege_level/README.md @@ -33,16 +33,14 @@ architectures, please have a look at the following links: - [RISC-V privilege modes](https://content.riscv.org/wp-content/uploads/2017/12/Tue0942-riscv-hypervisor-waterman.pdf). At this point, I strongly recommend that you glimpse over `Chapter 3` of the [Programmer’s Guide for -ARMv8-A](http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf) -before you continue. It gives a concise overview about the topic. +ARMv8-A] before you continue. It gives a concise overview about the topic. -## Scope of this tutorial +[Programmer’s Guide forARMv8-A]: http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf -If you set up your SD Card exactly like mentioned in [tutorial 06], the Rpi will always start -executing in `EL2`. Since we are writing a traditional `Kernel`, we have to transition into the more -appropriate `EL1`. +## Scope of this tutorial -[tutorial 06]: https://github.com/rust-embedded/rust-raspberrypi-OS-tutorials/tree/master/06_drivers_gpio_uart#boot-it-from-sd-card +By default, the Rpi will always start executing in `EL2`. Since we are writing a traditional +`Kernel`, we have to transition into the more appropriate `EL1`. ## Checking for EL2 in the entrypoint @@ -50,7 +48,8 @@ First of all, we need to ensure that we actually execute in `EL2` before we can to transition to `EL1`: ```rust -pub unsafe extern "C" fn _start() -> ! { +#[no_mangle] +pub unsafe fn _start() -> ! { // Expect the boot core to start in EL2. if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) && (CurrentEL.get() == CurrentEL::EL::EL2.value) @@ -58,7 +57,7 @@ pub unsafe extern "C" fn _start() -> ! { el2_to_el1_transition() } else { // If not core0, infinitely wait for events. - wait_forever() + cpu::wait_forever() } } ``` @@ -74,7 +73,7 @@ We are already using them since [tutorial 08](../08_timestamps/), so of course w Therefore we set the respective flags in the [Counter-timer Hypervisor Control register] and additionally set the virtual offset to zero so that we get the real physical value everytime: -[Counter-timer Hypervisor Control register]: https://docs.rs/cortex-a/2.4.0/src/cortex_a/regs/cnthctl_el2.rs.html +[Counter-timer Hypervisor Control register]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/regs/cnthctl_el2.rs.html ```rust // Enable timer counter registers for EL1. @@ -84,10 +83,10 @@ CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); CNTVOFF_EL2.set(0); ``` -Next, we configure the [Hypervisor Configuration Register] such that `EL1` should actually run in -`AArch64` mode, and not in `AArch32`, which would also be possible. +Next, we configure the [Hypervisor Configuration Register] such that `EL1` runs in `AArch64` mode, +and not in `AArch32`, which would also be possible. -[Hypervisor Configuration Register]: https://docs.rs/cortex-a/2.4.0/src/cortex_a/regs/hcr_el2.rs.html +[Hypervisor Configuration Register]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/regs/hcr_el2.rs.html ```rust // Set EL1 execution state to AArch64. @@ -99,7 +98,7 @@ HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); There is actually only one way to transition from a higher EL to a lower EL, which is by way of executing the [ERET] instruction. -[ERET]: https://docs.rs/cortex-a/2.4.0/src/cortex_a/asm.rs.html#49-62 +[ERET]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/asm.rs.html#87-96 This instruction will copy the contents of the [Saved Program Status Register - EL2] to `Current Program Status Register - EL1` and jump to the instruction address that is stored in the [Exception @@ -108,8 +107,8 @@ Link Register - EL2]. This is basically the reverse of what is happening when an exception is taken. You'll learn about it in an upcoming tutorial. -[Saved Program Status Register - EL2]: https://docs.rs/cortex-a/2.4.0/src/cortex_a/regs/spsr_el2.rs.html -[Exception Link Register - EL2]: https://docs.rs/cortex-a/2.4.0/src/cortex_a/regs/elr_el2.rs.html +[Saved Program Status Register - EL2]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/regs/spsr_el2.rs.html +[Exception Link Register - EL2]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/regs/elr_el2.rs.html ```rust // Set up a simulated exception return. @@ -156,23 +155,23 @@ Disassembly of section .text: 0000000000080000 <_start>: 80000: d53800a8 mrs x8, mpidr_el1 80004: f240051f tst x8, #0x3 - 80008: 54000081 b.ne 80018 <_start+0x18> + 80008: 54000081 b.ne 80018 <_start+0x18> // b.any 8000c: d5384248 mrs x8, currentel 80010: f100211f cmp x8, #0x8 - 80014: 54000060 b.eq 80020 <_start+0x20> + 80014: 54000060 b.eq 80020 <_start+0x20> // b.none 80018: d503205f wfe 8001c: 17ffffff b 80018 <_start+0x18> 80020: aa1f03e8 mov x8, xzr - 80024: 52800069 mov w9, #0x3 + 80024: 52800069 mov w9, #0x3 // #3 80028: d51ce109 msr cnthctl_el2, x9 8002c: d51ce068 msr cntvoff_el2, x8 - 80030: 90000008 adrp x8, 80000 <_start> - 80034: 52b0000a mov w10, #0x80000000 - 80038: 528078ab mov w11, #0x3c5 - 8003c: 52a0010c mov w12, #0x80000 + 80030: d0000008 adrp x8, 82000 + 80034: 52b0000a mov w10, #0x80000000 // #-2147483648 + 80038: 528078ab mov w11, #0x3c5 // #965 + 8003c: 52a0010c mov w12, #0x80000 // #524288 80040: d51c110a msr hcr_el2, x10 80044: d51c400b msr spsr_el2, x11 - 80048: 91374108 add x8, x8, #0xdd0 + 80048: 9114c108 add x8, x8, #0x530 8004c: d51c4028 msr elr_el2, x8 80050: d51c410c msr sp_el1, x12 80054: d69f03e0 eret @@ -223,35 +222,20 @@ Minipush 1.0 ## Diff to previous ```diff -diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs 10_privilege_level/src/_arch/aarch64/cpu.rs ---- 09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs -+++ 10_privilege_level/src/_arch/aarch64/cpu.rs -@@ -18,21 +18,65 @@ - /// # Safety - /// - /// - Linker script must ensure to place this function where it is expected by the target machine. --/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is --/// actually set (`SP.set()`). -+/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -+/// a stack for EL2. - #[no_mangle] - pub unsafe fn _start() -> ! { -- use crate::runtime_init; -- -- if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { -- SP.set(bsp::memory::boot_core_stack_end() as u64); -- runtime_init::runtime_init() -+ // Expect the boot core to start in EL2. -+ if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) -+ && (CurrentEL.get() == CurrentEL::EL::EL2.value) -+ { -+ el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } - } +diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 10_privilege_level/src/_arch/aarch64/cpu/boot.rs +--- 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs ++++ 10_privilege_level/src/_arch/aarch64/cpu/boot.rs +@@ -12,7 +12,55 @@ + //! crate::cpu::boot::arch_boot + use crate::{bsp, cpu}; +-use cortex_a::regs::*; ++use cortex_a::{asm, regs::*}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++ +/// Transition from EL2 to EL1. +/// +/// # Safety @@ -295,20 +279,49 @@ diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/cpu.rs 10_privilege_level/src/_arch + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} -+ + //-------------------------------------------------------------------------------------------------- // Public Code - //-------------------------------------------------------------------------------------------------- +@@ -25,15 +73,15 @@ + /// # Safety + /// + /// - Linker script must ensure to place this function where it is expected by the target machine. +-/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +-/// actually set (`SP.set()`). ++/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up ++/// a stack for EL2. + #[no_mangle] + pub unsafe fn _start() -> ! { +- use crate::runtime_init; +- +- if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { +- SP.set(bsp::memory::boot_core_stack_end() as u64); +- runtime_init::runtime_init() ++ // Expect the boot core to start in EL2. ++ if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) ++ && (CurrentEL.get() == CurrentEL::EL::EL2.value) ++ { ++ el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs 10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs --- 09_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs +++ 10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs -@@ -0,0 +1,74 @@ +@@ -0,0 +1,81 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//! Architectural asynchronous exception handling. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::exception::asynchronous::arch_asynchronous + +use cortex_a::regs::*; + @@ -382,12 +395,19 @@ diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs 10_privil diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/exception.rs 10_privilege_level/src/_arch/aarch64/exception.rs --- 09_hw_debug_JTAG/src/_arch/aarch64/exception.rs +++ 10_privilege_level/src/_arch/aarch64/exception.rs -@@ -0,0 +1,23 @@ +@@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//! Architectural synchronous and asynchronous exception handling. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::exception::arch_exception + +use cortex_a::regs::*; + @@ -410,7 +430,7 @@ diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/exception.rs 10_privilege_level/src diff -uNr 09_hw_debug_JTAG/src/exception/asynchronous.rs 10_privilege_level/src/exception/asynchronous.rs --- 09_hw_debug_JTAG/src/exception/asynchronous.rs +++ 10_privilege_level/src/exception/asynchronous.rs -@@ -0,0 +1,10 @@ +@@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2021 Andre Richter @@ -419,13 +439,17 @@ diff -uNr 09_hw_debug_JTAG/src/exception/asynchronous.rs 10_privilege_level/src/ + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/exception/asynchronous.rs"] -+mod arch_exception_async; -+pub use arch_exception_async::*; ++mod arch_asynchronous; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_asynchronous::print_state; diff -uNr 09_hw_debug_JTAG/src/exception.rs 10_privilege_level/src/exception.rs --- 09_hw_debug_JTAG/src/exception.rs +++ 10_privilege_level/src/exception.rs -@@ -0,0 +1,26 @@ +@@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2021 Andre Richter @@ -435,11 +459,15 @@ diff -uNr 09_hw_debug_JTAG/src/exception.rs 10_privilege_level/src/exception.rs +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/exception.rs"] +mod arch_exception; -+pub use arch_exception::*; + +pub mod asynchronous; + +//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_exception::current_privilege_level; ++ ++//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + @@ -456,7 +484,7 @@ diff -uNr 09_hw_debug_JTAG/src/exception.rs 10_privilege_level/src/exception.rs diff -uNr 09_hw_debug_JTAG/src/main.rs 10_privilege_level/src/main.rs --- 09_hw_debug_JTAG/src/main.rs +++ 10_privilege_level/src/main.rs -@@ -116,6 +116,7 @@ +@@ -117,6 +117,7 @@ mod console; mod cpu; mod driver; @@ -464,7 +492,7 @@ diff -uNr 09_hw_debug_JTAG/src/main.rs 10_privilege_level/src/main.rs mod memory; mod panic_wait; mod print; -@@ -146,12 +147,20 @@ +@@ -147,12 +148,20 @@ /// The main function running after the early init. fn kernel_main() -> ! { @@ -485,7 +513,7 @@ diff -uNr 09_hw_debug_JTAG/src/main.rs 10_privilege_level/src/main.rs info!( "Architectural timer resolution: {} ns", time::time_manager().resolution().as_nanos() -@@ -166,11 +175,15 @@ +@@ -167,11 +176,15 @@ info!(" {}. {}", i + 1, driver.compatible()); } diff --git a/10_privilege_level/src/_arch/aarch64/cpu.rs b/10_privilege_level/src/_arch/aarch64/cpu.rs index 3846968b..d3b07461 100644 --- a/10_privilege_level/src/_arch/aarch64/cpu.rs +++ b/10_privilege_level/src/_arch/aarch64/cpu.rs @@ -3,79 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} - -/// Transition from EL2 to EL1. -/// -/// # Safety -/// -/// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - - // Enable timer counter registers for EL1. - CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - - // No offset for reading the counters. - CNTVOFF_EL2.set(0); - - // Set EL1 execution state to AArch64. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Set up a simulated exception return. - // - // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a - // stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, - ); - - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/10_privilege_level/src/_arch/aarch64/cpu/boot.rs b/10_privilege_level/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c2f5fe0b --- /dev/null +++ b/10_privilege_level/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[no_mangle] +pub unsafe fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/10_privilege_level/src/_arch/aarch64/cpu/smp.rs b/10_privilege_level/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/10_privilege_level/src/_arch/aarch64/cpu/smp.rs +++ b/10_privilege_level/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/10_privilege_level/src/_arch/aarch64/exception.rs b/10_privilege_level/src/_arch/aarch64/exception.rs index b6ee28b8..d8c617b2 100644 --- a/10_privilege_level/src/_arch/aarch64/exception.rs +++ b/10_privilege_level/src/_arch/aarch64/exception.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception use cortex_a::regs::*; diff --git a/10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs b/10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs index 445f490e..b63b00fe 100644 --- a/10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs +++ b/10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous use cortex_a::regs::*; diff --git a/10_privilege_level/src/_arch/aarch64/time.rs b/10_privilege_level/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/10_privilege_level/src/_arch/aarch64/time.rs +++ b/10_privilege_level/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/10_privilege_level/src/bsp.rs b/10_privilege_level/src/bsp.rs index 25750249..c558922f 100644 --- a/10_privilege_level/src/bsp.rs +++ b/10_privilege_level/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/10_privilege_level/src/console.rs b/10_privilege_level/src/console.rs index 3552823c..c3154ba2 100644 --- a/10_privilege_level/src/console.rs +++ b/10_privilege_level/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/10_privilege_level/src/cpu.rs b/10_privilege_level/src/cpu.rs index c9e5af72..3834f183 100644 --- a/10_privilege_level/src/cpu.rs +++ b/10_privilege_level/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; diff --git a/10_privilege_level/src/cpu/boot.rs b/10_privilege_level/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/10_privilege_level/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/10_privilege_level/src/cpu/smp.rs b/10_privilege_level/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/10_privilege_level/src/cpu/smp.rs +++ b/10_privilege_level/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/10_privilege_level/src/exception.rs b/10_privilege_level/src/exception.rs index 432c606b..87d4db63 100644 --- a/10_privilege_level/src/exception.rs +++ b/10_privilege_level/src/exception.rs @@ -7,10 +7,14 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/exception.rs"] mod arch_exception; -pub use arch_exception::*; pub mod asynchronous; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::current_privilege_level; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- diff --git a/10_privilege_level/src/exception/asynchronous.rs b/10_privilege_level/src/exception/asynchronous.rs index fbdba957..566efe32 100644 --- a/10_privilege_level/src/exception/asynchronous.rs +++ b/10_privilege_level/src/exception/asynchronous.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/exception/asynchronous.rs"] -mod arch_exception_async; -pub use arch_exception_async::*; +mod arch_asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::print_state; diff --git a/10_privilege_level/src/main.rs b/10_privilege_level/src/main.rs index fc2bd691..74188343 100644 --- a/10_privilege_level/src/main.rs +++ b/10_privilege_level/src/main.rs @@ -7,16 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -30,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -47,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -101,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] @@ -109,9 +113,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/10_privilege_level/src/print.rs b/10_privilege_level/src/print.rs index 1ea96b6a..5a563811 100644 --- a/10_privilege_level/src/print.rs +++ b/10_privilege_level/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/10_privilege_level/src/time.rs b/10_privilege_level/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/10_privilege_level/src/time.rs +++ b/10_privilege_level/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/11_virtual_mem_part1_identity_mapping/Makefile b/11_virtual_mem_part1_identity_mapping/Makefile index 9254018c..9105d297 100644 --- a/11_virtual_mem_part1_identity_mapping/Makefile +++ b/11_virtual_mem_part1_identity_mapping/Makefile @@ -44,8 +44,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/11_virtual_mem_part1_identity_mapping/README.md b/11_virtual_mem_part1_identity_mapping/README.md index c6444296..0d24f17f 100644 --- a/11_virtual_mem_part1_identity_mapping/README.md +++ b/11_virtual_mem_part1_identity_mapping/README.md @@ -3,7 +3,7 @@ ## tl;dr - The `MMU` is turned on. -- A simple scheme is used: static `64 KiB` translation tables. +- A simple scheme is used: Static `64 KiB` translation tables. - For educational purposes, we write to a remapped `UART`, and `identity map` everything else. ## Table of Contents @@ -13,7 +13,7 @@ - [Approach](#approach) * [Generic Kernel code: `memory/mmu.rs`](#generic-kernel-code-memorymmurs) * [BSP: `bsp/raspberrypi/memory/mmu.rs`](#bsp-bspraspberrypimemorymmurs) - * [AArch64: `_arch/aarch64/memory/mmu.rs`](#aarch64-_archaarch64memorymmurs) + * [AArch64: `_arch/aarch64/memory/*`](#aarch64-_archaarch64memory) * [`link.ld`](#linkld) - [Address translation examples](#address-translation-examples) * [Address translation using a 64 KiB page descriptor](#address-translation-using-a-64-kib-page-descriptor) @@ -25,8 +25,8 @@ Virtual memory is an immensely complex, but important and powerful topic. In this tutorial, we start slow and easy by switching on the `MMU`, using static translation tables and `identity-map` -everything at once (except for the `UART`, which we remap for educational purposes; This will be -gone again in the next tutorial). +everything at once (except for the `UART`, which we also remap a second time for educational +purposes; This will be gone again in the next tutorial). ## MMU and paging theory @@ -44,14 +44,15 @@ Back from reading `Chapter 12` already? Good job :+1:! ## Approach -1. The generic `kernel` part: `src/memory/mmu.rs` provides architecture-agnostic descriptor types - for composing a high-level data structure that describes the kernel's virtual memory layout: - `memory::mmu::KernelVirtualLayout`. +1. The generic `kernel` part: `src/memory/mmu.rs` and its submodules provide architecture-agnostic + descriptor types for composing a high-level data structure that describes the kernel's virtual + memory layout: `memory::mmu::KernelVirtualLayout`. 2. The `BSP` part: `src/bsp/raspberrypi/memory/mmu.rs` contains a static instance of `KernelVirtualLayout` and makes it accessible through the function `bsp::memory::mmu::virt_mem_layout()`. -3. The `aarch64` part: `src/_arch/aarch64/memory/mmu.rs` contains the actual `MMU` driver. It picks - up the `BSP`'s high-level `KernelVirtualLayout` and maps it using a `64 KiB` granule. +3. The `aarch64` part: `src/_arch/aarch64/memory/mmu.rs` and its submodules contain the actual `MMU` + driver. It picks up the `BSP`'s high-level `KernelVirtualLayout` and maps it using a `64 KiB` + granule. ### Generic Kernel code: `memory/mmu.rs` @@ -97,42 +98,63 @@ pub fn virt_addr_properties( ) -> Result<(usize, AttributeFields), &'static str> ``` -It will be used by the `_arch/aarch64`'s `MMU` code to request attributes for a virtual address and -the translation, which delivers the physical output address (the `usize` in the return-tuple). The +It will be used by `_arch/aarch64`'s `MMU` code to request attributes for a virtual address and the +translation, which delivers the physical output address (the `usize` in the return-tuple). The function scans for a descriptor that contains the queried address, and returns the respective findings for the first entry that is a hit. If no entry is found, it returns default attributes for normal chacheable DRAM and the input address, hence telling the `MMU` code that the requested address should be `identity mapped`. -Due to this default return, it is technicall not needed to define normal cacheable DRAM regions. +Due to this default behavior, it is not needed to define normal cacheable DRAM regions. -### AArch64: `_arch/aarch64/memory/mmu.rs` +### AArch64: `_arch/aarch64/memory/*` -This file contains the `AArch64` `MMU` driver. The granule is hardcoded here (`64 KiB` page +These modules contain the `AArch64` `MMU` driver. The granule is hardcoded here (`64 KiB` page descriptors). -The actual translation tables are stored in a global instance of the `ArchTranslationTable` struct: +In `translation_table.rs`, there is a definition of the actual translation table struct which is +generic over the number of `LVL2` tables. The latter depends on the size of the target board's +memory. Naturally, the `BSP` knows these details about the target board, and provides the size +through the constant `bsp::memory::mmu::KernelAddrSpaceSize::SIZE`. + +This information is used by `translation_table.rs` to calculate the number of needed `LVL2` tables. +Since one `LVL2` table in a `64 KiB` configuration covers `512 MiB`, all that needs to be done is to +divide `KernelAddrSpaceSize::SIZE` by `512 MiB` (there are several compile-time checks in place that +ensure that `KernelAddrSpaceSize` is a multiple of `512 MiB`). + +The final table type is exported as `KernelTranslationTable`. Below is the respective excerpt from +`translation_table.rs`: ```rust /// A table descriptor for 64 KiB aperture. /// /// The output points to the next table. #[derive(Copy, Clone)] -#[repr(transparent)] -struct TableDescriptor(InMemoryRegister); +#[repr(C)] +struct TableDescriptor { + value: u64, +} /// A page descriptor with 64 KiB aperture. /// /// The output points to physical memory. #[derive(Copy, Clone)] -#[repr(transparent)] -struct PageDescriptor(InMemoryRegister); +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB /// aligned, hence the "reverse" order of appearance. #[repr(C)] #[repr(align(65536))] -struct FixedSizeTranslationTable { +pub struct FixedSizeTranslationTable { /// Page descriptors, covering 64 KiB windows per entry. lvl3: [[PageDescriptor; 8192]; NUM_TABLES], @@ -140,55 +162,69 @@ struct FixedSizeTranslationTable { lvl2: [TableDescriptor; NUM_TABLES], } -const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -type ArchTranslationTable = FixedSizeTranslationTable; +/// A translation table type for the kernel space. +pub type KernelTranslationTable = FixedSizeTranslationTable; +``` + +In `mmu.rs`, `KernelTranslationTable` is then used to create the final instance of the kernel's +tables: +```rust //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -/// The translation tables. +/// The kernel translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); +static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); ``` -They are populated using `bsp::memory::mmu::virt_mem_layout().virt_addr_properties()` and a bunch of -utility functions that convert our own descriptors to the actual `64 bit` integer entries needed by -the `MMU` hardware for the translation table arrays. +They are populated during `MMU::init()` by calling `KERNEL_TABLES.populate_tt_entries()`, which +utilizes `bsp::memory::mmu::virt_mem_layout().virt_addr_properties()` and a bunch of utility +functions that convert the kernel generic descriptors to the actual `64 bit` integer entries needed +by the `AArch64 MMU` hardware for the translation table arrays. -Each page descriptor has an entry (`AttrIndex`) that indexes into the [MAIR_EL1] register, which -holds information about the cacheability of the respective page. We currently define normal -cacheable memory and device memory (which is not cached). +One notable thing is that each page descriptor has an entry (`AttrIndex`) that indexes into the +[MAIR_EL1] register, which holds information about the cacheability of the respective page. We +currently define normal cacheable memory and device memory (which is not cached). [MAIR_EL1]: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500d/CIHDHJBB.html ```rust -/// Setup function for the MAIR_EL1 register. -fn set_up_mair() { - // Define the memory types being mapped. - MAIR_EL1.write( - // Attribute 1 - Cacheable normal DRAM. - MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + // Attribute 0 - Device. MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, - ); -} + ); + } ``` Afterwards, the [Translation Table Base Register 0 - EL1] is set up with the base address of the -`lvl2` tables and the [Translation Control Register - EL1] is configured. +`lvl2` tables and the [Translation Control Register - EL1] is configured: + +```rust + // Set the "Translation Table Base Register". + TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); + + self.configure_translation_control(); +``` Finally, the `MMU` is turned on through the [System Control Register - EL1]. The last step also enables caching for data and instructions. -[Translation Table Base Register 0 - EL1]: https://docs.rs/crate/cortex-a/2.4.0/source/src/regs/ttbr0_el1.rs -[Translation Control Register - EL1]: https://docs.rs/crate/cortex-a/2.4.0/source/src/regs/tcr_el1.rs -[System Control Register - EL1]: https://docs.rs/crate/cortex-a/2.4.0/source/src/regs/sctlr_el1.rs +[Translation Table Base Register 0 - EL1]: https://docs.rs/crate/cortex-a/5.1.2/source/src/regs/ttbr0_el1.rs +[Translation Control Register - EL1]: https://docs.rs/crate/cortex-a/5.1.2/source/src/regs/tcr_el1.rs +[System Control Register - EL1]: https://docs.rs/crate/cortex-a/5.1.2/source/src/regs/sctlr_el1.rs ### `link.ld` @@ -227,15 +263,15 @@ Let's take a look again at the piece of code for setting up the `MAIR_EL1` regis ```rust /// Setup function for the MAIR_EL1 register. -fn set_up_mair() { +fn set_up_mair(&self) { // Define the memory types being mapped. MAIR_EL1.write( // Attribute 1 - Cacheable normal DRAM. MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + - MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + - // Attribute 0 - Device. - MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, + // Attribute 0 - Device. + MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, ); } ``` @@ -248,15 +284,26 @@ data sheet. Looking at the generated code, we can see that despite all the type- abstractions, it boils down to two assembly instructions: ```text -0000000000081660 <::init>: - ... - 816bc: 529fe088 mov w8, #0xff04 - ... - 816c4: d518a208 msr mair_el1, x8 + 800a8: 529fe089 mov w9, #0xff04 // #65284 + 800ac: d518a209 msr mair_el1, x9 ``` ## Test it +Turning on virtual memory is now the first thing we do during kernel init: + +```rust +unsafe fn kernel_init() -> ! { + use driver::interface::DriverManager; + use memory::mmu::interface::MMU; + + if let Err(string) = memory::mmu::mmu().init() { + panic!("MMU: {}", string); + } +``` + +Later in the boot process, prints about the mappings can be observed: + ```console $ make chainboot [...] @@ -299,22 +346,33 @@ Minipush 1.0 ## Diff to previous ```diff -diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs ---- 10_privilege_level/src/_arch/aarch64/memory/mmu.rs -+++ 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs -@@ -0,0 +1,343 @@ +diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu/translation_table.rs 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs +--- 10_privilege_level/src/_arch/aarch64/memory/mmu/translation_table.rs ++++ 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs +@@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// -+// Copyright (c) 2018-2021 Andre Richter ++// Copyright (c) 2021 Andre Richter + -+//! Memory Management Unit Driver. ++//! Architectural translation table. +//! -+//! Static translation tables, compiled on boot; Everything 64 KiB granule. -+ -+use super::{AccessPermissions, AttributeFields, MemAttributes}; -+use crate::{bsp, memory}; ++//! Only 64 KiB granule is supported. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::memory::mmu::translation_table::arch_translation_table ++ ++use crate::{ ++ bsp, memory, ++ memory::mmu::{ ++ arch_mmu::{Granule512MiB, Granule64KiB}, ++ AccessPermissions, AttributeFields, MemAttributes, ++ }, ++}; +use core::convert; -+use cortex_a::{barrier, regs::*}; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- @@ -392,65 +450,49 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part + ] +} + -+const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) -+const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) -+ +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] -+#[repr(transparent)] -+struct TableDescriptor(u64); ++#[repr(C)] ++struct TableDescriptor { ++ value: u64, ++} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] -+#[repr(transparent)] -+struct PageDescriptor(u64); -+ -+/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -+/// aligned, hence the "reverse" order of appearance. +#[repr(C)] -+#[repr(align(65536))] -+struct FixedSizeTranslationTable { -+ /// Page descriptors, covering 64 KiB windows per entry. -+ lvl3: [[PageDescriptor; 8192]; NUM_TABLES], -+ -+ /// Table descriptors, covering 512 MiB windows. -+ lvl2: [TableDescriptor; NUM_TABLES], ++struct PageDescriptor { ++ value: u64, +} + -+const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -+type ArchTranslationTable = FixedSizeTranslationTable; -+ +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + -+/// Constants for indexing the MAIR_EL1. -+#[allow(dead_code)] -+mod mair { -+ pub const DEVICE: u64 = 0; -+ pub const NORMAL: u64 = 1; -+} -+ -+/// Memory Management Unit type. -+struct MemoryManagementUnit; ++const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- -+// Global instances ++// Public Definitions +//-------------------------------------------------------------------------------------------------- + -+/// The translation tables. -+/// -+/// # Safety -+/// -+/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -+static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); ++/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB ++/// aligned, hence the "reverse" order of appearance. ++#[repr(C)] ++#[repr(align(65536))] ++pub struct FixedSizeTranslationTable { ++ /// Page descriptors, covering 64 KiB windows per entry. ++ lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + -+static MMU: MemoryManagementUnit = MemoryManagementUnit; ++ /// Table descriptors, covering 512 MiB windows. ++ lvl2: [TableDescriptor; NUM_TABLES], ++} ++ ++/// A translation table type for the kernel space. ++pub type KernelTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Private Code @@ -466,18 +508,26 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part + } +} + -+impl convert::From for TableDescriptor { -+ fn from(next_lvl_table_addr: usize) -> Self { ++impl TableDescriptor { ++ /// Create an instance. ++ /// ++ /// Descriptor is invalid by default. ++ pub const fn new_zeroed() -> Self { ++ Self { value: 0 } ++ } ++ ++ /// Create an instance pointing to the supplied address. ++ pub fn from_next_lvl_table_addr(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + -+ let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; ++ let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + -+ TableDescriptor(val.get()) ++ TableDescriptor { value: val.get() } + } +} + @@ -490,11 +540,11 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable -+ + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) ++ + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable -+ + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) ++ + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + @@ -520,10 +570,17 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part + +impl PageDescriptor { + /// Create an instance. -+ fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { ++ /// ++ /// Descriptor is invalid by default. ++ pub const fn new_zeroed() -> Self { ++ Self { value: 0 } ++ } ++ ++ /// Create an instance. ++ pub fn from_output_addr(output_addr: usize, attribute_fields: AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + -+ let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; ++ let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True @@ -532,73 +589,159 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + -+ Self(val.get()) ++ Self { value: val.get() } + } +} + -+impl FixedSizeTranslationTable<{ NUM_TABLES }> { ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl FixedSizeTranslationTable { + /// Create an instance. ++ #[allow(clippy::assertions_on_constants)] + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); ++ assert!((bsp::memory::mmu::KernelAddrSpaceSize::SIZE modulo Granule512MiB::SIZE) == 0); + + Self { -+ lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], -+ lvl2: [TableDescriptor(0); NUM_TABLES], ++ lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], ++ lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + } + } ++ ++ /// Iterates over all static translation table entries and fills them at once. ++ /// ++ /// # Safety ++ /// ++ /// - Modifies a `static mut`. Ensure it only happens from here. ++ pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { ++ for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { ++ *l2_entry = ++ TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); ++ ++ for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { ++ let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); ++ ++ let (output_addr, attribute_fields) = ++ bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; ++ ++ *l3_entry = PageDescriptor::from_output_addr(output_addr, attribute_fields); ++ } ++ } ++ ++ Ok(()) ++ } ++ ++ /// The translation table's base address to be used for programming the MMU. ++ pub fn base_address(&self) -> u64 { ++ self.lvl2.base_addr_u64() ++ } +} + +diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs +--- 10_privilege_level/src/_arch/aarch64/memory/mmu.rs ++++ 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs +@@ -0,0 +1,146 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter + -+/// Setup function for the MAIR_EL1 register. -+fn set_up_mair() { -+ // Define the memory types being mapped. -+ MAIR_EL1.write( -+ // Attribute 1 - Cacheable normal DRAM. -+ MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + -+ MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + ++//! Memory Management Unit Driver. ++//! ++//! Only 64 KiB granule is supported. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::memory::mmu::arch_mmu + -+ // Attribute 0 - Device. -+ MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, -+ ); ++use crate::{ ++ bsp, memory, ++ memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule}, ++}; ++use cortex_a::{barrier, regs::*}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Memory Management Unit type. ++struct MemoryManagementUnit; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; ++pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; ++ ++/// The min supported address space size. ++pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB ++ ++/// The max supported address space size. ++pub const MAX_ADDR_SPACE_SIZE: usize = 32 * 1024 * 1024 * 1024; // 32 GiB ++ ++/// The supported address space size granule. ++pub type AddrSpaceSizeGranule = Granule512MiB; ++ ++/// Constants for indexing the MAIR_EL1. ++#[allow(dead_code)] ++pub mod mair { ++ pub const DEVICE: u64 = 0; ++ pub const NORMAL: u64 = 1; +} + -+/// Iterates over all static translation table entries and fills them at once. ++//-------------------------------------------------------------------------------------------------- ++// Global instances ++//-------------------------------------------------------------------------------------------------- ++ ++/// The kernel translation tables. +/// +/// # Safety +/// -+/// - Modifies a `static mut`. Ensure it only happens from here. -+unsafe fn populate_tt_entries() -> Result<(), &'static str> { -+ for (l2_nr, l2_entry) in KERNEL_TABLES.lvl2.iter_mut().enumerate() { -+ *l2_entry = KERNEL_TABLES.lvl3[l2_nr].base_addr_usize().into(); ++/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". ++static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); ++ ++static MMU: MemoryManagementUnit = MemoryManagementUnit; + -+ for (l3_nr, l3_entry) in KERNEL_TABLES.lvl3[l2_nr].iter_mut().enumerate() { -+ let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- + -+ let (output_addr, attribute_fields) = -+ bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; ++impl MemoryManagementUnit { ++ /// Setup function for the MAIR_EL1 register. ++ fn set_up_mair(&self) { ++ // Define the memory types being mapped. ++ MAIR_EL1.write( ++ // Attribute 1 - Cacheable normal DRAM. ++ MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + ++ MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + -+ *l3_entry = PageDescriptor::new(output_addr, attribute_fields); -+ } ++ // Attribute 0 - Device. ++ MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, ++ ); + } + -+ Ok(()) -+} -+ -+/// Configure various settings of stage 1 of the EL1 translation regime. -+fn configure_translation_control() { -+ let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); -+ let t0sz: u64 = bsp::memory::mmu::addr_space_size().trailing_zeros().into(); -+ -+ TCR_EL1.write( -+ TCR_EL1::TBI0::Ignored -+ + TCR_EL1::IPS.val(ips) -+ + TCR_EL1::EPD1::DisableTTBR1Walks -+ + TCR_EL1::TG0::KiB_64 -+ + TCR_EL1::SH0::Inner -+ + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable -+ + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable -+ + TCR_EL1::EPD0::EnableTTBR0Walks -+ + TCR_EL1::T0SZ.val(t0sz), -+ ); ++ /// Configure various settings of stage 1 of the EL1 translation regime. ++ fn configure_translation_control(&self) { ++ let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); ++ let t0sz = (64 - bsp::memory::mmu::KernelAddrSpaceSize::SHIFT) as u64; ++ ++ TCR_EL1.write( ++ TCR_EL1::TBI0::Ignored ++ + TCR_EL1::IPS.val(ips) ++ + TCR_EL1::EPD1::DisableTTBR1Walks ++ + TCR_EL1::TG0::KiB_64 ++ + TCR_EL1::SH0::Inner ++ + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable ++ + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable ++ + TCR_EL1::EPD0::EnableTTBR0Walks ++ + TCR_EL1::T0SZ.val(t0sz), ++ ); ++ } +} + +//-------------------------------------------------------------------------------------------------- @@ -622,15 +765,15 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part + } + + // Prepare the memory attribute indirection register. -+ set_up_mair(); ++ self.set_up_mair(); + + // Populate translation tables. -+ populate_tt_entries()?; ++ KERNEL_TABLES.populate_tt_entries()?; + + // Set the "Translation Table Base Register". -+ TTBR0_EL1.set_baddr(KERNEL_TABLES.lvl2.base_addr_u64()); ++ TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); + -+ configure_translation_control(); ++ self.configure_translation_control(); + + // Switch the MMU on. + // @@ -671,7 +814,7 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/link.ld 11_virtual_mem_part1_id diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs --- 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs +++ 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs -@@ -0,0 +1,93 @@ +@@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -686,13 +829,16 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 11_virtual_mem_pa +// Public Definitions +//-------------------------------------------------------------------------------------------------- + ++/// The address space size chosen by this BSP. ++pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; ++ +const NUM_MEM_RANGES: usize = 3; + +/// The virtual memory layout. +/// +/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. +/// It is agnostic of the paging granularity that the architecture's MMU will use. -+pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( ++pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( + memory_map::END_INCLUSIVE, + [ + TranslationDescriptor { @@ -751,18 +897,8 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 11_virtual_mem_pa +// Public Code +//-------------------------------------------------------------------------------------------------- + -+/// Return the address space size in bytes. -+/// -+/// Guarantees size to be a power of two. -+pub const fn addr_space_size() -> usize { -+ let size = memory_map::END_INCLUSIVE + 1; -+ assert!(size.is_power_of_two()); -+ -+ size -+} -+ +/// Return a reference to the virtual memory layout. -+pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { ++pub fn virt_mem_layout() -> &'static KernelVirtualLayout { + &LAYOUT +} @@ -859,7 +995,7 @@ diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_mem_part1_identity_mapping/sr +++ 11_virtual_mem_part1_identity_mapping/src/bsp.rs @@ -4,7 +4,7 @@ - //! Conditional re-exporting of Board Support Packages. + //! Conditional reexporting of Board Support Packages. -mod device_driver; +pub mod device_driver; @@ -870,22 +1006,9 @@ diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_mem_part1_identity_mapping/sr diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/src/main.rs --- 10_privilege_level/src/main.rs +++ 11_virtual_mem_part1_identity_mapping/src/main.rs -@@ -11,10 +11,12 @@ - //! - //! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. - //! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -+//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. - //! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. +@@ -106,7 +106,10 @@ //! - //! [console interface]: ../libkernel/console/interface/index.html - //! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -+//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html - //! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html - //! - //! # Code organization and architecture -@@ -102,7 +104,10 @@ - //! - `crate::memory::*` - //! - `crate::bsp::memory::*` + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html +#![allow(incomplete_features)] #![feature(const_fn_fn_ptr_basics)] @@ -894,7 +1017,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/s #![feature(format_args_nl)] #![feature(panic_info_message)] #![feature(trait_alias)] -@@ -129,9 +134,18 @@ +@@ -130,9 +133,18 @@ /// # Safety /// /// - Only a single core must be active and running this function. @@ -914,7 +1037,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/s for i in bsp::driver::driver_manager().all_device_drivers().iter() { if let Err(x) = i.init() { -@@ -155,6 +169,9 @@ +@@ -156,6 +168,9 @@ info!("Booting on: {}", bsp::board_name()); @@ -924,7 +1047,7 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/s let (_, privilege_level) = exception::current_privilege_level(); info!("Current privilege level: {}", privilege_level); -@@ -178,6 +195,13 @@ +@@ -179,6 +194,13 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); @@ -939,10 +1062,29 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/s // Discard any spurious received characters before going into echo mode. +diff -uNr 10_privilege_level/src/memory/mmu/translation_table.rs 11_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs +--- 10_privilege_level/src/memory/mmu/translation_table.rs ++++ 11_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs +@@ -0,0 +1,14 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2021 Andre Richter ++ ++//! Translation table. ++ ++#[cfg(target_arch = "aarch64")] ++#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] ++mod arch_translation_table; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_translation_table::KernelTranslationTable; + diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs --- 10_privilege_level/src/memory/mmu.rs +++ 11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs -@@ -0,0 +1,199 @@ +@@ -0,0 +1,247 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2021 Andre Richter @@ -950,8 +1092,8 @@ diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_mem_part1_identity_map +//! Memory Management Unit. +//! +//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file -+//! provides types for composing an architecture-agnostic description of the kernel 's virtual -+//! memory layout. ++//! provides types for composing an architecture-agnostic description of the kernel's virtual memory ++//! layout. +//! +//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` +//! function. @@ -962,11 +1104,17 @@ diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_mem_part1_identity_map +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/memory/mmu.rs"] +mod arch_mmu; -+pub use arch_mmu::*; ++ ++mod translation_table; + +use core::{fmt, ops::RangeInclusive}; + +//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_mmu::mmu; ++ ++//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + @@ -985,8 +1133,15 @@ diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_mem_part1_identity_map + } +} + ++/// Describes the characteristics of a translation granule. ++pub struct TranslationGranule; ++ ++/// Describes the size of an address space. ++pub struct AddressSpaceSize; ++ +/// Architecture agnostic translation types. +#[allow(missing_docs)] ++#[allow(dead_code)] +#[derive(Copy, Clone)] +pub enum Translation { + Identity, @@ -1040,6 +1195,41 @@ diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_mem_part1_identity_map +// Public Code +//-------------------------------------------------------------------------------------------------- + ++impl TranslationGranule { ++ /// The granule's size. ++ pub const SIZE: usize = Self::size_checked(); ++ ++ /// The granule's shift, aka log2(size). ++ pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; ++ ++ const fn size_checked() -> usize { ++ assert!(GRANULE_SIZE.is_power_of_two()); ++ ++ GRANULE_SIZE ++ } ++} ++ ++impl AddressSpaceSize { ++ /// The address space size. ++ pub const SIZE: usize = Self::size_checked(); ++ ++ /// The address space shift, aka log2(size). ++ pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; ++ ++ const fn size_checked() -> usize { ++ assert!(AS_SIZE.is_power_of_two()); ++ assert!(arch_mmu::MIN_ADDR_SPACE_SIZE.is_power_of_two()); ++ assert!(arch_mmu::MAX_ADDR_SPACE_SIZE.is_power_of_two()); ++ ++ // Must adhere to architectural restrictions. ++ assert!(AS_SIZE >= arch_mmu::MIN_ADDR_SPACE_SIZE); ++ assert!(AS_SIZE <= arch_mmu::MAX_ADDR_SPACE_SIZE); ++ assert!((AS_SIZE modulo arch_mmu::AddrSpaceSizeGranule::SIZE) == 0); ++ ++ AS_SIZE ++ } ++} ++ +impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu.rs index 3846968b..d3b07461 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu.rs @@ -3,79 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} - -/// Transition from EL2 to EL1. -/// -/// # Safety -/// -/// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - - // Enable timer counter registers for EL1. - CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - - // No offset for reading the counters. - CNTVOFF_EL2.set(0); - - // Set EL1 execution state to AArch64. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Set up a simulated exception return. - // - // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a - // stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, - ); - - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c2f5fe0b --- /dev/null +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[no_mangle] +pub unsafe fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs index b6ee28b8..d8c617b2 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception use cortex_a::regs::*; diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs index 445f490e..b63b00fe 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous use cortex_a::regs::*; diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs index 69023cef..3504d257 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs @@ -4,146 +4,61 @@ //! Memory Management Unit Driver. //! -//! Static translation tables, compiled on boot; Everything 64 KiB granule. +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::arch_mmu -use super::{AccessPermissions, AttributeFields, MemAttributes}; -use crate::{bsp, memory}; -use core::convert; +use crate::{ + bsp, memory, + memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule}, +}; use cortex_a::{barrier, regs::*}; -use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions //-------------------------------------------------------------------------------------------------- -// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. -register_bitfields! {u64, - STAGE1_TABLE_DESCRIPTOR [ - /// Physical address of the next descriptor. - NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. -register_bitfields! {u64, - STAGE1_PAGE_DESCRIPTOR [ - /// Unprivileged execute-never. - UXN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Privileged execute-never. - PXN OFFSET(53) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - /// Access flag. - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field. - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions. - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register. - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) -const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) - -/// A table descriptor for 64 KiB aperture. -/// -/// The output points to the next table. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct TableDescriptor(u64); +/// Memory Management Unit type. +struct MemoryManagementUnit; -/// A page descriptor with 64 KiB aperture. -/// -/// The output points to physical memory. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct PageDescriptor(u64); +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- -/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -/// aligned, hence the "reverse" order of appearance. -#[repr(C)] -#[repr(align(65536))] -struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; - /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; NUM_TABLES], -} +/// The min supported address space size. +pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB -const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -type ArchTranslationTable = FixedSizeTranslationTable; +/// The max supported address space size. +pub const MAX_ADDR_SPACE_SIZE: usize = 32 * 1024 * 1024 * 1024; // 32 GiB -trait BaseAddr { - fn base_addr_u64(&self) -> u64; - fn base_addr_usize(&self) -> usize; -} +/// The supported address space size granule. +pub type AddrSpaceSizeGranule = Granule512MiB; /// Constants for indexing the MAIR_EL1. #[allow(dead_code)] -mod mair { +pub mod mair { pub const DEVICE: u64 = 0; pub const NORMAL: u64 = 1; } -/// Memory Management Unit type. -struct MemoryManagementUnit; - //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -/// The translation tables. +/// The kernel translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); +static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); static MMU: MemoryManagementUnit = MemoryManagementUnit; @@ -151,149 +66,37 @@ static MMU: MemoryManagementUnit = MemoryManagementUnit; // Private Code //-------------------------------------------------------------------------------------------------- -impl BaseAddr for [T; N] { - fn base_addr_u64(&self) -> u64 { - self as *const T as u64 - } - - fn base_addr_usize(&self) -> usize { - self as *const _ as usize - } -} - -impl convert::From for TableDescriptor { - fn from(next_lvl_table_addr: usize) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), - ); - - TableDescriptor(val.get()) - } -} - -/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. -impl convert::From - for register::FieldValue -{ - fn from(attribute_fields: AttributeFields) -> Self { - // Memory attributes. - let mut desc = match attribute_fields.mem_attributes { - MemAttributes::CacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - } - MemAttributes::Device => { - STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } - }; - - // Access Permissions. - desc += match attribute_fields.acc_perms { - AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, - AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, - }; - - // The execute-never attribute is mapped to PXN in AArch64. - desc += if attribute_fields.execute_never { - STAGE1_PAGE_DESCRIPTOR::PXN::True - } else { - STAGE1_PAGE_DESCRIPTOR::PXN::False - }; - - // Always set unprivileged exectue-never as long as userspace is not implemented yet. - desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; - - desc - } -} - -impl PageDescriptor { - /// Create an instance. - fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), - ); - - Self(val.get()) - } -} - -impl FixedSizeTranslationTable<{ NUM_TABLES }> { - /// Create an instance. - pub const fn new() -> Self { - assert!(NUM_TABLES > 0); - - Self { - lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], - lvl2: [TableDescriptor(0); NUM_TABLES], - } - } -} - -/// Setup function for the MAIR_EL1 register. -fn set_up_mair() { - // Define the memory types being mapped. - MAIR_EL1.write( - // Attribute 1 - Cacheable normal DRAM. - MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + // Attribute 0 - Device. MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, - ); -} - -/// Iterates over all static translation table entries and fills them at once. -/// -/// # Safety -/// -/// - Modifies a `static mut`. Ensure it only happens from here. -unsafe fn populate_tt_entries() -> Result<(), &'static str> { - for (l2_nr, l2_entry) in KERNEL_TABLES.lvl2.iter_mut().enumerate() { - *l2_entry = KERNEL_TABLES.lvl3[l2_nr].base_addr_usize().into(); - - for (l3_nr, l3_entry) in KERNEL_TABLES.lvl3[l2_nr].iter_mut().enumerate() { - let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); - - let (output_addr, attribute_fields) = - bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; - - *l3_entry = PageDescriptor::new(output_addr, attribute_fields); - } + ); } - Ok(()) -} - -/// Configure various settings of stage 1 of the EL1 translation regime. -fn configure_translation_control() { - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - let t0sz: u64 = bsp::memory::mmu::addr_space_size().trailing_zeros().into(); - - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::EPD1::DisableTTBR1Walks - + TCR_EL1::TG0::KiB_64 - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(t0sz), - ); + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + let t0sz = (64 - bsp::memory::mmu::KernelAddrSpaceSize::SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::EPD1::DisableTTBR1Walks + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(t0sz), + ); + } } //-------------------------------------------------------------------------------------------------- @@ -317,15 +120,15 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { } // Prepare the memory attribute indirection register. - set_up_mair(); + self.set_up_mair(); // Populate translation tables. - populate_tt_entries()?; + KERNEL_TABLES.populate_tt_entries()?; // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(KERNEL_TABLES.lvl2.base_addr_u64()); + TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); - configure_translation_control(); + self.configure_translation_control(); // Switch the MMU on. // diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs new file mode 100644 index 00000000..cbe1d783 --- /dev/null +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural translation table. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::translation_table::arch_translation_table + +use crate::{ + bsp, memory, + memory::mmu::{ + arch_mmu::{Granule512MiB, Granule64KiB}, + AccessPermissions, AttributeFields, MemAttributes, + }, +}; +use core::convert; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Unprivileged execute-never. + UXN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(C)] +struct TableDescriptor { + value: u64, +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +pub struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], +} + +/// A translation table type for the kernel space. +pub type KernelTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl BaseAddr for [T; N] { + fn base_addr_u64(&self) -> u64 { + self as *const T as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const _ as usize + } +} + +impl TableDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance pointing to the supplied address. + pub fn from_next_lvl_table_addr(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + + TableDescriptor { value: val.get() } + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // The execute-never attribute is mapped to PXN in AArch64. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + // Always set unprivileged exectue-never as long as userspace is not implemented yet. + desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; + + desc + } +} + +impl PageDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance. + pub fn from_output_addr(output_addr: usize, attribute_fields: AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self { value: val.get() } + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl FixedSizeTranslationTable { + /// Create an instance. + #[allow(clippy::assertions_on_constants)] + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + assert!((bsp::memory::mmu::KernelAddrSpaceSize::SIZE % Granule512MiB::SIZE) == 0); + + Self { + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + } + } + + /// Iterates over all static translation table entries and fills them at once. + /// + /// # Safety + /// + /// - Modifies a `static mut`. Ensure it only happens from here. + pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { + for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { + *l2_entry = + TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + + for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { + let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + + let (output_addr, attribute_fields) = + bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + + *l3_entry = PageDescriptor::from_output_addr(output_addr, attribute_fields); + } + } + + Ok(()) + } + + /// The translation table's base address to be used for programming the MMU. + pub fn base_address(&self) -> u64 { + self.lvl2.base_addr_u64() + } +} diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/time.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/time.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/11_virtual_mem_part1_identity_mapping/src/bsp.rs b/11_virtual_mem_part1_identity_mapping/src/bsp.rs index 3d758767..91ce3c49 100644 --- a/11_virtual_mem_part1_identity_mapping/src/bsp.rs +++ b/11_virtual_mem_part1_identity_mapping/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. pub mod device_driver; diff --git a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs index 982fc065..911d1054 100644 --- a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs @@ -12,13 +12,16 @@ use core::ops::RangeInclusive; // Public Definitions //-------------------------------------------------------------------------------------------------- +/// The address space size chosen by this BSP. +pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; + const NUM_MEM_RANGES: usize = 3; /// The virtual memory layout. /// /// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. /// It is agnostic of the paging granularity that the architecture's MMU will use. -pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( +pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( memory_map::END_INCLUSIVE, [ TranslationDescriptor { @@ -77,17 +80,7 @@ fn mmio_range_inclusive() -> RangeInclusive { // Public Code //-------------------------------------------------------------------------------------------------- -/// Return the address space size in bytes. -/// -/// Guarantees size to be a power of two. -pub const fn addr_space_size() -> usize { - let size = memory_map::END_INCLUSIVE + 1; - assert!(size.is_power_of_two()); - - size -} - /// Return a reference to the virtual memory layout. -pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { +pub fn virt_mem_layout() -> &'static KernelVirtualLayout { &LAYOUT } diff --git a/11_virtual_mem_part1_identity_mapping/src/console.rs b/11_virtual_mem_part1_identity_mapping/src/console.rs index 3552823c..c3154ba2 100644 --- a/11_virtual_mem_part1_identity_mapping/src/console.rs +++ b/11_virtual_mem_part1_identity_mapping/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/11_virtual_mem_part1_identity_mapping/src/cpu.rs b/11_virtual_mem_part1_identity_mapping/src/cpu.rs index c9e5af72..3834f183 100644 --- a/11_virtual_mem_part1_identity_mapping/src/cpu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; diff --git a/11_virtual_mem_part1_identity_mapping/src/cpu/boot.rs b/11_virtual_mem_part1_identity_mapping/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/11_virtual_mem_part1_identity_mapping/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs b/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs +++ b/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/11_virtual_mem_part1_identity_mapping/src/exception.rs b/11_virtual_mem_part1_identity_mapping/src/exception.rs index 432c606b..87d4db63 100644 --- a/11_virtual_mem_part1_identity_mapping/src/exception.rs +++ b/11_virtual_mem_part1_identity_mapping/src/exception.rs @@ -7,10 +7,14 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/exception.rs"] mod arch_exception; -pub use arch_exception::*; pub mod asynchronous; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::current_privilege_level; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- diff --git a/11_virtual_mem_part1_identity_mapping/src/exception/asynchronous.rs b/11_virtual_mem_part1_identity_mapping/src/exception/asynchronous.rs index fbdba957..566efe32 100644 --- a/11_virtual_mem_part1_identity_mapping/src/exception/asynchronous.rs +++ b/11_virtual_mem_part1_identity_mapping/src/exception/asynchronous.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/exception/asynchronous.rs"] -mod arch_exception_async; -pub use arch_exception_async::*; +mod arch_asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::print_state; diff --git a/11_virtual_mem_part1_identity_mapping/src/main.rs b/11_virtual_mem_part1_identity_mapping/src/main.rs index 25971535..83ee1de5 100644 --- a/11_virtual_mem_part1_identity_mapping/src/main.rs +++ b/11_virtual_mem_part1_identity_mapping/src/main.rs @@ -7,18 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -32,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -49,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -103,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![allow(incomplete_features)] #![feature(const_fn_fn_ptr_basics)] @@ -114,9 +116,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs b/11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs index 4880ce91..efc9c447 100644 --- a/11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs @@ -5,8 +5,8 @@ //! Memory Management Unit. //! //! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file -//! provides types for composing an architecture-agnostic description of the kernel 's virtual -//! memory layout. +//! provides types for composing an architecture-agnostic description of the kernel's virtual memory +//! layout. //! //! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` //! function. @@ -17,10 +17,16 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; -pub use arch_mmu::*; + +mod translation_table; use core::{fmt, ops::RangeInclusive}; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_mmu::mmu; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -40,8 +46,15 @@ pub mod interface { } } +/// Describes the characteristics of a translation granule. +pub struct TranslationGranule; + +/// Describes the size of an address space. +pub struct AddressSpaceSize; + /// Architecture agnostic translation types. #[allow(missing_docs)] +#[allow(dead_code)] #[derive(Copy, Clone)] pub enum Translation { Identity, @@ -95,6 +108,41 @@ pub struct KernelVirtualLayout { // Public Code //-------------------------------------------------------------------------------------------------- +impl TranslationGranule { + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(GRANULE_SIZE.is_power_of_two()); + + GRANULE_SIZE + } +} + +impl AddressSpaceSize { + /// The address space size. + pub const SIZE: usize = Self::size_checked(); + + /// The address space shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(AS_SIZE.is_power_of_two()); + assert!(arch_mmu::MIN_ADDR_SPACE_SIZE.is_power_of_two()); + assert!(arch_mmu::MAX_ADDR_SPACE_SIZE.is_power_of_two()); + + // Must adhere to architectural restrictions. + assert!(AS_SIZE >= arch_mmu::MIN_ADDR_SPACE_SIZE); + assert!(AS_SIZE <= arch_mmu::MAX_ADDR_SPACE_SIZE); + assert!((AS_SIZE % arch_mmu::AddrSpaceSizeGranule::SIZE) == 0); + + AS_SIZE + } +} + impl Default for AttributeFields { fn default() -> AttributeFields { AttributeFields { diff --git a/11_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs b/11_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs new file mode 100644 index 00000000..bbb6f939 --- /dev/null +++ b/11_virtual_mem_part1_identity_mapping/src/memory/mmu/translation_table.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Translation table. + +#[cfg(target_arch = "aarch64")] +#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] +mod arch_translation_table; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_translation_table::KernelTranslationTable; diff --git a/11_virtual_mem_part1_identity_mapping/src/print.rs b/11_virtual_mem_part1_identity_mapping/src/print.rs index 1ea96b6a..5a563811 100644 --- a/11_virtual_mem_part1_identity_mapping/src/print.rs +++ b/11_virtual_mem_part1_identity_mapping/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/11_virtual_mem_part1_identity_mapping/src/time.rs b/11_virtual_mem_part1_identity_mapping/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/11_virtual_mem_part1_identity_mapping/src/time.rs +++ b/11_virtual_mem_part1_identity_mapping/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/12_exceptions_part1_groundwork/Makefile b/12_exceptions_part1_groundwork/Makefile index 9254018c..9105d297 100644 --- a/12_exceptions_part1_groundwork/Makefile +++ b/12_exceptions_part1_groundwork/Makefile @@ -44,8 +44,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/12_exceptions_part1_groundwork/README.md b/12_exceptions_part1_groundwork/README.md index be16673f..829ed092 100644 --- a/12_exceptions_part1_groundwork/README.md +++ b/12_exceptions_part1_groundwork/README.md @@ -2,8 +2,8 @@ ## tl;dr -- We lay the groundwork for all the architectural `CPU exceptions`. For now, only print an elaborate - system state through a `panic!` call, and halt execution +- We lay the groundwork for all the architectural `CPU exceptions`. +- For now, only print an elaborate system state through a `panic!` call, and halt execution - This will help finding bugs during development and runtime. - For demo purposes, MMU `page faults` are used to demonstrate (i) returning from an exception and (ii) the default `panic!` behavior. @@ -27,8 +27,8 @@ Now that we are executing in `EL1`, and have activated the `MMU`, time is due for implementing `CPU exceptions`. For now, we only set up a scaffold with very basic functionality that will help us to -find bugs along the way. A follow-up `Interrupt` tutorial in the future will continue the work we -start here. +find bugs along the way. A follow-up `Interrupt` tutorial later will continue the work we start +here. Please note that this tutorial is specific to the `AArch64` architecture. It does not contain any generic exception handling code yet. @@ -38,7 +38,7 @@ generic exception handling code yet. In `AArch64`, it is differentiated between four types of exceptions. These are: - Synchronous - For example, a `data abort` (e.g. `page fault`) or a `system call`. They happen in direct - consequence of executing a certain instruction, hence _synchronously_. + consequence of executing a certain CPU instruction, hence _synchronously_. - Interrupt Request (`IRQ`) - For example, an external device, like a timer, is asserting a physical interrupt line. IRQs happen _asynchronously_. @@ -176,8 +176,8 @@ resources in its own code without bothering, and as a last action before returni handling code, restore the context, so that the processor can continue where it left off before taking the exception. -Context save and restore is one of the few places in system software where it is strongly advised to -to use some hand-crafted assembly. Introducing `exception.S`: +Context save and restore is one of the few places in system software where there is no way around +some hand-crafted assembly. Introducing `exception.S`: ```asm /// Call the function provided by parameter `\handler` after saving the exception context. Provide @@ -356,7 +356,7 @@ unsafe { core::ptr::read_volatile(big_addr as *mut u64) }; ``` This triggers our exception code, because we try to read from a virtual address for which no mapping -has been installed. Remember, we only installed up to `4 GiB` of address space in the previous +has been installed. Remember, we only mapped up to `4 GiB` of address space in the previous tutorial. To survive this exception, the respective handler has a special demo case: @@ -391,8 +391,6 @@ catch, eventually triggering the `panic!` call from the default handler. ## Test it -Emphasis on the events at timestamps > `4.xxxxxx`. - ```console $ make chainboot [...] @@ -482,9 +480,9 @@ General purpose register: diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs --- 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs +++ 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs -@@ -4,7 +4,230 @@ - - //! Architectural synchronous and asynchronous exception handling. +@@ -11,7 +11,230 @@ + //! + //! crate::exception::arch_exception -use cortex_a::regs::*; +use core::{cell::UnsafeCell, fmt}; @@ -714,7 +712,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs 1 //-------------------------------------------------------------------------------------------------- // Public Code -@@ -21,3 +244,23 @@ +@@ -28,3 +251,23 @@ _ => (PrivilegeLevel::Unknown, "Unknown"), } } @@ -901,16 +899,16 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld 12_e diff -uNr 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs --- 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs +++ 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs -@@ -12,7 +12,7 @@ - // Public Definitions - //-------------------------------------------------------------------------------------------------- +@@ -15,7 +15,7 @@ + /// The address space size chosen by this BSP. + pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; -const NUM_MEM_RANGES: usize = 3; +const NUM_MEM_RANGES: usize = 2; /// The virtual memory layout. /// -@@ -32,16 +32,6 @@ +@@ -35,16 +35,6 @@ }, }, TranslationDescriptor { @@ -927,7 +925,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.r name: "Device MMIO", virtual_range: mmio_range_inclusive, physical_range_translation: Translation::Identity, -@@ -64,11 +54,6 @@ +@@ -67,11 +57,6 @@ RangeInclusive::new(super::ro_start(), super::ro_end() - 1) } @@ -945,7 +943,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/bsp.rs 12_exceptions_part1_g +++ 12_exceptions_part1_groundwork/src/bsp.rs @@ -4,7 +4,7 @@ - //! Conditional re-exporting of Board Support Packages. + //! Conditional reexporting of Board Support Packages. -pub mod device_driver; +mod device_driver; @@ -953,10 +951,23 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/bsp.rs 12_exceptions_part1_g #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; +diff -uNr 11_virtual_mem_part1_identity_mapping/src/exception.rs 12_exceptions_part1_groundwork/src/exception.rs +--- 11_virtual_mem_part1_identity_mapping/src/exception.rs ++++ 12_exceptions_part1_groundwork/src/exception.rs +@@ -13,7 +13,7 @@ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- +-pub use arch_exception::current_privilege_level; ++pub use arch_exception::{current_privilege_level, handling_init}; + + //-------------------------------------------------------------------------------------------------- + // Public Definitions + diff -uNr 11_virtual_mem_part1_identity_mapping/src/main.rs 12_exceptions_part1_groundwork/src/main.rs --- 11_virtual_mem_part1_identity_mapping/src/main.rs +++ 12_exceptions_part1_groundwork/src/main.rs -@@ -109,6 +109,7 @@ +@@ -111,6 +111,7 @@ #![feature(const_generics)] #![feature(const_panic)] #![feature(format_args_nl)] @@ -964,7 +975,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/main.rs 12_exceptions_part1_ #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] -@@ -143,6 +144,8 @@ +@@ -142,6 +143,8 @@ use driver::interface::DriverManager; use memory::mmu::interface::MMU; @@ -973,7 +984,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/main.rs 12_exceptions_part1_ if let Err(string) = memory::mmu::mmu().init() { panic!("MMU: {}", string); } -@@ -195,13 +198,28 @@ +@@ -194,13 +197,28 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); @@ -1009,16 +1020,4 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/main.rs 12_exceptions_part1_ // Discard any spurious received characters before going into echo mode. -diff -uNr 11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs 12_exceptions_part1_groundwork/src/memory/mmu.rs ---- 11_virtual_mem_part1_identity_mapping/src/memory/mmu.rs -+++ 12_exceptions_part1_groundwork/src/memory/mmu.rs -@@ -42,6 +42,7 @@ - - /// Architecture agnostic translation types. - #[allow(missing_docs)] -+#[allow(dead_code)] - #[derive(Copy, Clone)] - pub enum Translation { - Identity, - ``` diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs index 3846968b..d3b07461 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs @@ -3,79 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} - -/// Transition from EL2 to EL1. -/// -/// # Safety -/// -/// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - - // Enable timer counter registers for EL1. - CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - - // No offset for reading the counters. - CNTVOFF_EL2.set(0); - - // Set EL1 execution state to AArch64. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Set up a simulated exception return. - // - // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a - // stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, - ); - - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c2f5fe0b --- /dev/null +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[no_mangle] +pub unsafe fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs index 10d00354..64f1fd23 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception use core::{cell::UnsafeCell, fmt}; use cortex_a::{asm, barrier, regs::*}; diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception/asynchronous.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception/asynchronous.rs index 445f490e..b63b00fe 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception/asynchronous.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception/asynchronous.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous use cortex_a::regs::*; diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs index 69023cef..3504d257 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs @@ -4,146 +4,61 @@ //! Memory Management Unit Driver. //! -//! Static translation tables, compiled on boot; Everything 64 KiB granule. +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::arch_mmu -use super::{AccessPermissions, AttributeFields, MemAttributes}; -use crate::{bsp, memory}; -use core::convert; +use crate::{ + bsp, memory, + memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule}, +}; use cortex_a::{barrier, regs::*}; -use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions //-------------------------------------------------------------------------------------------------- -// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. -register_bitfields! {u64, - STAGE1_TABLE_DESCRIPTOR [ - /// Physical address of the next descriptor. - NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. -register_bitfields! {u64, - STAGE1_PAGE_DESCRIPTOR [ - /// Unprivileged execute-never. - UXN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Privileged execute-never. - PXN OFFSET(53) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - /// Access flag. - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field. - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions. - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register. - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) -const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) - -/// A table descriptor for 64 KiB aperture. -/// -/// The output points to the next table. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct TableDescriptor(u64); +/// Memory Management Unit type. +struct MemoryManagementUnit; -/// A page descriptor with 64 KiB aperture. -/// -/// The output points to physical memory. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct PageDescriptor(u64); +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- -/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -/// aligned, hence the "reverse" order of appearance. -#[repr(C)] -#[repr(align(65536))] -struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; - /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; NUM_TABLES], -} +/// The min supported address space size. +pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB -const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -type ArchTranslationTable = FixedSizeTranslationTable; +/// The max supported address space size. +pub const MAX_ADDR_SPACE_SIZE: usize = 32 * 1024 * 1024 * 1024; // 32 GiB -trait BaseAddr { - fn base_addr_u64(&self) -> u64; - fn base_addr_usize(&self) -> usize; -} +/// The supported address space size granule. +pub type AddrSpaceSizeGranule = Granule512MiB; /// Constants for indexing the MAIR_EL1. #[allow(dead_code)] -mod mair { +pub mod mair { pub const DEVICE: u64 = 0; pub const NORMAL: u64 = 1; } -/// Memory Management Unit type. -struct MemoryManagementUnit; - //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -/// The translation tables. +/// The kernel translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); +static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); static MMU: MemoryManagementUnit = MemoryManagementUnit; @@ -151,149 +66,37 @@ static MMU: MemoryManagementUnit = MemoryManagementUnit; // Private Code //-------------------------------------------------------------------------------------------------- -impl BaseAddr for [T; N] { - fn base_addr_u64(&self) -> u64 { - self as *const T as u64 - } - - fn base_addr_usize(&self) -> usize { - self as *const _ as usize - } -} - -impl convert::From for TableDescriptor { - fn from(next_lvl_table_addr: usize) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), - ); - - TableDescriptor(val.get()) - } -} - -/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. -impl convert::From - for register::FieldValue -{ - fn from(attribute_fields: AttributeFields) -> Self { - // Memory attributes. - let mut desc = match attribute_fields.mem_attributes { - MemAttributes::CacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - } - MemAttributes::Device => { - STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } - }; - - // Access Permissions. - desc += match attribute_fields.acc_perms { - AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, - AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, - }; - - // The execute-never attribute is mapped to PXN in AArch64. - desc += if attribute_fields.execute_never { - STAGE1_PAGE_DESCRIPTOR::PXN::True - } else { - STAGE1_PAGE_DESCRIPTOR::PXN::False - }; - - // Always set unprivileged exectue-never as long as userspace is not implemented yet. - desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; - - desc - } -} - -impl PageDescriptor { - /// Create an instance. - fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), - ); - - Self(val.get()) - } -} - -impl FixedSizeTranslationTable<{ NUM_TABLES }> { - /// Create an instance. - pub const fn new() -> Self { - assert!(NUM_TABLES > 0); - - Self { - lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], - lvl2: [TableDescriptor(0); NUM_TABLES], - } - } -} - -/// Setup function for the MAIR_EL1 register. -fn set_up_mair() { - // Define the memory types being mapped. - MAIR_EL1.write( - // Attribute 1 - Cacheable normal DRAM. - MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + // Attribute 0 - Device. MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, - ); -} - -/// Iterates over all static translation table entries and fills them at once. -/// -/// # Safety -/// -/// - Modifies a `static mut`. Ensure it only happens from here. -unsafe fn populate_tt_entries() -> Result<(), &'static str> { - for (l2_nr, l2_entry) in KERNEL_TABLES.lvl2.iter_mut().enumerate() { - *l2_entry = KERNEL_TABLES.lvl3[l2_nr].base_addr_usize().into(); - - for (l3_nr, l3_entry) in KERNEL_TABLES.lvl3[l2_nr].iter_mut().enumerate() { - let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); - - let (output_addr, attribute_fields) = - bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; - - *l3_entry = PageDescriptor::new(output_addr, attribute_fields); - } + ); } - Ok(()) -} - -/// Configure various settings of stage 1 of the EL1 translation regime. -fn configure_translation_control() { - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - let t0sz: u64 = bsp::memory::mmu::addr_space_size().trailing_zeros().into(); - - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::EPD1::DisableTTBR1Walks - + TCR_EL1::TG0::KiB_64 - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(t0sz), - ); + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + let t0sz = (64 - bsp::memory::mmu::KernelAddrSpaceSize::SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::EPD1::DisableTTBR1Walks + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(t0sz), + ); + } } //-------------------------------------------------------------------------------------------------- @@ -317,15 +120,15 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { } // Prepare the memory attribute indirection register. - set_up_mair(); + self.set_up_mair(); // Populate translation tables. - populate_tt_entries()?; + KERNEL_TABLES.populate_tt_entries()?; // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(KERNEL_TABLES.lvl2.base_addr_u64()); + TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); - configure_translation_control(); + self.configure_translation_control(); // Switch the MMU on. // diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translation_table.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translation_table.rs new file mode 100644 index 00000000..cbe1d783 --- /dev/null +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural translation table. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::translation_table::arch_translation_table + +use crate::{ + bsp, memory, + memory::mmu::{ + arch_mmu::{Granule512MiB, Granule64KiB}, + AccessPermissions, AttributeFields, MemAttributes, + }, +}; +use core::convert; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Unprivileged execute-never. + UXN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(C)] +struct TableDescriptor { + value: u64, +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +pub struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], +} + +/// A translation table type for the kernel space. +pub type KernelTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl BaseAddr for [T; N] { + fn base_addr_u64(&self) -> u64 { + self as *const T as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const _ as usize + } +} + +impl TableDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance pointing to the supplied address. + pub fn from_next_lvl_table_addr(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + + TableDescriptor { value: val.get() } + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // The execute-never attribute is mapped to PXN in AArch64. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + // Always set unprivileged exectue-never as long as userspace is not implemented yet. + desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; + + desc + } +} + +impl PageDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance. + pub fn from_output_addr(output_addr: usize, attribute_fields: AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self { value: val.get() } + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl FixedSizeTranslationTable { + /// Create an instance. + #[allow(clippy::assertions_on_constants)] + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + assert!((bsp::memory::mmu::KernelAddrSpaceSize::SIZE % Granule512MiB::SIZE) == 0); + + Self { + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + } + } + + /// Iterates over all static translation table entries and fills them at once. + /// + /// # Safety + /// + /// - Modifies a `static mut`. Ensure it only happens from here. + pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { + for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { + *l2_entry = + TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + + for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { + let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + + let (output_addr, attribute_fields) = + bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + + *l3_entry = PageDescriptor::from_output_addr(output_addr, attribute_fields); + } + } + + Ok(()) + } + + /// The translation table's base address to be used for programming the MMU. + pub fn base_address(&self) -> u64 { + self.lvl2.base_addr_u64() + } +} diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/time.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/time.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/12_exceptions_part1_groundwork/src/bsp.rs b/12_exceptions_part1_groundwork/src/bsp.rs index 25750249..c558922f 100644 --- a/12_exceptions_part1_groundwork/src/bsp.rs +++ b/12_exceptions_part1_groundwork/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs index 59d736a7..fe98604d 100644 --- a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs +++ b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs @@ -12,13 +12,16 @@ use core::ops::RangeInclusive; // Public Definitions //-------------------------------------------------------------------------------------------------- +/// The address space size chosen by this BSP. +pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; + const NUM_MEM_RANGES: usize = 2; /// The virtual memory layout. /// /// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. /// It is agnostic of the paging granularity that the architecture's MMU will use. -pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( +pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( memory_map::END_INCLUSIVE, [ TranslationDescriptor { @@ -62,17 +65,7 @@ fn mmio_range_inclusive() -> RangeInclusive { // Public Code //-------------------------------------------------------------------------------------------------- -/// Return the address space size in bytes. -/// -/// Guarantees size to be a power of two. -pub const fn addr_space_size() -> usize { - let size = memory_map::END_INCLUSIVE + 1; - assert!(size.is_power_of_two()); - - size -} - /// Return a reference to the virtual memory layout. -pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { +pub fn virt_mem_layout() -> &'static KernelVirtualLayout { &LAYOUT } diff --git a/12_exceptions_part1_groundwork/src/console.rs b/12_exceptions_part1_groundwork/src/console.rs index 3552823c..c3154ba2 100644 --- a/12_exceptions_part1_groundwork/src/console.rs +++ b/12_exceptions_part1_groundwork/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/12_exceptions_part1_groundwork/src/cpu.rs b/12_exceptions_part1_groundwork/src/cpu.rs index c9e5af72..3834f183 100644 --- a/12_exceptions_part1_groundwork/src/cpu.rs +++ b/12_exceptions_part1_groundwork/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; diff --git a/12_exceptions_part1_groundwork/src/cpu/boot.rs b/12_exceptions_part1_groundwork/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/12_exceptions_part1_groundwork/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/12_exceptions_part1_groundwork/src/cpu/smp.rs b/12_exceptions_part1_groundwork/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/12_exceptions_part1_groundwork/src/cpu/smp.rs +++ b/12_exceptions_part1_groundwork/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/12_exceptions_part1_groundwork/src/exception.rs b/12_exceptions_part1_groundwork/src/exception.rs index 432c606b..8e91d8bb 100644 --- a/12_exceptions_part1_groundwork/src/exception.rs +++ b/12_exceptions_part1_groundwork/src/exception.rs @@ -7,10 +7,14 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/exception.rs"] mod arch_exception; -pub use arch_exception::*; pub mod asynchronous; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::{current_privilege_level, handling_init}; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- diff --git a/12_exceptions_part1_groundwork/src/exception/asynchronous.rs b/12_exceptions_part1_groundwork/src/exception/asynchronous.rs index fbdba957..566efe32 100644 --- a/12_exceptions_part1_groundwork/src/exception/asynchronous.rs +++ b/12_exceptions_part1_groundwork/src/exception/asynchronous.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/exception/asynchronous.rs"] -mod arch_exception_async; -pub use arch_exception_async::*; +mod arch_asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::print_state; diff --git a/12_exceptions_part1_groundwork/src/main.rs b/12_exceptions_part1_groundwork/src/main.rs index a2cf7752..80706a24 100644 --- a/12_exceptions_part1_groundwork/src/main.rs +++ b/12_exceptions_part1_groundwork/src/main.rs @@ -7,18 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -32,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -49,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -103,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![allow(incomplete_features)] #![feature(const_fn_fn_ptr_basics)] @@ -115,9 +117,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/12_exceptions_part1_groundwork/src/memory/mmu.rs b/12_exceptions_part1_groundwork/src/memory/mmu.rs index 826cda6c..efc9c447 100644 --- a/12_exceptions_part1_groundwork/src/memory/mmu.rs +++ b/12_exceptions_part1_groundwork/src/memory/mmu.rs @@ -5,8 +5,8 @@ //! Memory Management Unit. //! //! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file -//! provides types for composing an architecture-agnostic description of the kernel 's virtual -//! memory layout. +//! provides types for composing an architecture-agnostic description of the kernel's virtual memory +//! layout. //! //! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` //! function. @@ -17,10 +17,16 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; -pub use arch_mmu::*; + +mod translation_table; use core::{fmt, ops::RangeInclusive}; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_mmu::mmu; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -40,6 +46,12 @@ pub mod interface { } } +/// Describes the characteristics of a translation granule. +pub struct TranslationGranule; + +/// Describes the size of an address space. +pub struct AddressSpaceSize; + /// Architecture agnostic translation types. #[allow(missing_docs)] #[allow(dead_code)] @@ -96,6 +108,41 @@ pub struct KernelVirtualLayout { // Public Code //-------------------------------------------------------------------------------------------------- +impl TranslationGranule { + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(GRANULE_SIZE.is_power_of_two()); + + GRANULE_SIZE + } +} + +impl AddressSpaceSize { + /// The address space size. + pub const SIZE: usize = Self::size_checked(); + + /// The address space shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(AS_SIZE.is_power_of_two()); + assert!(arch_mmu::MIN_ADDR_SPACE_SIZE.is_power_of_two()); + assert!(arch_mmu::MAX_ADDR_SPACE_SIZE.is_power_of_two()); + + // Must adhere to architectural restrictions. + assert!(AS_SIZE >= arch_mmu::MIN_ADDR_SPACE_SIZE); + assert!(AS_SIZE <= arch_mmu::MAX_ADDR_SPACE_SIZE); + assert!((AS_SIZE % arch_mmu::AddrSpaceSizeGranule::SIZE) == 0); + + AS_SIZE + } +} + impl Default for AttributeFields { fn default() -> AttributeFields { AttributeFields { diff --git a/12_exceptions_part1_groundwork/src/memory/mmu/translation_table.rs b/12_exceptions_part1_groundwork/src/memory/mmu/translation_table.rs new file mode 100644 index 00000000..bbb6f939 --- /dev/null +++ b/12_exceptions_part1_groundwork/src/memory/mmu/translation_table.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Translation table. + +#[cfg(target_arch = "aarch64")] +#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] +mod arch_translation_table; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_translation_table::KernelTranslationTable; diff --git a/12_exceptions_part1_groundwork/src/print.rs b/12_exceptions_part1_groundwork/src/print.rs index 1ea96b6a..5a563811 100644 --- a/12_exceptions_part1_groundwork/src/print.rs +++ b/12_exceptions_part1_groundwork/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/12_exceptions_part1_groundwork/src/time.rs b/12_exceptions_part1_groundwork/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/12_exceptions_part1_groundwork/src/time.rs +++ b/12_exceptions_part1_groundwork/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/13_integrated_testing/Makefile b/13_integrated_testing/Makefile index 9f7ea9a3..4f7561db 100644 --- a/13_integrated_testing/Makefile +++ b/13_integrated_testing/Makefile @@ -57,8 +57,9 @@ QEMU_MISSING_STRING = "This board is not yet supported for QEMU." RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/13_integrated_testing/README.md b/13_integrated_testing/README.md index 849cbbfb..9a99cbdf 100644 --- a/13_integrated_testing/README.md +++ b/13_integrated_testing/README.md @@ -138,7 +138,6 @@ In `lib.rs`, we add the following headers to get started with `custom_test_frame ```rust // Testing #![cfg_attr(test, no_main)] -#![cfg_attr(test, feature(slice_ptr_range))] #![feature(custom_test_frameworks)] #![reexport_test_harness_main = "test_main"] #![test_runner(crate::test_runner)] @@ -255,9 +254,9 @@ this is an opportunity to cut down on setup code. [tutorial 03]: ../03_hacky_hello_world -As a matter of fact, for the `Raspberrys`, nothing needs to be done and the function is empy. But +As a matter of fact, for the `Raspberrys`, nothing needs to be done, so the function is empy. But this might be different for other hardware emulated by QEMU, so it makes sense to introduce the -function now to make it easier in case new `BSPs` are added to the kernel in the future. +function now to make it easier in case new `BSPs` are added to the kernel in the future. Next, the reexported `test_main()` is called, which will call our `test_runner()` which finally prints the unit test names and executes them. @@ -875,7 +874,7 @@ diff -uNr 12_exceptions_part1_groundwork/Makefile 13_integrated_testing/Makefile RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs -@@ -52,6 +65,7 @@ +@@ -53,6 +66,7 @@ DOC_CMD = cargo doc $(COMPILER_ARGS) CLIPPY_CMD = cargo clippy $(COMPILER_ARGS) CHECK_CMD = cargo check $(COMPILER_ARGS) @@ -883,7 +882,7 @@ diff -uNr 12_exceptions_part1_groundwork/Makefile 13_integrated_testing/Makefile OBJCOPY_CMD = rust-objcopy \ --strip-all \ -O binary -@@ -68,6 +82,7 @@ +@@ -69,6 +83,7 @@ DOCKER_QEMU = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE) DOCKER_GDB = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) @@ -891,7 +890,7 @@ diff -uNr 12_exceptions_part1_groundwork/Makefile 13_integrated_testing/Makefile DOCKER_ELFTOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE) # Dockerize commands that require USB device passthrough only on Linux -@@ -84,8 +99,8 @@ +@@ -85,8 +100,8 @@ EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) EXEC_MINIPUSH = ruby ../utils/minipush.rb @@ -902,7 +901,7 @@ diff -uNr 12_exceptions_part1_groundwork/Makefile 13_integrated_testing/Makefile all: $(KERNEL_BIN) -@@ -99,11 +114,26 @@ +@@ -100,11 +115,26 @@ $(DOC_CMD) --document-private-items --open ifeq ($(QEMU_MACHINE_TYPE),) @@ -935,7 +934,7 @@ diff -uNr 12_exceptions_part1_groundwork/Makefile 13_integrated_testing/Makefile diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs 13_integrated_testing/src/_arch/aarch64/cpu.rs --- 12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs +++ 13_integrated_testing/src/_arch/aarch64/cpu.rs -@@ -90,3 +90,20 @@ +@@ -26,3 +26,20 @@ asm::wfe() } } @@ -960,8 +959,8 @@ diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/cpu.rs 13_integrated_ diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs 13_integrated_testing/src/_arch/aarch64/exception.rs --- 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs +++ 13_integrated_testing/src/_arch/aarch64/exception.rs -@@ -5,7 +5,7 @@ - //! Architectural synchronous and asynchronous exception handling. +@@ -12,7 +12,7 @@ + //! crate::exception::arch_exception use core::{cell::UnsafeCell, fmt}; -use cortex_a::{asm, barrier, regs::*}; @@ -969,7 +968,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs 13_integ use register::InMemoryRegister; // Assembly counterpart to this file. -@@ -80,16 +80,6 @@ +@@ -87,16 +87,6 @@ #[no_mangle] unsafe extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { @@ -987,11 +986,11 @@ diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs 13_integ } -diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 13_integrated_testing/src/_arch/aarch64/memory/mmu.rs ---- 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs -+++ 13_integrated_testing/src/_arch/aarch64/memory/mmu.rs -@@ -341,3 +341,40 @@ - Ok(()) +diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translation_table.rs 13_integrated_testing/src/_arch/aarch64/memory/mmu/translation_table.rs +--- 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translation_table.rs ++++ 13_integrated_testing/src/_arch/aarch64/memory/mmu/translation_table.rs +@@ -286,3 +286,31 @@ + self.lvl2.base_addr_u64() } } + @@ -1021,6 +1020,24 @@ diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 13_inte + core::mem::size_of::() + ); + } ++} + +diff -uNr 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 13_integrated_testing/src/_arch/aarch64/memory/mmu.rs +--- 12_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs ++++ 13_integrated_testing/src/_arch/aarch64/memory/mmu.rs +@@ -144,3 +144,22 @@ + Ok(()) + } + } ++ ++//-------------------------------------------------------------------------------------------------- ++// Testing ++//-------------------------------------------------------------------------------------------------- ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use test_macros::kernel_test; + + /// Check if KERNEL_TABLES is in .bss. + #[kernel_test] @@ -1053,8 +1070,8 @@ diff -uNr 12_exceptions_part1_groundwork/src/bsp/raspberrypi/console.rs 13_integ diff -uNr 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs 13_integrated_testing/src/bsp/raspberrypi/memory/mmu.rs --- 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs +++ 13_integrated_testing/src/bsp/raspberrypi/memory/mmu.rs -@@ -76,3 +76,46 @@ - pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { +@@ -69,3 +69,46 @@ + pub fn virt_mem_layout() -> &'static KernelVirtualLayout { &LAYOUT } + @@ -1101,10 +1118,20 @@ diff -uNr 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs 13_in + } +} +diff -uNr 12_exceptions_part1_groundwork/src/cpu.rs 13_integrated_testing/src/cpu.rs +--- 12_exceptions_part1_groundwork/src/cpu.rs ++++ 13_integrated_testing/src/cpu.rs +@@ -15,4 +15,4 @@ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- +-pub use arch_cpu::{nop, wait_forever}; ++pub use arch_cpu::{nop, qemu_exit_failure, qemu_exit_success, wait_forever}; + diff -uNr 12_exceptions_part1_groundwork/src/exception.rs 13_integrated_testing/src/exception.rs --- 12_exceptions_part1_groundwork/src/exception.rs +++ 13_integrated_testing/src/exception.rs -@@ -24,3 +24,21 @@ +@@ -28,3 +28,21 @@ Hypervisor, Unknown, } @@ -1130,7 +1157,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/exception.rs 13_integrated_testing/ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/lib.rs --- 12_exceptions_part1_groundwork/src/lib.rs +++ 13_integrated_testing/src/lib.rs -@@ -0,0 +1,169 @@ +@@ -0,0 +1,168 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -1140,19 +1167,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li + +//! The `kernel` library. +//! -+//! Used by `main.rs` to compose the final kernel binary. -+//! -+//! # TL;DR - Overview of important Kernel entities -+//! -+//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -+//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -+//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -+//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -+//! -+//! [console interface]: ../libkernel/console/interface/index.html -+//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -+//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -+//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html ++//! Used to compose the final kernel binary. +//! +//! # Code organization and architecture +//! @@ -1167,15 +1182,22 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li +//! `src/_arch`, for example, `src/_arch/aarch64`. +//! +//! The architecture folders mirror the subsystem modules laid out in `src`. For example, -+//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -+//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -+//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -+//! module organization. That means a public function `foo()` defined in -+//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. ++//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go ++//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in ++//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic ++//! module's name prefixed with `arch_`. +//! -+//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -+//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -+//! "_arch/xxx/yyy.rs"]` attribute. ++//! For example, this is the top of `src/memory/mmu.rs`: ++//! ++//! ``` ++//! #[cfg(target_arch = "aarch64")] ++//! #[path = "../_arch/aarch64/memory/mmu.rs"] ++//! mod arch_mmu; ++//! ``` ++//! ++//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. ++//! This way, each architecture specific module can provide its implementation of an item, while the ++//! caller must not be concerned which architecture has been conditionally compiled. +//! +//! ## BSP code +//! @@ -1184,9 +1206,8 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li +//! or instances of drivers for devices that are featured on the respective board. +//! +//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -+//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -+//! whatever is provided must be called starting from the `bsp` namespace, e.g. -+//! `bsp::driver::driver_manager()`. ++//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is ++//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. +//! +//! ## Kernel interfaces +//! @@ -1238,6 +1259,14 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li +//! +//! - `crate::memory::*` +//! - `crate::bsp::memory::*` ++//! ++//! # Boot flow ++//! ++//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. ++//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. ++//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. ++//! ++//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html + +#![allow(incomplete_features)] +#![feature(const_fn_fn_ptr_basics)] @@ -1255,9 +1284,6 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li +#![reexport_test_harness_main = "test_main"] +#![test_runner(crate::test_runner)] + -+// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -+// `runtime_init()`, which jumps to `kernel_init()` (defined in `main.rs`). -+ +mod panic_wait; +mod runtime_init; +mod synchronization; @@ -1304,23 +1330,11 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/main.rs --- 12_exceptions_part1_groundwork/src/main.rs +++ 13_integrated_testing/src/main.rs -@@ -6,129 +6,12 @@ +@@ -6,128 +6,12 @@ #![doc(html_logo_url = "https://git.io/JeGIp")] //! The `kernel` binary. -//! --//! # TL;DR - Overview of important Kernel entities --//! --//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. --//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. --//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. --//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. --//! --//! [console interface]: ../libkernel/console/interface/index.html --//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html --//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html --//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html --//! -//! # Code organization and architecture -//! -//! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -1334,15 +1348,22 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m -//! `src/_arch`, for example, `src/_arch/aarch64`. -//! -//! The architecture folders mirror the subsystem modules laid out in `src`. For example, --//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go --//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in --//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's --//! module organization. That means a public function `foo()` defined in --//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +-//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +-//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +-//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +-//! module's name prefixed with `arch_`. -//! --//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. --//! Rather, it's contents are conditionally pulled into respective files using the `#[path = --//! "_arch/xxx/yyy.rs"]` attribute. +-//! For example, this is the top of `src/memory/mmu.rs`: +-//! +-//! ``` +-//! #[cfg(target_arch = "aarch64")] +-//! #[path = "../_arch/aarch64/memory/mmu.rs"] +-//! mod arch_mmu; +-//! ``` +-//! +-//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +-//! This way, each architecture specific module can provide its implementation of an item, while the +-//! caller must not be concerned which architecture has been conditionally compiled. -//! -//! ## BSP code -//! @@ -1351,9 +1372,8 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m -//! or instances of drivers for devices that are featured on the respective board. -//! -//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the --//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means --//! whatever is provided must be called starting from the `bsp` namespace, e.g. --//! `bsp::driver::driver_manager()`. +-//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +-//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. -//! -//! ## Kernel interfaces -//! @@ -1405,6 +1425,14 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m -//! -//! - `crate::memory::*` -//! - `crate::bsp::memory::*` +-//! +-//! # Boot flow +-//! +-//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +-//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +-//! +-//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html - -#![allow(incomplete_features)] -#![feature(const_fn_fn_ptr_basics)] @@ -1418,9 +1446,6 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m #![no_main] #![no_std] --// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls --// `runtime_init()`, which jumps to `kernel_init()`. -- -mod bsp; -mod console; -mod cpu; @@ -1436,7 +1461,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m /// Early init code. /// -@@ -140,6 +23,7 @@ +@@ -139,6 +23,7 @@ /// - Without it, any atomic operations, e.g. the yet-to-be-introduced spinlocks in the device /// drivers (which currently employ NullLocks instead of spinlocks), will fail to work on /// the RPi SoCs. @@ -1444,7 +1469,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m unsafe fn kernel_init() -> ! { use driver::interface::DriverManager; use memory::mmu::interface::MMU; -@@ -166,9 +50,7 @@ +@@ -165,9 +50,7 @@ fn kernel_main() -> ! { use bsp::console::console; use console::interface::All; @@ -1454,7 +1479,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m info!("Booting on: {}", bsp::board_name()); -@@ -195,31 +77,6 @@ +@@ -194,31 +77,6 @@ info!(" {}. {}", i + 1, driver.compatible()); } @@ -1490,7 +1515,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m diff -uNr 12_exceptions_part1_groundwork/src/memory/mmu.rs 13_integrated_testing/src/memory/mmu.rs --- 12_exceptions_part1_groundwork/src/memory/mmu.rs +++ 13_integrated_testing/src/memory/mmu.rs -@@ -42,7 +42,6 @@ +@@ -54,7 +54,6 @@ /// Architecture agnostic translation types. #[allow(missing_docs)] @@ -1498,7 +1523,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/memory/mmu.rs 13_integrated_testing #[derive(Copy, Clone)] pub enum Translation { Identity, -@@ -197,4 +196,9 @@ +@@ -244,4 +243,9 @@ info!("{}", i); } } diff --git a/13_integrated_testing/src/_arch/aarch64/cpu.rs b/13_integrated_testing/src/_arch/aarch64/cpu.rs index 491f17ca..948bad74 100644 --- a/13_integrated_testing/src/_arch/aarch64/cpu.rs +++ b/13_integrated_testing/src/_arch/aarch64/cpu.rs @@ -3,79 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} - -/// Transition from EL2 to EL1. -/// -/// # Safety -/// -/// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - - // Enable timer counter registers for EL1. - CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - - // No offset for reading the counters. - CNTVOFF_EL2.set(0); - - // Set EL1 execution state to AArch64. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Set up a simulated exception return. - // - // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a - // stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, - ); - - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs b/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c2f5fe0b --- /dev/null +++ b/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[no_mangle] +pub unsafe fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs b/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs +++ b/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/13_integrated_testing/src/_arch/aarch64/exception.rs b/13_integrated_testing/src/_arch/aarch64/exception.rs index 7070d814..0e640ec6 100644 --- a/13_integrated_testing/src/_arch/aarch64/exception.rs +++ b/13_integrated_testing/src/_arch/aarch64/exception.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception use core::{cell::UnsafeCell, fmt}; use cortex_a::{barrier, regs::*}; diff --git a/13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs b/13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs index 445f490e..b63b00fe 100644 --- a/13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs +++ b/13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous use cortex_a::regs::*; diff --git a/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs b/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs index 9b658e86..42ba0519 100644 --- a/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs +++ b/13_integrated_testing/src/_arch/aarch64/memory/mmu.rs @@ -4,146 +4,61 @@ //! Memory Management Unit Driver. //! -//! Static translation tables, compiled on boot; Everything 64 KiB granule. +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::arch_mmu -use super::{AccessPermissions, AttributeFields, MemAttributes}; -use crate::{bsp, memory}; -use core::convert; +use crate::{ + bsp, memory, + memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule}, +}; use cortex_a::{barrier, regs::*}; -use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions //-------------------------------------------------------------------------------------------------- -// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. -register_bitfields! {u64, - STAGE1_TABLE_DESCRIPTOR [ - /// Physical address of the next descriptor. - NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. -register_bitfields! {u64, - STAGE1_PAGE_DESCRIPTOR [ - /// Unprivileged execute-never. - UXN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Privileged execute-never. - PXN OFFSET(53) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - /// Access flag. - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field. - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions. - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register. - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} +/// Memory Management Unit type. +struct MemoryManagementUnit; -const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) -const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- -/// A table descriptor for 64 KiB aperture. -/// -/// The output points to the next table. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct TableDescriptor(u64); +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; -/// A page descriptor with 64 KiB aperture. -/// -/// The output points to physical memory. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct PageDescriptor(u64); - -/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -/// aligned, hence the "reverse" order of appearance. -#[repr(C)] -#[repr(align(65536))] -struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], - - /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; NUM_TABLES], -} +/// The min supported address space size. +pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB -const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -type ArchTranslationTable = FixedSizeTranslationTable; +/// The max supported address space size. +pub const MAX_ADDR_SPACE_SIZE: usize = 32 * 1024 * 1024 * 1024; // 32 GiB -trait BaseAddr { - fn base_addr_u64(&self) -> u64; - fn base_addr_usize(&self) -> usize; -} +/// The supported address space size granule. +pub type AddrSpaceSizeGranule = Granule512MiB; /// Constants for indexing the MAIR_EL1. #[allow(dead_code)] -mod mair { +pub mod mair { pub const DEVICE: u64 = 0; pub const NORMAL: u64 = 1; } -/// Memory Management Unit type. -struct MemoryManagementUnit; - //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -/// The translation tables. +/// The kernel translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); +static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); static MMU: MemoryManagementUnit = MemoryManagementUnit; @@ -151,149 +66,37 @@ static MMU: MemoryManagementUnit = MemoryManagementUnit; // Private Code //-------------------------------------------------------------------------------------------------- -impl BaseAddr for [T; N] { - fn base_addr_u64(&self) -> u64 { - self as *const T as u64 - } - - fn base_addr_usize(&self) -> usize { - self as *const _ as usize - } -} - -impl convert::From for TableDescriptor { - fn from(next_lvl_table_addr: usize) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), - ); - - TableDescriptor(val.get()) - } -} - -/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. -impl convert::From - for register::FieldValue -{ - fn from(attribute_fields: AttributeFields) -> Self { - // Memory attributes. - let mut desc = match attribute_fields.mem_attributes { - MemAttributes::CacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - } - MemAttributes::Device => { - STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } - }; - - // Access Permissions. - desc += match attribute_fields.acc_perms { - AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, - AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, - }; - - // The execute-never attribute is mapped to PXN in AArch64. - desc += if attribute_fields.execute_never { - STAGE1_PAGE_DESCRIPTOR::PXN::True - } else { - STAGE1_PAGE_DESCRIPTOR::PXN::False - }; - - // Always set unprivileged exectue-never as long as userspace is not implemented yet. - desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; - - desc - } -} - -impl PageDescriptor { - /// Create an instance. - fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), - ); - - Self(val.get()) - } -} - -impl FixedSizeTranslationTable<{ NUM_TABLES }> { - /// Create an instance. - pub const fn new() -> Self { - assert!(NUM_TABLES > 0); - - Self { - lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], - lvl2: [TableDescriptor(0); NUM_TABLES], - } - } -} - -/// Setup function for the MAIR_EL1 register. -fn set_up_mair() { - // Define the memory types being mapped. - MAIR_EL1.write( - // Attribute 1 - Cacheable normal DRAM. - MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + // Attribute 0 - Device. MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, - ); -} - -/// Iterates over all static translation table entries and fills them at once. -/// -/// # Safety -/// -/// - Modifies a `static mut`. Ensure it only happens from here. -unsafe fn populate_tt_entries() -> Result<(), &'static str> { - for (l2_nr, l2_entry) in KERNEL_TABLES.lvl2.iter_mut().enumerate() { - *l2_entry = KERNEL_TABLES.lvl3[l2_nr].base_addr_usize().into(); - - for (l3_nr, l3_entry) in KERNEL_TABLES.lvl3[l2_nr].iter_mut().enumerate() { - let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); - - let (output_addr, attribute_fields) = - bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; - - *l3_entry = PageDescriptor::new(output_addr, attribute_fields); - } + ); } - Ok(()) -} - -/// Configure various settings of stage 1 of the EL1 translation regime. -fn configure_translation_control() { - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - let t0sz: u64 = bsp::memory::mmu::addr_space_size().trailing_zeros().into(); - - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::EPD1::DisableTTBR1Walks - + TCR_EL1::TG0::KiB_64 - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(t0sz), - ); + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + let t0sz = (64 - bsp::memory::mmu::KernelAddrSpaceSize::SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::EPD1::DisableTTBR1Walks + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(t0sz), + ); + } } //-------------------------------------------------------------------------------------------------- @@ -317,15 +120,15 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { } // Prepare the memory attribute indirection register. - set_up_mair(); + self.set_up_mair(); // Populate translation tables. - populate_tt_entries()?; + KERNEL_TABLES.populate_tt_entries()?; // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(KERNEL_TABLES.lvl2.base_addr_u64()); + TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); - configure_translation_control(); + self.configure_translation_control(); // Switch the MMU on. // @@ -351,24 +154,6 @@ mod tests { use super::*; use test_macros::kernel_test; - /// Check if the size of `struct TableDescriptor` is as expected. - #[kernel_test] - fn size_of_tabledescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); - } - - /// Check if the size of `struct PageDescriptor` is as expected. - #[kernel_test] - fn size_of_pagedescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); - } - /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { diff --git a/13_integrated_testing/src/_arch/aarch64/memory/mmu/translation_table.rs b/13_integrated_testing/src/_arch/aarch64/memory/mmu/translation_table.rs new file mode 100644 index 00000000..337f9aed --- /dev/null +++ b/13_integrated_testing/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural translation table. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::translation_table::arch_translation_table + +use crate::{ + bsp, memory, + memory::mmu::{ + arch_mmu::{Granule512MiB, Granule64KiB}, + AccessPermissions, AttributeFields, MemAttributes, + }, +}; +use core::convert; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Unprivileged execute-never. + UXN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(C)] +struct TableDescriptor { + value: u64, +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +pub struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], +} + +/// A translation table type for the kernel space. +pub type KernelTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl BaseAddr for [T; N] { + fn base_addr_u64(&self) -> u64 { + self as *const T as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const _ as usize + } +} + +impl TableDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance pointing to the supplied address. + pub fn from_next_lvl_table_addr(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + + TableDescriptor { value: val.get() } + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // The execute-never attribute is mapped to PXN in AArch64. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + // Always set unprivileged exectue-never as long as userspace is not implemented yet. + desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; + + desc + } +} + +impl PageDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance. + pub fn from_output_addr(output_addr: usize, attribute_fields: AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self { value: val.get() } + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl FixedSizeTranslationTable { + /// Create an instance. + #[allow(clippy::assertions_on_constants)] + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + assert!((bsp::memory::mmu::KernelAddrSpaceSize::SIZE % Granule512MiB::SIZE) == 0); + + Self { + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + } + } + + /// Iterates over all static translation table entries and fills them at once. + /// + /// # Safety + /// + /// - Modifies a `static mut`. Ensure it only happens from here. + pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { + for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { + *l2_entry = + TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + + for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { + let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + + let (output_addr, attribute_fields) = + bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + + *l3_entry = PageDescriptor::from_output_addr(output_addr, attribute_fields); + } + } + + Ok(()) + } + + /// The translation table's base address to be used for programming the MMU. + pub fn base_address(&self) -> u64 { + self.lvl2.base_addr_u64() + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check if the size of `struct TableDescriptor` is as expected. + #[kernel_test] + fn size_of_tabledescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if the size of `struct PageDescriptor` is as expected. + #[kernel_test] + fn size_of_pagedescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } +} diff --git a/13_integrated_testing/src/_arch/aarch64/time.rs b/13_integrated_testing/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/13_integrated_testing/src/_arch/aarch64/time.rs +++ b/13_integrated_testing/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/13_integrated_testing/src/bsp.rs b/13_integrated_testing/src/bsp.rs index 25750249..c558922f 100644 --- a/13_integrated_testing/src/bsp.rs +++ b/13_integrated_testing/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/13_integrated_testing/src/bsp/raspberrypi/memory/mmu.rs b/13_integrated_testing/src/bsp/raspberrypi/memory/mmu.rs index 485b8a46..451d927b 100644 --- a/13_integrated_testing/src/bsp/raspberrypi/memory/mmu.rs +++ b/13_integrated_testing/src/bsp/raspberrypi/memory/mmu.rs @@ -12,13 +12,16 @@ use core::ops::RangeInclusive; // Public Definitions //-------------------------------------------------------------------------------------------------- +/// The address space size chosen by this BSP. +pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; + const NUM_MEM_RANGES: usize = 2; /// The virtual memory layout. /// /// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. /// It is agnostic of the paging granularity that the architecture's MMU will use. -pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( +pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( memory_map::END_INCLUSIVE, [ TranslationDescriptor { @@ -62,18 +65,8 @@ fn mmio_range_inclusive() -> RangeInclusive { // Public Code //-------------------------------------------------------------------------------------------------- -/// Return the address space size in bytes. -/// -/// Guarantees size to be a power of two. -pub const fn addr_space_size() -> usize { - let size = memory_map::END_INCLUSIVE + 1; - assert!(size.is_power_of_two()); - - size -} - /// Return a reference to the virtual memory layout. -pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { +pub fn virt_mem_layout() -> &'static KernelVirtualLayout { &LAYOUT } diff --git a/13_integrated_testing/src/console.rs b/13_integrated_testing/src/console.rs index 3552823c..c3154ba2 100644 --- a/13_integrated_testing/src/console.rs +++ b/13_integrated_testing/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/13_integrated_testing/src/cpu.rs b/13_integrated_testing/src/cpu.rs index c9e5af72..9c8eb6f6 100644 --- a/13_integrated_testing/src/cpu.rs +++ b/13_integrated_testing/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, qemu_exit_failure, qemu_exit_success, wait_forever}; diff --git a/13_integrated_testing/src/cpu/boot.rs b/13_integrated_testing/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/13_integrated_testing/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/13_integrated_testing/src/cpu/smp.rs b/13_integrated_testing/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/13_integrated_testing/src/cpu/smp.rs +++ b/13_integrated_testing/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/13_integrated_testing/src/exception.rs b/13_integrated_testing/src/exception.rs index dfa852a8..3c5e7bc8 100644 --- a/13_integrated_testing/src/exception.rs +++ b/13_integrated_testing/src/exception.rs @@ -7,10 +7,14 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/exception.rs"] mod arch_exception; -pub use arch_exception::*; pub mod asynchronous; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::{current_privilege_level, handling_init}; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- diff --git a/13_integrated_testing/src/exception/asynchronous.rs b/13_integrated_testing/src/exception/asynchronous.rs index fbdba957..566efe32 100644 --- a/13_integrated_testing/src/exception/asynchronous.rs +++ b/13_integrated_testing/src/exception/asynchronous.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/exception/asynchronous.rs"] -mod arch_exception_async; -pub use arch_exception_async::*; +mod arch_asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::print_state; diff --git a/13_integrated_testing/src/lib.rs b/13_integrated_testing/src/lib.rs index f958ae5d..0bc8d329 100644 --- a/13_integrated_testing/src/lib.rs +++ b/13_integrated_testing/src/lib.rs @@ -7,19 +7,7 @@ //! The `kernel` library. //! -//! Used by `main.rs` to compose the final kernel binary. -//! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html +//! Used to compose the final kernel binary. //! //! # Code organization and architecture //! @@ -34,15 +22,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -51,9 +46,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -105,6 +99,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![allow(incomplete_features)] #![feature(const_fn_fn_ptr_basics)] @@ -122,9 +124,6 @@ #![reexport_test_harness_main = "test_main"] #![test_runner(crate::test_runner)] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()` (defined in `main.rs`). - mod panic_wait; mod runtime_init; mod synchronization; diff --git a/13_integrated_testing/src/memory/mmu.rs b/13_integrated_testing/src/memory/mmu.rs index 163419ec..cca2951a 100644 --- a/13_integrated_testing/src/memory/mmu.rs +++ b/13_integrated_testing/src/memory/mmu.rs @@ -5,8 +5,8 @@ //! Memory Management Unit. //! //! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file -//! provides types for composing an architecture-agnostic description of the kernel 's virtual -//! memory layout. +//! provides types for composing an architecture-agnostic description of the kernel's virtual memory +//! layout. //! //! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` //! function. @@ -17,10 +17,16 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; -pub use arch_mmu::*; + +mod translation_table; use core::{fmt, ops::RangeInclusive}; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_mmu::mmu; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -40,6 +46,12 @@ pub mod interface { } } +/// Describes the characteristics of a translation granule. +pub struct TranslationGranule; + +/// Describes the size of an address space. +pub struct AddressSpaceSize; + /// Architecture agnostic translation types. #[allow(missing_docs)] #[derive(Copy, Clone)] @@ -95,6 +107,41 @@ pub struct KernelVirtualLayout { // Public Code //-------------------------------------------------------------------------------------------------- +impl TranslationGranule { + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(GRANULE_SIZE.is_power_of_two()); + + GRANULE_SIZE + } +} + +impl AddressSpaceSize { + /// The address space size. + pub const SIZE: usize = Self::size_checked(); + + /// The address space shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(AS_SIZE.is_power_of_two()); + assert!(arch_mmu::MIN_ADDR_SPACE_SIZE.is_power_of_two()); + assert!(arch_mmu::MAX_ADDR_SPACE_SIZE.is_power_of_two()); + + // Must adhere to architectural restrictions. + assert!(AS_SIZE >= arch_mmu::MIN_ADDR_SPACE_SIZE); + assert!(AS_SIZE <= arch_mmu::MAX_ADDR_SPACE_SIZE); + assert!((AS_SIZE % arch_mmu::AddrSpaceSizeGranule::SIZE) == 0); + + AS_SIZE + } +} + impl Default for AttributeFields { fn default() -> AttributeFields { AttributeFields { diff --git a/13_integrated_testing/src/memory/mmu/translation_table.rs b/13_integrated_testing/src/memory/mmu/translation_table.rs new file mode 100644 index 00000000..bbb6f939 --- /dev/null +++ b/13_integrated_testing/src/memory/mmu/translation_table.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Translation table. + +#[cfg(target_arch = "aarch64")] +#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] +mod arch_translation_table; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_translation_table::KernelTranslationTable; diff --git a/13_integrated_testing/src/print.rs b/13_integrated_testing/src/print.rs index 1ea96b6a..5a563811 100644 --- a/13_integrated_testing/src/print.rs +++ b/13_integrated_testing/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/13_integrated_testing/src/time.rs b/13_integrated_testing/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/13_integrated_testing/src/time.rs +++ b/13_integrated_testing/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/14_exceptions_part2_peripheral_IRQs/README.md b/14_exceptions_part2_peripheral_IRQs/README.md index 8e68edcb..2c74961d 100644 --- a/14_exceptions_part2_peripheral_IRQs/README.md +++ b/14_exceptions_part2_peripheral_IRQs/README.md @@ -744,25 +744,10 @@ Minipush 1.0 ## Diff to previous ```diff -diff -uNr 13_integrated_testing/Makefile 14_exceptions_part2_peripheral_IRQs/Makefile ---- 13_integrated_testing/Makefile -+++ 14_exceptions_part2_peripheral_IRQs/Makefile -@@ -57,8 +57,9 @@ - RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) - RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs - -+FEATURES = bsp_$(BSP) - COMPILER_ARGS = --target=$(TARGET) \ -- --features bsp_$(BSP) \ -+ --features $(FEATURES) \ - --release - - RUSTC_CMD = cargo rustc $(COMPILER_ARGS) - diff -uNr 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs --- 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs +++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs -@@ -10,6 +10,10 @@ +@@ -17,6 +17,10 @@ // Private Definitions //-------------------------------------------------------------------------------------------------- @@ -773,7 +758,7 @@ diff -uNr 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs 14_e trait DaifField { fn daif_field() -> register::Field; } -@@ -58,6 +62,71 @@ +@@ -65,6 +69,71 @@ // Public Code //-------------------------------------------------------------------------------------------------- @@ -849,15 +834,15 @@ diff -uNr 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs 14_e diff -uNr 13_integrated_testing/src/_arch/aarch64/exception.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs --- 13_integrated_testing/src/_arch/aarch64/exception.rs +++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs -@@ -4,6 +4,7 @@ - - //! Architectural synchronous and asynchronous exception handling. +@@ -11,6 +11,7 @@ + //! + //! crate::exception::arch_exception +use crate::{bsp, exception}; use core::{cell::UnsafeCell, fmt}; use cortex_a::{barrier, regs::*}; use register::InMemoryRegister; -@@ -84,8 +85,11 @@ +@@ -91,8 +92,11 @@ } #[no_mangle] @@ -2156,12 +2141,20 @@ diff -uNr 13_integrated_testing/src/driver.rs 14_exceptions_part2_peripheral_IRQ diff -uNr 13_integrated_testing/src/exception/asynchronous.rs 14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs --- 13_integrated_testing/src/exception/asynchronous.rs +++ 14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs -@@ -8,3 +8,138 @@ +@@ -8,7 +8,145 @@ #[path = "../_arch/aarch64/exception/asynchronous.rs"] - mod arch_exception_async; - pub use arch_exception_async::*; -+ + mod arch_asynchronous; + +use core::{fmt, marker::PhantomData}; ++ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- +-pub use arch_asynchronous::print_state; ++pub use arch_asynchronous::{ ++ is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask, ++ print_state, ++}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions @@ -2268,7 +2261,7 @@ diff -uNr 13_integrated_testing/src/exception/asynchronous.rs 14_exceptions_part + } + + /// Return the wrapped number. -+ pub fn get(self) -> usize { ++ pub const fn get(self) -> usize { + self.0 + } +} @@ -2299,26 +2292,8 @@ diff -uNr 13_integrated_testing/src/exception/asynchronous.rs 14_exceptions_part diff -uNr 13_integrated_testing/src/lib.rs 14_exceptions_part2_peripheral_IRQs/src/lib.rs --- 13_integrated_testing/src/lib.rs +++ 14_exceptions_part2_peripheral_IRQs/src/lib.rs -@@ -13,12 +13,17 @@ - //! - //! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. - //! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -+//! - [`bsp::exception::asynchronous::irq_manager()`] - Returns a reference to the kernel's [IRQ -+//! Handling interface]. - //! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -+//! - [`state::state_manager()`] - Returns a reference to the kernel's [state management] instance. - //! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. - //! - //! [console interface]: ../libkernel/console/interface/index.html - //! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -+//! [IRQ Handling interface]: ../libkernel/exception/asynchronous/interface/trait.IRQManager.html - //! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -+//! [state management]: ../libkernel/state/struct.StateManager.html - //! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html - //! - //! # Code organization and architecture -@@ -107,9 +112,11 @@ - //! - `crate::bsp::memory::*` +@@ -109,9 +109,11 @@ + //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![allow(incomplete_features)] +#![feature(asm)] @@ -2329,7 +2304,7 @@ diff -uNr 13_integrated_testing/src/lib.rs 14_exceptions_part2_peripheral_IRQs/s #![feature(format_args_nl)] #![feature(global_asm)] #![feature(linkage)] -@@ -136,6 +143,7 @@ +@@ -135,6 +137,7 @@ pub mod exception; pub mod memory; pub mod print; @@ -2506,7 +2481,7 @@ diff -uNr 13_integrated_testing/src/state.rs 14_exceptions_part2_peripheral_IRQs + } + } + -+ /// Return if the kernel is still in an init state. ++ /// Return if the kernel is init state. + pub fn is_init(&self) -> bool { + self.state() == State::Init + } diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs index 491f17ca..948bad74 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs @@ -3,79 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} - -/// Transition from EL2 to EL1. -/// -/// # Safety -/// -/// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - - // Enable timer counter registers for EL1. - CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - - // No offset for reading the counters. - CNTVOFF_EL2.set(0); - - // Set EL1 execution state to AArch64. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Set up a simulated exception return. - // - // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a - // stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, - ); - - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c2f5fe0b --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[no_mangle] +pub unsafe fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs index 99fbd85b..5cf9eb5c 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception use crate::{bsp, exception}; use core::{cell::UnsafeCell, fmt}; diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs index 968eedf3..a4b1a548 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous use cortex_a::regs::*; diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs index 9b658e86..42ba0519 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs @@ -4,146 +4,61 @@ //! Memory Management Unit Driver. //! -//! Static translation tables, compiled on boot; Everything 64 KiB granule. +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::arch_mmu -use super::{AccessPermissions, AttributeFields, MemAttributes}; -use crate::{bsp, memory}; -use core::convert; +use crate::{ + bsp, memory, + memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule}, +}; use cortex_a::{barrier, regs::*}; -use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions //-------------------------------------------------------------------------------------------------- -// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. -register_bitfields! {u64, - STAGE1_TABLE_DESCRIPTOR [ - /// Physical address of the next descriptor. - NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. -register_bitfields! {u64, - STAGE1_PAGE_DESCRIPTOR [ - /// Unprivileged execute-never. - UXN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Privileged execute-never. - PXN OFFSET(53) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - /// Access flag. - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field. - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions. - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register. - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} +/// Memory Management Unit type. +struct MemoryManagementUnit; -const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) -const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- -/// A table descriptor for 64 KiB aperture. -/// -/// The output points to the next table. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct TableDescriptor(u64); +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; -/// A page descriptor with 64 KiB aperture. -/// -/// The output points to physical memory. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct PageDescriptor(u64); - -/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -/// aligned, hence the "reverse" order of appearance. -#[repr(C)] -#[repr(align(65536))] -struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], - - /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; NUM_TABLES], -} +/// The min supported address space size. +pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB -const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; -type ArchTranslationTable = FixedSizeTranslationTable; +/// The max supported address space size. +pub const MAX_ADDR_SPACE_SIZE: usize = 32 * 1024 * 1024 * 1024; // 32 GiB -trait BaseAddr { - fn base_addr_u64(&self) -> u64; - fn base_addr_usize(&self) -> usize; -} +/// The supported address space size granule. +pub type AddrSpaceSizeGranule = Granule512MiB; /// Constants for indexing the MAIR_EL1. #[allow(dead_code)] -mod mair { +pub mod mair { pub const DEVICE: u64 = 0; pub const NORMAL: u64 = 1; } -/// Memory Management Unit type. -struct MemoryManagementUnit; - //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -/// The translation tables. +/// The kernel translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); +static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); static MMU: MemoryManagementUnit = MemoryManagementUnit; @@ -151,149 +66,37 @@ static MMU: MemoryManagementUnit = MemoryManagementUnit; // Private Code //-------------------------------------------------------------------------------------------------- -impl BaseAddr for [T; N] { - fn base_addr_u64(&self) -> u64 { - self as *const T as u64 - } - - fn base_addr_usize(&self) -> usize { - self as *const _ as usize - } -} - -impl convert::From for TableDescriptor { - fn from(next_lvl_table_addr: usize) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), - ); - - TableDescriptor(val.get()) - } -} - -/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. -impl convert::From - for register::FieldValue -{ - fn from(attribute_fields: AttributeFields) -> Self { - // Memory attributes. - let mut desc = match attribute_fields.mem_attributes { - MemAttributes::CacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - } - MemAttributes::Device => { - STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } - }; - - // Access Permissions. - desc += match attribute_fields.acc_perms { - AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, - AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, - }; - - // The execute-never attribute is mapped to PXN in AArch64. - desc += if attribute_fields.execute_never { - STAGE1_PAGE_DESCRIPTOR::PXN::True - } else { - STAGE1_PAGE_DESCRIPTOR::PXN::False - }; - - // Always set unprivileged exectue-never as long as userspace is not implemented yet. - desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; - - desc - } -} - -impl PageDescriptor { - /// Create an instance. - fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; - val.write( - STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), - ); - - Self(val.get()) - } -} - -impl FixedSizeTranslationTable<{ NUM_TABLES }> { - /// Create an instance. - pub const fn new() -> Self { - assert!(NUM_TABLES > 0); - - Self { - lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], - lvl2: [TableDescriptor(0); NUM_TABLES], - } - } -} - -/// Setup function for the MAIR_EL1 register. -fn set_up_mair() { - // Define the memory types being mapped. - MAIR_EL1.write( - // Attribute 1 - Cacheable normal DRAM. - MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + // Attribute 0 - Device. MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, - ); -} - -/// Iterates over all static translation table entries and fills them at once. -/// -/// # Safety -/// -/// - Modifies a `static mut`. Ensure it only happens from here. -unsafe fn populate_tt_entries() -> Result<(), &'static str> { - for (l2_nr, l2_entry) in KERNEL_TABLES.lvl2.iter_mut().enumerate() { - *l2_entry = KERNEL_TABLES.lvl3[l2_nr].base_addr_usize().into(); - - for (l3_nr, l3_entry) in KERNEL_TABLES.lvl3[l2_nr].iter_mut().enumerate() { - let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); - - let (output_addr, attribute_fields) = - bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; - - *l3_entry = PageDescriptor::new(output_addr, attribute_fields); - } + ); } - Ok(()) -} - -/// Configure various settings of stage 1 of the EL1 translation regime. -fn configure_translation_control() { - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - let t0sz: u64 = bsp::memory::mmu::addr_space_size().trailing_zeros().into(); - - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::EPD1::DisableTTBR1Walks - + TCR_EL1::TG0::KiB_64 - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(t0sz), - ); + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + let t0sz = (64 - bsp::memory::mmu::KernelAddrSpaceSize::SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::EPD1::DisableTTBR1Walks + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(t0sz), + ); + } } //-------------------------------------------------------------------------------------------------- @@ -317,15 +120,15 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { } // Prepare the memory attribute indirection register. - set_up_mair(); + self.set_up_mair(); // Populate translation tables. - populate_tt_entries()?; + KERNEL_TABLES.populate_tt_entries()?; // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(KERNEL_TABLES.lvl2.base_addr_u64()); + TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); - configure_translation_control(); + self.configure_translation_control(); // Switch the MMU on. // @@ -351,24 +154,6 @@ mod tests { use super::*; use test_macros::kernel_test; - /// Check if the size of `struct TableDescriptor` is as expected. - #[kernel_test] - fn size_of_tabledescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); - } - - /// Check if the size of `struct PageDescriptor` is as expected. - #[kernel_test] - fn size_of_pagedescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); - } - /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/translation_table.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/translation_table.rs new file mode 100644 index 00000000..337f9aed --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural translation table. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::translation_table::arch_translation_table + +use crate::{ + bsp, memory, + memory::mmu::{ + arch_mmu::{Granule512MiB, Granule64KiB}, + AccessPermissions, AttributeFields, MemAttributes, + }, +}; +use core::convert; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Unprivileged execute-never. + UXN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(C)] +struct TableDescriptor { + value: u64, +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +pub struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], +} + +/// A translation table type for the kernel space. +pub type KernelTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl BaseAddr for [T; N] { + fn base_addr_u64(&self) -> u64 { + self as *const T as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const _ as usize + } +} + +impl TableDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance pointing to the supplied address. + pub fn from_next_lvl_table_addr(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + + TableDescriptor { value: val.get() } + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // The execute-never attribute is mapped to PXN in AArch64. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + // Always set unprivileged exectue-never as long as userspace is not implemented yet. + desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; + + desc + } +} + +impl PageDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance. + pub fn from_output_addr(output_addr: usize, attribute_fields: AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self { value: val.get() } + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl FixedSizeTranslationTable { + /// Create an instance. + #[allow(clippy::assertions_on_constants)] + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + assert!((bsp::memory::mmu::KernelAddrSpaceSize::SIZE % Granule512MiB::SIZE) == 0); + + Self { + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + } + } + + /// Iterates over all static translation table entries and fills them at once. + /// + /// # Safety + /// + /// - Modifies a `static mut`. Ensure it only happens from here. + pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { + for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { + *l2_entry = + TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + + for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { + let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + + let (output_addr, attribute_fields) = + bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + + *l3_entry = PageDescriptor::from_output_addr(output_addr, attribute_fields); + } + } + + Ok(()) + } + + /// The translation table's base address to be used for programming the MMU. + pub fn base_address(&self) -> u64 { + self.lvl2.base_addr_u64() + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check if the size of `struct TableDescriptor` is as expected. + #[kernel_test] + fn size_of_tabledescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if the size of `struct PageDescriptor` is as expected. + #[kernel_test] + fn size_of_pagedescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp.rs index 25750249..c558922f 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/bsp.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs index 485b8a46..451d927b 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs @@ -12,13 +12,16 @@ use core::ops::RangeInclusive; // Public Definitions //-------------------------------------------------------------------------------------------------- +/// The address space size chosen by this BSP. +pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; + const NUM_MEM_RANGES: usize = 2; /// The virtual memory layout. /// /// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. /// It is agnostic of the paging granularity that the architecture's MMU will use. -pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( +pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( memory_map::END_INCLUSIVE, [ TranslationDescriptor { @@ -62,18 +65,8 @@ fn mmio_range_inclusive() -> RangeInclusive { // Public Code //-------------------------------------------------------------------------------------------------- -/// Return the address space size in bytes. -/// -/// Guarantees size to be a power of two. -pub const fn addr_space_size() -> usize { - let size = memory_map::END_INCLUSIVE + 1; - assert!(size.is_power_of_two()); - - size -} - /// Return a reference to the virtual memory layout. -pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { +pub fn virt_mem_layout() -> &'static KernelVirtualLayout { &LAYOUT } diff --git a/14_exceptions_part2_peripheral_IRQs/src/console.rs b/14_exceptions_part2_peripheral_IRQs/src/console.rs index 3552823c..c3154ba2 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/console.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/14_exceptions_part2_peripheral_IRQs/src/cpu.rs b/14_exceptions_part2_peripheral_IRQs/src/cpu.rs index c9e5af72..9c8eb6f6 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/cpu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, qemu_exit_failure, qemu_exit_success, wait_forever}; diff --git a/14_exceptions_part2_peripheral_IRQs/src/cpu/boot.rs b/14_exceptions_part2_peripheral_IRQs/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs b/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/14_exceptions_part2_peripheral_IRQs/src/exception.rs b/14_exceptions_part2_peripheral_IRQs/src/exception.rs index dfa852a8..3c5e7bc8 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/exception.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/exception.rs @@ -7,10 +7,14 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/exception.rs"] mod arch_exception; -pub use arch_exception::*; pub mod asynchronous; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::{current_privilege_level, handling_init}; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- diff --git a/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs b/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs index c16ce007..2b9ef05f 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs @@ -6,11 +6,18 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/exception/asynchronous.rs"] -mod arch_exception_async; -pub use arch_exception_async::*; +mod arch_asynchronous; use core::{fmt, marker::PhantomData}; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::{ + is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask, + print_state, +}; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -116,7 +123,7 @@ impl IRQNumber<{ MAX_INCLUSIVE }> { } /// Return the wrapped number. - pub fn get(self) -> usize { + pub const fn get(self) -> usize { self.0 } } diff --git a/14_exceptions_part2_peripheral_IRQs/src/lib.rs b/14_exceptions_part2_peripheral_IRQs/src/lib.rs index 635732bd..bc50cabd 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/lib.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/lib.rs @@ -7,24 +7,7 @@ //! The `kernel` library. //! -//! Used by `main.rs` to compose the final kernel binary. -//! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`bsp::exception::asynchronous::irq_manager()`] - Returns a reference to the kernel's [IRQ -//! Handling interface]. -//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -//! - [`state::state_manager()`] - Returns a reference to the kernel's [state management] instance. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [IRQ Handling interface]: ../libkernel/exception/asynchronous/interface/trait.IRQManager.html -//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -//! [state management]: ../libkernel/state/struct.StateManager.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html +//! Used to compose the final kernel binary. //! //! # Code organization and architecture //! @@ -39,15 +22,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -56,9 +46,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -110,6 +99,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![allow(incomplete_features)] #![feature(asm)] @@ -129,9 +126,6 @@ #![reexport_test_harness_main = "test_main"] #![test_runner(crate::test_runner)] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()` (defined in `main.rs`). - mod panic_wait; mod runtime_init; mod synchronization; diff --git a/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs index 163419ec..cca2951a 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs @@ -5,8 +5,8 @@ //! Memory Management Unit. //! //! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file -//! provides types for composing an architecture-agnostic description of the kernel 's virtual -//! memory layout. +//! provides types for composing an architecture-agnostic description of the kernel's virtual memory +//! layout. //! //! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` //! function. @@ -17,10 +17,16 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; -pub use arch_mmu::*; + +mod translation_table; use core::{fmt, ops::RangeInclusive}; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_mmu::mmu; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -40,6 +46,12 @@ pub mod interface { } } +/// Describes the characteristics of a translation granule. +pub struct TranslationGranule; + +/// Describes the size of an address space. +pub struct AddressSpaceSize; + /// Architecture agnostic translation types. #[allow(missing_docs)] #[derive(Copy, Clone)] @@ -95,6 +107,41 @@ pub struct KernelVirtualLayout { // Public Code //-------------------------------------------------------------------------------------------------- +impl TranslationGranule { + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(GRANULE_SIZE.is_power_of_two()); + + GRANULE_SIZE + } +} + +impl AddressSpaceSize { + /// The address space size. + pub const SIZE: usize = Self::size_checked(); + + /// The address space shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(AS_SIZE.is_power_of_two()); + assert!(arch_mmu::MIN_ADDR_SPACE_SIZE.is_power_of_two()); + assert!(arch_mmu::MAX_ADDR_SPACE_SIZE.is_power_of_two()); + + // Must adhere to architectural restrictions. + assert!(AS_SIZE >= arch_mmu::MIN_ADDR_SPACE_SIZE); + assert!(AS_SIZE <= arch_mmu::MAX_ADDR_SPACE_SIZE); + assert!((AS_SIZE % arch_mmu::AddrSpaceSizeGranule::SIZE) == 0); + + AS_SIZE + } +} + impl Default for AttributeFields { fn default() -> AttributeFields { AttributeFields { diff --git a/14_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.rs b/14_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.rs new file mode 100644 index 00000000..bbb6f939 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Translation table. + +#[cfg(target_arch = "aarch64")] +#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] +mod arch_translation_table; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_translation_table::KernelTranslationTable; diff --git a/14_exceptions_part2_peripheral_IRQs/src/print.rs b/14_exceptions_part2_peripheral_IRQs/src/print.rs index 1ea96b6a..5a563811 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/print.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/14_exceptions_part2_peripheral_IRQs/src/state.rs b/14_exceptions_part2_peripheral_IRQs/src/state.rs index d08e67d6..c94d04c8 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/state.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/state.rs @@ -69,7 +69,7 @@ impl StateManager { } } - /// Return if the kernel is still in an init state. + /// Return if the kernel is init state. pub fn is_init(&self) -> bool { self.state() == State::Init } diff --git a/14_exceptions_part2_peripheral_IRQs/src/time.rs b/14_exceptions_part2_peripheral_IRQs/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/time.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/15_virtual_mem_part2_mmio_remap/README.md b/15_virtual_mem_part2_mmio_remap/README.md index acb48ded..beb2cffc 100644 --- a/15_virtual_mem_part2_mmio_remap/README.md +++ b/15_virtual_mem_part2_mmio_remap/README.md @@ -15,7 +15,7 @@ - [Introduction](#introduction) - [Implementation](#implementation) - - [A New Mapping API in `src/memory/mmu.rs`](#a-new-mapping-api-in-srcmemorymmurs) + - [A New Mapping API in `src/memory/mmu.rs`](#a-new-mapping-api-in-srcmemorymmutranslationtablers) - [Using the new API in `bsp` code and drivers](#using-the-new-api-in-bsp-code-and-drivers) - [Additional Changes](#additional-changes) - [Test it](#test-it) @@ -77,7 +77,7 @@ Until now, the whole address space of the board was identity mapped at once. The together directly while setting up the translation tables, without any indirection through **generic kernel code** (`src/memory/**`). -The way it worked was that the `architectural MMU driver` would query the `bsp code` about the start +The way it worked was that the `architectural MMU code` would query the `bsp code` about the start and end of the physical address space, and any special regions in this space that need a mapping that _is not_ normal chacheable DRAM. It would then go ahead and map the whole address space at once and never touch the page tables again during runtime. @@ -85,7 +85,7 @@ and never touch the page tables again during runtime. Changing in this tutorial, **architecture** and **bsp** code will no longer talk to each other directly. Instead, this is decoupled now through the kernel's **generic MMU subsystem code**. -### A New Mapping API in `src/memory/mmu.rs` +### A New Mapping API in `src/memory/mmu/translation_table.rs` First, we define an interface for operating on `translation tables`: @@ -98,11 +98,11 @@ pub trait TranslationTable { /// The translation table's base address to be used for programming the MMU. fn phys_base_address(&self) -> Address; - /// Map the given physical pages to the given virtual pages. + /// Map the given virtual pages to the given physical pages. unsafe fn map_pages_at( &mut self, - phys_pages: &PageSliceDescriptor, virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, attr: &AttributeFields, ) -> Result<(), &'static str>; @@ -126,7 +126,7 @@ pub trait TranslationTable { The MMU driver (`src/_arch/_/memory/mmu.rs`) has one global instance for the kernel tables which implements this interface, and which can be accessed by calling `arch_mmu::kernel_translation_tables()` in the generic kernel code (`src/memory/mmu.rs`). From -there, we provice a couple of memory mapping functions that wrap around this interface , and which +there, we provide a couple of memory mapping functions that wrap around this interface , and which are exported for the rest of the kernel to use: ```rust @@ -163,8 +163,8 @@ provides a dedicated call to **map the kernel binary** (because it is the `BSP` pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { kernel_mmu::kernel_map_pages_at( "Kernel boot-core stack", - &phys_stack_page_desc(), &virt_stack_page_desc(), + &phys_stack_page_desc(), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadWrite, @@ -225,7 +225,7 @@ There's a couple of changes not covered in this tutorial text, but the reader sh through them: - [`src/memory/mmu/types.rs`](src/memory/mmu/types.rs) introduces a couple of supporting types, like - `Address`, which is used to differentiate between `Physical` and `Virtual` addresses. + `Page`. - [`src/memory/mmu/mapping_record.rs`](src/memory/mmu/mapping_record.rs) provides the generic kernel code's way of tracking previous memory mappings for use cases such as reusing existing mappings (in case of drivers that have their MMIO ranges in the same `64 KiB` page) or printing mappings @@ -310,10 +310,10 @@ Minipush 1.0 ## Diff to previous ```diff -diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu.rs ---- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs -+++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu.rs -@@ -71,7 +71,7 @@ +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +--- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs ++++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +@@ -56,7 +56,7 @@ ELR_EL2.set(runtime_init::runtime_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. @@ -323,182 +323,81 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs 15_virtua // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. asm::eret() -diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs ---- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs -+++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs -@@ -4,10 +4,19 @@ - - //! Memory Management Unit Driver. - //! --//! Static translation tables, compiled on boot; Everything 64 KiB granule. -+//! Only 64 KiB granule is supported. +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/translation_table.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs +--- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu/translation_table.rs ++++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs +@@ -15,9 +15,12 @@ --use super::{AccessPermissions, AttributeFields, MemAttributes}; --use crate::{bsp, memory}; -+use crate::{ -+ bsp, + use crate::{ + bsp, memory, +- memory::mmu::{ +- arch_mmu::{Granule512MiB, Granule64KiB}, +- AccessPermissions, AttributeFields, MemAttributes, + memory::{ -+ mmu, + mmu::{ -+ AccessPermissions, Address, AddressType, AttributeFields, MemAttributes, Page, -+ PageSliceDescriptor, Physical, Virtual, ++ arch_mmu::{Granule512MiB, Granule64KiB}, ++ AccessPermissions, AttributeFields, MemAttributes, Page, PageSliceDescriptor, + }, -+ }, -+ synchronization::InitStateLock, -+}; ++ Address, AddressType, Physical, Virtual, + }, + }; use core::convert; - use cortex_a::{barrier, regs::*}; - use register::{register_bitfields, InMemoryRegister}; -@@ -15,6 +24,7 @@ - //-------------------------------------------------------------------------------------------------- - // Private Definitions - //-------------------------------------------------------------------------------------------------- -+use mmu::interface::TranslationGranule; +@@ -117,11 +120,11 @@ + } - // A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. - register_bitfields! {u64, -@@ -87,9 +97,6 @@ - ] + trait BaseAddr { +- fn base_addr_u64(&self) -> u64; +- fn base_addr_usize(&self) -> usize; ++ fn phys_base_addr(&self) -> Address; } --const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) --const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) -- - /// A table descriptor for 64 KiB aperture. - /// - /// The output points to the next table. -@@ -104,35 +111,65 @@ - #[repr(transparent)] - struct PageDescriptor(u64); +-const NUM_LVL2_TABLES: usize = bsp::memory::mmu::KernelAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; ++const NUM_LVL2_TABLES: usize = ++ bsp::memory::mmu::KernelVirtAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; -+#[derive(Copy, Clone)] -+enum Granule512MiB {} -+ -+trait BaseAddr { -+ fn phys_base_addr(&self) -> Address; -+} -+ -+/// Constants for indexing the MAIR_EL1. -+#[allow(dead_code)] -+mod mair { -+ pub const DEVICE: u64 = 0; -+ pub const NORMAL: u64 = 1; -+} -+ -+/// Memory Management Unit type. -+struct MemoryManagementUnit; -+ -+/// This constant is the power-of-two exponent that defines the virtual address space size. -+/// -+/// Values tested and known to be working: -+/// - 30 (1 GiB) -+/// - 31 (2 GiB) -+/// - 32 (4 GiB) -+/// - 33 (8 GiB) -+const ADDR_SPACE_SIZE_EXPONENT: usize = 33; -+ -+const NUM_LVL2_TABLES: usize = (1 << ADDR_SPACE_SIZE_EXPONENT) >> Granule512MiB::SHIFT; -+const T0SZ: u64 = (64 - ADDR_SPACE_SIZE_EXPONENT) as u64; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Definitions -+//-------------------------------------------------------------------------------------------------- -+ - /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB - /// aligned, hence the "reverse" order of appearance. - #[repr(C)] - #[repr(align(65536))] --struct FixedSizeTranslationTable { -+pub(in crate::memory::mmu) struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + //-------------------------------------------------------------------------------------------------- + // Public Definitions +@@ -137,6 +140,12 @@ /// Table descriptors, covering 512 MiB windows. lvl2: [TableDescriptor; NUM_TABLES], --} - --const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; --type ArchTranslationTable = FixedSizeTranslationTable; ++ + /// Index of the next free MMIO page. + cur_l3_mmio_index: usize, - --trait BaseAddr { -- fn base_addr_u64(&self) -> u64; -- fn base_addr_usize(&self) -> usize; ++ + /// Have the tables been initialized? + initialized: bool, } --/// Constants for indexing the MAIR_EL1. --#[allow(dead_code)] --mod mair { -- pub const DEVICE: u64 = 0; -- pub const NORMAL: u64 = 1; --} -+pub(in crate::memory::mmu) type ArchTranslationTable = FixedSizeTranslationTable; - --/// Memory Management Unit type. --struct MemoryManagementUnit; -+// Supported translation granules are exported below, so that BSP code can pick between the options. -+// This driver only supports 64 KiB at the moment. -+ -+#[derive(Copy, Clone)] -+/// 64 KiB translation granule. -+pub enum Granule64KiB {} - - //-------------------------------------------------------------------------------------------------- - // Global instances -@@ -143,7 +180,8 @@ - /// # Safety - /// - /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". --static mut KERNEL_TABLES: ArchTranslationTable = ArchTranslationTable::new(); -+static KERNEL_TABLES: InitStateLock = -+ InitStateLock::new(ArchTranslationTable::new()); - - static MMU: MemoryManagementUnit = MemoryManagementUnit; - -@@ -151,13 +189,15 @@ - // Private Code + /// A translation table type for the kernel space. +@@ -147,12 +156,9 @@ //-------------------------------------------------------------------------------------------------- --impl BaseAddr for [T; N] { + impl BaseAddr for [T; N] { - fn base_addr_u64(&self) -> u64 { - self as *const T as u64 - } -+impl mmu::interface::TranslationGranule for Granule512MiB { -+ const SIZE: usize = 512 * 1024 * 1024; -+ const SHIFT: usize = 29; // log2(SIZE) -+} - +- - fn base_addr_usize(&self) -> usize { - self as *const _ as usize -+impl BaseAddr for [T; N] { + fn phys_base_addr(&self) -> Address { + // The binary is still identity mapped, so we don't need to convert here. + Address::new(self as *const _ as usize) } } -@@ -165,7 +205,7 @@ - fn from(next_lvl_table_addr: usize) -> Self { - let val = InMemoryRegister::::new(0); - -- let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; -+ let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table -@@ -215,23 +255,33 @@ +@@ -225,20 +231,29 @@ + } - impl PageDescriptor { /// Create an instance. -- fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { -+ fn new(output_addr: *const Page, attribute_fields: &AttributeFields) -> Self { +- pub fn from_output_addr(output_addr: usize, attribute_fields: AttributeFields) -> Self { ++ pub fn from_output_addr( ++ output_addr: *const Page, ++ attribute_fields: &AttributeFields, ++ ) -> Self { let val = InMemoryRegister::::new(0); -- let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; -+ let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; val.write( STAGE1_PAGE_DESCRIPTOR::VALID::True + STAGE1_PAGE_DESCRIPTOR::AF::True @@ -508,36 +407,53 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), ); - Self(val.get()) + Self { value: val.get() } } + + /// Returns the valid bit. + fn is_valid(&self) -> bool { -+ InMemoryRegister::::new(self.0) ++ InMemoryRegister::::new(self.value) + .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) + } } - impl FixedSizeTranslationTable<{ NUM_TABLES }> { + //-------------------------------------------------------------------------------------------------- +@@ -246,44 +261,172 @@ + //-------------------------------------------------------------------------------------------------- + + impl FixedSizeTranslationTable { + // Reserve the last 256 MiB of the address space for MMIO mappings. + const L2_MMIO_START_INDEX: usize = NUM_TABLES - 1; + const L3_MMIO_START_INDEX: usize = 8192 / 2; + /// Create an instance. + #[allow(clippy::assertions_on_constants)] pub const fn new() -> Self { ++ assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); assert!(NUM_TABLES > 0); -@@ -239,7 +289,55 @@ +- assert!((bsp::memory::mmu::KernelAddrSpaceSize::SIZE modulo Granule512MiB::SIZE) == 0); ++ assert!((bsp::memory::mmu::KernelVirtAddrSpaceSize::SIZE modulo Granule512MiB::SIZE) == 0); + Self { - lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], - lvl2: [TableDescriptor(0); NUM_TABLES], + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + cur_l3_mmio_index: 0, + initialized: false, -+ } -+ } -+ + } + } + +- /// Iterates over all static translation table entries and fills them at once. +- /// +- /// # Safety +- /// +- /// - Modifies a `static mut`. Ensure it only happens from here. +- pub unsafe fn populate_tt_entries(&mut self) -> Result<(), &'static str> { +- for (l2_nr, l2_entry) in self.lvl2.iter_mut().enumerate() { +- *l2_entry = +- TableDescriptor::from_next_lvl_table_addr(self.lvl3[l2_nr].base_addr_usize()); + /// The start address of the table's MMIO range. + #[inline(always)] -+ fn mmio_start_addr(&self) -> Address { ++ const fn mmio_start_addr(&self) -> Address { + Address::new( + (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (Self::L3_MMIO_START_INDEX << Granule64KiB::SHIFT), @@ -546,7 +462,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + + /// The inclusive end address of the table's MMIO range. + #[inline(always)] -+ fn mmio_end_addr_inclusive(&self) -> Address { ++ const fn mmio_end_addr_inclusive(&self) -> Address { + Address::new( + (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (8191 << Granule64KiB::SHIFT) @@ -565,7 +481,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + + if lvl2_index > (NUM_TABLES - 1) { + return Err("Virtual page is out of bounds of translation table"); - } ++ } + + Ok((lvl2_index, lvl3_index)) + } @@ -579,100 +495,46 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + + Ok(&mut self.lvl3[lvl2_index][lvl3_index]) - } - } - -@@ -256,32 +354,9 @@ - ); - } - --/// Iterates over all static translation table entries and fills them at once. --/// --/// # Safety --/// --/// - Modifies a `static mut`. Ensure it only happens from here. --unsafe fn populate_tt_entries() -> Result<(), &'static str> { -- for (l2_nr, l2_entry) in KERNEL_TABLES.lvl2.iter_mut().enumerate() { -- *l2_entry = KERNEL_TABLES.lvl3[l2_nr].base_addr_usize().into(); -- -- for (l3_nr, l3_entry) in KERNEL_TABLES.lvl3[l2_nr].iter_mut().enumerate() { -- let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); -- -- let (output_addr, attribute_fields) = -- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; -- -- *l3_entry = PageDescriptor::new(output_addr, attribute_fields); -- } -- } -- -- Ok(()) --} -- - /// Configure various settings of stage 1 of the EL1 translation regime. - fn configure_translation_control() { - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); -- let t0sz: u64 = bsp::memory::mmu::addr_space_size().trailing_zeros().into(); - - TCR_EL1.write( - TCR_EL1::TBI0::Ignored -@@ -292,7 +367,7 @@ - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks -- + TCR_EL1::T0SZ.val(t0sz), -+ + TCR_EL1::T0SZ.val(T0SZ), - ); - } - -@@ -300,17 +375,126 @@ - // Public Code - //-------------------------------------------------------------------------------------------------- - -+/// Return a guarded reference to the kernel's translation tables. -+pub(in crate::memory::mmu) fn kernel_translation_tables( -+) -> &'static InitStateLock { -+ &KERNEL_TABLES ++ } +} + - /// Return a reference to the MMU instance. --pub fn mmu() -> &'static impl memory::mmu::interface::MMU { -+pub(in crate::memory::mmu) fn mmu() -> &'static impl mmu::interface::MMU { - &MMU - } - - //------------------------------------------------------------------------------ - // OS Interface Code - //------------------------------------------------------------------------------ -+impl mmu::interface::TranslationGranule for Granule64KiB { -+ const SIZE: usize = 64 * 1024; -+ const SHIFT: usize = 16; // log2(SIZE) -+} ++//------------------------------------------------------------------------------ ++// OS Interface Code ++//------------------------------------------------------------------------------ + -+impl mmu::interface::TranslationTable -+ for FixedSizeTranslationTable<{ NUM_TABLES }> ++impl memory::mmu::translation_table::interface::TranslationTable ++ for FixedSizeTranslationTable +{ + unsafe fn init(&mut self) { + if self.initialized { + return; + } -+ + +- for (l3_nr, l3_entry) in self.lvl3[l2_nr].iter_mut().enumerate() { +- let virt_addr = (l2_nr << Granule512MiB::SHIFT) + (l3_nr << Granule64KiB::SHIFT); + // Populate the l2 entries. + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { -+ *lvl2_entry = self.lvl3[lvl2_nr].phys_base_addr().into_usize().into(); ++ let desc = TableDescriptor::from_next_lvl_table_addr( ++ self.lvl3[lvl2_nr].phys_base_addr().into_usize(), ++ ); ++ *lvl2_entry = desc; + } + + self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; + self.initialized = true; + } -+ + +- let (output_addr, attribute_fields) = +- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; + fn phys_base_address(&self) -> Address { + self.lvl2.phys_base_addr() + } -+ + +- *l3_entry = PageDescriptor::from_output_addr(output_addr, attribute_fields); + unsafe fn map_pages_at( + &mut self, -+ phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + assert_eq!(self.initialized, true, "Translation tables not initialized"); @@ -683,9 +545,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + if p.len() != v.len() { + return Err("Tried to map page slices with unequal sizes"); + } - --impl memory::mmu::interface::MMU for MemoryManagementUnit { -- unsafe fn init(&self) -> Result<(), &'static str> { ++ + // No work to do for empty slices. + if p.is_empty() { + return Ok(()); @@ -700,14 +560,17 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; + if page_descriptor.is_valid() { + return Err("Virtual page is already mapped"); -+ } -+ -+ *page_descriptor = PageDescriptor::new(phys_page.as_ptr(), &attr); -+ } -+ -+ Ok(()) -+ } + } + ++ *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), &attr); + } + + Ok(()) + } + +- /// The translation table's base address to be used for programming the MMU. +- pub fn base_address(&self) -> u64 { +- self.lvl2.base_addr_u64() + fn next_mmio_virt_page_slice( + &mut self, + num_pages: usize, @@ -743,41 +606,102 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 + } + + false -+ } + } + } + +@@ -292,6 +435,9 @@ + //-------------------------------------------------------------------------------------------------- + + #[cfg(test)] ++pub type MinSizeKernelTranslationTable = FixedSizeTranslationTable<1>; ++ ++#[cfg(test)] + mod tests { + use super::*; + use test_macros::kernel_test; + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs +--- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs ++++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs +@@ -15,7 +15,11 @@ + + use crate::{ + bsp, memory, +- memory::mmu::{translation_table::KernelTranslationTable, TranslationGranule}, ++ memory::{ ++ mmu::{translation_table::KernelTranslationTable, TranslationGranule}, ++ Address, Physical, ++ }, ++ synchronization::InitStateLock, + }; + use cortex_a::{barrier, regs::*}; + +@@ -37,7 +41,7 @@ + pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB + + /// The max supported address space size. +-pub const MAX_ADDR_SPACE_SIZE: usize = 32 * 1024 * 1024 * 1024; // 32 GiB ++pub const MAX_ADDR_SPACE_SIZE: usize = 8 * 1024 * 1024 * 1024; // 8 GiB + + /// The supported address space size granule. + pub type AddrSpaceSizeGranule = Granule512MiB; +@@ -58,7 +62,8 @@ + /// # Safety + /// + /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". +-static mut KERNEL_TABLES: KernelTranslationTable = KernelTranslationTable::new(); ++static KERNEL_TABLES: InitStateLock = ++ InitStateLock::new(KernelTranslationTable::new()); + + static MMU: MemoryManagementUnit = MemoryManagementUnit; + +@@ -83,7 +88,7 @@ + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); +- let t0sz = (64 - bsp::memory::mmu::KernelAddrSpaceSize::SHIFT) as u64; ++ let t0sz = (64 - bsp::memory::mmu::KernelVirtAddrSpaceSize::SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored +@@ -103,6 +108,11 @@ + // Public Code + //-------------------------------------------------------------------------------------------------- + ++/// Return a guarded reference to the kernel's translation tables. ++pub fn kernel_translation_tables() -> &'static InitStateLock { ++ &KERNEL_TABLES +} + -+impl mmu::interface::MMU for MemoryManagementUnit { + /// Return a reference to the MMU instance. + pub fn mmu() -> &'static impl memory::mmu::interface::MMU { + &MMU +@@ -113,7 +123,10 @@ + //------------------------------------------------------------------------------ + + impl memory::mmu::interface::MMU for MemoryManagementUnit { +- unsafe fn init(&self) -> Result<(), &'static str> { + unsafe fn enable( + &self, -+ phys_kernel_table_base_addr: Address, ++ kernel_table_phys_base_addr: Address, + ) -> Result<(), &'static str> { // Fail early if translation granule is not supported. Both RPis support it, though. if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { return Err("Translation granule not supported in HW"); -@@ -319,11 +503,8 @@ +@@ -122,11 +135,8 @@ // Prepare the memory attribute indirection register. - set_up_mair(); + self.set_up_mair(); - // Populate translation tables. -- populate_tt_entries()?; +- KERNEL_TABLES.populate_tt_entries()?; - // Set the "Translation Table Base Register". -- TTBR0_EL1.set_baddr(KERNEL_TABLES.lvl2.base_addr_u64()); -+ TTBR0_EL1.set_baddr(phys_kernel_table_base_addr.into_usize() as u64); - - configure_translation_control(); +- TTBR0_EL1.set_baddr(KERNEL_TABLES.base_address()); ++ TTBR0_EL1.set_baddr(kernel_table_phys_base_addr.into_usize() as u64); -@@ -347,6 +528,9 @@ - //-------------------------------------------------------------------------------------------------- + self.configure_translation_control(); - #[cfg(test)] -+pub(in crate::memory::mmu) type MinSizeArchTranslationTable = FixedSizeTranslationTable<1>; -+ -+#[cfg(test)] - mod tests { - use super::*; - use test_macros::kernel_test; -@@ -373,7 +557,7 @@ +@@ -158,7 +168,7 @@ #[kernel_test] fn kernel_tables_in_bss() { let bss_range = bsp::memory::bss_range_inclusive(); @@ -787,19 +711,6 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 assert!(bss_range.contains(&kernel_tables_addr)); } -diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/time.rs ---- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs -+++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/time.rs -@@ -55,7 +55,7 @@ - } - - // Calculate the register compare value. -- let frq = CNTFRQ_EL0.get() as u64; -+ let frq = CNTFRQ_EL0.get(); - let x = match frq.checked_mul(duration.as_nanos() as u64) { - None => { - warn!("Spin duration too long, skipping"); - diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs 15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs --- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs +++ 15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs @@ -974,7 +885,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs -use crate::{bsp, cpu, driver, exception, synchronization, synchronization::InitStateLock}; +use crate::{ -+ bsp, cpu, driver, exception, memory, memory::mmu::Physical, synchronization, ++ bsp, cpu, driver, exception, memory, memory::Physical, synchronization, + synchronization::InitStateLock, +}; +use core::sync::atomic::{AtomicBool, Ordering}; @@ -1055,7 +966,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_ use crate::{ - bsp::device_driver::common::MMIODerefWrapper, driver, synchronization, - synchronization::IRQSafeNullLock, -+ bsp::device_driver::common::MMIODerefWrapper, driver, memory, memory::mmu::Physical, ++ bsp::device_driver::common::MMIODerefWrapper, driver, memory, memory::Physical, + synchronization, synchronization::IRQSafeNullLock, }; +use core::sync::atomic::{AtomicUsize, Ordering}; @@ -1153,7 +1064,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_ bsp::device_driver::common::MMIODerefWrapper, - exception, synchronization, + driver, exception, memory, -+ memory::mmu::Physical, ++ memory::Physical, + synchronization, synchronization::{IRQSafeNullLock, InitStateLock}, }; @@ -1242,7 +1153,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_ mod peripheral_ic; -use crate::{driver, exception}; -+use crate::{driver, exception, memory, memory::mmu::Physical}; ++use crate::{driver, exception, memory, memory::Physical}; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -1285,7 +1196,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_ - bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, - synchronization, synchronization::IRQSafeNullLock, + bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, memory, -+ memory::mmu::Physical, synchronization, synchronization::IRQSafeNullLock, ++ memory::Physical, synchronization, synchronization::IRQSafeNullLock, +}; +use core::{ + fmt, @@ -1468,7 +1379,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 15_vir diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs --- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +++ 15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs -@@ -4,77 +4,128 @@ +@@ -4,70 +4,131 @@ //! BSP Memory Management Unit. @@ -1480,9 +1391,10 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + memory::{ + mmu as kernel_mmu, + mmu::{ -+ interface, AccessPermissions, AttributeFields, Granule64KiB, MemAttributes, Page, -+ PageSliceDescriptor, Physical, Virtual, ++ AccessPermissions, AddressSpaceSize, AttributeFields, MemAttributes, Page, ++ PageSliceDescriptor, TranslationGranule, + }, ++ Physical, Virtual, + }, +}; @@ -1490,13 +1402,19 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs // Public Definitions //-------------------------------------------------------------------------------------------------- --const NUM_MEM_RANGES: usize = 2; +-/// The address space size chosen by this BSP. +-pub type KernelAddrSpaceSize = AddressSpaceSize<{ memory_map::END_INCLUSIVE + 1 }>; - +-const NUM_MEM_RANGES: usize = 2; ++/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to ++/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. ++pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; + -/// The virtual memory layout. -/// -/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. -/// It is agnostic of the paging granularity that the architecture's MMU will use. --pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( +-pub static LAYOUT: KernelVirtualLayout = KernelVirtualLayout::new( - memory_map::END_INCLUSIVE, - [ - TranslationDescriptor { @@ -1521,15 +1439,17 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs - }, - ], -); -+/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to -+/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. -+pub type KernelGranule = Granule64KiB; ++/// The address space size chosen by this BSP. ++pub type KernelVirtAddrSpaceSize = AddressSpaceSize<{ 8 * 1024 * 1024 * 1024 }>; //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -+use interface::TranslationGranule; -+ + +-fn ro_range_inclusive() -> RangeInclusive { +- // Notice the subtraction to turn the exclusive end into an inclusive end. +- #[allow(clippy::range_minus_one)] +- RangeInclusive::new(super::ro_start(), super::ro_end() - 1) +/// Helper function for calculating the number of pages the given parameter spans. +const fn size_to_num_pages(size: usize) -> usize { + assert!(size > 0); @@ -1555,11 +1475,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// The data pages of the kernel binary. +fn virt_data_page_desc() -> PageSliceDescriptor { + let num_pages = size_to_num_pages(super::data_size()); - --fn ro_range_inclusive() -> RangeInclusive { -- // Notice the subtraction to turn the exclusive end into an inclusive end. -- #[allow(clippy::range_minus_one)] -- RangeInclusive::new(super::ro_start(), super::ro_end() - 1) ++ + PageSliceDescriptor::from_addr(super::virt_data_start(), num_pages) } @@ -1586,7 +1502,9 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs // Public Code //-------------------------------------------------------------------------------------------------- --/// Return the address space size in bytes. +-/// Return a reference to the virtual memory layout. +-pub fn virt_mem_layout() -> &'static KernelVirtualLayout { +- &LAYOUT +/// Pointer to the last page of the physical address space. +pub fn phys_addr_space_end_page() -> *const Page { + common::align_down( @@ -1598,44 +1516,35 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// Map the kernel binary. +/// +/// # Safety - /// --/// Guarantees size to be a power of two. --pub const fn addr_space_size() -> usize { -- let size = memory_map::END_INCLUSIVE + 1; -- assert!(size.is_power_of_two()); ++/// +/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. +pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { + kernel_mmu::kernel_map_pages_at( + "Kernel boot-core stack", -+ &phys_stack_page_desc(), + &virt_stack_page_desc(), ++ &phys_stack_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; - -- size --} ++ + kernel_mmu::kernel_map_pages_at( + "Kernel code and RO data", -+ &phys_ro_page_desc(), + &virt_ro_page_desc(), ++ &phys_ro_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + )?; - --/// Return a reference to the virtual memory layout. --pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { -- &LAYOUT ++ + kernel_mmu::kernel_map_pages_at( + "Kernel data and bss", -+ &phys_data_page_desc(), + &virt_data_page_desc(), ++ &phys_data_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, @@ -1647,19 +1556,19 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs } //-------------------------------------------------------------------------------------------------- -@@ -89,14 +140,12 @@ +@@ -82,14 +143,12 @@ /// Check alignment of the kernel's virtual memory layout sections. #[kernel_test] fn virt_mem_layout_sections_are_64KiB_aligned() { - const SIXTYFOUR_KIB: usize = 65536; -- -- for i in LAYOUT.inner().iter() { -- let start: usize = *(i.virtual_range)().start(); -- let end: usize = *(i.virtual_range)().end() + 1; + for i in [virt_stack_page_desc, virt_ro_page_desc, virt_data_page_desc].iter() { + let start: usize = i().start_addr().into_usize(); + let end: usize = i().end_addr().into_usize(); +- for i in LAYOUT.inner().iter() { +- let start: usize = *(i.virtual_range)().start(); +- let end: usize = *(i.virtual_range)().end() + 1; +- - assert_eq!(start modulo SIXTYFOUR_KIB, 0); - assert_eq!(end modulo SIXTYFOUR_KIB, 0); + assert_eq!(start modulo KernelGranule::SIZE, 0); @@ -1667,7 +1576,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs assert!(end >= start); } } -@@ -104,17 +153,18 @@ +@@ -97,17 +156,18 @@ /// Ensure the kernel's virtual memory layout is free of overlaps. #[kernel_test] fn virt_mem_layout_has_no_overlaps() { @@ -1739,7 +1648,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 15_v pub mod mmu; -+use crate::memory::mmu::{Address, Physical, Virtual}; ++use crate::memory::{Address, Physical, Virtual}; use core::{cell::UnsafeCell, ops::RangeInclusive}; //-------------------------------------------------------------------------------------------------- @@ -2030,7 +1939,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/driver.rs 15_virtual_mem_part2 diff -uNr 14_exceptions_part2_peripheral_IRQs/src/lib.rs 15_virtual_mem_part2_mmio_remap/src/lib.rs --- 14_exceptions_part2_peripheral_IRQs/src/lib.rs +++ 15_virtual_mem_part2_mmio_remap/src/lib.rs -@@ -113,6 +113,7 @@ +@@ -110,6 +110,7 @@ #![allow(incomplete_features)] #![feature(asm)] @@ -2038,7 +1947,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/lib.rs 15_virtual_mem_part2_mm #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -137,6 +138,7 @@ +@@ -131,6 +132,7 @@ mod synchronization; pub mod bsp; @@ -2151,8 +2060,8 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/mapping_record.rs 1 +impl MappingRecordEntry { + pub fn new( + name: &'static str, -+ phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Self { + Self { @@ -2206,13 +2115,13 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/mapping_record.rs 1 + pub fn add( + &mut self, + name: &'static str, -+ phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + let x = self.find_next_free()?; + -+ *x = Some(MappingRecordEntry::new(name, phys_pages, virt_pages, attr)); ++ *x = Some(MappingRecordEntry::new(name, virt_pages, phys_pages, attr)); + Ok(()) + } + @@ -2300,11 +2209,11 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/mapping_record.rs 1 +/// Add an entry to the mapping info record. +pub fn kernel_add( + name: &'static str, -+ phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, +) -> Result<(), &'static str> { -+ KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, phys_pages, virt_pages, attr)) ++ KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_pages, phys_pages, attr)) +} + +pub fn kernel_find_and_insert_mmio_duplicate( @@ -2329,41 +2238,134 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/mapping_record.rs 1 + KERNEL_MAPPING_RECORD.read(|mr| mr.print()); +} -diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs ---- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs -+++ 15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs -@@ -0,0 +1,283 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2020-2021 Andre Richter -+ -+//! Memory Management Unit Types. +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.rs 15_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs +--- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/translation_table.rs ++++ 15_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs +@@ -8,7 +8,104 @@ + #[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] + mod arch_translation_table; + ++use crate::memory::{ ++ mmu::{AttributeFields, PageSliceDescriptor}, ++ Address, Physical, Virtual, ++}; + -+use crate::{bsp, common}; -+use core::{convert::From, marker::PhantomData, ops::RangeInclusive}; + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- + pub use arch_translation_table::KernelTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- -+use super::interface::TranslationGranule; + -+/// Metadata trait for marking the type of an address. -+pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} ++/// Translation table interfaces. ++pub mod interface { ++ use super::*; + -+/// Zero-sized type to mark a physical address. -+#[derive(Copy, Clone, PartialOrd, PartialEq)] -+pub enum Physical {} ++ /// Translation table operations. ++ pub trait TranslationTable { ++ /// Anything that needs to run before any of the other provided functions can be used. ++ /// ++ /// # Safety ++ /// ++ /// - Implementor must ensure that this function can run only once or is harmless if invoked ++ /// multiple times. ++ unsafe fn init(&mut self); + -+/// Zero-sized type to mark a virtual address. -+#[derive(Copy, Clone, PartialOrd, PartialEq)] -+pub enum Virtual {} ++ /// The translation table's base address to be used for programming the MMU. ++ fn phys_base_address(&self) -> Address; + -+/// Generic address type. -+#[derive(Copy, Clone, PartialOrd, PartialEq)] -+pub struct Address { -+ value: usize, -+ _address_type: PhantomData, ++ /// Map the given virtual pages to the given physical pages. ++ /// ++ /// # Safety ++ /// ++ /// - Using wrong attributes can cause multiple issues of different nature in the system. ++ /// - It is not required that the architectural implementation prevents aliasing. That is, ++ /// mapping to the same physical memory using multiple virtual addresses, which would ++ /// break Rust's ownership assumptions. This should be protected against in the kernel's ++ /// generic MMU code. ++ unsafe fn map_pages_at( ++ &mut self, ++ virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++ ) -> Result<(), &'static str>; ++ ++ /// Obtain a free virtual page slice in the MMIO region. ++ /// ++ /// The "MMIO region" is a distinct region of the implementor's choice, which allows ++ /// differentiating MMIO addresses from others. This can speed up debugging efforts. ++ /// Ideally, those MMIO addresses are also standing out visually so that a human eye can ++ /// identify them. For example, by allocating them from near the end of the virtual address ++ /// space. ++ fn next_mmio_virt_page_slice( ++ &mut self, ++ num_pages: usize, ++ ) -> Result, &'static str>; ++ ++ /// Check if a virtual page splice is in the "MMIO region". ++ fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; ++ } ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Testing ++//-------------------------------------------------------------------------------------------------- ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use crate::bsp; ++ use arch_translation_table::MinSizeKernelTranslationTable; ++ use interface::TranslationTable; ++ use test_macros::kernel_test; ++ ++ /// Sanity checks for the kernel TranslationTable implementation. ++ #[kernel_test] ++ fn translationtable_implementation_sanity() { ++ // Need to take care that `tables` fits into the stack. ++ let mut tables = MinSizeKernelTranslationTable::new(); ++ ++ unsafe { tables.init() }; ++ ++ let x = tables.next_mmio_virt_page_slice(0); ++ assert!(x.is_err()); ++ ++ let x = tables.next_mmio_virt_page_slice(1_0000_0000); ++ assert!(x.is_err()); ++ ++ let x = tables.next_mmio_virt_page_slice(2).unwrap(); ++ assert_eq!(x.size(), bsp::memory::mmu::KernelGranule::SIZE * 2); ++ ++ assert_eq!(tables.is_virt_page_slice_mmio(&x), true); ++ ++ assert_eq!( ++ tables.is_virt_page_slice_mmio(&PageSliceDescriptor::from_addr(Address::new(0), 1)), ++ false ++ ); ++ } +} + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs +--- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs ++++ 15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs +@@ -0,0 +1,213 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020-2021 Andre Richter ++ ++//! Memory Management Unit types. ++ ++use crate::{ ++ bsp, common, ++ memory::{Address, AddressType, Physical, Virtual}, ++}; ++use core::{convert::From, marker::PhantomData, ops::RangeInclusive}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- + +/// Generic page type. +#[repr(C)] @@ -2415,60 +2417,6 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 15_virtual +// Public Code +//-------------------------------------------------------------------------------------------------- + -+impl AddressType for Physical {} -+impl AddressType for Virtual {} -+ -+//------------------------------------------------------------------------------ -+// Address -+//------------------------------------------------------------------------------ -+ -+impl Address { -+ /// Create an instance. -+ pub const fn new(value: usize) -> Self { -+ Self { -+ value, -+ _address_type: PhantomData, -+ } -+ } -+ -+ /// Align down. -+ pub const fn align_down(self, alignment: usize) -> Self { -+ let aligned = common::align_down(self.value, alignment); -+ -+ Self { -+ value: aligned, -+ _address_type: PhantomData, -+ } -+ } -+ -+ /// Converts `Address` into an usize. -+ pub const fn into_usize(self) -> usize { -+ self.value -+ } -+} -+ -+impl core::ops::Add for Address { -+ type Output = Self; -+ -+ fn add(self, other: usize) -> Self { -+ Self { -+ value: self.value + other, -+ _address_type: PhantomData, -+ } -+ } -+} -+ -+impl core::ops::Sub for Address { -+ type Output = Self; -+ -+ fn sub(self, other: usize) -> Self { -+ Self { -+ value: self.value - other, -+ _address_type: PhantomData, -+ } -+ } -+} -+ +//------------------------------------------------------------------------------ +// Page +//------------------------------------------------------------------------------ @@ -2620,14 +2568,14 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 15_virtual diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_part2_mmio_remap/src/memory/mmu.rs --- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs +++ 15_virtual_mem_part2_mmio_remap/src/memory/mmu.rs -@@ -3,23 +3,18 @@ +@@ -3,29 +3,22 @@ // Copyright (c) 2020-2021 Andre Richter //! Memory Management Unit. -//! -//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file --//! provides types for composing an architecture-agnostic description of the kernel 's virtual --//! memory layout. +-//! provides types for composing an architecture-agnostic description of the kernel's virtual memory +-//! layout. -//! -//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` -//! function. @@ -2638,76 +2586,31 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; - pub use arch_mmu::*; --use core::{fmt, ops::RangeInclusive}; +mod mapping_record; + mod translation_table; +mod types; -+ -+use crate::{bsp, synchronization, warn}; -+ + +-use core::{fmt, ops::RangeInclusive}; ++use crate::{ ++ bsp, ++ memory::{Address, Physical, Virtual}, ++ synchronization, warn, ++}; + +-//-------------------------------------------------------------------------------------------------- +-// Architectural Public Reexports +-//-------------------------------------------------------------------------------------------------- +-pub use arch_mmu::mmu; +pub use types::*; //-------------------------------------------------------------------------------------------------- // Public Definitions -@@ -27,178 +22,229 @@ +@@ -33,16 +26,20 @@ /// Memory Management interfaces. pub mod interface { + use super::*; -+ -+ /// Describes the characteristics of a translation granule. -+ #[allow(missing_docs)] -+ pub trait TranslationGranule { -+ const SIZE: usize; -+ const MASK: usize = Self::SIZE - 1; -+ const SHIFT: usize; -+ } -+ -+ /// Translation table operations. -+ pub trait TranslationTable { -+ /// Anything that needs to run before any of the other provided functions can be used. -+ /// -+ /// # Safety -+ /// -+ /// - Implementor must ensure that this function can run only once or is harmless if invoked -+ /// multiple times. -+ unsafe fn init(&mut self); -+ -+ /// The translation table's base address to be used for programming the MMU. -+ fn phys_base_address(&self) -> Address; -+ -+ /// Map the given physical pages to the given virtual pages. -+ /// -+ /// # Safety -+ /// -+ /// - Using wrong attributes can cause multiple issues of different nature in the system. -+ /// - It is not required that the architectural implementation prevents aliasing. That is, -+ /// mapping to the same physical memory using multiple virtual addresses, which would -+ /// break Rust's ownership assumptions. This should be protected against in this module -+ /// (the kernel's generic MMU code). -+ unsafe fn map_pages_at( -+ &mut self, -+ phys_pages: &PageSliceDescriptor, -+ virt_pages: &PageSliceDescriptor, -+ attr: &AttributeFields, -+ ) -> Result<(), &'static str>; -+ -+ /// Obtain a free virtual page slice in the MMIO region. -+ /// -+ /// The "MMIO region" is a distinct region of the implementor's choice, which allows -+ /// differentiating MMIO addresses from others. This can speed up debugging efforts. -+ /// Ideally, those MMIO addresses are also standing out visually so that a human eye can -+ /// identify them. For example, by allocating them from near the end of the virtual address -+ /// space. -+ fn next_mmio_virt_page_slice( -+ &mut self, -+ num_pages: usize, -+ ) -> Result, &'static str>; -+ -+ /// Check if a virtual page splice is in the "MMIO region". -+ fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; -+ } /// MMU functions. pub trait MMU { @@ -2722,11 +2625,15 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p - unsafe fn init(&self) -> Result<(), &'static str>; + unsafe fn enable( + &self, -+ phys_kernel_table_base_addr: Address, ++ kernel_table_phys_base_addr: Address, + ) -> Result<(), &'static str>; } } +@@ -52,55 +49,35 @@ + /// Describes the size of an address space. + pub struct AddressSpaceSize; + -/// Architecture agnostic translation types. -#[allow(missing_docs)] -#[derive(Copy, Clone)] @@ -2742,12 +2649,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p - CacheableDRAM, - Device, -} -+//-------------------------------------------------------------------------------------------------- -+// Private Code -+//-------------------------------------------------------------------------------------------------- -+use interface::{TranslationTable, MMU}; -+use synchronization::interface::ReadWriteEx; - +- -/// Architecture agnostic access permissions. -#[allow(missing_docs)] -#[derive(Copy, Clone)] @@ -2755,23 +2657,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p - ReadOnly, - ReadWrite, -} -+/// Map pages in the kernel's translation tables. -+/// -+/// No input checks done, input is passed through to the architectural implementation. -+/// -+/// # Safety -+/// -+/// - See `map_pages_at()`. -+/// - Does not prevent aliasing. -+unsafe fn kernel_map_pages_at_unchecked( -+ name: &'static str, -+ phys_pages: &PageSliceDescriptor, -+ virt_pages: &PageSliceDescriptor, -+ attr: &AttributeFields, -+) -> Result<(), &'static str> { -+ arch_mmu::kernel_translation_tables() -+ .write(|tables| tables.map_pages_at(phys_pages, virt_pages, attr))?; - +- -/// Collection of memory attributes. -#[allow(missing_docs)] -#[derive(Copy, Clone)] @@ -2780,10 +2666,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p - pub acc_perms: AccessPermissions, - pub execute_never: bool, -} -+ if let Err(x) = mapping_record::kernel_add(name, phys_pages, virt_pages, attr) { -+ warn!("{}", x); -+ } - +- -/// Architecture agnostic descriptor for a memory range. -#[allow(missing_docs)] -pub struct TranslationDescriptor { @@ -2791,111 +2674,59 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p - pub virtual_range: fn() -> RangeInclusive, - pub physical_range_translation: Translation, - pub attribute_fields: AttributeFields, -+ Ok(()) - } - +-} +- -/// Type for expressing the kernel's virtual memory layout. -pub struct KernelVirtualLayout { - /// The last (inclusive) address of the address space. - max_virt_addr_inclusive: usize, +//-------------------------------------------------------------------------------------------------- -+// Public Code ++// Private Code +//-------------------------------------------------------------------------------------------------- -+use interface::TranslationGranule; - -- /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. -- inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], -+/// Raw mapping of virtual to physical pages in the kernel translation tables. ++use interface::MMU; ++use synchronization::interface::ReadWriteEx; ++use translation_table::interface::TranslationTable; ++ ++/// Map pages in the kernel's translation tables. +/// -+/// Prevents mapping into the MMIO range of the tables. ++/// No input checks done, input is passed through to the architectural implementation. +/// +/// # Safety +/// -+/// - See `kernel_map_pages_at_unchecked()`. -+/// - Does not prevent aliasing. Currently, we have to trust the callers. -+pub unsafe fn kernel_map_pages_at( -+ name: &'static str, -+ phys_pages: &PageSliceDescriptor, ++/// - See `map_pages_at()`. ++/// - Does not prevent aliasing. ++unsafe fn kernel_map_pages_at_unchecked( ++ name: &'static str, + virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, +) -> Result<(), &'static str> { -+ let is_mmio = arch_mmu::kernel_translation_tables() -+ .read(|tables| tables.is_virt_page_slice_mmio(virt_pages)); -+ if is_mmio { -+ return Err("Attempt to manually map into MMIO region"); -+ } -+ -+ kernel_map_pages_at_unchecked(name, phys_pages, virt_pages, attr)?; ++ arch_mmu::kernel_translation_tables() ++ .write(|tables| tables.map_pages_at(virt_pages, phys_pages, attr))?; + ++ if let Err(x) = mapping_record::kernel_add(name, virt_pages, phys_pages, attr) { ++ warn!("{}", x); ++ } + +- /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. +- inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], + Ok(()) -+} -+ -+/// MMIO remapping in the kernel translation tables. -+/// -+/// Typically used by device drivers. -+/// -+/// # Safety -+/// -+/// - Same as `kernel_map_pages_at_unchecked()`, minus the aliasing part. -+pub unsafe fn kernel_map_mmio( -+ name: &'static str, -+ phys_mmio_descriptor: &MMIODescriptor, -+) -> Result, &'static str> { -+ let phys_pages: PageSliceDescriptor = phys_mmio_descriptor.clone().into(); -+ let offset_into_start_page = -+ phys_mmio_descriptor.start_addr().into_usize() & bsp::memory::mmu::KernelGranule::MASK; -+ -+ // Check if an identical page slice has been mapped for another driver. If so, reuse it. -+ let virt_addr = if let Some(addr) = -+ mapping_record::kernel_find_and_insert_mmio_duplicate(phys_mmio_descriptor, name) -+ { -+ addr -+ // Otherwise, allocate a new virtual page slice and map it. -+ } else { -+ let virt_pages: PageSliceDescriptor = arch_mmu::kernel_translation_tables() -+ .write(|tables| tables.next_mmio_virt_page_slice(phys_pages.num_pages()))?; -+ -+ kernel_map_pages_at_unchecked( -+ name, -+ &phys_pages, -+ &virt_pages, -+ &AttributeFields { -+ mem_attributes: MemAttributes::Device, -+ acc_perms: AccessPermissions::ReadWrite, -+ execute_never: true, -+ }, -+ )?; -+ -+ virt_pages.start_addr() -+ }; -+ -+ Ok(virt_addr + offset_into_start_page) -+} -+ -+/// Map the kernel's binary and enable the MMU. -+/// -+/// # Safety -+/// -+/// - Crucial function during kernel init. Changes the the complete memory view of the processor. -+pub unsafe fn kernel_map_binary_and_enable_mmu() -> Result<(), &'static str> { -+ let phys_base_addr = arch_mmu::kernel_translation_tables().write(|tables| { -+ tables.init(); -+ tables.phys_base_address() -+ }); -+ -+ bsp::memory::mmu::kernel_map_binary()?; -+ arch_mmu::mmu().enable(phys_base_addr) -+} -+ -+/// Human-readable print of all recorded kernel mappings. -+pub fn kernel_print_mappings() { -+ mapping_record::kernel_print() } //-------------------------------------------------------------------------------------------------- --// Public Code -+// Testing - //-------------------------------------------------------------------------------------------------- +@@ -111,6 +88,9 @@ + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + ++ /// The granule's mask. ++ pub const MASK: usize = Self::SIZE - 1; ++ + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + +@@ -142,110 +122,89 @@ + } + } -impl Default for AttributeFields { - fn default() -> AttributeFields { @@ -3002,37 +2833,185 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_mem_p - #[cfg(test)] - pub fn inner(&self) -> &[TranslationDescriptor; NUM_SPECIAL_RANGES] { - &self.inner -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use test_macros::kernel_test; +- } ++/// Raw mapping of virtual to physical pages in the kernel translation tables. ++/// ++/// Prevents mapping into the MMIO range of the tables. ++/// ++/// # Safety ++/// ++/// - See `kernel_map_pages_at_unchecked()`. ++/// - Does not prevent aliasing. Currently, the callers must be trusted. ++pub unsafe fn kernel_map_pages_at( ++ name: &'static str, ++ virt_pages: &PageSliceDescriptor, ++ phys_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++) -> Result<(), &'static str> { ++ let is_mmio = arch_mmu::kernel_translation_tables() ++ .read(|tables| tables.is_virt_page_slice_mmio(virt_pages)); ++ if is_mmio { ++ return Err("Attempt to manually map into MMIO region"); ++ } + -+ /// Sanity checks for the kernel TranslationTable implementation. -+ #[kernel_test] -+ fn translationtable_implementation_sanity() { -+ // Need to take care that `tables` fits into the stack. -+ let mut tables = MinSizeArchTranslationTable::new(); ++ kernel_map_pages_at_unchecked(name, virt_pages, phys_pages, attr)?; + -+ unsafe { tables.init() }; ++ Ok(()) ++} + -+ let x = tables.next_mmio_virt_page_slice(0); -+ assert!(x.is_err()); ++/// MMIO remapping in the kernel translation tables. ++/// ++/// Typically used by device drivers. ++/// ++/// # Safety ++/// ++/// - Same as `kernel_map_pages_at_unchecked()`, minus the aliasing part. ++pub unsafe fn kernel_map_mmio( ++ name: &'static str, ++ phys_mmio_descriptor: &MMIODescriptor, ++) -> Result, &'static str> { ++ let phys_pages: PageSliceDescriptor = phys_mmio_descriptor.clone().into(); ++ let offset_into_start_page = ++ phys_mmio_descriptor.start_addr().into_usize() & bsp::memory::mmu::KernelGranule::MASK; + -+ let x = tables.next_mmio_virt_page_slice(1_0000_0000); -+ assert!(x.is_err()); ++ // Check if an identical page slice has been mapped for another driver. If so, reuse it. ++ let virt_addr = if let Some(addr) = ++ mapping_record::kernel_find_and_insert_mmio_duplicate(phys_mmio_descriptor, name) ++ { ++ addr ++ // Otherwise, allocate a new virtual page slice and map it. ++ } else { ++ let virt_pages: PageSliceDescriptor = arch_mmu::kernel_translation_tables() ++ .write(|tables| tables.next_mmio_virt_page_slice(phys_pages.num_pages()))?; + -+ let x = tables.next_mmio_virt_page_slice(2).unwrap(); -+ assert_eq!(x.size(), bsp::memory::mmu::KernelGranule::SIZE * 2); ++ kernel_map_pages_at_unchecked( ++ name, ++ &virt_pages, ++ &phys_pages, ++ &AttributeFields { ++ mem_attributes: MemAttributes::Device, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ }, ++ )?; + -+ assert_eq!(tables.is_virt_page_slice_mmio(&x), true); ++ virt_pages.start_addr() ++ }; + -+ assert_eq!( -+ tables.is_virt_page_slice_mmio(&PageSliceDescriptor::from_addr(Address::new(0), 1)), -+ false -+ ); - } ++ Ok(virt_addr + offset_into_start_page) ++} ++ ++/// Map the kernel's binary and enable the MMU. ++/// ++/// # Safety ++/// ++/// - Crucial function during kernel init. Changes the the complete memory view of the processor. ++pub unsafe fn kernel_map_binary_and_enable_mmu() -> Result<(), &'static str> { ++ let phys_base_addr = arch_mmu::kernel_translation_tables().write(|tables| { ++ tables.init(); ++ tables.phys_base_address() ++ }); ++ ++ bsp::memory::mmu::kernel_map_binary()?; ++ arch_mmu::mmu().enable(phys_base_addr) ++} ++ ++/// Human-readable print of all recorded kernel mappings. ++pub fn kernel_print_mappings() { ++ mapping_record::kernel_print() } +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory.rs 15_virtual_mem_part2_mmio_remap/src/memory.rs +--- 14_exceptions_part2_peripheral_IRQs/src/memory.rs ++++ 15_virtual_mem_part2_mmio_remap/src/memory.rs +@@ -6,12 +6,85 @@ + + pub mod mmu; + +-use core::ops::RangeInclusive; ++use crate::common; ++use core::{marker::PhantomData, ops::RangeInclusive}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Metadata trait for marking the type of an address. ++pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} ++ ++/// Zero-sized type to mark a physical address. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub enum Physical {} ++ ++/// Zero-sized type to mark a virtual address. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub enum Virtual {} ++ ++/// Generic address type. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub struct Address { ++ value: usize, ++ _address_type: PhantomData ATYPE>, ++} + + //-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- + ++impl AddressType for Physical {} ++impl AddressType for Virtual {} ++ ++impl Address { ++ /// Create an instance. ++ pub const fn new(value: usize) -> Self { ++ Self { ++ value, ++ _address_type: PhantomData, ++ } ++ } ++ ++ /// Align down. ++ pub const fn align_down(self, alignment: usize) -> Self { ++ let aligned = common::align_down(self.value, alignment); ++ ++ Self { ++ value: aligned, ++ _address_type: PhantomData, ++ } ++ } ++ ++ /// Converts `Address` into an usize. ++ pub const fn into_usize(self) -> usize { ++ self.value ++ } ++} ++ ++impl core::ops::Add for Address { ++ type Output = Self; ++ ++ fn add(self, other: usize) -> Self { ++ Self { ++ value: self.value + other, ++ _address_type: PhantomData, ++ } ++ } ++} ++ ++impl core::ops::Sub for Address { ++ type Output = Self; ++ ++ fn sub(self, other: usize) -> Self { ++ Self { ++ value: self.value - other, ++ _address_type: PhantomData, ++ } ++ } ++} ++ + /// Zero out an inclusive memory range. + /// + /// # Safety + diff -uNr 14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs 15_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs --- 14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs +++ 15_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu.rs index 6ecea182..948bad74 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu.rs @@ -3,79 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} - -/// Transition from EL2 to EL1. -/// -/// # Safety -/// -/// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. -#[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - - // Enable timer counter registers for EL1. - CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); - - // No offset for reading the counters. - CNTVOFF_EL2.set(0); - - // Set EL1 execution state to AArch64. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Set up a simulated exception return. - // - // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a - // stack pointer. - SPSR_EL2.write( - SPSR_EL2::D::Masked - + SPSR_EL2::A::Masked - + SPSR_EL2::I::Masked - + SPSR_EL2::F::Masked - + SPSR_EL2::M::EL1h, - ); - - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::phys_boot_core_stack_end().into_usize() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..27696514 --- /dev/null +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::phys_boot_core_stack_end().into_usize() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up +/// a stack for EL2. +#[no_mangle] +pub unsafe fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs index 99fbd85b..5cf9eb5c 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception use crate::{bsp, exception}; use core::{cell::UnsafeCell, fmt}; diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs index 968eedf3..a4b1a548 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous use cortex_a::regs::*; diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs index 8abb997e..ebf454ce 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs @@ -5,183 +5,65 @@ //! Memory Management Unit Driver. //! //! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::arch_mmu use crate::{ - bsp, + bsp, memory, memory::{ - mmu, - mmu::{ - AccessPermissions, Address, AddressType, AttributeFields, MemAttributes, Page, - PageSliceDescriptor, Physical, Virtual, - }, + mmu::{translation_table::KernelTranslationTable, TranslationGranule}, + Address, Physical, }, synchronization::InitStateLock, }; -use core::convert; use cortex_a::{barrier, regs::*}; -use register::{register_bitfields, InMemoryRegister}; //-------------------------------------------------------------------------------------------------- // Private Definitions //-------------------------------------------------------------------------------------------------- -use mmu::interface::TranslationGranule; - -// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. -register_bitfields! {u64, - STAGE1_TABLE_DESCRIPTOR [ - /// Physical address of the next descriptor. - NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. -register_bitfields! {u64, - STAGE1_PAGE_DESCRIPTOR [ - /// Unprivileged execute-never. - UXN OFFSET(54) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Privileged execute-never. - PXN OFFSET(53) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). - OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] - - /// Access flag. - AF OFFSET(10) NUMBITS(1) [ - False = 0, - True = 1 - ], - - /// Shareability field. - SH OFFSET(8) NUMBITS(2) [ - OuterShareable = 0b10, - InnerShareable = 0b11 - ], - - /// Access Permissions. - AP OFFSET(6) NUMBITS(2) [ - RW_EL1 = 0b00, - RW_EL1_EL0 = 0b01, - RO_EL1 = 0b10, - RO_EL1_EL0 = 0b11 - ], - - /// Memory attributes index into the MAIR_EL1 register. - AttrIndx OFFSET(2) NUMBITS(3) [], - - TYPE OFFSET(1) NUMBITS(1) [ - Block = 0, - Table = 1 - ], - - VALID OFFSET(0) NUMBITS(1) [ - False = 0, - True = 1 - ] - ] -} - -/// A table descriptor for 64 KiB aperture. -/// -/// The output points to the next table. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct TableDescriptor(u64); - -/// A page descriptor with 64 KiB aperture. -/// -/// The output points to physical memory. -#[derive(Copy, Clone)] -#[repr(transparent)] -struct PageDescriptor(u64); - -#[derive(Copy, Clone)] -enum Granule512MiB {} - -trait BaseAddr { - fn phys_base_addr(&self) -> Address; -} - -/// Constants for indexing the MAIR_EL1. -#[allow(dead_code)] -mod mair { - pub const DEVICE: u64 = 0; - pub const NORMAL: u64 = 1; -} /// Memory Management Unit type. struct MemoryManagementUnit; -/// This constant is the power-of-two exponent that defines the virtual address space size. -/// -/// Values tested and known to be working: -/// - 30 (1 GiB) -/// - 31 (2 GiB) -/// - 32 (4 GiB) -/// - 33 (8 GiB) -const ADDR_SPACE_SIZE_EXPONENT: usize = 33; - -const NUM_LVL2_TABLES: usize = (1 << ADDR_SPACE_SIZE_EXPONENT) >> Granule512MiB::SHIFT; -const T0SZ: u64 = (64 - ADDR_SPACE_SIZE_EXPONENT) as u64; - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- -/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB -/// aligned, hence the "reverse" order of appearance. -#[repr(C)] -#[repr(align(65536))] -pub(in crate::memory::mmu) struct FixedSizeTranslationTable { - /// Page descriptors, covering 64 KiB windows per entry. - lvl3: [[PageDescriptor; 8192]; NUM_TABLES], +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; - /// Table descriptors, covering 512 MiB windows. - lvl2: [TableDescriptor; NUM_TABLES], +/// The min supported address space size. +pub const MIN_ADDR_SPACE_SIZE: usize = 1024 * 1024 * 1024; // 1 GiB - /// Index of the next free MMIO page. - cur_l3_mmio_index: usize, - - /// Have the tables been initialized? - initialized: bool, -} +/// The max supported address space size. +pub const MAX_ADDR_SPACE_SIZE: usize = 8 * 1024 * 1024 * 1024; // 8 GiB -pub(in crate::memory::mmu) type ArchTranslationTable = FixedSizeTranslationTable; +/// The supported address space size granule. +pub type AddrSpaceSizeGranule = Granule512MiB; -// Supported translation granules are exported below, so that BSP code can pick between the options. -// This driver only supports 64 KiB at the moment. - -#[derive(Copy, Clone)] -/// 64 KiB translation granule. -pub enum Granule64KiB {} +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +pub mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; +} //-------------------------------------------------------------------------------------------------- // Global instances //-------------------------------------------------------------------------------------------------- -/// The translation tables. +/// The kernel translation tables. /// /// # Safety /// /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". -static KERNEL_TABLES: InitStateLock = - InitStateLock::new(ArchTranslationTable::new()); +static KERNEL_TABLES: InitStateLock = + InitStateLock::new(KernelTranslationTable::new()); static MMU: MemoryManagementUnit = MemoryManagementUnit; @@ -189,311 +71,61 @@ static MMU: MemoryManagementUnit = MemoryManagementUnit; // Private Code //-------------------------------------------------------------------------------------------------- -impl mmu::interface::TranslationGranule for Granule512MiB { - const SIZE: usize = 512 * 1024 * 1024; - const SHIFT: usize = 29; // log2(SIZE) -} - -impl BaseAddr for [T; N] { - fn phys_base_addr(&self) -> Address { - // The binary is still identity mapped, so we don't need to convert here. - Address::new(self as *const _ as usize) - } -} - -impl convert::From for TableDescriptor { - fn from(next_lvl_table_addr: usize) -> Self { - let val = InMemoryRegister::::new(0); +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + - let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; - val.write( - STAGE1_TABLE_DESCRIPTOR::VALID::True - + STAGE1_TABLE_DESCRIPTOR::TYPE::Table - + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + // Attribute 0 - Device. + MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, ); - - TableDescriptor(val.get()) } -} - -/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. -impl convert::From - for register::FieldValue -{ - fn from(attribute_fields: AttributeFields) -> Self { - // Memory attributes. - let mut desc = match attribute_fields.mem_attributes { - MemAttributes::CacheableDRAM => { - STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) - } - MemAttributes::Device => { - STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable - + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) - } - }; - - // Access Permissions. - desc += match attribute_fields.acc_perms { - AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, - AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, - }; - - // The execute-never attribute is mapped to PXN in AArch64. - desc += if attribute_fields.execute_never { - STAGE1_PAGE_DESCRIPTOR::PXN::True - } else { - STAGE1_PAGE_DESCRIPTOR::PXN::False - }; - - // Always set unprivileged exectue-never as long as userspace is not implemented yet. - desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; - - desc - } -} -impl PageDescriptor { - /// Create an instance. - fn new(output_addr: *const Page, attribute_fields: &AttributeFields) -> Self { - let val = InMemoryRegister::::new(0); - - let shifted = output_addr as u64 >> Granule64KiB::SHIFT; - val.write( - STAGE1_PAGE_DESCRIPTOR::VALID::True - + STAGE1_PAGE_DESCRIPTOR::AF::True - + attribute_fields.clone().into() - + STAGE1_PAGE_DESCRIPTOR::TYPE::Table - + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control(&self) { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + let t0sz = (64 - bsp::memory::mmu::KernelVirtAddrSpaceSize::SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::EPD1::DisableTTBR1Walks + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(t0sz), ); - - Self(val.get()) - } - - /// Returns the valid bit. - fn is_valid(&self) -> bool { - InMemoryRegister::::new(self.0) - .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) } } -impl FixedSizeTranslationTable<{ NUM_TABLES }> { - // Reserve the last 256 MiB of the address space for MMIO mappings. - const L2_MMIO_START_INDEX: usize = NUM_TABLES - 1; - const L3_MMIO_START_INDEX: usize = 8192 / 2; - - /// Create an instance. - pub const fn new() -> Self { - assert!(NUM_TABLES > 0); - - Self { - lvl3: [[PageDescriptor(0); 8192]; NUM_TABLES], - lvl2: [TableDescriptor(0); NUM_TABLES], - cur_l3_mmio_index: 0, - initialized: false, - } - } - - /// The start address of the table's MMIO range. - #[inline(always)] - fn mmio_start_addr(&self) -> Address { - Address::new( - (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) - | (Self::L3_MMIO_START_INDEX << Granule64KiB::SHIFT), - ) - } - - /// The inclusive end address of the table's MMIO range. - #[inline(always)] - fn mmio_end_addr_inclusive(&self) -> Address { - Address::new( - (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) - | (8191 << Granule64KiB::SHIFT) - | (Granule64KiB::SIZE - 1), - ) - } - - /// Helper to calculate the lvl2 and lvl3 indices from an address. - #[inline(always)] - fn lvl2_lvl3_index_from( - &self, - addr: *const Page, - ) -> Result<(usize, usize), &'static str> { - let lvl2_index = addr as usize >> Granule512MiB::SHIFT; - let lvl3_index = (addr as usize & Granule512MiB::MASK) >> Granule64KiB::SHIFT; - - if lvl2_index > (NUM_TABLES - 1) { - return Err("Virtual page is out of bounds of translation table"); - } - - Ok((lvl2_index, lvl3_index)) - } - - /// Returns the PageDescriptor corresponding to the supplied Page. - #[inline(always)] - fn page_descriptor_from( - &mut self, - addr: *const Page, - ) -> Result<&mut PageDescriptor, &'static str> { - let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; - - Ok(&mut self.lvl3[lvl2_index][lvl3_index]) - } -} - -/// Setup function for the MAIR_EL1 register. -fn set_up_mair() { - // Define the memory types being mapped. - MAIR_EL1.write( - // Attribute 1 - Cacheable normal DRAM. - MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + - MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + - - // Attribute 0 - Device. - MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, - ); -} - -/// Configure various settings of stage 1 of the EL1 translation regime. -fn configure_translation_control() { - let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); - - TCR_EL1.write( - TCR_EL1::TBI0::Ignored - + TCR_EL1::IPS.val(ips) - + TCR_EL1::EPD1::DisableTTBR1Walks - + TCR_EL1::TG0::KiB_64 - + TCR_EL1::SH0::Inner - + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL1::EPD0::EnableTTBR0Walks - + TCR_EL1::T0SZ.val(T0SZ), - ); -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- /// Return a guarded reference to the kernel's translation tables. -pub(in crate::memory::mmu) fn kernel_translation_tables( -) -> &'static InitStateLock { +pub fn kernel_translation_tables() -> &'static InitStateLock { &KERNEL_TABLES } /// Return a reference to the MMU instance. -pub(in crate::memory::mmu) fn mmu() -> &'static impl mmu::interface::MMU { +pub fn mmu() -> &'static impl memory::mmu::interface::MMU { &MMU } //------------------------------------------------------------------------------ // OS Interface Code //------------------------------------------------------------------------------ -impl mmu::interface::TranslationGranule for Granule64KiB { - const SIZE: usize = 64 * 1024; - const SHIFT: usize = 16; // log2(SIZE) -} - -impl mmu::interface::TranslationTable - for FixedSizeTranslationTable<{ NUM_TABLES }> -{ - unsafe fn init(&mut self) { - if self.initialized { - return; - } - - // Populate the l2 entries. - for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { - *lvl2_entry = self.lvl3[lvl2_nr].phys_base_addr().into_usize().into(); - } - - self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; - self.initialized = true; - } - - fn phys_base_address(&self) -> Address { - self.lvl2.phys_base_addr() - } - - unsafe fn map_pages_at( - &mut self, - phys_pages: &PageSliceDescriptor, - virt_pages: &PageSliceDescriptor, - attr: &AttributeFields, - ) -> Result<(), &'static str> { - assert_eq!(self.initialized, true, "Translation tables not initialized"); - - let p = phys_pages.as_slice(); - let v = virt_pages.as_slice(); - - if p.len() != v.len() { - return Err("Tried to map page slices with unequal sizes"); - } - - // No work to do for empty slices. - if p.is_empty() { - return Ok(()); - } - - if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() { - return Err("Tried to map outside of physical address space"); - } - let iter = p.iter().zip(v.iter()); - for (phys_page, virt_page) in iter { - let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; - if page_descriptor.is_valid() { - return Err("Virtual page is already mapped"); - } - - *page_descriptor = PageDescriptor::new(phys_page.as_ptr(), &attr); - } - - Ok(()) - } - - fn next_mmio_virt_page_slice( - &mut self, - num_pages: usize, - ) -> Result, &'static str> { - assert_eq!(self.initialized, true, "Translation tables not initialized"); - - if num_pages == 0 { - return Err("num_pages == 0"); - } - - if (self.cur_l3_mmio_index + num_pages) > 8191 { - return Err("Not enough MMIO space left"); - } - - let addr = (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) - | (self.cur_l3_mmio_index << Granule64KiB::SHIFT); - self.cur_l3_mmio_index += num_pages; - - Ok(PageSliceDescriptor::from_addr( - Address::new(addr), - num_pages, - )) - } - - fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool { - let start_addr = virt_pages.start_addr(); - let end_addr_inclusive = virt_pages.end_addr_inclusive(); - - for i in [start_addr, end_addr_inclusive].iter() { - if (*i >= self.mmio_start_addr()) && (*i <= self.mmio_end_addr_inclusive()) { - return true; - } - } - - false - } -} - -impl mmu::interface::MMU for MemoryManagementUnit { +impl memory::mmu::interface::MMU for MemoryManagementUnit { unsafe fn enable( &self, - phys_kernel_table_base_addr: Address, + kernel_table_phys_base_addr: Address, ) -> Result<(), &'static str> { // Fail early if translation granule is not supported. Both RPis support it, though. if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { @@ -501,12 +133,12 @@ impl mmu::interface::MMU for MemoryManagementUnit { } // Prepare the memory attribute indirection register. - set_up_mair(); + self.set_up_mair(); // Set the "Translation Table Base Register". - TTBR0_EL1.set_baddr(phys_kernel_table_base_addr.into_usize() as u64); + TTBR0_EL1.set_baddr(kernel_table_phys_base_addr.into_usize() as u64); - configure_translation_control(); + self.configure_translation_control(); // Switch the MMU on. // @@ -527,32 +159,11 @@ impl mmu::interface::MMU for MemoryManagementUnit { // Testing //-------------------------------------------------------------------------------------------------- -#[cfg(test)] -pub(in crate::memory::mmu) type MinSizeArchTranslationTable = FixedSizeTranslationTable<1>; - #[cfg(test)] mod tests { use super::*; use test_macros::kernel_test; - /// Check if the size of `struct TableDescriptor` is as expected. - #[kernel_test] - fn size_of_tabledescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); - } - - /// Check if the size of `struct PageDescriptor` is as expected. - #[kernel_test] - fn size_of_pagedescriptor_equals_64_bit() { - assert_eq!( - core::mem::size_of::(), - core::mem::size_of::() - ); - } - /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs new file mode 100644 index 00000000..f682d6a4 --- /dev/null +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural translation table. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::translation_table::arch_translation_table + +use crate::{ + bsp, memory, + memory::{ + mmu::{ + arch_mmu::{Granule512MiB, Granule64KiB}, + AccessPermissions, AttributeFields, MemAttributes, Page, PageSliceDescriptor, + }, + Address, AddressType, Physical, Virtual, + }, +}; +use core::convert; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Unprivileged execute-never. + UXN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(C)] +struct TableDescriptor { + value: u64, +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +trait BaseAddr { + fn phys_base_addr(&self) -> Address; +} + +const NUM_LVL2_TABLES: usize = + bsp::memory::mmu::KernelVirtAddrSpaceSize::SIZE >> Granule512MiB::SHIFT; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +pub struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], + + /// Index of the next free MMIO page. + cur_l3_mmio_index: usize, + + /// Have the tables been initialized? + initialized: bool, +} + +/// A translation table type for the kernel space. +pub type KernelTranslationTable = FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl BaseAddr for [T; N] { + fn phys_base_addr(&self) -> Address { + // The binary is still identity mapped, so we don't need to convert here. + Address::new(self as *const _ as usize) + } +} + +impl TableDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance pointing to the supplied address. + pub fn from_next_lvl_table_addr(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + + TableDescriptor { value: val.get() } + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // The execute-never attribute is mapped to PXN in AArch64. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + // Always set unprivileged exectue-never as long as userspace is not implemented yet. + desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; + + desc + } +} + +impl PageDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance. + pub fn from_output_addr( + output_addr: *const Page, + attribute_fields: &AttributeFields, + ) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.clone().into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self { value: val.get() } + } + + /// Returns the valid bit. + fn is_valid(&self) -> bool { + InMemoryRegister::::new(self.value) + .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl FixedSizeTranslationTable { + // Reserve the last 256 MiB of the address space for MMIO mappings. + const L2_MMIO_START_INDEX: usize = NUM_TABLES - 1; + const L3_MMIO_START_INDEX: usize = 8192 / 2; + + /// Create an instance. + #[allow(clippy::assertions_on_constants)] + pub const fn new() -> Self { + assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); + assert!(NUM_TABLES > 0); + assert!((bsp::memory::mmu::KernelVirtAddrSpaceSize::SIZE % Granule512MiB::SIZE) == 0); + + Self { + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + cur_l3_mmio_index: 0, + initialized: false, + } + } + + /// The start address of the table's MMIO range. + #[inline(always)] + const fn mmio_start_addr(&self) -> Address { + Address::new( + (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (Self::L3_MMIO_START_INDEX << Granule64KiB::SHIFT), + ) + } + + /// The inclusive end address of the table's MMIO range. + #[inline(always)] + const fn mmio_end_addr_inclusive(&self) -> Address { + Address::new( + (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (8191 << Granule64KiB::SHIFT) + | (Granule64KiB::SIZE - 1), + ) + } + + /// Helper to calculate the lvl2 and lvl3 indices from an address. + #[inline(always)] + fn lvl2_lvl3_index_from( + &self, + addr: *const Page, + ) -> Result<(usize, usize), &'static str> { + let lvl2_index = addr as usize >> Granule512MiB::SHIFT; + let lvl3_index = (addr as usize & Granule512MiB::MASK) >> Granule64KiB::SHIFT; + + if lvl2_index > (NUM_TABLES - 1) { + return Err("Virtual page is out of bounds of translation table"); + } + + Ok((lvl2_index, lvl3_index)) + } + + /// Returns the PageDescriptor corresponding to the supplied Page. + #[inline(always)] + fn page_descriptor_from( + &mut self, + addr: *const Page, + ) -> Result<&mut PageDescriptor, &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + + Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl memory::mmu::translation_table::interface::TranslationTable + for FixedSizeTranslationTable +{ + unsafe fn init(&mut self) { + if self.initialized { + return; + } + + // Populate the l2 entries. + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { + let desc = TableDescriptor::from_next_lvl_table_addr( + self.lvl3[lvl2_nr].phys_base_addr().into_usize(), + ); + *lvl2_entry = desc; + } + + self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; + self.initialized = true; + } + + fn phys_base_address(&self) -> Address { + self.lvl2.phys_base_addr() + } + + unsafe fn map_pages_at( + &mut self, + virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + assert_eq!(self.initialized, true, "Translation tables not initialized"); + + let p = phys_pages.as_slice(); + let v = virt_pages.as_slice(); + + if p.len() != v.len() { + return Err("Tried to map page slices with unequal sizes"); + } + + // No work to do for empty slices. + if p.is_empty() { + return Ok(()); + } + + if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() { + return Err("Tried to map outside of physical address space"); + } + + let iter = p.iter().zip(v.iter()); + for (phys_page, virt_page) in iter { + let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; + if page_descriptor.is_valid() { + return Err("Virtual page is already mapped"); + } + + *page_descriptor = PageDescriptor::from_output_addr(phys_page.as_ptr(), &attr); + } + + Ok(()) + } + + fn next_mmio_virt_page_slice( + &mut self, + num_pages: usize, + ) -> Result, &'static str> { + assert_eq!(self.initialized, true, "Translation tables not initialized"); + + if num_pages == 0 { + return Err("num_pages == 0"); + } + + if (self.cur_l3_mmio_index + num_pages) > 8191 { + return Err("Not enough MMIO space left"); + } + + let addr = (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (self.cur_l3_mmio_index << Granule64KiB::SHIFT); + self.cur_l3_mmio_index += num_pages; + + Ok(PageSliceDescriptor::from_addr( + Address::new(addr), + num_pages, + )) + } + + fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool { + let start_addr = virt_pages.start_addr(); + let end_addr_inclusive = virt_pages.end_addr_inclusive(); + + for i in [start_addr, end_addr_inclusive].iter() { + if (*i >= self.mmio_start_addr()) && (*i <= self.mmio_end_addr_inclusive()) { + return true; + } + } + + false + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +pub type MinSizeKernelTranslationTable = FixedSizeTranslationTable<1>; + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check if the size of `struct TableDescriptor` is as expected. + #[kernel_test] + fn size_of_tabledescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if the size of `struct PageDescriptor` is as expected. + #[kernel_test] + fn size_of_pagedescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } +} diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/time.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/time.rs index a2c57eb0..3a766009 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/time.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp.rs b/15_virtual_mem_part2_mmio_remap/src/bsp.rs index 25750249..c558922f 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs index e7a37697..4c0e1124 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs @@ -80,7 +80,7 @@ mod gicc; mod gicd; use crate::{ - bsp, cpu, driver, exception, memory, memory::mmu::Physical, synchronization, + bsp, cpu, driver, exception, memory, memory::Physical, synchronization, synchronization::InitStateLock, }; use core::sync::atomic::{AtomicBool, Ordering}; diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs index c1fa1d14..45dbe75c 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -5,7 +5,7 @@ //! GPIO Driver. use crate::{ - bsp::device_driver::common::MMIODerefWrapper, driver, memory, memory::mmu::Physical, + bsp::device_driver::common::MMIODerefWrapper, driver, memory, memory::Physical, synchronization, synchronization::IRQSafeNullLock, }; use core::sync::atomic::{AtomicUsize, Ordering}; diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs index dfc5b66e..69c001ad 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs @@ -6,7 +6,7 @@ mod peripheral_ic; -use crate::{driver, exception, memory, memory::mmu::Physical}; +use crate::{driver, exception, memory, memory::Physical}; //-------------------------------------------------------------------------------------------------- // Private Definitions diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs index 8fe915b3..654ac3c7 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs @@ -8,7 +8,7 @@ use super::{InterruptController, PendingIRQs, PeripheralIRQ}; use crate::{ bsp::device_driver::common::MMIODerefWrapper, driver, exception, memory, - memory::mmu::Physical, + memory::Physical, synchronization, synchronization::{IRQSafeNullLock, InitStateLock}, }; diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs index e404f11b..d7d83840 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs @@ -6,7 +6,7 @@ use crate::{ bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, memory, - memory::mmu::Physical, synchronization, synchronization::IRQSafeNullLock, + memory::Physical, synchronization, synchronization::IRQSafeNullLock, }; use core::{ fmt, diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs index 9d359cb7..7b48d7b5 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; -use crate::memory::mmu::{Address, Physical, Virtual}; +use crate::memory::{Address, Physical, Virtual}; use core::{cell::UnsafeCell, ops::RangeInclusive}; //-------------------------------------------------------------------------------------------------- diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs index fc16c87c..76b93e47 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs @@ -9,9 +9,10 @@ use crate::{ memory::{ mmu as kernel_mmu, mmu::{ - interface, AccessPermissions, AttributeFields, Granule64KiB, MemAttributes, Page, - PageSliceDescriptor, Physical, Virtual, + AccessPermissions, AddressSpaceSize, AttributeFields, MemAttributes, Page, + PageSliceDescriptor, TranslationGranule, }, + Physical, Virtual, }, }; @@ -21,12 +22,14 @@ use crate::{ /// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to /// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. -pub type KernelGranule = Granule64KiB; +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; + +/// The address space size chosen by this BSP. +pub type KernelVirtAddrSpaceSize = AddressSpaceSize<{ 8 * 1024 * 1024 * 1024 }>; //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -use interface::TranslationGranule; /// Helper function for calculating the number of pages the given parameter spans. const fn size_to_num_pages(size: usize) -> usize { @@ -94,8 +97,8 @@ pub fn phys_addr_space_end_page() -> *const Page { pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { kernel_mmu::kernel_map_pages_at( "Kernel boot-core stack", - &phys_stack_page_desc(), &virt_stack_page_desc(), + &phys_stack_page_desc(), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadWrite, @@ -105,8 +108,8 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { kernel_mmu::kernel_map_pages_at( "Kernel code and RO data", - &phys_ro_page_desc(), &virt_ro_page_desc(), + &phys_ro_page_desc(), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadOnly, @@ -116,8 +119,8 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { kernel_mmu::kernel_map_pages_at( "Kernel data and bss", - &phys_data_page_desc(), &virt_data_page_desc(), + &phys_data_page_desc(), &AttributeFields { mem_attributes: MemAttributes::CacheableDRAM, acc_perms: AccessPermissions::ReadWrite, diff --git a/15_virtual_mem_part2_mmio_remap/src/console.rs b/15_virtual_mem_part2_mmio_remap/src/console.rs index 3552823c..c3154ba2 100644 --- a/15_virtual_mem_part2_mmio_remap/src/console.rs +++ b/15_virtual_mem_part2_mmio_remap/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/15_virtual_mem_part2_mmio_remap/src/cpu.rs b/15_virtual_mem_part2_mmio_remap/src/cpu.rs index c9e5af72..9c8eb6f6 100644 --- a/15_virtual_mem_part2_mmio_remap/src/cpu.rs +++ b/15_virtual_mem_part2_mmio_remap/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, qemu_exit_failure, qemu_exit_success, wait_forever}; diff --git a/15_virtual_mem_part2_mmio_remap/src/cpu/boot.rs b/15_virtual_mem_part2_mmio_remap/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/15_virtual_mem_part2_mmio_remap/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/15_virtual_mem_part2_mmio_remap/src/cpu/smp.rs b/15_virtual_mem_part2_mmio_remap/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/15_virtual_mem_part2_mmio_remap/src/cpu/smp.rs +++ b/15_virtual_mem_part2_mmio_remap/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/15_virtual_mem_part2_mmio_remap/src/exception.rs b/15_virtual_mem_part2_mmio_remap/src/exception.rs index dfa852a8..3c5e7bc8 100644 --- a/15_virtual_mem_part2_mmio_remap/src/exception.rs +++ b/15_virtual_mem_part2_mmio_remap/src/exception.rs @@ -7,10 +7,14 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/exception.rs"] mod arch_exception; -pub use arch_exception::*; pub mod asynchronous; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::{current_privilege_level, handling_init}; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- diff --git a/15_virtual_mem_part2_mmio_remap/src/exception/asynchronous.rs b/15_virtual_mem_part2_mmio_remap/src/exception/asynchronous.rs index c16ce007..2b9ef05f 100644 --- a/15_virtual_mem_part2_mmio_remap/src/exception/asynchronous.rs +++ b/15_virtual_mem_part2_mmio_remap/src/exception/asynchronous.rs @@ -6,11 +6,18 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/exception/asynchronous.rs"] -mod arch_exception_async; -pub use arch_exception_async::*; +mod arch_asynchronous; use core::{fmt, marker::PhantomData}; +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::{ + is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask, + print_state, +}; + //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -116,7 +123,7 @@ impl IRQNumber<{ MAX_INCLUSIVE }> { } /// Return the wrapped number. - pub fn get(self) -> usize { + pub const fn get(self) -> usize { self.0 } } diff --git a/15_virtual_mem_part2_mmio_remap/src/lib.rs b/15_virtual_mem_part2_mmio_remap/src/lib.rs index e66dd3e8..89df0dd5 100644 --- a/15_virtual_mem_part2_mmio_remap/src/lib.rs +++ b/15_virtual_mem_part2_mmio_remap/src/lib.rs @@ -7,24 +7,7 @@ //! The `kernel` library. //! -//! Used by `main.rs` to compose the final kernel binary. -//! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`bsp::exception::asynchronous::irq_manager()`] - Returns a reference to the kernel's [IRQ -//! Handling interface]. -//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. -//! - [`state::state_manager()`] - Returns a reference to the kernel's [state management] instance. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [IRQ Handling interface]: ../libkernel/exception/asynchronous/interface/trait.IRQManager.html -//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html -//! [state management]: ../libkernel/state/struct.StateManager.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html +//! Used to compose the final kernel binary. //! //! # Code organization and architecture //! @@ -39,15 +22,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -56,9 +46,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -110,6 +99,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![allow(incomplete_features)] #![feature(asm)] @@ -130,9 +127,6 @@ #![reexport_test_harness_main = "test_main"] #![test_runner(crate::test_runner)] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()` (defined in `main.rs`). - mod panic_wait; mod runtime_init; mod synchronization; diff --git a/15_virtual_mem_part2_mmio_remap/src/memory.rs b/15_virtual_mem_part2_mmio_remap/src/memory.rs index 1ef0285a..1493b1a9 100644 --- a/15_virtual_mem_part2_mmio_remap/src/memory.rs +++ b/15_virtual_mem_part2_mmio_remap/src/memory.rs @@ -6,12 +6,85 @@ pub mod mmu; -use core::ops::RangeInclusive; +use crate::common; +use core::{marker::PhantomData, ops::RangeInclusive}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Metadata trait for marking the type of an address. +pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} + +/// Zero-sized type to mark a physical address. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub enum Physical {} + +/// Zero-sized type to mark a virtual address. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub enum Virtual {} + +/// Generic address type. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub struct Address { + value: usize, + _address_type: PhantomData ATYPE>, +} //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- +impl AddressType for Physical {} +impl AddressType for Virtual {} + +impl Address { + /// Create an instance. + pub const fn new(value: usize) -> Self { + Self { + value, + _address_type: PhantomData, + } + } + + /// Align down. + pub const fn align_down(self, alignment: usize) -> Self { + let aligned = common::align_down(self.value, alignment); + + Self { + value: aligned, + _address_type: PhantomData, + } + } + + /// Converts `Address` into an usize. + pub const fn into_usize(self) -> usize { + self.value + } +} + +impl core::ops::Add for Address { + type Output = Self; + + fn add(self, other: usize) -> Self { + Self { + value: self.value + other, + _address_type: PhantomData, + } + } +} + +impl core::ops::Sub for Address { + type Output = Self; + + fn sub(self, other: usize) -> Self { + Self { + value: self.value - other, + _address_type: PhantomData, + } + } +} + /// Zero out an inclusive memory range. /// /// # Safety diff --git a/15_virtual_mem_part2_mmio_remap/src/memory/mmu.rs b/15_virtual_mem_part2_mmio_remap/src/memory/mmu.rs index 9204874a..b0205593 100644 --- a/15_virtual_mem_part2_mmio_remap/src/memory/mmu.rs +++ b/15_virtual_mem_part2_mmio_remap/src/memory/mmu.rs @@ -7,12 +7,16 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/memory/mmu.rs"] mod arch_mmu; -pub use arch_mmu::*; mod mapping_record; +mod translation_table; mod types; -use crate::{bsp, synchronization, warn}; +use crate::{ + bsp, + memory::{Address, Physical, Virtual}, + synchronization, warn, +}; pub use types::*; @@ -24,59 +28,6 @@ pub use types::*; pub mod interface { use super::*; - /// Describes the characteristics of a translation granule. - #[allow(missing_docs)] - pub trait TranslationGranule { - const SIZE: usize; - const MASK: usize = Self::SIZE - 1; - const SHIFT: usize; - } - - /// Translation table operations. - pub trait TranslationTable { - /// Anything that needs to run before any of the other provided functions can be used. - /// - /// # Safety - /// - /// - Implementor must ensure that this function can run only once or is harmless if invoked - /// multiple times. - unsafe fn init(&mut self); - - /// The translation table's base address to be used for programming the MMU. - fn phys_base_address(&self) -> Address; - - /// Map the given physical pages to the given virtual pages. - /// - /// # Safety - /// - /// - Using wrong attributes can cause multiple issues of different nature in the system. - /// - It is not required that the architectural implementation prevents aliasing. That is, - /// mapping to the same physical memory using multiple virtual addresses, which would - /// break Rust's ownership assumptions. This should be protected against in this module - /// (the kernel's generic MMU code). - unsafe fn map_pages_at( - &mut self, - phys_pages: &PageSliceDescriptor, - virt_pages: &PageSliceDescriptor, - attr: &AttributeFields, - ) -> Result<(), &'static str>; - - /// Obtain a free virtual page slice in the MMIO region. - /// - /// The "MMIO region" is a distinct region of the implementor's choice, which allows - /// differentiating MMIO addresses from others. This can speed up debugging efforts. - /// Ideally, those MMIO addresses are also standing out visually so that a human eye can - /// identify them. For example, by allocating them from near the end of the virtual address - /// space. - fn next_mmio_virt_page_slice( - &mut self, - num_pages: usize, - ) -> Result, &'static str>; - - /// Check if a virtual page splice is in the "MMIO region". - fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; - } - /// MMU functions. pub trait MMU { /// Turns on the MMU. @@ -87,16 +38,23 @@ pub mod interface { /// - Changes the HW's global state. unsafe fn enable( &self, - phys_kernel_table_base_addr: Address, + kernel_table_phys_base_addr: Address, ) -> Result<(), &'static str>; } } +/// Describes the characteristics of a translation granule. +pub struct TranslationGranule; + +/// Describes the size of an address space. +pub struct AddressSpaceSize; + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -use interface::{TranslationTable, MMU}; +use interface::MMU; use synchronization::interface::ReadWriteEx; +use translation_table::interface::TranslationTable; /// Map pages in the kernel's translation tables. /// @@ -108,14 +66,14 @@ use synchronization::interface::ReadWriteEx; /// - Does not prevent aliasing. unsafe fn kernel_map_pages_at_unchecked( name: &'static str, - phys_pages: &PageSliceDescriptor, virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, attr: &AttributeFields, ) -> Result<(), &'static str> { arch_mmu::kernel_translation_tables() - .write(|tables| tables.map_pages_at(phys_pages, virt_pages, attr))?; + .write(|tables| tables.map_pages_at(virt_pages, phys_pages, attr))?; - if let Err(x) = mapping_record::kernel_add(name, phys_pages, virt_pages, attr) { + if let Err(x) = mapping_record::kernel_add(name, virt_pages, phys_pages, attr) { warn!("{}", x); } @@ -125,7 +83,44 @@ unsafe fn kernel_map_pages_at_unchecked( //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -use interface::TranslationGranule; + +impl TranslationGranule { + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + + /// The granule's mask. + pub const MASK: usize = Self::SIZE - 1; + + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(GRANULE_SIZE.is_power_of_two()); + + GRANULE_SIZE + } +} + +impl AddressSpaceSize { + /// The address space size. + pub const SIZE: usize = Self::size_checked(); + + /// The address space shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(AS_SIZE.is_power_of_two()); + assert!(arch_mmu::MIN_ADDR_SPACE_SIZE.is_power_of_two()); + assert!(arch_mmu::MAX_ADDR_SPACE_SIZE.is_power_of_two()); + + // Must adhere to architectural restrictions. + assert!(AS_SIZE >= arch_mmu::MIN_ADDR_SPACE_SIZE); + assert!(AS_SIZE <= arch_mmu::MAX_ADDR_SPACE_SIZE); + assert!((AS_SIZE % arch_mmu::AddrSpaceSizeGranule::SIZE) == 0); + + AS_SIZE + } +} /// Raw mapping of virtual to physical pages in the kernel translation tables. /// @@ -134,11 +129,11 @@ use interface::TranslationGranule; /// # Safety /// /// - See `kernel_map_pages_at_unchecked()`. -/// - Does not prevent aliasing. Currently, we have to trust the callers. +/// - Does not prevent aliasing. Currently, the callers must be trusted. pub unsafe fn kernel_map_pages_at( name: &'static str, - phys_pages: &PageSliceDescriptor, virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, attr: &AttributeFields, ) -> Result<(), &'static str> { let is_mmio = arch_mmu::kernel_translation_tables() @@ -147,7 +142,7 @@ pub unsafe fn kernel_map_pages_at( return Err("Attempt to manually map into MMIO region"); } - kernel_map_pages_at_unchecked(name, phys_pages, virt_pages, attr)?; + kernel_map_pages_at_unchecked(name, virt_pages, phys_pages, attr)?; Ok(()) } @@ -179,8 +174,8 @@ pub unsafe fn kernel_map_mmio( kernel_map_pages_at_unchecked( name, - &phys_pages, &virt_pages, + &phys_pages, &AttributeFields { mem_attributes: MemAttributes::Device, acc_perms: AccessPermissions::ReadWrite, @@ -213,38 +208,3 @@ pub unsafe fn kernel_map_binary_and_enable_mmu() -> Result<(), &'static str> { pub fn kernel_print_mappings() { mapping_record::kernel_print() } - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Sanity checks for the kernel TranslationTable implementation. - #[kernel_test] - fn translationtable_implementation_sanity() { - // Need to take care that `tables` fits into the stack. - let mut tables = MinSizeArchTranslationTable::new(); - - unsafe { tables.init() }; - - let x = tables.next_mmio_virt_page_slice(0); - assert!(x.is_err()); - - let x = tables.next_mmio_virt_page_slice(1_0000_0000); - assert!(x.is_err()); - - let x = tables.next_mmio_virt_page_slice(2).unwrap(); - assert_eq!(x.size(), bsp::memory::mmu::KernelGranule::SIZE * 2); - - assert_eq!(tables.is_virt_page_slice_mmio(&x), true); - - assert_eq!( - tables.is_virt_page_slice_mmio(&PageSliceDescriptor::from_addr(Address::new(0), 1)), - false - ); - } -} diff --git a/15_virtual_mem_part2_mmio_remap/src/memory/mmu/mapping_record.rs b/15_virtual_mem_part2_mmio_remap/src/memory/mmu/mapping_record.rs index cd337808..cd46403f 100644 --- a/15_virtual_mem_part2_mmio_remap/src/memory/mmu/mapping_record.rs +++ b/15_virtual_mem_part2_mmio_remap/src/memory/mmu/mapping_record.rs @@ -42,8 +42,8 @@ static KERNEL_MAPPING_RECORD: InitStateLock = impl MappingRecordEntry { pub fn new( name: &'static str, - phys_pages: &PageSliceDescriptor, virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, attr: &AttributeFields, ) -> Self { Self { @@ -97,13 +97,13 @@ impl MappingRecord { pub fn add( &mut self, name: &'static str, - phys_pages: &PageSliceDescriptor, virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, attr: &AttributeFields, ) -> Result<(), &'static str> { let x = self.find_next_free()?; - *x = Some(MappingRecordEntry::new(name, phys_pages, virt_pages, attr)); + *x = Some(MappingRecordEntry::new(name, virt_pages, phys_pages, attr)); Ok(()) } @@ -191,11 +191,11 @@ use synchronization::interface::ReadWriteEx; /// Add an entry to the mapping info record. pub fn kernel_add( name: &'static str, - phys_pages: &PageSliceDescriptor, virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, attr: &AttributeFields, ) -> Result<(), &'static str> { - KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, phys_pages, virt_pages, attr)) + KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_pages, phys_pages, attr)) } pub fn kernel_find_and_insert_mmio_duplicate( diff --git a/15_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs b/15_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs new file mode 100644 index 00000000..7e16e606 --- /dev/null +++ b/15_virtual_mem_part2_mmio_remap/src/memory/mmu/translation_table.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Translation table. + +#[cfg(target_arch = "aarch64")] +#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] +mod arch_translation_table; + +use crate::memory::{ + mmu::{AttributeFields, PageSliceDescriptor}, + Address, Physical, Virtual, +}; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_translation_table::KernelTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Translation table interfaces. +pub mod interface { + use super::*; + + /// Translation table operations. + pub trait TranslationTable { + /// Anything that needs to run before any of the other provided functions can be used. + /// + /// # Safety + /// + /// - Implementor must ensure that this function can run only once or is harmless if invoked + /// multiple times. + unsafe fn init(&mut self); + + /// The translation table's base address to be used for programming the MMU. + fn phys_base_address(&self) -> Address; + + /// Map the given virtual pages to the given physical pages. + /// + /// # Safety + /// + /// - Using wrong attributes can cause multiple issues of different nature in the system. + /// - It is not required that the architectural implementation prevents aliasing. That is, + /// mapping to the same physical memory using multiple virtual addresses, which would + /// break Rust's ownership assumptions. This should be protected against in the kernel's + /// generic MMU code. + unsafe fn map_pages_at( + &mut self, + virt_pages: &PageSliceDescriptor, + phys_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str>; + + /// Obtain a free virtual page slice in the MMIO region. + /// + /// The "MMIO region" is a distinct region of the implementor's choice, which allows + /// differentiating MMIO addresses from others. This can speed up debugging efforts. + /// Ideally, those MMIO addresses are also standing out visually so that a human eye can + /// identify them. For example, by allocating them from near the end of the virtual address + /// space. + fn next_mmio_virt_page_slice( + &mut self, + num_pages: usize, + ) -> Result, &'static str>; + + /// Check if a virtual page splice is in the "MMIO region". + fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::bsp; + use arch_translation_table::MinSizeKernelTranslationTable; + use interface::TranslationTable; + use test_macros::kernel_test; + + /// Sanity checks for the kernel TranslationTable implementation. + #[kernel_test] + fn translationtable_implementation_sanity() { + // Need to take care that `tables` fits into the stack. + let mut tables = MinSizeKernelTranslationTable::new(); + + unsafe { tables.init() }; + + let x = tables.next_mmio_virt_page_slice(0); + assert!(x.is_err()); + + let x = tables.next_mmio_virt_page_slice(1_0000_0000); + assert!(x.is_err()); + + let x = tables.next_mmio_virt_page_slice(2).unwrap(); + assert_eq!(x.size(), bsp::memory::mmu::KernelGranule::SIZE * 2); + + assert_eq!(tables.is_virt_page_slice_mmio(&x), true); + + assert_eq!( + tables.is_virt_page_slice_mmio(&PageSliceDescriptor::from_addr(Address::new(0), 1)), + false + ); + } +} diff --git a/15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs b/15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs index cea7bb39..59431588 100644 --- a/15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs +++ b/15_virtual_mem_part2_mmio_remap/src/memory/mmu/types.rs @@ -2,33 +2,17 @@ // // Copyright (c) 2020-2021 Andre Richter -//! Memory Management Unit Types. +//! Memory Management Unit types. -use crate::{bsp, common}; +use crate::{ + bsp, common, + memory::{Address, AddressType, Physical, Virtual}, +}; use core::{convert::From, marker::PhantomData, ops::RangeInclusive}; //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- -use super::interface::TranslationGranule; - -/// Metadata trait for marking the type of an address. -pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} - -/// Zero-sized type to mark a physical address. -#[derive(Copy, Clone, PartialOrd, PartialEq)] -pub enum Physical {} - -/// Zero-sized type to mark a virtual address. -#[derive(Copy, Clone, PartialOrd, PartialEq)] -pub enum Virtual {} - -/// Generic address type. -#[derive(Copy, Clone, PartialOrd, PartialEq)] -pub struct Address { - value: usize, - _address_type: PhantomData, -} /// Generic page type. #[repr(C)] @@ -80,60 +64,6 @@ pub struct MMIODescriptor { // Public Code //-------------------------------------------------------------------------------------------------- -impl AddressType for Physical {} -impl AddressType for Virtual {} - -//------------------------------------------------------------------------------ -// Address -//------------------------------------------------------------------------------ - -impl Address { - /// Create an instance. - pub const fn new(value: usize) -> Self { - Self { - value, - _address_type: PhantomData, - } - } - - /// Align down. - pub const fn align_down(self, alignment: usize) -> Self { - let aligned = common::align_down(self.value, alignment); - - Self { - value: aligned, - _address_type: PhantomData, - } - } - - /// Converts `Address` into an usize. - pub const fn into_usize(self) -> usize { - self.value - } -} - -impl core::ops::Add for Address { - type Output = Self; - - fn add(self, other: usize) -> Self { - Self { - value: self.value + other, - _address_type: PhantomData, - } - } -} - -impl core::ops::Sub for Address { - type Output = Self; - - fn sub(self, other: usize) -> Self { - Self { - value: self.value - other, - _address_type: PhantomData, - } - } -} - //------------------------------------------------------------------------------ // Page //------------------------------------------------------------------------------ diff --git a/15_virtual_mem_part2_mmio_remap/src/print.rs b/15_virtual_mem_part2_mmio_remap/src/print.rs index 1ea96b6a..5a563811 100644 --- a/15_virtual_mem_part2_mmio_remap/src/print.rs +++ b/15_virtual_mem_part2_mmio_remap/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/15_virtual_mem_part2_mmio_remap/src/state.rs b/15_virtual_mem_part2_mmio_remap/src/state.rs index d08e67d6..c94d04c8 100644 --- a/15_virtual_mem_part2_mmio_remap/src/state.rs +++ b/15_virtual_mem_part2_mmio_remap/src/state.rs @@ -69,7 +69,7 @@ impl StateManager { } } - /// Return if the kernel is still in an init state. + /// Return if the kernel is init state. pub fn is_init(&self) -> bool { self.state() == State::Init } diff --git a/15_virtual_mem_part2_mmio_remap/src/time.rs b/15_virtual_mem_part2_mmio_remap/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/15_virtual_mem_part2_mmio_remap/src/time.rs +++ b/15_virtual_mem_part2_mmio_remap/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/X1_JTAG_boot/Cargo.lock b/X1_JTAG_boot/Cargo.lock index 102e5730..e87b55af 100644 --- a/X1_JTAG_boot/Cargo.lock +++ b/X1_JTAG_boot/Cargo.lock @@ -2,9 +2,9 @@ # It is not intended for manual editing. [[package]] name = "cortex-a" -version = "4.1.0" +version = "5.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a389514c1229e12a03c0e8de8b357671b71e1d1bdab3a4f5b591abc94169cba8" +checksum = "d33a61602f83c3fff3e092967ad4d5d820a9fea7bc418e119616872085fe731d" dependencies = [ "register", ] @@ -19,15 +19,15 @@ dependencies = [ [[package]] name = "register" -version = "0.5.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaba5b0e477d21f61a57504bb5cef4a1e86de30300b457d38971c1cfc98b815" +checksum = "f4a247de29ab7cc8f5006cfe775c4a81c704f9914c5e2a79696862e643135433" dependencies = [ "tock-registers", ] [[package]] name = "tock-registers" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70323afdb8082186c0986da0e10f6e4ed103d681c921c00597e98d9806dac20f" +checksum = "f521a79accce68c417c9c77ce22108056b626126da1932f7e2e9b5bbffee0cea" diff --git a/X1_JTAG_boot/Cargo.toml b/X1_JTAG_boot/Cargo.toml index 8a1a364e..b8906da1 100644 --- a/X1_JTAG_boot/Cargo.toml +++ b/X1_JTAG_boot/Cargo.toml @@ -20,9 +20,9 @@ bsp_rpi4 = ["register"] [dependencies] # Optional dependencies -register = { version = "0.5.x", optional = true } +register = { version = "1.x.x", optional = true } # Platform specific dependencies [target.'cfg(target_arch = "aarch64")'.dependencies] -cortex-a = { version = "4.x.x" } +cortex-a = { version = "5.x.x" } diff --git a/X1_JTAG_boot/Makefile b/X1_JTAG_boot/Makefile index 071167b9..27944c93 100644 --- a/X1_JTAG_boot/Makefile +++ b/X1_JTAG_boot/Makefile @@ -40,8 +40,9 @@ export LINKER_FILE RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) COMPILER_ARGS = --target=$(TARGET) \ - --features bsp_$(BSP) \ + --features $(FEATURES) \ --release RUSTC_CMD = cargo rustc $(COMPILER_ARGS) diff --git a/X1_JTAG_boot/jtag_boot_rpi3.img b/X1_JTAG_boot/jtag_boot_rpi3.img index 42480957..6fe15429 100755 Binary files a/X1_JTAG_boot/jtag_boot_rpi3.img and b/X1_JTAG_boot/jtag_boot_rpi3.img differ diff --git a/X1_JTAG_boot/jtag_boot_rpi4.img b/X1_JTAG_boot/jtag_boot_rpi4.img index 0813b2ba..d5573467 100755 Binary files a/X1_JTAG_boot/jtag_boot_rpi4.img and b/X1_JTAG_boot/jtag_boot_rpi4.img differ diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu.rs b/X1_JTAG_boot/src/_arch/aarch64/cpu.rs index 6072751d..d3b07461 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu.rs +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu.rs @@ -3,36 +3,15 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural processor code. - -use crate::{bsp, cpu}; -use cortex_a::{asm, regs::*}; - -//-------------------------------------------------------------------------------------------------- -// Boot Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - // Expect the boot core to start in EL2. - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - wait_forever() - } -} +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; //-------------------------------------------------------------------------------------------------- // Public Code diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..549b5927 --- /dev/null +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{bsp, cpu}; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function where it is expected by the target machine. +/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is +/// actually set (`SP.set()`). +#[no_mangle] +pub unsafe fn _start() -> ! { + use crate::runtime_init; + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + SP.set(bsp::memory::boot_core_stack_end() as u64); + runtime_init::runtime_init() + } else { + // If not core0, infinitely wait for events. + cpu::wait_forever() + } +} diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs b/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs index c80f7e78..b9fdd0f7 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp use cortex_a::regs::*; diff --git a/X1_JTAG_boot/src/_arch/aarch64/time.rs b/X1_JTAG_boot/src/_arch/aarch64/time.rs index 7f1bc696..3a766009 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/time.rs +++ b/X1_JTAG_boot/src/_arch/aarch64/time.rs @@ -3,6 +3,13 @@ // Copyright (c) 2018-2021 Andre Richter //! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time use crate::{time, warn}; use core::time::Duration; @@ -55,7 +62,7 @@ impl time::interface::TimeManager for GenericTimer { } // Calculate the register compare value. - let frq = CNTFRQ_EL0.get() as u64; + let frq = CNTFRQ_EL0.get(); let x = match frq.checked_mul(duration.as_nanos() as u64) { None => { warn!("Spin duration too long, skipping"); diff --git a/X1_JTAG_boot/src/bsp.rs b/X1_JTAG_boot/src/bsp.rs index 25750249..c558922f 100644 --- a/X1_JTAG_boot/src/bsp.rs +++ b/X1_JTAG_boot/src/bsp.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Conditional re-exporting of Board Support Packages. +//! Conditional reexporting of Board Support Packages. mod device_driver; diff --git a/X1_JTAG_boot/src/console.rs b/X1_JTAG_boot/src/console.rs index 3552823c..c3154ba2 100644 --- a/X1_JTAG_boot/src/console.rs +++ b/X1_JTAG_boot/src/console.rs @@ -20,8 +20,7 @@ pub mod interface { /// Write a Rust format string. fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; - /// Block execution until the last buffered character has been physically put on the TX - /// wire. + /// Block until the last buffered character has been physically put on the TX wire. fn flush(&self); } diff --git a/X1_JTAG_boot/src/cpu.rs b/X1_JTAG_boot/src/cpu.rs index c9e5af72..3834f183 100644 --- a/X1_JTAG_boot/src/cpu.rs +++ b/X1_JTAG_boot/src/cpu.rs @@ -7,6 +7,12 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/cpu.rs"] mod arch_cpu; -pub use arch_cpu::*; + +mod boot; pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; diff --git a/X1_JTAG_boot/src/cpu/boot.rs b/X1_JTAG_boot/src/cpu/boot.rs new file mode 100644 index 00000000..1dc5c180 --- /dev/null +++ b/X1_JTAG_boot/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/X1_JTAG_boot/src/cpu/smp.rs b/X1_JTAG_boot/src/cpu/smp.rs index 90ecbdf3..38230ce1 100644 --- a/X1_JTAG_boot/src/cpu/smp.rs +++ b/X1_JTAG_boot/src/cpu/smp.rs @@ -6,5 +6,9 @@ #[cfg(target_arch = "aarch64")] #[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_cpu_smp; -pub use arch_cpu_smp::*; +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/X1_JTAG_boot/src/main.rs b/X1_JTAG_boot/src/main.rs index 340acd4e..fa5ba64a 100644 --- a/X1_JTAG_boot/src/main.rs +++ b/X1_JTAG_boot/src/main.rs @@ -7,16 +7,6 @@ //! The `kernel` binary. //! -//! # TL;DR - Overview of important Kernel entities -//! -//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. -//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. -//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. -//! -//! [console interface]: ../libkernel/console/interface/index.html -//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html -//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html -//! //! # Code organization and architecture //! //! The code is divided into different *modules*, each representing a typical **subsystem** of the @@ -30,15 +20,22 @@ //! `src/_arch`, for example, `src/_arch/aarch64`. //! //! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go -//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in -//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's -//! module organization. That means a public function `foo()` defined in -//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. //! -//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. -//! Rather, it's contents are conditionally pulled into respective files using the `#[path = -//! "_arch/xxx/yyy.rs"]` attribute. +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. //! //! ## BSP code //! @@ -47,9 +44,8 @@ //! or instances of drivers for devices that are featured on the respective board. //! //! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means -//! whatever is provided must be called starting from the `bsp` namespace, e.g. -//! `bsp::driver::driver_manager()`. +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. //! //! ## Kernel interfaces //! @@ -101,6 +97,14 @@ //! //! - `crate::memory::*` //! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! +//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] @@ -109,9 +113,6 @@ #![no_main] #![no_std] -// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls -// `runtime_init()`, which jumps to `kernel_init()`. - mod bsp; mod console; mod cpu; diff --git a/X1_JTAG_boot/src/print.rs b/X1_JTAG_boot/src/print.rs index 1ea96b6a..5a563811 100644 --- a/X1_JTAG_boot/src/print.rs +++ b/X1_JTAG_boot/src/print.rs @@ -2,7 +2,7 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Printing facilities. +//! Printing. use crate::{bsp, console}; use core::fmt; diff --git a/X1_JTAG_boot/src/time.rs b/X1_JTAG_boot/src/time.rs index 4f2f4e38..953b6f0d 100644 --- a/X1_JTAG_boot/src/time.rs +++ b/X1_JTAG_boot/src/time.rs @@ -7,7 +7,11 @@ #[cfg(target_arch = "aarch64")] #[path = "_arch/aarch64/time.rs"] mod arch_time; -pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; //-------------------------------------------------------------------------------------------------- // Public Definitions @@ -18,8 +22,6 @@ pub mod interface { use core::time::Duration; /// Time management functions. - /// - /// The `BSP` is supposed to supply one global instance. pub trait TimeManager { /// The timer's resolution. fn resolution(&self) -> Duration; diff --git a/rust-toolchain b/rust-toolchain index b50e4d14..26668a71 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2021-01-01" +channel = "nightly-2021-01-08" components = ["llvm-tools-preview"] targets = ["aarch64-unknown-none-softfloat"]