diff --git a/02_runtime_init/README.md b/02_runtime_init/README.md index 5cba10fb..3dc0b3d3 100644 --- a/02_runtime_init/README.md +++ b/02_runtime_init/README.md @@ -2,8 +2,9 @@ ## tl;dr -- We extend `boot.s` to call into Rust code for the first time. There, we zero the [bss] section - before execution is halted with a call to `panic()`. +- We extend `boot.s` to call into Rust code for the first time. Before the jump + to Rust happens, a bit of runtime init work is done. +- The Rust code being called just halts execution with a call to `panic!()`. - Check out `make qemu` again to see the additional code run. ## Notable additions @@ -12,11 +13,11 @@ - New sections: `.rodata`, `.got`, `.data`, `.bss`. - A dedicated place for linking boot-time arguments that need to be read by `_start()`. - `_start()` in `_arch/__arch_name__/cpu/boot.s`: - 1. Halt core if core != core0. - 1. Set up the `stack pointer`. - 1. Jump to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`. -- `runtime_init()` in `runtime_init.rs`: - - Zeros the `.bss` section. + 1. Halts core if core != core0. + 1. Initializes the `DRAM` by zeroing the [bss] section. + 1. Sets up the `stack pointer`. + 1. Jumps to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`. +- `_start_rust()`: - Calls `kernel_init()`, which calls `panic!()`, which eventually halts core0 as well. - The library now uses the [cortex-a] crate, which provides zero-overhead abstractions and wraps `unsafe` parts when dealing with the CPU's resources. @@ -64,12 +65,8 @@ diff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arch/aarch64/cpu/boot.rs --- 01_wait_forever/src/_arch/aarch64/cpu/boot.rs +++ 02_runtime_init/src/_arch/aarch64/cpu/boot.rs -@@ -11,5 +11,23 @@ - //! - //! crate::cpu::boot::arch_boot +@@ -13,3 +13,15 @@ -+use crate::runtime_init; -+ // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); + @@ -80,13 +77,9 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arc +/// The Rust entry of the `kernel` binary. +/// +/// The function is called from the assembly `_start` function. -+/// -+/// # Safety -+/// -+/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +#[no_mangle] +pub unsafe fn _start_rust() -> ! { -+ runtime_init::runtime_init() ++ crate::kernel_init() +} diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch/aarch64/cpu/boot.s @@ -117,7 +110,7 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch // Public Code //-------------------------------------------------------------------------------------------------- .section .text._start -@@ -11,6 +29,22 @@ +@@ -11,9 +29,38 @@ // fn _start() //------------------------------------------------------------------------------ _start: @@ -126,10 +119,22 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 -+ b.ne 1f ++ b.ne parking_loop ++ ++ // If execution reaches here, it is the boot core. ++ ++ // Initialize DRAM. ++ ADR_REL x0, __bss_start ++ ADR_REL x1, __bss_end_exclusive + -+ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. ++bss_init_loop: ++ cmp x0, x1 ++ b.eq prepare_rust ++ stp xzr, xzr, [x0], #16 ++ b bss_init_loop + ++ // Prepare the jump to Rust code. ++prepare_rust: + // Set the stack pointer. + ADR_REL x0, __boot_core_stack_end_exclusive + mov sp, x0 @@ -138,8 +143,14 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch + b _start_rust + // Infinitely wait for events (aka "park the core"). - 1: wfe - b 1b +-1: wfe +- b 1b ++parking_loop: ++ wfe ++ b parking_loop + + .size _start, . - _start + .type _start, function diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs --- 01_wait_forever/src/_arch/aarch64/cpu.rs @@ -194,7 +205,7 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/cpu.rs 02_runtime_init/src/bsp/ras diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld --- 01_wait_forever/src/bsp/raspberrypi/link.ld +++ 02_runtime_init/src/bsp/raspberrypi/link.ld -@@ -11,17 +11,45 @@ +@@ -11,17 +11,43 @@ PHDRS { segment_rx PT_LOAD FLAGS(5); /* 5 == RX */ @@ -230,70 +241,25 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra + ***********************************************************************************************/ + .data : { *(.data*) } :segment_rw + -+ /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -+ .bss : ALIGN(8) ++ /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ ++ .bss : ALIGN(16) + { + __bss_start = .; + *(.bss*); -+ . = ALIGN(8); -+ -+ . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ -+ __bss_end_inclusive = . - 8; ++ . = ALIGN(16); ++ __bss_end_exclusive = .; + } :NONE } -diff -uNr 01_wait_forever/src/bsp/raspberrypi/memory.rs 02_runtime_init/src/bsp/raspberrypi/memory.rs ---- 01_wait_forever/src/bsp/raspberrypi/memory.rs -+++ 02_runtime_init/src/bsp/raspberrypi/memory.rs -@@ -0,0 +1,37 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! BSP Memory Management. -+ -+use core::{cell::UnsafeCell, ops::RangeInclusive}; -+ -+//-------------------------------------------------------------------------------------------------- -+// Private Definitions -+//-------------------------------------------------------------------------------------------------- -+ -+// Symbols from the linker script. -+extern "Rust" { -+ static __bss_start: UnsafeCell; -+ static __bss_end_inclusive: UnsafeCell; -+} -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Return the inclusive range spanning the .bss section. -+/// -+/// # Safety -+/// -+/// - Values are provided by the linker script and must be trusted as-is. -+/// - The linker-provided addresses must be u64 aligned. -+pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { -+ let range; -+ unsafe { -+ range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); -+ } -+ assert!(!range.is_empty()); -+ -+ range -+} - diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs --- 01_wait_forever/src/bsp/raspberrypi.rs +++ 02_runtime_init/src/bsp/raspberrypi.rs -@@ -4,4 +4,5 @@ +@@ -4,4 +4,4 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. -// Coming soon. +pub mod cpu; -+pub mod memory; diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs --- 01_wait_forever/src/cpu.rs @@ -316,24 +282,19 @@ diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs --- 01_wait_forever/src/main.rs +++ 02_runtime_init/src/main.rs -@@ -102,14 +102,25 @@ +@@ -102,8 +102,8 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -+//! -+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html ++//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. -#![feature(asm)] #![feature(global_asm)] #![no_main] #![no_std] - - mod bsp; +@@ -112,4 +112,11 @@ mod cpu; -+mod memory; mod panic_wait; -+mod runtime_init; -// Kernel code coming next tutorial. +/// Early init code. @@ -345,41 +306,6 @@ diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs + panic!() +} -diff -uNr 01_wait_forever/src/memory.rs 02_runtime_init/src/memory.rs ---- 01_wait_forever/src/memory.rs -+++ 02_runtime_init/src/memory.rs -@@ -0,0 +1,30 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Memory Management. -+ -+use core::ops::RangeInclusive; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Zero out an inclusive memory range. -+/// -+/// # Safety -+/// -+/// - `range.start` and `range.end` must be valid. -+/// - `range.start` and `range.end` must be `T` aligned. -+pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -+where -+ T: From, -+{ -+ let mut ptr = *range.start(); -+ let end_inclusive = *range.end(); -+ -+ while ptr <= end_inclusive { -+ core::ptr::write_volatile(ptr, T::from(0)); -+ ptr = ptr.offset(1); -+ } -+} - diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs --- 01_wait_forever/src/panic_wait.rs +++ 02_runtime_init/src/panic_wait.rs @@ -396,46 +322,4 @@ diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs + cpu::wait_forever() } -diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.rs ---- 01_wait_forever/src/runtime_init.rs -+++ 02_runtime_init/src/runtime_init.rs -@@ -0,0 +1,37 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Rust runtime initialization code. -+ -+use crate::{bsp, memory}; -+ -+//-------------------------------------------------------------------------------------------------- -+// Private Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Zero out the .bss section. -+/// -+/// # Safety -+/// -+/// - Must only be called pre `kernel_init()`. -+#[inline(always)] -+unsafe fn zero_bss() { -+ memory::zero_volatile(bsp::memory::bss_range_inclusive()); -+} -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -+/// init code. -+/// -+/// # Safety -+/// -+/// - Only a single core must be active and running this function. -+pub unsafe fn runtime_init() -> ! { -+ zero_bss(); -+ -+ crate::kernel_init() -+} - ``` diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.s b/02_runtime_init/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu/boot.s +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/02_runtime_init/src/bsp/raspberrypi.rs b/02_runtime_init/src/bsp/raspberrypi.rs index 10535d7b..8fed4b83 100644 --- a/02_runtime_init/src/bsp/raspberrypi.rs +++ b/02_runtime_init/src/bsp/raspberrypi.rs @@ -5,4 +5,3 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. pub mod cpu; -pub mod memory; diff --git a/02_runtime_init/src/bsp/raspberrypi/link.ld b/02_runtime_init/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/02_runtime_init/src/bsp/raspberrypi/link.ld +++ b/02_runtime_init/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/02_runtime_init/src/bsp/raspberrypi/memory.rs b/02_runtime_init/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index 5c6b1ea5..00000000 --- a/02_runtime_init/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/02_runtime_init/src/main.rs b/02_runtime_init/src/main.rs index faad6b09..cf15402f 100644 --- a/02_runtime_init/src/main.rs +++ b/02_runtime_init/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![feature(global_asm)] #![no_main] @@ -112,9 +110,7 @@ mod bsp; mod cpu; -mod memory; mod panic_wait; -mod runtime_init; /// Early init code. /// diff --git a/02_runtime_init/src/memory.rs b/02_runtime_init/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/02_runtime_init/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/02_runtime_init/src/runtime_init.rs b/02_runtime_init/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/02_runtime_init/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/03_hacky_hello_world/README.md b/03_hacky_hello_world/README.md index 6958b8bc..01bbf54c 100644 --- a/03_hacky_hello_world/README.md +++ b/03_hacky_hello_world/README.md @@ -117,13 +117,12 @@ diff -uNr 02_runtime_init/src/bsp/raspberrypi/console.rs 03_hacky_hello_world/sr diff -uNr 02_runtime_init/src/bsp/raspberrypi.rs 03_hacky_hello_world/src/bsp/raspberrypi.rs --- 02_runtime_init/src/bsp/raspberrypi.rs +++ 03_hacky_hello_world/src/bsp/raspberrypi.rs -@@ -4,5 +4,6 @@ +@@ -4,4 +4,5 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. +pub mod console; pub mod cpu; - pub mod memory; diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs --- 02_runtime_init/src/console.rs @@ -152,9 +151,9 @@ diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs --- 02_runtime_init/src/main.rs +++ 03_hacky_hello_world/src/main.rs -@@ -106,14 +106,18 @@ - //! - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -104,13 +104,17 @@ + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. +#![feature(format_args_nl)] #![feature(global_asm)] @@ -165,13 +164,12 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs mod bsp; +mod console; mod cpu; - mod memory; mod panic_wait; +mod print; - mod runtime_init; /// Early init code. -@@ -122,5 +126,7 @@ + /// +@@ -118,5 +122,7 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/03_hacky_hello_world/src/bsp/raspberrypi.rs b/03_hacky_hello_world/src/bsp/raspberrypi.rs index fdb7551a..73863b17 100644 --- a/03_hacky_hello_world/src/bsp/raspberrypi.rs +++ b/03_hacky_hello_world/src/bsp/raspberrypi.rs @@ -6,4 +6,3 @@ pub mod console; pub mod cpu; -pub mod memory; diff --git a/03_hacky_hello_world/src/bsp/raspberrypi/link.ld b/03_hacky_hello_world/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/03_hacky_hello_world/src/bsp/raspberrypi/link.ld +++ b/03_hacky_hello_world/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/03_hacky_hello_world/src/bsp/raspberrypi/memory.rs b/03_hacky_hello_world/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index 5c6b1ea5..00000000 --- a/03_hacky_hello_world/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/03_hacky_hello_world/src/main.rs b/03_hacky_hello_world/src/main.rs index bc290ddd..de9a0576 100644 --- a/03_hacky_hello_world/src/main.rs +++ b/03_hacky_hello_world/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![feature(format_args_nl)] #![feature(global_asm)] @@ -115,10 +113,8 @@ mod bsp; mod console; mod cpu; -mod memory; mod panic_wait; mod print; -mod runtime_init; /// Early init code. /// diff --git a/03_hacky_hello_world/src/memory.rs b/03_hacky_hello_world/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/03_hacky_hello_world/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/03_hacky_hello_world/src/runtime_init.rs b/03_hacky_hello_world/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/03_hacky_hello_world/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/04_safe_globals/README.md b/04_safe_globals/README.md index 272edfce..6b7f52c4 100644 --- a/04_safe_globals/README.md +++ b/04_safe_globals/README.md @@ -224,7 +224,7 @@ diff -uNr 03_hacky_hello_world/src/console.rs 04_safe_globals/src/console.rs diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs --- 03_hacky_hello_world/src/main.rs +++ 04_safe_globals/src/main.rs -@@ -109,6 +109,7 @@ +@@ -107,6 +107,7 @@ #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] @@ -232,15 +232,15 @@ diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs #![no_main] #![no_std] -@@ -119,6 +120,7 @@ +@@ -115,6 +116,7 @@ + mod cpu; mod panic_wait; mod print; - mod runtime_init; +mod synchronization; /// Early init code. /// -@@ -126,7 +128,15 @@ +@@ -122,7 +124,15 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/04_safe_globals/src/_arch/aarch64/cpu/boot.rs b/04_safe_globals/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/04_safe_globals/src/_arch/aarch64/cpu/boot.rs +++ b/04_safe_globals/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/04_safe_globals/src/_arch/aarch64/cpu/boot.s b/04_safe_globals/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/04_safe_globals/src/_arch/aarch64/cpu/boot.s +++ b/04_safe_globals/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/04_safe_globals/src/bsp/raspberrypi.rs b/04_safe_globals/src/bsp/raspberrypi.rs index fdb7551a..73863b17 100644 --- a/04_safe_globals/src/bsp/raspberrypi.rs +++ b/04_safe_globals/src/bsp/raspberrypi.rs @@ -6,4 +6,3 @@ pub mod console; pub mod cpu; -pub mod memory; diff --git a/04_safe_globals/src/bsp/raspberrypi/link.ld b/04_safe_globals/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/04_safe_globals/src/bsp/raspberrypi/link.ld +++ b/04_safe_globals/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/04_safe_globals/src/bsp/raspberrypi/memory.rs b/04_safe_globals/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index 5c6b1ea5..00000000 --- a/04_safe_globals/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/04_safe_globals/src/main.rs b/04_safe_globals/src/main.rs index 48687ace..82262ea1 100644 --- a/04_safe_globals/src/main.rs +++ b/04_safe_globals/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![feature(format_args_nl)] #![feature(global_asm)] @@ -116,10 +114,8 @@ mod bsp; mod console; mod cpu; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; /// Early init code. diff --git a/04_safe_globals/src/memory.rs b/04_safe_globals/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/04_safe_globals/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/04_safe_globals/src/runtime_init.rs b/04_safe_globals/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/04_safe_globals/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/05_drivers_gpio_uart/README.md b/05_drivers_gpio_uart/README.md index b5895cd7..4125583e 100644 --- a/05_drivers_gpio_uart/README.md +++ b/05_drivers_gpio_uart/README.md @@ -1120,10 +1120,14 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/driver.rs 05_drivers_gpio_uart/src diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs --- 04_safe_globals/src/bsp/raspberrypi/memory.rs +++ 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs -@@ -17,6 +17,38 @@ - } - - //-------------------------------------------------------------------------------------------------- +@@ -0,0 +1,37 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! BSP Memory Management. ++ ++//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + @@ -1154,21 +1158,16 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src + pub const PL011_UART_START: usize = START + UART_OFFSET; + } +} -+ -+//-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - diff -uNr 04_safe_globals/src/bsp/raspberrypi.rs 05_drivers_gpio_uart/src/bsp/raspberrypi.rs --- 04_safe_globals/src/bsp/raspberrypi.rs +++ 05_drivers_gpio_uart/src/bsp/raspberrypi.rs -@@ -6,4 +6,33 @@ +@@ -6,3 +6,33 @@ pub mod console; pub mod cpu; +pub mod driver; - pub mod memory; ++pub mod memory; + +//-------------------------------------------------------------------------------------------------- +// Global instances @@ -1321,24 +1320,24 @@ diff -uNr 04_safe_globals/src/driver.rs 05_drivers_gpio_uart/src/driver.rs diff -uNr 04_safe_globals/src/main.rs 05_drivers_gpio_uart/src/main.rs --- 04_safe_globals/src/main.rs +++ 05_drivers_gpio_uart/src/main.rs -@@ -106,6 +106,8 @@ - //! - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -104,6 +104,8 @@ + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. +#![allow(clippy::upper_case_acronyms)] +#![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] -@@ -116,6 +118,7 @@ +@@ -114,6 +116,7 @@ mod bsp; mod console; mod cpu; +mod driver; - mod memory; mod panic_wait; mod print; -@@ -127,16 +130,54 @@ + mod synchronization; +@@ -123,16 +126,54 @@ /// # Safety /// /// - Only a single core must be active and running this function. diff --git a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs +++ b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s +++ b/05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld b/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld +++ b/05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs b/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs index c6d65e36..ef397a6f 100644 --- a/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs +++ b/05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/05_drivers_gpio_uart/src/main.rs b/05_drivers_gpio_uart/src/main.rs index d7291911..0261a136 100644 --- a/05_drivers_gpio_uart/src/main.rs +++ b/05_drivers_gpio_uart/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; /// Early init code. diff --git a/05_drivers_gpio_uart/src/memory.rs b/05_drivers_gpio_uart/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/05_drivers_gpio_uart/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/05_drivers_gpio_uart/src/runtime_init.rs b/05_drivers_gpio_uart/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/05_drivers_gpio_uart/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/06_uart_chainloader/README.md b/06_uart_chainloader/README.md index 44fc1028..18741f04 100644 --- a/06_uart_chainloader/README.md +++ b/06_uart_chainloader/README.md @@ -232,26 +232,36 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/ .equ _core_id_mask, 0b11 //-------------------------------------------------------------------------------------------------- -@@ -34,20 +45,31 @@ - and x1, x1, _core_id_mask - ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs - cmp x1, x2 -- b.ne 1f -+ b.ne 2f -+ -+ // If execution reaches here, it is the boot core. +@@ -39,23 +50,35 @@ + // If execution reaches here, it is the boot core. + + // Initialize DRAM. +- ADR_REL x0, __bss_start +- ADR_REL x1, __bss_end_exclusive ++ ADR_ABS x0, __bss_start ++ ADR_ABS x1, __bss_end_exclusive + + bss_init_loop: + cmp x0, x1 +- b.eq prepare_rust ++ b.eq relocate_binary + stp xzr, xzr, [x0], #16 + b bss_init_loop -- // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // Next, relocate the binary. ++relocate_binary: + ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to. + ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to. + ADR_ABS x2, __binary_nonzero_end_exclusive + -+1: ldr x3, [x0], #8 ++copy_loop: ++ ldr x3, [x0], #8 + str x3, [x1], #8 + cmp x1, x2 -+ b.lo 1b - ++ b.lo copy_loop ++ + // Prepare the jump to Rust code. +-prepare_rust: // Set the stack pointer. - ADR_REL x0, __boot_core_stack_end_exclusive + ADR_ABS x0, __boot_core_stack_end_exclusive @@ -264,13 +274,7 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/ + br x1 // Infinitely wait for events (aka "park the core"). --1: wfe -- b 1b -+2: wfe -+ b 2b - - .size _start, . - _start - .type _start, function + parking_loop: diff -uNr 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs --- 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -350,7 +354,7 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s .text : { KEEP(*(.text._start)) -@@ -42,8 +44,12 @@ +@@ -42,6 +44,10 @@ ***********************************************************************************************/ .data : { *(.data*) } :segment_rw @@ -358,17 +362,14 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s + . = ALIGN(8); + __binary_nonzero_end_exclusive = .; + - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -- .bss : ALIGN(8) -+ .bss : + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { - __bss_start = .; - *(.bss*); diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader/src/bsp/raspberrypi/memory.rs --- 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs +++ 06_uart_chainloader/src/bsp/raspberrypi/memory.rs -@@ -23,9 +23,10 @@ +@@ -11,9 +11,10 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -381,34 +382,33 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader /// Physical devices. #[cfg(feature = "bsp_rpi3")] -@@ -52,7 +53,13 @@ - // Public Code - //-------------------------------------------------------------------------------------------------- - --/// Return the inclusive range spanning the .bss section. +@@ -35,3 +36,13 @@ + pub const PL011_UART_START: usize = START + UART_OFFSET; + } + } ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ +/// The address on which the Raspberry firmware loads every binary by default. +#[inline(always)] +pub fn board_default_load_addr() -> *const u64 { + map::BOARD_DEFAULT_LOAD_ADDRESS as _ +} -+ -+/// Return the inclusive range spanning the relocated .bss section. - /// - /// # Safety - /// diff -uNr 05_drivers_gpio_uart/src/main.rs 06_uart_chainloader/src/main.rs --- 05_drivers_gpio_uart/src/main.rs +++ 06_uart_chainloader/src/main.rs -@@ -107,6 +107,7 @@ - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -105,6 +105,7 @@ + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] +#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(global_asm)] -@@ -146,38 +147,56 @@ +@@ -142,38 +143,56 @@ kernel_main() } diff --git a/06_uart_chainloader/demo_payload_rpi3.img b/06_uart_chainloader/demo_payload_rpi3.img index a8d7f680..4e640c63 100755 Binary files a/06_uart_chainloader/demo_payload_rpi3.img and b/06_uart_chainloader/demo_payload_rpi3.img differ diff --git a/06_uart_chainloader/demo_payload_rpi4.img b/06_uart_chainloader/demo_payload_rpi4.img index 2d181336..7e017a42 100755 Binary files a/06_uart_chainloader/demo_payload_rpi4.img and b/06_uart_chainloader/demo_payload_rpi4.img differ diff --git a/06_uart_chainloader/src/_arch/aarch64/cpu/boot.rs b/06_uart_chainloader/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/06_uart_chainloader/src/_arch/aarch64/cpu/boot.rs +++ b/06_uart_chainloader/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s b/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s index 10aebb34..c4ae4219 100644 --- a/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s +++ b/06_uart_chainloader/src/_arch/aarch64/cpu/boot.s @@ -45,20 +45,33 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 2f + b.ne parking_loop // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_ABS x0, __bss_start + ADR_ABS x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq relocate_binary + stp xzr, xzr, [x0], #16 + b bss_init_loop + // Next, relocate the binary. +relocate_binary: ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to. ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to. ADR_ABS x2, __binary_nonzero_end_exclusive -1: ldr x3, [x0], #8 +copy_loop: + ldr x3, [x0], #8 str x3, [x1], #8 cmp x1, x2 - b.lo 1b + b.lo copy_loop + // Prepare the jump to Rust code. // Set the stack pointer. ADR_ABS x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -68,8 +81,9 @@ _start: br x1 // Infinitely wait for events (aka "park the core"). -2: wfe - b 2b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/06_uart_chainloader/src/bsp/raspberrypi/link.ld b/06_uart_chainloader/src/bsp/raspberrypi/link.ld index 324172b1..40added4 100644 --- a/06_uart_chainloader/src/bsp/raspberrypi/link.ld +++ b/06_uart_chainloader/src/bsp/raspberrypi/link.ld @@ -48,14 +48,12 @@ SECTIONS . = ALIGN(8); __binary_nonzero_end_exclusive = .; - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/06_uart_chainloader/src/bsp/raspberrypi/memory.rs b/06_uart_chainloader/src/bsp/raspberrypi/memory.rs index 9a66cc8a..852b8905 100644 --- a/06_uart_chainloader/src/bsp/raspberrypi/memory.rs +++ b/06_uart_chainloader/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -58,19 +46,3 @@ pub(super) mod map { pub fn board_default_load_addr() -> *const u64 { map::BOARD_DEFAULT_LOAD_ADDRESS as _ } - -/// Return the inclusive range spanning the relocated .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/06_uart_chainloader/src/main.rs b/06_uart_chainloader/src/main.rs index aed10f7a..5c45cb5f 100644 --- a/06_uart_chainloader/src/main.rs +++ b/06_uart_chainloader/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(asm)] @@ -120,10 +118,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; /// Early init code. diff --git a/06_uart_chainloader/src/memory.rs b/06_uart_chainloader/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/06_uart_chainloader/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/06_uart_chainloader/src/runtime_init.rs b/06_uart_chainloader/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/06_uart_chainloader/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/07_timestamps/README.md b/07_timestamps/README.md index 88b82908..25d08f84 100644 --- a/07_timestamps/README.md +++ b/07_timestamps/README.md @@ -154,26 +154,36 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar .equ _core_id_mask, 0b11 //-------------------------------------------------------------------------------------------------- -@@ -45,31 +34,20 @@ - and x1, x1, _core_id_mask - ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs - cmp x1, x2 -- b.ne 2f -- -- // If execution reaches here, it is the boot core. -+ b.ne 1f +@@ -50,35 +39,23 @@ + // If execution reaches here, it is the boot core. + + // Initialize DRAM. +- ADR_ABS x0, __bss_start +- ADR_ABS x1, __bss_end_exclusive ++ ADR_REL x0, __bss_start ++ ADR_REL x1, __bss_end_exclusive + + bss_init_loop: + cmp x0, x1 +- b.eq relocate_binary ++ b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop - // Next, relocate the binary. +-relocate_binary: - ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to. - ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to. - ADR_ABS x2, __binary_nonzero_end_exclusive - --1: ldr x3, [x0], #8 +-copy_loop: +- ldr x3, [x0], #8 - str x3, [x1], #8 - cmp x1, x2 -- b.lo 1b -+ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. - +- b.lo copy_loop +- + // Prepare the jump to Rust code. ++prepare_rust: // Set the stack pointer. - ADR_ABS x0, __boot_core_stack_end_exclusive + ADR_REL x0, __boot_core_stack_end_exclusive @@ -186,13 +196,7 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar + b _start_rust // Infinitely wait for events (aka "park the core"). --2: wfe -- b 2b -+1: wfe -+ b 1b - - .size _start, . - _start - .type _start, function + parking_loop: diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu.rs 07_timestamps/src/_arch/aarch64/cpu.rs --- 06_uart_chainloader/src/_arch/aarch64/cpu.rs @@ -438,7 +442,7 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/ .text : { KEEP(*(.text._start)) -@@ -44,12 +42,8 @@ +@@ -44,10 +42,6 @@ ***********************************************************************************************/ .data : { *(.data*) } :segment_rw @@ -446,17 +450,14 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/ - . = ALIGN(8); - __binary_nonzero_end_exclusive = .; - - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -- .bss : -+ .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { - __bss_start = .; - *(.bss*); diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bsp/raspberrypi/memory.rs --- 06_uart_chainloader/src/bsp/raspberrypi/memory.rs +++ 07_timestamps/src/bsp/raspberrypi/memory.rs -@@ -23,10 +23,9 @@ +@@ -11,10 +11,9 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -469,21 +470,20 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bs /// Physical devices. #[cfg(feature = "bsp_rpi3")] -@@ -53,13 +52,7 @@ - // Public Code - //-------------------------------------------------------------------------------------------------- - +@@ -36,13 +35,3 @@ + pub const PL011_UART_START: usize = START + UART_OFFSET; + } + } +- +-//-------------------------------------------------------------------------------------------------- +-// Public Code +-//-------------------------------------------------------------------------------------------------- +- -/// The address on which the Raspberry firmware loads every binary by default. -#[inline(always)] -pub fn board_default_load_addr() -> *const u64 { - map::BOARD_DEFAULT_LOAD_ADDRESS as _ -} -- --/// Return the inclusive range spanning the relocated .bss section. -+/// Return the inclusive range spanning the .bss section. - /// - /// # Safety - /// diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs --- 06_uart_chainloader/src/cpu.rs @@ -499,23 +499,23 @@ diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs diff -uNr 06_uart_chainloader/src/main.rs 07_timestamps/src/main.rs --- 06_uart_chainloader/src/main.rs +++ 07_timestamps/src/main.rs -@@ -107,7 +107,6 @@ - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -105,7 +105,6 @@ + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] -#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] #![feature(global_asm)] -@@ -125,6 +124,7 @@ +@@ -121,6 +120,7 @@ + mod panic_wait; mod print; - mod runtime_init; mod synchronization; +mod time; /// Early init code. /// -@@ -147,56 +147,38 @@ +@@ -143,56 +143,38 @@ kernel_main() } diff --git a/07_timestamps/src/_arch/aarch64/cpu/boot.rs b/07_timestamps/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/07_timestamps/src/_arch/aarch64/cpu/boot.rs +++ b/07_timestamps/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/07_timestamps/src/_arch/aarch64/cpu/boot.s b/07_timestamps/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/07_timestamps/src/_arch/aarch64/cpu/boot.s +++ b/07_timestamps/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/07_timestamps/src/bsp/raspberrypi/link.ld b/07_timestamps/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/07_timestamps/src/bsp/raspberrypi/link.ld +++ b/07_timestamps/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/07_timestamps/src/bsp/raspberrypi/memory.rs b/07_timestamps/src/bsp/raspberrypi/memory.rs index c6d65e36..ef397a6f 100644 --- a/07_timestamps/src/bsp/raspberrypi/memory.rs +++ b/07_timestamps/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/07_timestamps/src/main.rs b/07_timestamps/src/main.rs index 54b69275..1e6bf16a 100644 --- a/07_timestamps/src/main.rs +++ b/07_timestamps/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/07_timestamps/src/memory.rs b/07_timestamps/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/07_timestamps/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/07_timestamps/src/runtime_init.rs b/07_timestamps/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/07_timestamps/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs +++ b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s +++ b/08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld b/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld +++ b/08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs b/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs index c6d65e36..ef397a6f 100644 --- a/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs +++ b/08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/08_hw_debug_JTAG/src/main.rs b/08_hw_debug_JTAG/src/main.rs index 54b69275..1e6bf16a 100644 --- a/08_hw_debug_JTAG/src/main.rs +++ b/08_hw_debug_JTAG/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/08_hw_debug_JTAG/src/memory.rs b/08_hw_debug_JTAG/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/08_hw_debug_JTAG/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/08_hw_debug_JTAG/src/runtime_init.rs b/08_hw_debug_JTAG/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/08_hw_debug_JTAG/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/09_privilege_level/README.md b/09_privilege_level/README.md index 869c3960..7b4c0d3c 100644 --- a/09_privilege_level/README.md +++ b/09_privilege_level/README.md @@ -62,7 +62,7 @@ Afterwards, we continue with preparing the `EL2` -> `EL1` transition by calling pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } ``` @@ -125,19 +125,16 @@ SPSR_EL2.write( + SPSR_EL2::M::EL1h, ); -// Second, let the link register point to runtime_init(). -ELR_EL2.set(runtime_init::runtime_init as *const () as u64); +// Second, let the link register point to kernel_init(). +ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); ``` -As you can see, we are populating `ELR_EL2` with the address of the [runtime_init()] function that -we earlier used to call directly from the entrypoint. Finally, we set the stack pointer for -`SP_EL1`. - -[runtime_init()]: src/runtime_init.rs +As you can see, we are populating `ELR_EL2` with the address of the `kernel_init()` function that we +earlier used to call directly from the entrypoint. Finally, we set the stack pointer for `SP_EL1`. You might have noticed that the stack's address was supplied as a function argument. As you might remember, in `_start()` in `boot.s`, we are already setting up the stack for `EL2`. Since there @@ -151,7 +148,7 @@ Lastly, back in `_start_rust()` a call to `ERET` is made: pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } ``` @@ -214,12 +211,12 @@ diff -uNr 08_hw_debug_JTAG/Cargo.toml 09_privilege_level/Cargo.toml diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/_arch/aarch64/cpu/boot.rs --- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs +++ 09_privilege_level/src/_arch/aarch64/cpu/boot.rs -@@ -12,11 +12,53 @@ +@@ -11,17 +11,67 @@ + //! //! crate::cpu::boot::arch_boot - use crate::runtime_init; +use cortex_a::{asm, regs::*}; - ++ // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -256,8 +253,8 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/ + + SPSR_EL2::M::EL1h, + ); + -+ // Second, let the link register point to runtime_init(). -+ ELR_EL2.set(runtime_init::runtime_init as *const () as u64); ++ // Second, let the link register point to kernel_init(). ++ ELR_EL2.set(crate::kernel_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. @@ -268,18 +265,20 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/ // Public Code //-------------------------------------------------------------------------------------------------- -@@ -27,7 +69,11 @@ - /// # Safety + /// The Rust entry of the `kernel` binary. /// - /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -+/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. + /// The function is called from the assembly `_start` function. ++/// ++/// # Safety ++/// ++/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] -pub unsafe fn _start_rust() -> ! { -- runtime_init::runtime_init() +- crate::kernel_init() +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + -+ // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. ++ // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. + asm::eret() } @@ -301,15 +300,15 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s 09_privilege_level/src/_ + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 -+ b.ne 1f ++ b.ne parking_loop + // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask -@@ -38,11 +44,11 @@ - - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. +@@ -50,11 +56,11 @@ + // Prepare the jump to Rust code. + prepare_rust: - // Set the stack pointer. + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive @@ -499,15 +498,15 @@ diff -uNr 08_hw_debug_JTAG/src/exception.rs 09_privilege_level/src/exception.rs diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs --- 08_hw_debug_JTAG/src/main.rs +++ 09_privilege_level/src/main.rs -@@ -119,6 +119,7 @@ +@@ -117,6 +117,7 @@ mod console; mod cpu; mod driver; +mod exception; - mod memory; mod panic_wait; mod print; -@@ -149,6 +150,8 @@ + mod synchronization; +@@ -145,6 +146,8 @@ /// The main function running after the early init. fn kernel_main() -> ! { @@ -516,7 +515,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs use core::time::Duration; use driver::interface::DriverManager; use time::interface::TimeManager; -@@ -160,6 +163,12 @@ +@@ -156,6 +159,12 @@ ); info!("Booting on: {}", bsp::board_name()); @@ -529,7 +528,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs info!( "Architectural timer resolution: {} ns", time::time_manager().resolution().as_nanos() -@@ -174,11 +183,15 @@ +@@ -170,11 +179,15 @@ info!(" {}. {}", i + 1, driver.compatible()); } diff --git a/09_privilege_level/src/_arch/aarch64/cpu/boot.rs b/09_privilege_level/src/_arch/aarch64/cpu/boot.rs index 02798fdc..b743418e 100644 --- a/09_privilege_level/src/_arch/aarch64/cpu/boot.rs +++ b/09_privilege_level/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/09_privilege_level/src/_arch/aarch64/cpu/boot.s b/09_privilege_level/src/_arch/aarch64/cpu/boot.s index 5696220d..d1666919 100644 --- a/09_privilege_level/src/_arch/aarch64/cpu/boot.s +++ b/09_privilege_level/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/09_privilege_level/src/bsp/raspberrypi/link.ld b/09_privilege_level/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/09_privilege_level/src/bsp/raspberrypi/link.ld +++ b/09_privilege_level/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/09_privilege_level/src/bsp/raspberrypi/memory.rs b/09_privilege_level/src/bsp/raspberrypi/memory.rs index c6d65e36..ef397a6f 100644 --- a/09_privilege_level/src/bsp/raspberrypi/memory.rs +++ b/09_privilege_level/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/09_privilege_level/src/main.rs b/09_privilege_level/src/main.rs index 5cede7f9..74902def 100644 --- a/09_privilege_level/src/main.rs +++ b/09_privilege_level/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -120,10 +118,8 @@ mod console; mod cpu; mod driver; mod exception; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/09_privilege_level/src/memory.rs b/09_privilege_level/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/09_privilege_level/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/09_privilege_level/src/runtime_init.rs b/09_privilege_level/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/09_privilege_level/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/10_virtual_mem_part1_identity_mapping/README.md b/10_virtual_mem_part1_identity_mapping/README.md index 75ff9c05..f573de03 100644 --- a/10_virtual_mem_part1_identity_mapping/README.md +++ b/10_virtual_mem_part1_identity_mapping/README.md @@ -939,26 +939,28 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 10_virtual_mem_pa diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs --- 09_privilege_level/src/bsp/raspberrypi/memory.rs +++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs -@@ -4,6 +4,8 @@ +@@ -4,6 +4,20 @@ //! BSP Memory Management. +pub mod mmu; + - use core::{cell::UnsafeCell, ops::RangeInclusive}; - - //-------------------------------------------------------------------------------------------------- -@@ -12,6 +14,9 @@ - - // Symbols from the linker script. - extern "Rust" { ++use core::cell::UnsafeCell; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++// Symbols from the linker script. ++extern "Rust" { + static __rx_start: UnsafeCell<()>; + static __rx_end_exclusive: UnsafeCell<()>; ++} + - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; - } -@@ -23,6 +28,20 @@ + //-------------------------------------------------------------------------------------------------- + // Public Definitions + //-------------------------------------------------------------------------------------------------- +@@ -11,6 +25,20 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -979,7 +981,7 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_ pub const GPIO_OFFSET: usize = 0x0020_0000; pub const UART_OFFSET: usize = 0x0020_1000; -@@ -35,6 +54,7 @@ +@@ -23,6 +51,7 @@ pub const START: usize = 0x3F00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; @@ -987,15 +989,15 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_ } /// Physical devices. -@@ -45,10 +65,35 @@ +@@ -33,5 +62,30 @@ pub const START: usize = 0xFE00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; + pub const END_INCLUSIVE: usize = 0xFF84_FFFF; } } - - //-------------------------------------------------------------------------------------------------- ++ ++//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + @@ -1018,11 +1020,6 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_ +fn rx_end_exclusive() -> usize { + unsafe { __rx_end_exclusive.get() as usize } +} -+ -+//-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/src/bsp.rs --- 09_privilege_level/src/bsp.rs @@ -1040,8 +1037,8 @@ diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/sr diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/src/main.rs --- 09_privilege_level/src/main.rs +++ 10_virtual_mem_part1_identity_mapping/src/main.rs -@@ -107,7 +107,11 @@ - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +@@ -105,7 +105,11 @@ + //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] +#![allow(incomplete_features)] @@ -1052,7 +1049,15 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] -@@ -132,9 +136,17 @@ +@@ -118,6 +122,7 @@ + mod cpu; + mod driver; + mod exception; ++mod memory; + mod panic_wait; + mod print; + mod synchronization; +@@ -128,9 +133,17 @@ /// # Safety /// /// - Only a single core must be active and running this function. @@ -1071,7 +1076,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s for i in bsp::driver::driver_manager().all_device_drivers().iter() { if let Err(x) = i.init() { -@@ -163,6 +175,9 @@ +@@ -159,6 +172,9 @@ ); info!("Booting on: {}", bsp::board_name()); @@ -1081,7 +1086,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s let (_, privilege_level) = exception::current_privilege_level(); info!("Current privilege level: {}", privilege_level); -@@ -186,6 +201,13 @@ +@@ -182,6 +198,13 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); @@ -1387,14 +1392,13 @@ diff -uNr 09_privilege_level/src/memory/mmu.rs 10_virtual_mem_part1_identity_map diff -uNr 09_privilege_level/src/memory.rs 10_virtual_mem_part1_identity_mapping/src/memory.rs --- 09_privilege_level/src/memory.rs +++ 10_virtual_mem_part1_identity_mapping/src/memory.rs -@@ -4,6 +4,8 @@ - - //! Memory Management. - -+pub mod mmu; +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter + - use core::ops::RangeInclusive; - - //-------------------------------------------------------------------------------------------------- ++//! Memory Management. ++ ++pub mod mmu; ``` diff --git a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs index 02798fdc..b743418e 100644 --- a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs +++ b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s index 5696220d..d1666919 100644 --- a/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s +++ b/10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld index bda0da5e..485ba49b 100644 --- a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld +++ b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs index 81233775..48b1d603 100644 --- a/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs +++ b/10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -92,23 +89,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/10_virtual_mem_part1_identity_mapping/src/main.rs b/10_virtual_mem_part1_identity_mapping/src/main.rs index 35f41c0e..3a1958d6 100644 --- a/10_virtual_mem_part1_identity_mapping/src/main.rs +++ b/10_virtual_mem_part1_identity_mapping/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -127,7 +125,6 @@ mod exception; mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/10_virtual_mem_part1_identity_mapping/src/memory.rs b/10_virtual_mem_part1_identity_mapping/src/memory.rs index e8cc752f..7959c555 100644 --- a/10_virtual_mem_part1_identity_mapping/src/memory.rs +++ b/10_virtual_mem_part1_identity_mapping/src/memory.rs @@ -5,28 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/10_virtual_mem_part1_identity_mapping/src/runtime_init.rs b/10_virtual_mem_part1_identity_mapping/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/10_virtual_mem_part1_identity_mapping/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/11_exceptions_part1_groundwork/README.md b/11_exceptions_part1_groundwork/README.md index f9c70824..77a45ec6 100644 --- a/11_exceptions_part1_groundwork/README.md +++ b/11_exceptions_part1_groundwork/README.md @@ -967,7 +967,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/exception.rs 11_exceptions_p diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_groundwork/src/main.rs --- 10_virtual_mem_part1_identity_mapping/src/main.rs +++ 11_exceptions_part1_groundwork/src/main.rs -@@ -144,6 +144,8 @@ +@@ -141,6 +141,8 @@ use driver::interface::DriverManager; use memory::mmu::interface::MMU; @@ -976,7 +976,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_ if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() { panic!("MMU: {}", string); } -@@ -201,13 +203,28 @@ +@@ -198,13 +200,28 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); diff --git a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs index 02798fdc..b743418e 100644 --- a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs +++ b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s index 5696220d..d1666919 100644 --- a/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s +++ b/11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld index bda0da5e..485ba49b 100644 --- a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld +++ b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs index 81233775..48b1d603 100644 --- a/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs +++ b/11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -92,23 +89,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/11_exceptions_part1_groundwork/src/main.rs b/11_exceptions_part1_groundwork/src/main.rs index 5d00393c..cd9d0786 100644 --- a/11_exceptions_part1_groundwork/src/main.rs +++ b/11_exceptions_part1_groundwork/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -127,7 +125,6 @@ mod exception; mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/11_exceptions_part1_groundwork/src/memory.rs b/11_exceptions_part1_groundwork/src/memory.rs index e8cc752f..7959c555 100644 --- a/11_exceptions_part1_groundwork/src/memory.rs +++ b/11_exceptions_part1_groundwork/src/memory.rs @@ -5,28 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/11_exceptions_part1_groundwork/src/runtime_init.rs b/11_exceptions_part1_groundwork/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/11_exceptions_part1_groundwork/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/12_integrated_testing/README.md b/12_integrated_testing/README.md index cff8587b..d252d0c2 100644 --- a/12_integrated_testing/README.md +++ b/12_integrated_testing/README.md @@ -215,25 +215,24 @@ kernel boot: | | Function | File | | - | - | - | -| 1. | `_start()` | `lib.rs` | -| 2. | (some more aarch64 code) | `lib.rs` | -| 3. | `runtime_init()` | `lib.rs` | -| 4. | `kernel_init()` | `main.rs` | -| 5. | `kernel_main()` | `main.rs` | +| 1. | `_start()` | The library's `boot.s` | +| 2. | (some more aarch64 code) | The library's `boot.rs` | +| 3. | `kernel_init()` | `main.rs` | +| 4. | `kernel_main()` | `main.rs` | A function named `main` is never called. Hence, the `main()` function generated by `cargo test` -would be silently dropped, and therefore the tests would never be executed. As you can see, -`runtime_init()` is the last function residing in our carved-out `lib.rs`, and it calls into -`kernel_init()`. So in order to get the tests to execute, we add a test-environment version of -`kernel_init()` to `lib.rs` as well (conditional compilation ensures it is only present when the -test flag is set), and call the `cargo test` generated `main()` function from there. +would be silently dropped, and therefore the tests would never be executed. As you can see, the +first function getting called in our carved-out `main.rs` is `kernel_init()`. So in order to get the +tests to execute, we add a test-environment version of `kernel_init()` to `lib.rs` as well +(conditional compilation ensures it is only present when the test flag is set), and call the `cargo +test` generated `main()` function from there. This is where `#![reexport_test_harness_main = "test_main"]` finally comes into picture. It declares the name of the generated main function so that we can manually call it. Here is the final implementation in `lib.rs`: ```rust -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { @@ -1073,7 +1072,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translatio diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs --- 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs +++ 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs -@@ -162,3 +162,22 @@ +@@ -162,3 +162,33 @@ SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } } @@ -1085,12 +1084,23 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_inte +#[cfg(test)] +mod tests { + use super::*; ++ use core::{cell::UnsafeCell, ops::Range}; + use test_macros::kernel_test; + + /// Check if KERNEL_TABLES is in .bss. + #[kernel_test] + fn kernel_tables_in_bss() { -+ let bss_range = bsp::memory::bss_range_inclusive(); ++ extern "Rust" { ++ static __bss_start: UnsafeCell; ++ static __bss_end_exclusive: UnsafeCell; ++ } ++ ++ let bss_range = unsafe { ++ Range { ++ start: __bss_start.get(), ++ end: __bss_end_exclusive.get(), ++ } ++ }; + let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; + + assert!(bss_range.contains(&kernel_tables_addr)); @@ -1207,7 +1217,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/exception.rs 12_integrated_testing/ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/lib.rs --- 11_exceptions_part1_groundwork/src/lib.rs +++ 12_integrated_testing/src/lib.rs -@@ -0,0 +1,184 @@ +@@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -1314,9 +1324,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li +//! +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -+//! -+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html ++//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. + +#![allow(clippy::upper_case_acronyms)] +#![allow(incomplete_features)] @@ -1337,7 +1345,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li +#![test_runner(crate::test_runner)] + +mod panic_wait; -+mod runtime_init; +mod synchronization; + +pub mod bsp; @@ -1362,6 +1369,11 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li + ) +} + ++#[cfg(not(test))] ++extern "Rust" { ++ fn kernel_init() -> !; ++} ++ +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- @@ -1381,7 +1393,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li + } +} + -+/// The `kernel_init()` for unit tests. Called from `runtime_init()`. ++/// The `kernel_init()` for unit tests. +#[cfg(test)] +#[no_mangle] +unsafe fn kernel_init() -> ! { @@ -1396,7 +1408,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/main.rs --- 11_exceptions_part1_groundwork/src/main.rs +++ 12_integrated_testing/src/main.rs -@@ -6,130 +6,12 @@ +@@ -6,127 +6,12 @@ #![doc(html_logo_url = "https://git.io/JeGIp")] //! The `kernel` binary. @@ -1496,9 +1508,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m -//! -//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. --//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. --//! --//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +-//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. - -#![allow(clippy::upper_case_acronyms)] -#![allow(incomplete_features)] @@ -1522,14 +1532,13 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m -mod memory; -mod panic_wait; -mod print; --mod runtime_init; -mod synchronization; -mod time; +use libkernel::{bsp, console, driver, exception, info, memory, time}; /// Early init code. /// -@@ -140,6 +22,7 @@ +@@ -137,6 +22,7 @@ /// - MMU + Data caching must be activated at the earliest. Without it, any atomic operations, /// e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ /// NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs. @@ -1537,7 +1546,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m unsafe fn kernel_init() -> ! { use driver::interface::DriverManager; use memory::mmu::interface::MMU; -@@ -166,15 +49,9 @@ +@@ -163,15 +49,9 @@ fn kernel_main() -> ! { use bsp::console::console; use console::interface::All; @@ -1554,7 +1563,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m info!("Booting on: {}", bsp::board_name()); info!("MMU online. Special regions:"); -@@ -200,31 +77,6 @@ +@@ -197,31 +77,6 @@ info!(" {}. {}", i + 1, driver.compatible()); } @@ -1609,51 +1618,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/memory/mmu.rs 12_integrated_testing + } } -diff -uNr 11_exceptions_part1_groundwork/src/memory.rs 12_integrated_testing/src/memory.rs ---- 11_exceptions_part1_groundwork/src/memory.rs -+++ 12_integrated_testing/src/memory.rs -@@ -30,3 +30,40 @@ - ptr = ptr.offset(1); - } - } -+ -+//-------------------------------------------------------------------------------------------------- -+// Testing -+//-------------------------------------------------------------------------------------------------- -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use test_macros::kernel_test; -+ -+ /// Check `zero_volatile()`. -+ #[kernel_test] -+ fn zero_volatile_works() { -+ let mut x: [usize; 3] = [10, 11, 12]; -+ let x_range = x.as_mut_ptr_range(); -+ let x_range_inclusive = -+ RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); -+ -+ unsafe { zero_volatile(x_range_inclusive) }; -+ -+ assert_eq!(x, [0, 0, 0]); -+ } -+ -+ /// Check `bss` section layout. -+ #[kernel_test] -+ fn bss_section_is_sane() { -+ use crate::bsp::memory::bss_range_inclusive; -+ use core::mem; -+ -+ let start = *bss_range_inclusive().start() as usize; -+ let end = *bss_range_inclusive().end() as usize; -+ -+ assert_eq!(start modulo mem::size_of::(), 0); -+ assert_eq!(end modulo mem::size_of::(), 0); -+ assert!(end >= start); -+ } -+} - diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing/src/panic_wait.rs --- 11_exceptions_part1_groundwork/src/panic_wait.rs +++ 12_integrated_testing/src/panic_wait.rs @@ -1689,23 +1653,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing + _panic_exit() } -diff -uNr 11_exceptions_part1_groundwork/src/runtime_init.rs 12_integrated_testing/src/runtime_init.rs ---- 11_exceptions_part1_groundwork/src/runtime_init.rs -+++ 12_integrated_testing/src/runtime_init.rs -@@ -31,7 +31,10 @@ - /// - /// - Only a single core must be active and running this function. - pub unsafe fn runtime_init() -> ! { -- zero_bss(); -+ extern "Rust" { -+ fn kernel_init() -> !; -+ } - -- crate::kernel_init() -+ zero_bss(); -+ kernel_init() - } - diff -uNr 11_exceptions_part1_groundwork/test-macros/Cargo.toml 12_integrated_testing/test-macros/Cargo.toml --- 11_exceptions_part1_groundwork/test-macros/Cargo.toml +++ 12_integrated_testing/test-macros/Cargo.toml diff --git a/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs b/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs index 02798fdc..b743418e 100644 --- a/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs +++ b/12_integrated_testing/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/12_integrated_testing/src/_arch/aarch64/cpu/boot.s b/12_integrated_testing/src/_arch/aarch64/cpu/boot.s index 5696220d..d1666919 100644 --- a/12_integrated_testing/src/_arch/aarch64/cpu/boot.s +++ b/12_integrated_testing/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs b/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs index 29e8125d..a706290b 100644 --- a/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs +++ b/12_integrated_testing/src/_arch/aarch64/memory/mmu.rs @@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { #[cfg(test)] mod tests { use super::*; + use core::{cell::UnsafeCell, ops::Range}; use test_macros::kernel_test; /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { - let bss_range = bsp::memory::bss_range_inclusive(); + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; assert!(bss_range.contains(&kernel_tables_addr)); diff --git a/12_integrated_testing/src/bsp/raspberrypi/link.ld b/12_integrated_testing/src/bsp/raspberrypi/link.ld index bda0da5e..485ba49b 100644 --- a/12_integrated_testing/src/bsp/raspberrypi/link.ld +++ b/12_integrated_testing/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/12_integrated_testing/src/bsp/raspberrypi/memory.rs b/12_integrated_testing/src/bsp/raspberrypi/memory.rs index 81233775..48b1d603 100644 --- a/12_integrated_testing/src/bsp/raspberrypi/memory.rs +++ b/12_integrated_testing/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -92,23 +89,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/12_integrated_testing/src/lib.rs b/12_integrated_testing/src/lib.rs index 1e315b4d..9890351f 100644 --- a/12_integrated_testing/src/lib.rs +++ b/12_integrated_testing/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -127,7 +125,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -152,6 +149,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -171,7 +173,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/12_integrated_testing/src/memory.rs b/12_integrated_testing/src/memory.rs index 1ef0285a..7959c555 100644 --- a/12_integrated_testing/src/memory.rs +++ b/12_integrated_testing/src/memory.rs @@ -5,65 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/12_integrated_testing/src/runtime_init.rs b/12_integrated_testing/src/runtime_init.rs deleted file mode 100644 index 0a1c685c..00000000 --- a/12_integrated_testing/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/13_exceptions_part2_peripheral_IRQs/README.md b/13_exceptions_part2_peripheral_IRQs/README.md index 2a44ecda..1174ca9e 100644 --- a/13_exceptions_part2_peripheral_IRQs/README.md +++ b/13_exceptions_part2_peripheral_IRQs/README.md @@ -2104,7 +2104,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/exception.rs 13_exceptions_p diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs --- 12_integrated_testing/src/bsp/raspberrypi/memory.rs +++ 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs -@@ -51,10 +51,12 @@ +@@ -48,10 +48,12 @@ pub mod mmio { use super::*; @@ -2121,7 +2121,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part } /// Physical devices. -@@ -65,6 +67,8 @@ +@@ -62,6 +64,8 @@ pub const START: usize = 0xFE00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; @@ -2376,7 +2376,7 @@ diff -uNr 12_integrated_testing/src/exception/asynchronous.rs 13_exceptions_part diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/src/lib.rs --- 12_integrated_testing/src/lib.rs +++ 13_exceptions_part2_peripheral_IRQs/src/lib.rs -@@ -110,6 +110,7 @@ +@@ -108,6 +108,7 @@ #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -2384,7 +2384,7 @@ diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/s #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -137,6 +138,7 @@ +@@ -134,6 +135,7 @@ pub mod exception; pub mod memory; pub mod print; diff --git a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs index 02798fdc..b743418e 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s index 5696220d..d1666919 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s +++ b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs index 29e8125d..a706290b 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs @@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { #[cfg(test)] mod tests { use super::*; + use core::{cell::UnsafeCell, ops::Range}; use test_macros::kernel_test; /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { - let bss_range = bsp::memory::bss_range_inclusive(); + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; assert!(bss_range.contains(&kernel_tables_addr)); diff --git a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld index bda0da5e..485ba49b 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld +++ b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld @@ -46,14 +46,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs index 8a58a17e..ede648f6 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs @@ -6,7 +6,7 @@ pub mod mmu; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -96,23 +93,3 @@ fn rx_start() -> usize { fn rx_end_exclusive() -> usize { unsafe { __rx_end_exclusive.get() as usize } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/13_exceptions_part2_peripheral_IRQs/src/lib.rs b/13_exceptions_part2_peripheral_IRQs/src/lib.rs index 9a787e60..ca761361 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/lib.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -128,7 +126,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -154,6 +151,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -173,7 +175,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/13_exceptions_part2_peripheral_IRQs/src/memory.rs b/13_exceptions_part2_peripheral_IRQs/src/memory.rs index 1ef0285a..7959c555 100644 --- a/13_exceptions_part2_peripheral_IRQs/src/memory.rs +++ b/13_exceptions_part2_peripheral_IRQs/src/memory.rs @@ -5,65 +5,3 @@ //! Memory Management. pub mod mmu; - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs b/13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs deleted file mode 100644 index 0a1c685c..00000000 --- a/13_exceptions_part2_peripheral_IRQs/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/14_virtual_mem_part2_mmio_remap/README.md b/14_virtual_mem_part2_mmio_remap/README.md index 85aa5138..73ef83a3 100644 --- a/14_virtual_mem_part2_mmio_remap/README.md +++ b/14_virtual_mem_part2_mmio_remap/README.md @@ -806,7 +806,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 14 self.configure_translation_control(); -@@ -162,22 +153,3 @@ +@@ -162,33 +153,3 @@ SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } } @@ -818,12 +818,23 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 14 -#[cfg(test)] -mod tests { - use super::*; +- use core::{cell::UnsafeCell, ops::Range}; - use test_macros::kernel_test; - - /// Check if KERNEL_TABLES is in .bss. - #[kernel_test] - fn kernel_tables_in_bss() { -- let bss_range = bsp::memory::bss_range_inclusive(); +- extern "Rust" { +- static __bss_start: UnsafeCell; +- static __bss_end_exclusive: UnsafeCell; +- } +- +- let bss_range = unsafe { +- Range { +- start: __bss_start.get(), +- end: __bss_end_exclusive.get(), +- } +- }; - let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 }; - - assert!(bss_range.contains(&kernel_tables_addr)); @@ -1489,10 +1500,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 14_vir + __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -@@ -56,4 +52,23 @@ - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ +@@ -54,4 +50,23 @@ + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE + + . = ALIGN(64K); /* Align to page boundary */ @@ -1554,16 +1565,16 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to +/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; -+ + +-const NUM_MEM_RANGES: usize = 2; +/// The kernel's virtual address space defined by this BSP. +pub type KernelVirtAddrSpace = AddressSpace<{ 8 * 1024 * 1024 * 1024 }>; --const NUM_MEM_RANGES: usize = 2; +-/// The virtual memory layout. +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- - --/// The virtual memory layout. ++ +/// The kernel translation tables. /// -/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. @@ -1642,15 +1653,15 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs +/// The Read+Execute (RX) pages of the kernel binary. +fn phys_rx_page_desc() -> PageSliceDescriptor { + virt_rx_page_desc().into() ++} ++ ++/// The Read+Write (RW) pages of the kernel binary. ++fn phys_rw_page_desc() -> PageSliceDescriptor { ++ virt_rw_page_desc().into() } -fn mmio_range_inclusive() -> RangeInclusive { - RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE) -+/// The Read+Write (RW) pages of the kernel binary. -+fn phys_rw_page_desc() -> PageSliceDescriptor { -+ virt_rw_page_desc().into() -+} -+ +/// The boot core's stack. +fn phys_boot_core_stack_page_desc() -> PageSliceDescriptor { + virt_boot_core_stack_page_desc().into() @@ -1726,15 +1737,17 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs } //-------------------------------------------------------------------------------------------------- -@@ -82,14 +176,18 @@ +@@ -77,19 +171,24 @@ + #[cfg(test)] + mod tests { + use super::*; ++ use core::{cell::UnsafeCell, ops::Range}; + use test_macros::kernel_test; + /// Check alignment of the kernel's virtual memory layout sections. #[kernel_test] fn virt_mem_layout_sections_are_64KiB_aligned() { - const SIXTYFOUR_KIB: usize = 65536; -- -- for i in LAYOUT.inner().iter() { -- let start: usize = *(i.virtual_range)().start(); -- let end: usize = *(i.virtual_range)().end() + 1; + for i in [ + virt_rx_page_desc, + virt_rw_page_desc, @@ -1745,6 +1758,10 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + let start: usize = i().start_addr().into_usize(); + let end: usize = i().end_addr().into_usize(); +- for i in LAYOUT.inner().iter() { +- let start: usize = *(i.virtual_range)().start(); +- let end: usize = *(i.virtual_range)().end() + 1; +- - assert_eq!(start modulo SIXTYFOUR_KIB, 0); - assert_eq!(end modulo SIXTYFOUR_KIB, 0); + assert_eq!(start modulo KernelGranule::SIZE, 0); @@ -1752,7 +1769,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs assert!(end >= start); } } -@@ -97,18 +195,28 @@ +@@ -97,18 +196,38 @@ /// Ensure the kernel's virtual memory layout is free of overlaps. #[kernel_test] fn virt_mem_layout_has_no_overlaps() { @@ -1786,7 +1803,17 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs + /// Check if KERNEL_TABLES is in .bss. + #[kernel_test] + fn kernel_tables_in_bss() { -+ let bss_range = super::super::bss_range_inclusive(); ++ extern "Rust" { ++ static __bss_start: UnsafeCell; ++ static __bss_end_exclusive: UnsafeCell; ++ } ++ ++ let bss_range = unsafe { ++ Range { ++ start: __bss_start.get(), ++ end: __bss_end_exclusive.get(), ++ } ++ }; + let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; + + assert!(bss_range.contains(&kernel_tables_addr)); @@ -1834,16 +1861,15 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v pub mod mmu; +use crate::memory::{Address, Physical, Virtual}; - use core::{cell::UnsafeCell, ops::RangeInclusive}; + use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- -@@ -17,8 +48,16 @@ +@@ -16,6 +47,15 @@ + extern "Rust" { static __rx_start: UnsafeCell<()>; static __rx_end_exclusive: UnsafeCell<()>; - ++ + static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; + static __rw_end_exclusive: UnsafeCell<()>; + + static __boot_core_stack_start: UnsafeCell<()>; @@ -1854,7 +1880,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v } //-------------------------------------------------------------------------------------------------- -@@ -28,35 +67,26 @@ +@@ -25,35 +65,26 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -1904,7 +1930,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v } /// Physical devices. -@@ -64,13 +94,22 @@ +@@ -61,13 +92,22 @@ pub mod mmio { use super::*; @@ -1933,7 +1959,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v } //-------------------------------------------------------------------------------------------------- -@@ -83,18 +122,69 @@ +@@ -80,16 +120,67 @@ /// /// - Value is provided by the linker script and must be trusted as-is. #[inline(always)] @@ -1941,21 +1967,19 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v - unsafe { __rx_start.get() as usize } +fn virt_rx_start() -> Address { + Address::new(unsafe { __rx_start.get() as usize }) - } - --/// Exclusive end address of the Read+Execute (RX) range. ++} ++ +/// Size of the Read+Execute (RX) range. - /// - /// # Safety - /// - /// - Value is provided by the linker script and must be trusted as-is. - #[inline(always)] --fn rx_end_exclusive() -> usize { -- unsafe { __rx_end_exclusive.get() as usize } ++/// ++/// # Safety ++/// ++/// - Value is provided by the linker script and must be trusted as-is. ++#[inline(always)] +fn rx_size() -> usize { + unsafe { (__rx_end_exclusive.get() as usize) - (__rx_start.get() as usize) } -+} -+ + } + +-/// Exclusive end address of the Read+Execute (RX) range. +/// Start address of the Read+Write (RW) range. +#[inline(always)] +fn virt_rw_start() -> Address { @@ -1963,11 +1987,13 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v +} + +/// Size of the Read+Write (RW) range. -+/// -+/// # Safety -+/// -+/// - Value is provided by the linker script and must be trusted as-is. -+#[inline(always)] + /// + /// # Safety + /// + /// - Value is provided by the linker script and must be trusted as-is. + #[inline(always)] +-fn rx_end_exclusive() -> usize { +- unsafe { __rx_end_exclusive.get() as usize } +fn rw_size() -> usize { + unsafe { (__rw_end_exclusive.get() as usize) - (__rw_start.get() as usize) } +} @@ -2007,8 +2033,6 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 14_v + map::END } - //-------------------------------------------------------------------------------------------------- - diff -uNr 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi.rs --- 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs +++ 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi.rs @@ -2131,7 +2155,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/driver.rs 14_virtual_mem_part2 diff -uNr 13_exceptions_part2_peripheral_IRQs/src/lib.rs 14_virtual_mem_part2_mmio_remap/src/lib.rs --- 13_exceptions_part2_peripheral_IRQs/src/lib.rs +++ 14_virtual_mem_part2_mmio_remap/src/lib.rs -@@ -111,6 +111,8 @@ +@@ -109,6 +109,8 @@ #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] #![feature(asm)] @@ -2140,7 +2164,7 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/lib.rs 14_virtual_mem_part2_mm #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -132,6 +134,7 @@ +@@ -129,6 +131,7 @@ mod synchronization; pub mod bsp; @@ -3131,16 +3155,16 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 14_virtual_mem_p diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory.rs 14_virtual_mem_part2_mmio_remap/src/memory.rs --- 13_exceptions_part2_peripheral_IRQs/src/memory.rs +++ 14_virtual_mem_part2_mmio_remap/src/memory.rs -@@ -6,12 +6,136 @@ +@@ -5,3 +5,133 @@ + //! Memory Management. pub mod mmu; - --use core::ops::RangeInclusive; ++ +use crate::common; +use core::{ + fmt, + marker::PhantomData, -+ ops::{AddAssign, RangeInclusive, SubAssign}, ++ ops::{AddAssign, SubAssign}, +}; + +//-------------------------------------------------------------------------------------------------- @@ -3164,11 +3188,11 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory.rs 14_virtual_mem_part2 + value: usize, + _address_type: PhantomData ATYPE>, +} - - //-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ +impl AddressType for Physical {} +impl AddressType for Virtual {} + @@ -3265,10 +3289,6 @@ diff -uNr 13_exceptions_part2_peripheral_IRQs/src/memory.rs 14_virtual_mem_part2 + write!(f, "{:04x}", q1) + } +} -+ - /// Zero out an inclusive memory range. - /// - /// # Safety diff -uNr 13_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs 14_virtual_mem_part2_mmio_remap/tests/02_exception_sync_page_fault.rs --- 13_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs diff --git a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs index 02798fdc..b743418e 100644 --- a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +++ b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. @@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s index 5696220d..d1666919 100644 --- a/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s +++ b/14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -52,8 +64,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld index 64545edf..698ad85d 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld @@ -42,15 +42,13 @@ SECTIONS __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE . = ALIGN(64K); /* Align to page boundary */ diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs index 7f571e08..e51b00fa 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; use crate::memory::{Address, Physical, Virtual}; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -49,8 +49,6 @@ extern "Rust" { static __rx_end_exclusive: UnsafeCell<()>; static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; static __rw_end_exclusive: UnsafeCell<()>; static __boot_core_stack_start: UnsafeCell<()>; @@ -186,23 +184,3 @@ fn boot_core_stack_guard_page_size() -> usize { fn phys_addr_space_end() -> Address { map::END } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs index 88fc80bd..813fa0b4 100644 --- a/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +++ b/14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs @@ -171,6 +171,7 @@ pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { #[cfg(test)] mod tests { use super::*; + use core::{cell::UnsafeCell, ops::Range}; use test_macros::kernel_test; /// Check alignment of the kernel's virtual memory layout sections. @@ -214,7 +215,17 @@ mod tests { /// Check if KERNEL_TABLES is in .bss. #[kernel_test] fn kernel_tables_in_bss() { - let bss_range = super::super::bss_range_inclusive(); + extern "Rust" { + static __bss_start: UnsafeCell; + static __bss_end_exclusive: UnsafeCell; + } + + let bss_range = unsafe { + Range { + start: __bss_start.get(), + end: __bss_end_exclusive.get(), + } + }; let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; assert!(bss_range.contains(&kernel_tables_addr)); diff --git a/14_virtual_mem_part2_mmio_remap/src/lib.rs b/14_virtual_mem_part2_mmio_remap/src/lib.rs index f3ef2951..1b2cb367 100644 --- a/14_virtual_mem_part2_mmio_remap/src/lib.rs +++ b/14_virtual_mem_part2_mmio_remap/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -130,7 +128,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -157,6 +154,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -176,7 +178,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/14_virtual_mem_part2_mmio_remap/src/memory.rs b/14_virtual_mem_part2_mmio_remap/src/memory.rs index 515731eb..3de1d607 100644 --- a/14_virtual_mem_part2_mmio_remap/src/memory.rs +++ b/14_virtual_mem_part2_mmio_remap/src/memory.rs @@ -10,7 +10,7 @@ use crate::common; use core::{ fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, }; //-------------------------------------------------------------------------------------------------- @@ -135,59 +135,3 @@ impl fmt::Display for Address { write!(f, "{:04x}", q1) } } - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/14_virtual_mem_part2_mmio_remap/src/runtime_init.rs b/14_virtual_mem_part2_mmio_remap/src/runtime_init.rs deleted file mode 100644 index 0a1c685c..00000000 --- a/14_virtual_mem_part2_mmio_remap/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/15_virtual_mem_part3_precomputed_tables/README.md b/15_virtual_mem_part3_precomputed_tables/README.md index bb9a50bb..b06b222d 100644 --- a/15_virtual_mem_part3_precomputed_tables/README.md +++ b/15_virtual_mem_part3_precomputed_tables/README.md @@ -530,7 +530,7 @@ pub unsafe extern "C" fn _start_rust( cpu::wait_forever(); } - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } ``` @@ -796,19 +796,18 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/Makefile 15_virtual_mem_part3_precompu diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs --- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs -@@ -11,7 +11,8 @@ +@@ -11,6 +11,8 @@ //! //! crate::cpu::boot::arch_boot --use crate::runtime_init; -+use crate::{cpu, memory, memory::Address, runtime_init}; ++use crate::{cpu, memory, memory::Address}; +use core::intrinsics::unlikely; use cortex_a::{asm, regs::*}; // Assembly counterpart to this file. -@@ -71,9 +72,18 @@ - /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. - /// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +@@ -69,9 +71,18 @@ + /// + /// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] -pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { +pub unsafe extern "C" fn _start_rust( @@ -823,17 +822,17 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs 15_virtu + cpu::wait_forever(); + } + - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff -uNr 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s --- 14_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s +++ 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s -@@ -44,11 +44,14 @@ - - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. +@@ -56,11 +56,14 @@ + // Prepare the jump to Rust code. + prepare_rust: + // Load the base address of the kernel's translation tables. + ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs + @@ -1204,7 +1203,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v "Kernel boot-core stack", &virt_boot_core_stack_page_desc(), &phys_boot_core_stack_page_desc(), -@@ -159,64 +183,5 @@ +@@ -159,75 +183,5 @@ acc_perms: AccessPermissions::ReadWrite, execute_never: true, }, @@ -1220,6 +1219,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v -#[cfg(test)] -mod tests { - use super::*; +- use core::{cell::UnsafeCell, ops::Range}; - use test_macros::kernel_test; - - /// Check alignment of the kernel's virtual memory layout sections. @@ -1263,7 +1263,17 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs 15_v - /// Check if KERNEL_TABLES is in .bss. - #[kernel_test] - fn kernel_tables_in_bss() { -- let bss_range = super::super::bss_range_inclusive(); +- extern "Rust" { +- static __bss_start: UnsafeCell; +- static __bss_end_exclusive: UnsafeCell; +- } +- +- let bss_range = unsafe { +- Range { +- start: __bss_start.get(), +- end: __bss_end_exclusive.get(), +- } +- }; - let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; - - assert!(bss_range.contains(&kernel_tables_addr)); @@ -1511,7 +1521,7 @@ diff -uNr 14_virtual_mem_part2_mmio_remap/src/memory.rs 15_virtual_mem_part3_pre + convert::TryFrom, fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, @@ -67,6 +68,14 @@ } } diff --git a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs index 5e0ecfc6..f275c792 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs @@ -11,7 +11,7 @@ //! //! crate::cpu::boot::arch_boot -use crate::{cpu, memory, memory::Address, runtime_init}; +use crate::{cpu, memory, memory::Address}; use core::intrinsics::unlikely; use cortex_a::{asm, regs::*}; @@ -51,8 +51,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(crate::kernel_init as *const () as u64); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -69,8 +69,7 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust( phys_kernel_tables_base_addr: u64, @@ -84,6 +83,6 @@ pub unsafe extern "C" fn _start_rust( cpu::wait_forever(); } - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. asm::eret() } diff --git a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s index 28646261..9e7fc619 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s +++ b/15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s @@ -33,17 +33,29 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Load the base address of the kernel's translation tables. ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs @@ -55,8 +67,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld index 7ad75a40..aa127cc0 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld +++ b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/link.ld @@ -45,15 +45,13 @@ SECTIONS __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE . = ALIGN(64K); /* Align to page boundary */ diff --git a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs index 7f571e08..e51b00fa 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; use crate::memory::{Address, Physical, Virtual}; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -49,8 +49,6 @@ extern "Rust" { static __rx_end_exclusive: UnsafeCell<()>; static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; static __rw_end_exclusive: UnsafeCell<()>; static __boot_core_stack_start: UnsafeCell<()>; @@ -186,23 +184,3 @@ fn boot_core_stack_guard_page_size() -> usize { fn phys_addr_space_end() -> Address { map::END } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/15_virtual_mem_part3_precomputed_tables/src/lib.rs b/15_virtual_mem_part3_precomputed_tables/src/lib.rs index f3ef2951..1b2cb367 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/lib.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -130,7 +128,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -157,6 +154,11 @@ pub fn version() -> &'static str { ) } +#[cfg(not(test))] +extern "Rust" { + fn kernel_init() -> !; +} + //-------------------------------------------------------------------------------------------------- // Testing //-------------------------------------------------------------------------------------------------- @@ -176,7 +178,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/15_virtual_mem_part3_precomputed_tables/src/memory.rs b/15_virtual_mem_part3_precomputed_tables/src/memory.rs index 87ffa81f..4ac96904 100644 --- a/15_virtual_mem_part3_precomputed_tables/src/memory.rs +++ b/15_virtual_mem_part3_precomputed_tables/src/memory.rs @@ -11,7 +11,7 @@ use core::{ convert::TryFrom, fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, }; //-------------------------------------------------------------------------------------------------- @@ -144,59 +144,3 @@ impl fmt::Display for Address { write!(f, "{:04x}", q1) } } - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs b/15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs deleted file mode 100644 index 0a1c685c..00000000 --- a/15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/16_virtual_mem_part4_higher_half_kernel/README.md b/16_virtual_mem_part4_higher_half_kernel/README.md index 17d71e2b..0a438184 100644 --- a/16_virtual_mem_part4_higher_half_kernel/README.md +++ b/16_virtual_mem_part4_higher_half_kernel/README.md @@ -99,7 +99,7 @@ by the time the CPU enters `EL1`, virtual memory will be active, and the CPU mus new higher-half `virtual addresses` for everything it does. Specifically, this means the address from which the CPU should execute upon entering `EL1` (function -`runtime_init()`) must be a valid _virtual address_, same as the stack pointer's address. Both of +`kernel_init()`) must be a valid _virtual address_, same as the stack pointer's address. Both of them are programmed in function `fn prepare_el2_to_el1_transition(...)`, so we must ensure now that _link-time_ addresses are used here. For this reason, retrieval of these addresses happens in `assembly` in `boot.s`, where we can explicitly enforce generation of **absolute** addresses: @@ -108,7 +108,7 @@ _link-time_ addresses are used here. For this reason, retrieval of these address // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at // the top of the 64 bit address space, these are effectively virtual addresses. ADR_ABS x1, __boot_core_stack_end_exclusive -ADR_ABS x2, runtime_init +ADR_ABS x2, kernel_init ``` Both values are forwarded to the Rust entry point function `_start_rust()`, which in turn forwards @@ -246,15 +246,6 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/Cargo.toml 16_virtual_mem_part diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs 16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs --- 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs -@@ -11,7 +11,7 @@ - //! - //! crate::cpu::boot::arch_boot - --use crate::{cpu, memory, memory::Address, runtime_init}; -+use crate::{cpu, memory, memory::Address}; - use core::intrinsics::unlikely; - use cortex_a::{asm, regs::*}; - @@ -29,7 +29,10 @@ /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. @@ -262,7 +253,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs -unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { +unsafe fn prepare_el2_to_el1_transition( + virt_boot_core_stack_end_exclusive_addr: u64, -+ virt_runtime_init_addr: u64, ++ virt_kernel_init_addr: u64, +) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -270,9 +261,9 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs @@ -52,11 +55,11 @@ ); - // Second, let the link register point to runtime_init(). -- ELR_EL2.set(runtime_init::runtime_init as *const () as u64); -+ ELR_EL2.set(virt_runtime_init_addr); + // Second, let the link register point to kernel_init(). +- ELR_EL2.set(crate::kernel_init as *const () as u64); ++ ELR_EL2.set(virt_kernel_init_addr); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -281,29 +272,29 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.rs } //-------------------------------------------------------------------------------------------------- -@@ -74,9 +77,13 @@ +@@ -73,9 +76,13 @@ #[no_mangle] pub unsafe extern "C" fn _start_rust( phys_kernel_tables_base_addr: u64, - phys_boot_core_stack_end_exclusive_addr: u64, + virt_boot_core_stack_end_exclusive_addr: u64, -+ virt_runtime_init_addr: u64, ++ virt_kernel_init_addr: u64, ) -> ! { - prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + prepare_el2_to_el1_transition( + virt_boot_core_stack_end_exclusive_addr, -+ virt_runtime_init_addr, ++ virt_kernel_init_addr, + ); // Turn on the MMU for EL1. let addr = Address::new(phys_kernel_tables_base_addr as usize); -@@ -84,6 +91,7 @@ +@@ -83,6 +90,7 @@ cpu::wait_forever(); } -- // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. +- // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1. + // Use `eret` to "return" to EL1. Since virtual memory will already be enabled, this results in -+ // execution of runtime_init() in EL1 from its _virtual address_. ++ // execution of kernel_init() in EL1 from its _virtual address_. asm::eret() } @@ -329,7 +320,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s 1 .equ _EL2, 0x8 .equ _core_id_mask, 0b11 -@@ -47,11 +59,23 @@ +@@ -59,11 +71,23 @@ // Load the base address of the kernel's translation tables. ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs @@ -339,7 +330,7 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/_arch/aarch64/cpu/boot.s 1 + // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at + // the top of the 64 bit address space, these are effectively virtual addresses. + ADR_ABS x1, __boot_core_stack_end_exclusive -+ ADR_ABS x2, runtime_init ++ ADR_ABS x2, kernel_init + + // Load the PC-relative address of the stack and set the stack pointer. + // @@ -590,6 +581,22 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/bsp/raspberrypi/memory/mmu //-------------------------------------------------------------------------------------------------- // Public Definitions +diff -uNr 15_virtual_mem_part3_precomputed_tables/src/lib.rs 16_virtual_mem_part4_higher_half_kernel/src/lib.rs +--- 15_virtual_mem_part3_precomputed_tables/src/lib.rs ++++ 16_virtual_mem_part4_higher_half_kernel/src/lib.rs +@@ -154,11 +154,6 @@ + ) + } + +-#[cfg(not(test))] +-extern "Rust" { +- fn kernel_init() -> !; +-} +- + //-------------------------------------------------------------------------------------------------- + // Testing + //-------------------------------------------------------------------------------------------------- + diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs --- 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs +++ 16_virtual_mem_part4_higher_half_kernel/src/memory/mmu.rs @@ -606,18 +613,6 @@ diff -uNr 15_virtual_mem_part3_precomputed_tables/src/memory/mmu.rs 16_virtual_m type TableStartFromBottom; } -diff -uNr 15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs 16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs ---- 15_virtual_mem_part3_precomputed_tables/src/runtime_init.rs -+++ 16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs -@@ -30,6 +30,7 @@ - /// # Safety - /// - /// - Only a single core must be active and running this function. -+#[no_mangle] - pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - diff -uNr 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs 16_virtual_mem_part4_higher_half_kernel/tests/02_exception_sync_page_fault.rs --- 15_virtual_mem_part3_precomputed_tables/tests/02_exception_sync_page_fault.rs +++ 16_virtual_mem_part4_higher_half_kernel/tests/02_exception_sync_page_fault.rs diff --git a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs index 4f0ef0b4..fe8e9060 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.rs @@ -31,7 +31,7 @@ global_asm!(include_str!("boot.s")); #[inline(always)] unsafe fn prepare_el2_to_el1_transition( virt_boot_core_stack_end_exclusive_addr: u64, - virt_runtime_init_addr: u64, + virt_kernel_init_addr: u64, ) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -54,8 +54,8 @@ unsafe fn prepare_el2_to_el1_transition( + SPSR_EL2::M::EL1h, ); - // Second, let the link register point to runtime_init(). - ELR_EL2.set(virt_runtime_init_addr); + // Second, let the link register point to kernel_init(). + ELR_EL2.set(virt_kernel_init_addr); // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there // are no plans to ever return to EL2, just re-use the same stack. @@ -72,17 +72,16 @@ unsafe fn prepare_el2_to_el1_transition( /// /// # Safety /// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. -/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. #[no_mangle] pub unsafe extern "C" fn _start_rust( phys_kernel_tables_base_addr: u64, virt_boot_core_stack_end_exclusive_addr: u64, - virt_runtime_init_addr: u64, + virt_kernel_init_addr: u64, ) -> ! { prepare_el2_to_el1_transition( virt_boot_core_stack_end_exclusive_addr, - virt_runtime_init_addr, + virt_kernel_init_addr, ); // Turn on the MMU for EL1. @@ -92,6 +91,6 @@ pub unsafe extern "C" fn _start_rust( } // Use `eret` to "return" to EL1. Since virtual memory will already be enabled, this results in - // execution of runtime_init() in EL1 from its _virtual address_. + // execution of kernel_init() in EL1 from its _virtual address_. asm::eret() } diff --git a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s index 638259d4..32a5c3ff 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s +++ b/16_virtual_mem_part4_higher_half_kernel/src/_arch/aarch64/cpu/boot.s @@ -45,24 +45,36 @@ _start: // Only proceed if the core executes in EL2. Park it otherwise. mrs x0, CurrentEL cmp x0, _EL2 - b.ne 1f + b.ne parking_loop // Only proceed on the boot core. Park it otherwise. mrs x1, MPIDR_EL1 and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Load the base address of the kernel's translation tables. ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at // the top of the 64 bit address space, these are effectively virtual addresses. ADR_ABS x1, __boot_core_stack_end_exclusive - ADR_ABS x2, runtime_init + ADR_ABS x2, kernel_init // Load the PC-relative address of the stack and set the stack pointer. // @@ -79,8 +91,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld index 962638d0..a3179f1a 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld +++ b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/link.ld @@ -55,15 +55,13 @@ SECTIONS __rw_start = .; .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE . = ALIGN(64K); /* Align to page boundary */ diff --git a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs index 7f571e08..e51b00fa 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/bsp/raspberrypi/memory.rs @@ -37,7 +37,7 @@ pub mod mmu; use crate::memory::{Address, Physical, Virtual}; -use core::{cell::UnsafeCell, ops::RangeInclusive}; +use core::cell::UnsafeCell; //-------------------------------------------------------------------------------------------------- // Private Definitions @@ -49,8 +49,6 @@ extern "Rust" { static __rx_end_exclusive: UnsafeCell<()>; static __rw_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; static __rw_end_exclusive: UnsafeCell<()>; static __boot_core_stack_start: UnsafeCell<()>; @@ -186,23 +184,3 @@ fn boot_core_stack_guard_page_size() -> usize { fn phys_addr_space_end() -> Address { map::END } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/16_virtual_mem_part4_higher_half_kernel/src/lib.rs b/16_virtual_mem_part4_higher_half_kernel/src/lib.rs index f3ef2951..3e3e08d8 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/lib.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/lib.rs @@ -104,9 +104,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -130,7 +128,6 @@ #![test_runner(crate::test_runner)] mod panic_wait; -mod runtime_init; mod synchronization; pub mod bsp; @@ -176,7 +173,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) { } } -/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +/// The `kernel_init()` for unit tests. #[cfg(test)] #[no_mangle] unsafe fn kernel_init() -> ! { diff --git a/16_virtual_mem_part4_higher_half_kernel/src/memory.rs b/16_virtual_mem_part4_higher_half_kernel/src/memory.rs index 87ffa81f..4ac96904 100644 --- a/16_virtual_mem_part4_higher_half_kernel/src/memory.rs +++ b/16_virtual_mem_part4_higher_half_kernel/src/memory.rs @@ -11,7 +11,7 @@ use core::{ convert::TryFrom, fmt, marker::PhantomData, - ops::{AddAssign, RangeInclusive, SubAssign}, + ops::{AddAssign, SubAssign}, }; //-------------------------------------------------------------------------------------------------- @@ -144,59 +144,3 @@ impl fmt::Display for Address { write!(f, "{:04x}", q1) } } - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} - -//-------------------------------------------------------------------------------------------------- -// Testing -//-------------------------------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - use test_macros::kernel_test; - - /// Check `zero_volatile()`. - #[kernel_test] - fn zero_volatile_works() { - let mut x: [usize; 3] = [10, 11, 12]; - let x_range = x.as_mut_ptr_range(); - let x_range_inclusive = - RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) }); - - unsafe { zero_volatile(x_range_inclusive) }; - - assert_eq!(x, [0, 0, 0]); - } - - /// Check `bss` section layout. - #[kernel_test] - fn bss_section_is_sane() { - use crate::bsp::memory::bss_range_inclusive; - use core::mem; - - let start = *bss_range_inclusive().start() as usize; - let end = *bss_range_inclusive().end() as usize; - - assert_eq!(start % mem::size_of::(), 0); - assert_eq!(end % mem::size_of::(), 0); - assert!(end >= start); - } -} diff --git a/16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs b/16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs deleted file mode 100644 index 2c0a62e6..00000000 --- a/16_virtual_mem_part4_higher_half_kernel/src/runtime_init.rs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -#[no_mangle] -pub unsafe fn runtime_init() -> ! { - extern "Rust" { - fn kernel_init() -> !; - } - - zero_bss(); - kernel_init() -} diff --git a/X1_JTAG_boot/jtag_boot_rpi3.img b/X1_JTAG_boot/jtag_boot_rpi3.img index 96d8bfb1..91224331 100755 Binary files a/X1_JTAG_boot/jtag_boot_rpi3.img and b/X1_JTAG_boot/jtag_boot_rpi3.img differ diff --git a/X1_JTAG_boot/jtag_boot_rpi4.img b/X1_JTAG_boot/jtag_boot_rpi4.img index 76531e6f..25e8aedf 100755 Binary files a/X1_JTAG_boot/jtag_boot_rpi4.img and b/X1_JTAG_boot/jtag_boot_rpi4.img differ diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs index c85bb94b..7513df07 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs @@ -11,8 +11,6 @@ //! //! crate::cpu::boot::arch_boot -use crate::runtime_init; - // Assembly counterpart to this file. global_asm!(include_str!("boot.s")); @@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s")); /// The Rust entry of the `kernel` binary. /// /// The function is called from the assembly `_start` function. -/// -/// # Safety -/// -/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] pub unsafe fn _start_rust() -> ! { - runtime_init::runtime_init() + crate::kernel_init() } diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s index bfa94abf..f4162c87 100644 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s @@ -34,10 +34,22 @@ _start: and x1, x1, _core_id_mask ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs cmp x1, x2 - b.ne 1f + b.ne parking_loop - // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + // If execution reaches here, it is the boot core. + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +bss_init_loop: + cmp x0, x1 + b.eq prepare_rust + stp xzr, xzr, [x0], #16 + b bss_init_loop + + // Prepare the jump to Rust code. +prepare_rust: // Set the stack pointer. ADR_REL x0, __boot_core_stack_end_exclusive mov sp, x0 @@ -46,8 +58,9 @@ _start: b _start_rust // Infinitely wait for events (aka "park the core"). -1: wfe - b 1b +parking_loop: + wfe + b parking_loop .size _start, . - _start .type _start, function diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/link.ld b/X1_JTAG_boot/src/bsp/raspberrypi/link.ld index 97ea6d69..cf63a4a6 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/link.ld +++ b/X1_JTAG_boot/src/bsp/raspberrypi/link.ld @@ -42,14 +42,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss : ALIGN(16) { __bss_start = .; *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; + . = ALIGN(16); + __bss_end_exclusive = .; } :NONE } diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs b/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs index c6d65e36..ef397a6f 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs +++ b/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs @@ -4,18 +4,6 @@ //! BSP Memory Management. -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - //-------------------------------------------------------------------------------------------------- // Public Definitions //-------------------------------------------------------------------------------------------------- @@ -47,23 +35,3 @@ pub(super) mod map { pub const PL011_UART_START: usize = START + UART_OFFSET; } } - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/X1_JTAG_boot/src/main.rs b/X1_JTAG_boot/src/main.rs index 0dc87fa1..23221621 100644 --- a/X1_JTAG_boot/src/main.rs +++ b/X1_JTAG_boot/src/main.rs @@ -102,9 +102,7 @@ //! //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. #![allow(clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] @@ -119,10 +117,8 @@ mod bsp; mod console; mod cpu; mod driver; -mod memory; mod panic_wait; mod print; -mod runtime_init; mod synchronization; mod time; diff --git a/X1_JTAG_boot/src/memory.rs b/X1_JTAG_boot/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/X1_JTAG_boot/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/X1_JTAG_boot/src/runtime_init.rs b/X1_JTAG_boot/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/X1_JTAG_boot/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -}