Init DRAM in assembly instead of Rust

See https://github.com/rust-embedded/cortex-m-rt/issues/300
pull/115/head
Andre Richter 3 years ago
parent 39a066c246
commit 7f666000ce
No known key found for this signature in database
GPG Key ID: 2116C1AB102F615E

@ -2,8 +2,9 @@
## tl;dr
- We extend `boot.s` to call into Rust code for the first time. There, we zero the [bss] section
before execution is halted with a call to `panic()`.
- We extend `boot.s` to call into Rust code for the first time. Before the jump
to Rust happens, a bit of runtime init work is done.
- The Rust code being called just halts execution with a call to `panic!()`.
- Check out `make qemu` again to see the additional code run.
## Notable additions
@ -12,11 +13,11 @@
- New sections: `.rodata`, `.got`, `.data`, `.bss`.
- A dedicated place for linking boot-time arguments that need to be read by `_start()`.
- `_start()` in `_arch/__arch_name__/cpu/boot.s`:
1. Halt core if core != core0.
1. Set up the `stack pointer`.
1. Jump to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`.
- `runtime_init()` in `runtime_init.rs`:
- Zeros the `.bss` section.
1. Halts core if core != core0.
1. Initializes the `DRAM` by zeroing the [bss] section.
1. Sets up the `stack pointer`.
1. Jumps to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`.
- `_start_rust()`:
- Calls `kernel_init()`, which calls `panic!()`, which eventually halts core0 as well.
- The library now uses the [cortex-a] crate, which provides zero-overhead abstractions and wraps
`unsafe` parts when dealing with the CPU's resources.
@ -64,12 +65,8 @@ diff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile
diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arch/aarch64/cpu/boot.rs
--- 01_wait_forever/src/_arch/aarch64/cpu/boot.rs
+++ 02_runtime_init/src/_arch/aarch64/cpu/boot.rs
@@ -11,5 +11,23 @@
//!
//! crate::cpu::boot::arch_boot
@@ -13,3 +13,15 @@
+use crate::runtime_init;
+
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
+
@ -80,13 +77,9 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arc
+/// The Rust entry of the `kernel` binary.
+///
+/// The function is called from the assembly `_start` function.
+///
+/// # Safety
+///
+/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+#[no_mangle]
+pub unsafe fn _start_rust() -> ! {
+ runtime_init::runtime_init()
+ crate::kernel_init()
+}
diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch/aarch64/cpu/boot.s
@ -117,7 +110,7 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch
// Public Code
//--------------------------------------------------------------------------------------------------
.section .text._start
@@ -11,6 +29,22 @@
@@ -11,9 +29,38 @@
// fn _start()
//------------------------------------------------------------------------------
_start:
@ -126,10 +119,22 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch
+ and x1, x1, _core_id_mask
+ ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
+ cmp x1, x2
+ b.ne 1f
+ b.ne parking_loop
+
+ // If execution reaches here, it is the boot core.
+
+ // Initialize DRAM.
+ ADR_REL x0, __bss_start
+ ADR_REL x1, __bss_end_exclusive
+
+ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+bss_init_loop:
+ cmp x0, x1
+ b.eq prepare_rust
+ stp xzr, xzr, [x0], #16
+ b bss_init_loop
+
+ // Prepare the jump to Rust code.
+prepare_rust:
+ // Set the stack pointer.
+ ADR_REL x0, __boot_core_stack_end_exclusive
+ mov sp, x0
@ -138,8 +143,14 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch
+ b _start_rust
+
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
-1: wfe
- b 1b
+parking_loop:
+ wfe
+ b parking_loop
.size _start, . - _start
.type _start, function
diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs
--- 01_wait_forever/src/_arch/aarch64/cpu.rs
@ -194,7 +205,7 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/cpu.rs 02_runtime_init/src/bsp/ras
diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld
--- 01_wait_forever/src/bsp/raspberrypi/link.ld
+++ 02_runtime_init/src/bsp/raspberrypi/link.ld
@@ -11,17 +11,45 @@
@@ -11,17 +11,43 @@
PHDRS
{
segment_rx PT_LOAD FLAGS(5); /* 5 == RX */
@ -230,70 +241,25 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra
+ ***********************************************************************************************/
+ .data : { *(.data*) } :segment_rw
+
+ /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+ .bss : ALIGN(8)
+ /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
+ .bss : ALIGN(16)
+ {
+ __bss_start = .;
+ *(.bss*);
+ . = ALIGN(8);
+
+ . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
+ __bss_end_inclusive = . - 8;
+ . = ALIGN(16);
+ __bss_end_exclusive = .;
+ } :NONE
}
diff -uNr 01_wait_forever/src/bsp/raspberrypi/memory.rs 02_runtime_init/src/bsp/raspberrypi/memory.rs
--- 01_wait_forever/src/bsp/raspberrypi/memory.rs
+++ 02_runtime_init/src/bsp/raspberrypi/memory.rs
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//
+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
+
+//! BSP Memory Management.
+
+use core::{cell::UnsafeCell, ops::RangeInclusive};
+
+//--------------------------------------------------------------------------------------------------
+// Private Definitions
+//--------------------------------------------------------------------------------------------------
+
+// Symbols from the linker script.
+extern "Rust" {
+ static __bss_start: UnsafeCell<u64>;
+ static __bss_end_inclusive: UnsafeCell<u64>;
+}
+
+//--------------------------------------------------------------------------------------------------
+// Public Code
+//--------------------------------------------------------------------------------------------------
+
+/// Return the inclusive range spanning the .bss section.
+///
+/// # Safety
+///
+/// - Values are provided by the linker script and must be trusted as-is.
+/// - The linker-provided addresses must be u64 aligned.
+pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
+ let range;
+ unsafe {
+ range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
+ }
+ assert!(!range.is_empty());
+
+ range
+}
diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs
--- 01_wait_forever/src/bsp/raspberrypi.rs
+++ 02_runtime_init/src/bsp/raspberrypi.rs
@@ -4,4 +4,5 @@
@@ -4,4 +4,4 @@
//! Top-level BSP file for the Raspberry Pi 3 and 4.
-// Coming soon.
+pub mod cpu;
+pub mod memory;
diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs
--- 01_wait_forever/src/cpu.rs
@ -316,24 +282,19 @@ diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs
diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs
--- 01_wait_forever/src/main.rs
+++ 02_runtime_init/src/main.rs
@@ -102,14 +102,25 @@
@@ -102,8 +102,8 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//!
+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-#![feature(asm)]
#![feature(global_asm)]
#![no_main]
#![no_std]
mod bsp;
@@ -112,4 +112,11 @@
mod cpu;
+mod memory;
mod panic_wait;
+mod runtime_init;
-// Kernel code coming next tutorial.
+/// Early init code.
@ -345,41 +306,6 @@ diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs
+ panic!()
+}
diff -uNr 01_wait_forever/src/memory.rs 02_runtime_init/src/memory.rs
--- 01_wait_forever/src/memory.rs
+++ 02_runtime_init/src/memory.rs
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//
+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
+
+//! Memory Management.
+
+use core::ops::RangeInclusive;
+
+//--------------------------------------------------------------------------------------------------
+// Public Code
+//--------------------------------------------------------------------------------------------------
+
+/// Zero out an inclusive memory range.
+///
+/// # Safety
+///
+/// - `range.start` and `range.end` must be valid.
+/// - `range.start` and `range.end` must be `T` aligned.
+pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
+where
+ T: From<u8>,
+{
+ let mut ptr = *range.start();
+ let end_inclusive = *range.end();
+
+ while ptr <= end_inclusive {
+ core::ptr::write_volatile(ptr, T::from(0));
+ ptr = ptr.offset(1);
+ }
+}
diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs
--- 01_wait_forever/src/panic_wait.rs
+++ 02_runtime_init/src/panic_wait.rs
@ -396,46 +322,4 @@ diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs
+ cpu::wait_forever()
}
diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.rs
--- 01_wait_forever/src/runtime_init.rs
+++ 02_runtime_init/src/runtime_init.rs
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//
+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
+
+//! Rust runtime initialization code.
+
+use crate::{bsp, memory};
+
+//--------------------------------------------------------------------------------------------------
+// Private Code
+//--------------------------------------------------------------------------------------------------
+
+/// Zero out the .bss section.
+///
+/// # Safety
+///
+/// - Must only be called pre `kernel_init()`.
+#[inline(always)]
+unsafe fn zero_bss() {
+ memory::zero_volatile(bsp::memory::bss_range_inclusive());
+}
+
+//--------------------------------------------------------------------------------------------------
+// Public Code
+//--------------------------------------------------------------------------------------------------
+
+/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
+/// init code.
+///
+/// # Safety
+///
+/// - Only a single core must be active and running this function.
+pub unsafe fn runtime_init() -> ! {
+ zero_bss();
+
+ crate::kernel_init()
+}
```

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -34,10 +34,22 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -46,8 +58,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -5,4 +5,3 @@
//! Top-level BSP file for the Raspberry Pi 3 and 4.
pub mod cpu;
pub mod memory;

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![feature(global_asm)]
#![no_main]
@ -112,9 +110,7 @@
mod bsp;
mod cpu;
mod memory;
mod panic_wait;
mod runtime_init;
/// Early init code.
///

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -117,13 +117,12 @@ diff -uNr 02_runtime_init/src/bsp/raspberrypi/console.rs 03_hacky_hello_world/sr
diff -uNr 02_runtime_init/src/bsp/raspberrypi.rs 03_hacky_hello_world/src/bsp/raspberrypi.rs
--- 02_runtime_init/src/bsp/raspberrypi.rs
+++ 03_hacky_hello_world/src/bsp/raspberrypi.rs
@@ -4,5 +4,6 @@
@@ -4,4 +4,5 @@
//! Top-level BSP file for the Raspberry Pi 3 and 4.
+pub mod console;
pub mod cpu;
pub mod memory;
diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs
--- 02_runtime_init/src/console.rs
@ -152,9 +151,9 @@ diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs
diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs
--- 02_runtime_init/src/main.rs
+++ 03_hacky_hello_world/src/main.rs
@@ -106,14 +106,18 @@
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
@@ -104,13 +104,17 @@
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
+#![feature(format_args_nl)]
#![feature(global_asm)]
@ -165,13 +164,12 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs
mod bsp;
+mod console;
mod cpu;
mod memory;
mod panic_wait;
+mod print;
mod runtime_init;
/// Early init code.
@@ -122,5 +126,7 @@
///
@@ -118,5 +122,7 @@
///
/// - Only a single core must be active and running this function.
unsafe fn kernel_init() -> ! {

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -34,10 +34,22 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -46,8 +58,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -6,4 +6,3 @@
pub mod console;
pub mod cpu;
pub mod memory;

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![feature(format_args_nl)]
#![feature(global_asm)]
@ -115,10 +113,8 @@
mod bsp;
mod console;
mod cpu;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
/// Early init code.
///

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -224,7 +224,7 @@ diff -uNr 03_hacky_hello_world/src/console.rs 04_safe_globals/src/console.rs
diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs
--- 03_hacky_hello_world/src/main.rs
+++ 04_safe_globals/src/main.rs
@@ -109,6 +109,7 @@
@@ -107,6 +107,7 @@
#![feature(format_args_nl)]
#![feature(global_asm)]
#![feature(panic_info_message)]
@ -232,15 +232,15 @@ diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs
#![no_main]
#![no_std]
@@ -119,6 +120,7 @@
@@ -115,6 +116,7 @@
mod cpu;
mod panic_wait;
mod print;
mod runtime_init;
+mod synchronization;
/// Early init code.
///
@@ -126,7 +128,15 @@
@@ -122,7 +124,15 @@
///
/// - Only a single core must be active and running this function.
unsafe fn kernel_init() -> ! {

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -34,10 +34,22 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -46,8 +58,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -6,4 +6,3 @@
pub mod console;
pub mod cpu;
pub mod memory;

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![feature(format_args_nl)]
#![feature(global_asm)]
@ -116,10 +114,8 @@
mod bsp;
mod console;
mod cpu;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
/// Early init code.

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -1120,10 +1120,14 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/driver.rs 05_drivers_gpio_uart/src
diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs
--- 04_safe_globals/src/bsp/raspberrypi/memory.rs
+++ 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs
@@ -17,6 +17,38 @@
}
//--------------------------------------------------------------------------------------------------
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//
+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
+
+//! BSP Memory Management.
+
+//--------------------------------------------------------------------------------------------------
+// Public Definitions
+//--------------------------------------------------------------------------------------------------
+
@ -1154,21 +1158,16 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src
+ pub const PL011_UART_START: usize = START + UART_OFFSET;
+ }
+}
+
+//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
diff -uNr 04_safe_globals/src/bsp/raspberrypi.rs 05_drivers_gpio_uart/src/bsp/raspberrypi.rs
--- 04_safe_globals/src/bsp/raspberrypi.rs
+++ 05_drivers_gpio_uart/src/bsp/raspberrypi.rs
@@ -6,4 +6,33 @@
@@ -6,3 +6,33 @@
pub mod console;
pub mod cpu;
+pub mod driver;
pub mod memory;
+pub mod memory;
+
+//--------------------------------------------------------------------------------------------------
+// Global instances
@ -1321,24 +1320,24 @@ diff -uNr 04_safe_globals/src/driver.rs 05_drivers_gpio_uart/src/driver.rs
diff -uNr 04_safe_globals/src/main.rs 05_drivers_gpio_uart/src/main.rs
--- 04_safe_globals/src/main.rs
+++ 05_drivers_gpio_uart/src/main.rs
@@ -106,6 +106,8 @@
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
@@ -104,6 +104,8 @@
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
+#![allow(clippy::upper_case_acronyms)]
+#![feature(const_fn_fn_ptr_basics)]
#![feature(format_args_nl)]
#![feature(global_asm)]
#![feature(panic_info_message)]
@@ -116,6 +118,7 @@
@@ -114,6 +116,7 @@
mod bsp;
mod console;
mod cpu;
+mod driver;
mod memory;
mod panic_wait;
mod print;
@@ -127,16 +130,54 @@
mod synchronization;
@@ -123,16 +126,54 @@
/// # Safety
///
/// - Only a single core must be active and running this function.

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -34,10 +34,22 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -46,8 +58,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -4,18 +4,6 @@
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Definitions
//--------------------------------------------------------------------------------------------------
@ -47,23 +35,3 @@ pub(super) mod map {
pub const PL011_UART_START: usize = START + UART_OFFSET;
}
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![feature(const_fn_fn_ptr_basics)]
@ -119,10 +117,8 @@ mod bsp;
mod console;
mod cpu;
mod driver;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
/// Early init code.

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -232,26 +232,36 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/
.equ _core_id_mask, 0b11
//--------------------------------------------------------------------------------------------------
@@ -34,20 +45,31 @@
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
- b.ne 1f
+ b.ne 2f
+
+ // If execution reaches here, it is the boot core.
@@ -39,23 +50,35 @@
// If execution reaches here, it is the boot core.
// Initialize DRAM.
- ADR_REL x0, __bss_start
- ADR_REL x1, __bss_end_exclusive
+ ADR_ABS x0, __bss_start
+ ADR_ABS x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
- b.eq prepare_rust
+ b.eq relocate_binary
stp xzr, xzr, [x0], #16
b bss_init_loop
- // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+ // Next, relocate the binary.
+relocate_binary:
+ ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to.
+ ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to.
+ ADR_ABS x2, __binary_nonzero_end_exclusive
+
+1: ldr x3, [x0], #8
+copy_loop:
+ ldr x3, [x0], #8
+ str x3, [x1], #8
+ cmp x1, x2
+ b.lo 1b
+ b.lo copy_loop
+
// Prepare the jump to Rust code.
-prepare_rust:
// Set the stack pointer.
- ADR_REL x0, __boot_core_stack_end_exclusive
+ ADR_ABS x0, __boot_core_stack_end_exclusive
@ -264,13 +274,7 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/
+ br x1
// Infinitely wait for events (aka "park the core").
-1: wfe
- b 1b
+2: wfe
+ b 2b
.size _start, . - _start
.type _start, function
parking_loop:
diff -uNr 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs
--- 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs
@ -350,7 +354,7 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s
.text :
{
KEEP(*(.text._start))
@@ -42,8 +44,12 @@
@@ -42,6 +44,10 @@
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
@ -358,17 +362,14 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s
+ . = ALIGN(8);
+ __binary_nonzero_end_exclusive = .;
+
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
- .bss : ALIGN(8)
+ .bss :
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader/src/bsp/raspberrypi/memory.rs
--- 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs
+++ 06_uart_chainloader/src/bsp/raspberrypi/memory.rs
@@ -23,9 +23,10 @@
@@ -11,9 +11,10 @@
/// The board's physical memory map.
#[rustfmt::skip]
pub(super) mod map {
@ -381,34 +382,33 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader
/// Physical devices.
#[cfg(feature = "bsp_rpi3")]
@@ -52,7 +53,13 @@
// Public Code
//--------------------------------------------------------------------------------------------------
-/// Return the inclusive range spanning the .bss section.
@@ -35,3 +36,13 @@
pub const PL011_UART_START: usize = START + UART_OFFSET;
}
}
+
+//--------------------------------------------------------------------------------------------------
+// Public Code
+//--------------------------------------------------------------------------------------------------
+
+/// The address on which the Raspberry firmware loads every binary by default.
+#[inline(always)]
+pub fn board_default_load_addr() -> *const u64 {
+ map::BOARD_DEFAULT_LOAD_ADDRESS as _
+}
+
+/// Return the inclusive range spanning the relocated .bss section.
///
/// # Safety
///
diff -uNr 05_drivers_gpio_uart/src/main.rs 06_uart_chainloader/src/main.rs
--- 05_drivers_gpio_uart/src/main.rs
+++ 06_uart_chainloader/src/main.rs
@@ -107,6 +107,7 @@
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
@@ -105,6 +105,7 @@
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
+#![feature(asm)]
#![feature(const_fn_fn_ptr_basics)]
#![feature(format_args_nl)]
#![feature(global_asm)]
@@ -146,38 +147,56 @@
@@ -142,38 +143,56 @@
kernel_main()
}

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -45,20 +45,33 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 2f
b.ne parking_loop
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_ABS x0, __bss_start
ADR_ABS x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq relocate_binary
stp xzr, xzr, [x0], #16
b bss_init_loop
// Next, relocate the binary.
relocate_binary:
ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to.
ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to.
ADR_ABS x2, __binary_nonzero_end_exclusive
1: ldr x3, [x0], #8
copy_loop:
ldr x3, [x0], #8
str x3, [x1], #8
cmp x1, x2
b.lo 1b
b.lo copy_loop
// Prepare the jump to Rust code.
// Set the stack pointer.
ADR_ABS x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -68,8 +81,9 @@ _start:
br x1
// Infinitely wait for events (aka "park the core").
2: wfe
b 2b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -48,14 +48,12 @@ SECTIONS
. = ALIGN(8);
__binary_nonzero_end_exclusive = .;
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss :
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -4,18 +4,6 @@
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Definitions
//--------------------------------------------------------------------------------------------------
@ -58,19 +46,3 @@ pub(super) mod map {
pub fn board_default_load_addr() -> *const u64 {
map::BOARD_DEFAULT_LOAD_ADDRESS as _
}
/// Return the inclusive range spanning the relocated .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![feature(asm)]
@ -120,10 +118,8 @@ mod bsp;
mod console;
mod cpu;
mod driver;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
/// Early init code.

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -154,26 +154,36 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar
.equ _core_id_mask, 0b11
//--------------------------------------------------------------------------------------------------
@@ -45,31 +34,20 @@
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
- b.ne 2f
-
- // If execution reaches here, it is the boot core.
+ b.ne 1f
@@ -50,35 +39,23 @@
// If execution reaches here, it is the boot core.
// Initialize DRAM.
- ADR_ABS x0, __bss_start
- ADR_ABS x1, __bss_end_exclusive
+ ADR_REL x0, __bss_start
+ ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
- b.eq relocate_binary
+ b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
- // Next, relocate the binary.
-relocate_binary:
- ADR_REL x0, __binary_nonzero_start // The address the binary got loaded to.
- ADR_ABS x1, __binary_nonzero_start // The address the binary was linked to.
- ADR_ABS x2, __binary_nonzero_end_exclusive
-
-1: ldr x3, [x0], #8
-copy_loop:
- ldr x3, [x0], #8
- str x3, [x1], #8
- cmp x1, x2
- b.lo 1b
+ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
- b.lo copy_loop
-
// Prepare the jump to Rust code.
+prepare_rust:
// Set the stack pointer.
- ADR_ABS x0, __boot_core_stack_end_exclusive
+ ADR_REL x0, __boot_core_stack_end_exclusive
@ -186,13 +196,7 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar
+ b _start_rust
// Infinitely wait for events (aka "park the core").
-2: wfe
- b 2b
+1: wfe
+ b 1b
.size _start, . - _start
.type _start, function
parking_loop:
diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu.rs 07_timestamps/src/_arch/aarch64/cpu.rs
--- 06_uart_chainloader/src/_arch/aarch64/cpu.rs
@ -438,7 +442,7 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/
.text :
{
KEEP(*(.text._start))
@@ -44,12 +42,8 @@
@@ -44,10 +42,6 @@
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
@ -446,17 +450,14 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/
- . = ALIGN(8);
- __binary_nonzero_end_exclusive = .;
-
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
- .bss :
+ .bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bsp/raspberrypi/memory.rs
--- 06_uart_chainloader/src/bsp/raspberrypi/memory.rs
+++ 07_timestamps/src/bsp/raspberrypi/memory.rs
@@ -23,10 +23,9 @@
@@ -11,10 +11,9 @@
/// The board's physical memory map.
#[rustfmt::skip]
pub(super) mod map {
@ -469,21 +470,20 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bs
/// Physical devices.
#[cfg(feature = "bsp_rpi3")]
@@ -53,13 +52,7 @@
// Public Code
//--------------------------------------------------------------------------------------------------
@@ -36,13 +35,3 @@
pub const PL011_UART_START: usize = START + UART_OFFSET;
}
}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// The address on which the Raspberry firmware loads every binary by default.
-#[inline(always)]
-pub fn board_default_load_addr() -> *const u64 {
- map::BOARD_DEFAULT_LOAD_ADDRESS as _
-}
-
-/// Return the inclusive range spanning the relocated .bss section.
+/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs
--- 06_uart_chainloader/src/cpu.rs
@ -499,23 +499,23 @@ diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs
diff -uNr 06_uart_chainloader/src/main.rs 07_timestamps/src/main.rs
--- 06_uart_chainloader/src/main.rs
+++ 07_timestamps/src/main.rs
@@ -107,7 +107,6 @@
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
@@ -105,7 +105,6 @@
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
-#![feature(asm)]
#![feature(const_fn_fn_ptr_basics)]
#![feature(format_args_nl)]
#![feature(global_asm)]
@@ -125,6 +124,7 @@
@@ -121,6 +120,7 @@
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
+mod time;
/// Early init code.
///
@@ -147,56 +147,38 @@
@@ -143,56 +143,38 @@
kernel_main()
}

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -34,10 +34,22 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -46,8 +58,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -4,18 +4,6 @@
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Definitions
//--------------------------------------------------------------------------------------------------
@ -47,23 +35,3 @@ pub(super) mod map {
pub const PL011_UART_START: usize = START + UART_OFFSET;
}
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![feature(const_fn_fn_ptr_basics)]
@ -119,10 +117,8 @@ mod bsp;
mod console;
mod cpu;
mod driver;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
mod time;

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -11,8 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
/// The Rust entry of the `kernel` binary.
///
/// The function is called from the assembly `_start` function.
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
#[no_mangle]
pub unsafe fn _start_rust() -> ! {
runtime_init::runtime_init()
crate::kernel_init()
}

@ -34,10 +34,22 @@ _start:
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -46,8 +58,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -4,18 +4,6 @@
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Definitions
//--------------------------------------------------------------------------------------------------
@ -47,23 +35,3 @@ pub(super) mod map {
pub const PL011_UART_START: usize = START + UART_OFFSET;
}
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![feature(const_fn_fn_ptr_basics)]
@ -119,10 +117,8 @@ mod bsp;
mod console;
mod cpu;
mod driver;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
mod time;

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -62,7 +62,7 @@ Afterwards, we continue with preparing the `EL2` -> `EL1` transition by calling
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}
```
@ -125,19 +125,16 @@ SPSR_EL2.write(
+ SPSR_EL2::M::EL1h,
);
// Second, let the link register point to runtime_init().
ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
// Second, let the link register point to kernel_init().
ELR_EL2.set(crate::kernel_init as *const () as u64);
// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
// are no plans to ever return to EL2, just re-use the same stack.
SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);
```
As you can see, we are populating `ELR_EL2` with the address of the [runtime_init()] function that
we earlier used to call directly from the entrypoint. Finally, we set the stack pointer for
`SP_EL1`.
[runtime_init()]: src/runtime_init.rs
As you can see, we are populating `ELR_EL2` with the address of the `kernel_init()` function that we
earlier used to call directly from the entrypoint. Finally, we set the stack pointer for `SP_EL1`.
You might have noticed that the stack's address was supplied as a function argument. As you might
remember, in `_start()` in `boot.s`, we are already setting up the stack for `EL2`. Since there
@ -151,7 +148,7 @@ Lastly, back in `_start_rust()` a call to `ERET` is made:
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}
```
@ -214,12 +211,12 @@ diff -uNr 08_hw_debug_JTAG/Cargo.toml 09_privilege_level/Cargo.toml
diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/_arch/aarch64/cpu/boot.rs
--- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs
+++ 09_privilege_level/src/_arch/aarch64/cpu/boot.rs
@@ -12,11 +12,53 @@
@@ -11,17 +11,67 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
+use cortex_a::{asm, regs::*};
+
// Assembly counterpart to this file.
global_asm!(include_str!("boot.s"));
@ -256,8 +253,8 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/
+ + SPSR_EL2::M::EL1h,
+ );
+
+ // Second, let the link register point to runtime_init().
+ ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+ // Second, let the link register point to kernel_init().
+ ELR_EL2.set(crate::kernel_init as *const () as u64);
+
+ // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
+ // are no plans to ever return to EL2, just re-use the same stack.
@ -268,18 +265,20 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/
// Public Code
//--------------------------------------------------------------------------------------------------
@@ -27,7 +69,11 @@
/// # Safety
/// The Rust entry of the `kernel` binary.
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
/// The function is called from the assembly `_start` function.
+///
+/// # Safety
+///
+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
#[no_mangle]
-pub unsafe fn _start_rust() -> ! {
- runtime_init::runtime_init()
- crate::kernel_init()
+pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
+ prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
+
+ // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+ // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
+ asm::eret()
}
@ -301,15 +300,15 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s 09_privilege_level/src/_
+ // Only proceed if the core executes in EL2. Park it otherwise.
+ mrs x0, CurrentEL
+ cmp x0, _EL2
+ b.ne 1f
+ b.ne parking_loop
+
// Only proceed on the boot core. Park it otherwise.
mrs x1, MPIDR_EL1
and x1, x1, _core_id_mask
@@ -38,11 +44,11 @@
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
@@ -50,11 +56,11 @@
// Prepare the jump to Rust code.
prepare_rust:
- // Set the stack pointer.
+ // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
ADR_REL x0, __boot_core_stack_end_exclusive
@ -499,15 +498,15 @@ diff -uNr 08_hw_debug_JTAG/src/exception.rs 09_privilege_level/src/exception.rs
diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs
--- 08_hw_debug_JTAG/src/main.rs
+++ 09_privilege_level/src/main.rs
@@ -119,6 +119,7 @@
@@ -117,6 +117,7 @@
mod console;
mod cpu;
mod driver;
+mod exception;
mod memory;
mod panic_wait;
mod print;
@@ -149,6 +150,8 @@
mod synchronization;
@@ -145,6 +146,8 @@
/// The main function running after the early init.
fn kernel_main() -> ! {
@ -516,7 +515,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs
use core::time::Duration;
use driver::interface::DriverManager;
use time::interface::TimeManager;
@@ -160,6 +163,12 @@
@@ -156,6 +159,12 @@
);
info!("Booting on: {}", bsp::board_name());
@ -529,7 +528,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs
info!(
"Architectural timer resolution: {} ns",
time::time_manager().resolution().as_nanos()
@@ -174,11 +183,15 @@
@@ -170,11 +179,15 @@
info!(" {}. {}", i + 1, driver.compatible());
}

@ -11,7 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
use cortex_a::{asm, regs::*};
// Assembly counterpart to this file.
@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
+ SPSR_EL2::M::EL1h,
);
// Second, let the link register point to runtime_init().
ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
// Second, let the link register point to kernel_init().
ELR_EL2.set(crate::kernel_init as *const () as u64);
// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
// are no plans to ever return to EL2, just re-use the same stack.
@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
#[no_mangle]
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}

@ -33,17 +33,29 @@ _start:
// Only proceed if the core executes in EL2. Park it otherwise.
mrs x0, CurrentEL
cmp x0, _EL2
b.ne 1f
b.ne parking_loop
// Only proceed on the boot core. Park it otherwise.
mrs x1, MPIDR_EL1
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -52,8 +64,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -42,14 +42,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -4,18 +4,6 @@
//! BSP Memory Management.
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
// Private Definitions
//--------------------------------------------------------------------------------------------------
// Symbols from the linker script.
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
// Public Definitions
//--------------------------------------------------------------------------------------------------
@ -47,23 +35,3 @@ pub(super) mod map {
pub const PL011_UART_START: usize = START + UART_OFFSET;
}
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![feature(const_fn_fn_ptr_basics)]
@ -120,10 +118,8 @@ mod console;
mod cpu;
mod driver;
mod exception;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
mod time;

@ -1,30 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Memory Management.
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -939,26 +939,28 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 10_virtual_mem_pa
diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs
--- 09_privilege_level/src/bsp/raspberrypi/memory.rs
+++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs
@@ -4,6 +4,8 @@
@@ -4,6 +4,20 @@
//! BSP Memory Management.
+pub mod mmu;
+
use core::{cell::UnsafeCell, ops::RangeInclusive};
//--------------------------------------------------------------------------------------------------
@@ -12,6 +14,9 @@
// Symbols from the linker script.
extern "Rust" {
+use core::cell::UnsafeCell;
+
+//--------------------------------------------------------------------------------------------------
+// Private Definitions
+//--------------------------------------------------------------------------------------------------
+
+// Symbols from the linker script.
+extern "Rust" {
+ static __rx_start: UnsafeCell<()>;
+ static __rx_end_exclusive: UnsafeCell<()>;
+}
+
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
@@ -23,6 +28,20 @@
//--------------------------------------------------------------------------------------------------
// Public Definitions
//--------------------------------------------------------------------------------------------------
@@ -11,6 +25,20 @@
/// The board's physical memory map.
#[rustfmt::skip]
pub(super) mod map {
@ -979,7 +981,7 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_
pub const GPIO_OFFSET: usize = 0x0020_0000;
pub const UART_OFFSET: usize = 0x0020_1000;
@@ -35,6 +54,7 @@
@@ -23,6 +51,7 @@
pub const START: usize = 0x3F00_0000;
pub const GPIO_START: usize = START + GPIO_OFFSET;
pub const PL011_UART_START: usize = START + UART_OFFSET;
@ -987,15 +989,15 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_
}
/// Physical devices.
@@ -45,10 +65,35 @@
@@ -33,5 +62,30 @@
pub const START: usize = 0xFE00_0000;
pub const GPIO_START: usize = START + GPIO_OFFSET;
pub const PL011_UART_START: usize = START + UART_OFFSET;
+ pub const END_INCLUSIVE: usize = 0xFF84_FFFF;
}
}
//--------------------------------------------------------------------------------------------------
+
+//--------------------------------------------------------------------------------------------------
+// Private Code
+//--------------------------------------------------------------------------------------------------
+
@ -1018,11 +1020,6 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_
+fn rx_end_exclusive() -> usize {
+ unsafe { __rx_end_exclusive.get() as usize }
+}
+
+//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/src/bsp.rs
--- 09_privilege_level/src/bsp.rs
@ -1040,8 +1037,8 @@ diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/sr
diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/src/main.rs
--- 09_privilege_level/src/main.rs
+++ 10_virtual_mem_part1_identity_mapping/src/main.rs
@@ -107,7 +107,11 @@
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
@@ -105,7 +105,11 @@
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
+#![allow(incomplete_features)]
@ -1052,7 +1049,15 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s
#![feature(format_args_nl)]
#![feature(global_asm)]
#![feature(panic_info_message)]
@@ -132,9 +136,17 @@
@@ -118,6 +122,7 @@
mod cpu;
mod driver;
mod exception;
+mod memory;
mod panic_wait;
mod print;
mod synchronization;
@@ -128,9 +133,17 @@
/// # Safety
///
/// - Only a single core must be active and running this function.
@ -1071,7 +1076,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s
for i in bsp::driver::driver_manager().all_device_drivers().iter() {
if let Err(x) = i.init() {
@@ -163,6 +175,9 @@
@@ -159,6 +172,9 @@
);
info!("Booting on: {}", bsp::board_name());
@ -1081,7 +1086,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s
let (_, privilege_level) = exception::current_privilege_level();
info!("Current privilege level: {}", privilege_level);
@@ -186,6 +201,13 @@
@@ -182,6 +198,13 @@
info!("Timer test, spinning for 1 second");
time::time_manager().spin_for(Duration::from_secs(1));
@ -1387,14 +1392,13 @@ diff -uNr 09_privilege_level/src/memory/mmu.rs 10_virtual_mem_part1_identity_map
diff -uNr 09_privilege_level/src/memory.rs 10_virtual_mem_part1_identity_mapping/src/memory.rs
--- 09_privilege_level/src/memory.rs
+++ 10_virtual_mem_part1_identity_mapping/src/memory.rs
@@ -4,6 +4,8 @@
//! Memory Management.
+pub mod mmu;
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//
+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
+
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
+//! Memory Management.
+
+pub mod mmu;
```

@ -11,7 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
use cortex_a::{asm, regs::*};
// Assembly counterpart to this file.
@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
+ SPSR_EL2::M::EL1h,
);
// Second, let the link register point to runtime_init().
ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
// Second, let the link register point to kernel_init().
ELR_EL2.set(crate::kernel_init as *const () as u64);
// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
// are no plans to ever return to EL2, just re-use the same stack.
@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
#[no_mangle]
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}

@ -33,17 +33,29 @@ _start:
// Only proceed if the core executes in EL2. Park it otherwise.
mrs x0, CurrentEL
cmp x0, _EL2
b.ne 1f
b.ne parking_loop
// Only proceed on the boot core. Park it otherwise.
mrs x1, MPIDR_EL1
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -52,8 +64,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -46,14 +46,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -6,7 +6,7 @@
pub mod mmu;
use core::{cell::UnsafeCell, ops::RangeInclusive};
use core::cell::UnsafeCell;
//--------------------------------------------------------------------------------------------------
// Private Definitions
@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
extern "Rust" {
static __rx_start: UnsafeCell<()>;
static __rx_end_exclusive: UnsafeCell<()>;
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
@ -92,23 +89,3 @@ fn rx_start() -> usize {
fn rx_end_exclusive() -> usize {
unsafe { __rx_end_exclusive.get() as usize }
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![allow(incomplete_features)]
@ -127,7 +125,6 @@ mod exception;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
mod time;

@ -5,28 +5,3 @@
//! Memory Management.
pub mod mmu;
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -967,7 +967,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/exception.rs 11_exceptions_p
diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_groundwork/src/main.rs
--- 10_virtual_mem_part1_identity_mapping/src/main.rs
+++ 11_exceptions_part1_groundwork/src/main.rs
@@ -144,6 +144,8 @@
@@ -141,6 +141,8 @@
use driver::interface::DriverManager;
use memory::mmu::interface::MMU;
@ -976,7 +976,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_
if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {
panic!("MMU: {}", string);
}
@@ -201,13 +203,28 @@
@@ -198,13 +200,28 @@
info!("Timer test, spinning for 1 second");
time::time_manager().spin_for(Duration::from_secs(1));

@ -11,7 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
use cortex_a::{asm, regs::*};
// Assembly counterpart to this file.
@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
+ SPSR_EL2::M::EL1h,
);
// Second, let the link register point to runtime_init().
ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
// Second, let the link register point to kernel_init().
ELR_EL2.set(crate::kernel_init as *const () as u64);
// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
// are no plans to ever return to EL2, just re-use the same stack.
@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
#[no_mangle]
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}

@ -33,17 +33,29 @@ _start:
// Only proceed if the core executes in EL2. Park it otherwise.
mrs x0, CurrentEL
cmp x0, _EL2
b.ne 1f
b.ne parking_loop
// Only proceed on the boot core. Park it otherwise.
mrs x1, MPIDR_EL1
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -52,8 +64,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -46,14 +46,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -6,7 +6,7 @@
pub mod mmu;
use core::{cell::UnsafeCell, ops::RangeInclusive};
use core::cell::UnsafeCell;
//--------------------------------------------------------------------------------------------------
// Private Definitions
@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
extern "Rust" {
static __rx_start: UnsafeCell<()>;
static __rx_end_exclusive: UnsafeCell<()>;
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
@ -92,23 +89,3 @@ fn rx_start() -> usize {
fn rx_end_exclusive() -> usize {
unsafe { __rx_end_exclusive.get() as usize }
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -102,9 +102,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![allow(incomplete_features)]
@ -127,7 +125,6 @@ mod exception;
mod memory;
mod panic_wait;
mod print;
mod runtime_init;
mod synchronization;
mod time;

@ -5,28 +5,3 @@
//! Memory Management.
pub mod mmu;
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}

@ -1,37 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
zero_bss();
crate::kernel_init()
}

@ -215,25 +215,24 @@ kernel boot:
| | Function | File |
| - | - | - |
| 1. | `_start()` | `lib.rs` |
| 2. | (some more aarch64 code) | `lib.rs` |
| 3. | `runtime_init()` | `lib.rs` |
| 4. | `kernel_init()` | `main.rs` |
| 5. | `kernel_main()` | `main.rs` |
| 1. | `_start()` | The library's `boot.s` |
| 2. | (some more aarch64 code) | The library's `boot.rs` |
| 3. | `kernel_init()` | `main.rs` |
| 4. | `kernel_main()` | `main.rs` |
A function named `main` is never called. Hence, the `main()` function generated by `cargo test`
would be silently dropped, and therefore the tests would never be executed. As you can see,
`runtime_init()` is the last function residing in our carved-out `lib.rs`, and it calls into
`kernel_init()`. So in order to get the tests to execute, we add a test-environment version of
`kernel_init()` to `lib.rs` as well (conditional compilation ensures it is only present when the
test flag is set), and call the `cargo test` generated `main()` function from there.
would be silently dropped, and therefore the tests would never be executed. As you can see, the
first function getting called in our carved-out `main.rs` is `kernel_init()`. So in order to get the
tests to execute, we add a test-environment version of `kernel_init()` to `lib.rs` as well
(conditional compilation ensures it is only present when the test flag is set), and call the `cargo
test` generated `main()` function from there.
This is where `#![reexport_test_harness_main = "test_main"]` finally comes into picture. It declares
the name of the generated main function so that we can manually call it. Here is the final
implementation in `lib.rs`:
```rust
/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
/// The `kernel_init()` for unit tests.
#[cfg(test)]
#[no_mangle]
unsafe fn kernel_init() -> ! {
@ -1073,7 +1072,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translatio
diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs
--- 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs
+++ 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs
@@ -162,3 +162,22 @@
@@ -162,3 +162,33 @@
SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)
}
}
@ -1085,12 +1084,23 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_inte
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use core::{cell::UnsafeCell, ops::Range};
+ use test_macros::kernel_test;
+
+ /// Check if KERNEL_TABLES is in .bss.
+ #[kernel_test]
+ fn kernel_tables_in_bss() {
+ let bss_range = bsp::memory::bss_range_inclusive();
+ extern "Rust" {
+ static __bss_start: UnsafeCell<u64>;
+ static __bss_end_exclusive: UnsafeCell<u64>;
+ }
+
+ let bss_range = unsafe {
+ Range {
+ start: __bss_start.get(),
+ end: __bss_end_exclusive.get(),
+ }
+ };
+ let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };
+
+ assert!(bss_range.contains(&kernel_tables_addr));
@ -1207,7 +1217,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/exception.rs 12_integrated_testing/
diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/lib.rs
--- 11_exceptions_part1_groundwork/src/lib.rs
+++ 12_integrated_testing/src/lib.rs
@@ -0,0 +1,184 @@
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: MIT OR Apache-2.0
+//
+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
@ -1314,9 +1324,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
+//!
+//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
+//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//!
+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
+
+#![allow(clippy::upper_case_acronyms)]
+#![allow(incomplete_features)]
@ -1337,7 +1345,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
+#![test_runner(crate::test_runner)]
+
+mod panic_wait;
+mod runtime_init;
+mod synchronization;
+
+pub mod bsp;
@ -1362,6 +1369,11 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
+ )
+}
+
+#[cfg(not(test))]
+extern "Rust" {
+ fn kernel_init() -> !;
+}
+
+//--------------------------------------------------------------------------------------------------
+// Testing
+//--------------------------------------------------------------------------------------------------
@ -1381,7 +1393,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
+ }
+}
+
+/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
+/// The `kernel_init()` for unit tests.
+#[cfg(test)]
+#[no_mangle]
+unsafe fn kernel_init() -> ! {
@ -1396,7 +1408,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/main.rs
--- 11_exceptions_part1_groundwork/src/main.rs
+++ 12_integrated_testing/src/main.rs
@@ -6,130 +6,12 @@
@@ -6,127 +6,12 @@
#![doc(html_logo_url = "https://git.io/JeGIp")]
//! The `kernel` binary.
@ -1496,9 +1508,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
-//!
-//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
-//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
-//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-
-#![allow(clippy::upper_case_acronyms)]
-#![allow(incomplete_features)]
@ -1522,14 +1532,13 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
-mod memory;
-mod panic_wait;
-mod print;
-mod runtime_init;
-mod synchronization;
-mod time;
+use libkernel::{bsp, console, driver, exception, info, memory, time};
/// Early init code.
///
@@ -140,6 +22,7 @@
@@ -137,6 +22,7 @@
/// - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,
/// e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ
/// NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.
@ -1537,7 +1546,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
unsafe fn kernel_init() -> ! {
use driver::interface::DriverManager;
use memory::mmu::interface::MMU;
@@ -166,15 +49,9 @@
@@ -163,15 +49,9 @@
fn kernel_main() -> ! {
use bsp::console::console;
use console::interface::All;
@ -1554,7 +1563,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
info!("Booting on: {}", bsp::board_name());
info!("MMU online. Special regions:");
@@ -200,31 +77,6 @@
@@ -197,31 +77,6 @@
info!(" {}. {}", i + 1, driver.compatible());
}
@ -1609,51 +1618,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/memory/mmu.rs 12_integrated_testing
+ }
}
diff -uNr 11_exceptions_part1_groundwork/src/memory.rs 12_integrated_testing/src/memory.rs
--- 11_exceptions_part1_groundwork/src/memory.rs
+++ 12_integrated_testing/src/memory.rs
@@ -30,3 +30,40 @@
ptr = ptr.offset(1);
}
}
+
+//--------------------------------------------------------------------------------------------------
+// Testing
+//--------------------------------------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use test_macros::kernel_test;
+
+ /// Check `zero_volatile()`.
+ #[kernel_test]
+ fn zero_volatile_works() {
+ let mut x: [usize; 3] = [10, 11, 12];
+ let x_range = x.as_mut_ptr_range();
+ let x_range_inclusive =
+ RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) });
+
+ unsafe { zero_volatile(x_range_inclusive) };
+
+ assert_eq!(x, [0, 0, 0]);
+ }
+
+ /// Check `bss` section layout.
+ #[kernel_test]
+ fn bss_section_is_sane() {
+ use crate::bsp::memory::bss_range_inclusive;
+ use core::mem;
+
+ let start = *bss_range_inclusive().start() as usize;
+ let end = *bss_range_inclusive().end() as usize;
+
+ assert_eq!(start modulo mem::size_of::<usize>(), 0);
+ assert_eq!(end modulo mem::size_of::<usize>(), 0);
+ assert!(end >= start);
+ }
+}
diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing/src/panic_wait.rs
--- 11_exceptions_part1_groundwork/src/panic_wait.rs
+++ 12_integrated_testing/src/panic_wait.rs
@ -1689,23 +1653,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing
+ _panic_exit()
}
diff -uNr 11_exceptions_part1_groundwork/src/runtime_init.rs 12_integrated_testing/src/runtime_init.rs
--- 11_exceptions_part1_groundwork/src/runtime_init.rs
+++ 12_integrated_testing/src/runtime_init.rs
@@ -31,7 +31,10 @@
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
- zero_bss();
+ extern "Rust" {
+ fn kernel_init() -> !;
+ }
- crate::kernel_init()
+ zero_bss();
+ kernel_init()
}
diff -uNr 11_exceptions_part1_groundwork/test-macros/Cargo.toml 12_integrated_testing/test-macros/Cargo.toml
--- 11_exceptions_part1_groundwork/test-macros/Cargo.toml
+++ 12_integrated_testing/test-macros/Cargo.toml

@ -11,7 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
use cortex_a::{asm, regs::*};
// Assembly counterpart to this file.
@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
+ SPSR_EL2::M::EL1h,
);
// Second, let the link register point to runtime_init().
ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
// Second, let the link register point to kernel_init().
ELR_EL2.set(crate::kernel_init as *const () as u64);
// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
// are no plans to ever return to EL2, just re-use the same stack.
@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
#[no_mangle]
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}

@ -33,17 +33,29 @@ _start:
// Only proceed if the core executes in EL2. Park it otherwise.
mrs x0, CurrentEL
cmp x0, _EL2
b.ne 1f
b.ne parking_loop
// Only proceed on the boot core. Park it otherwise.
mrs x1, MPIDR_EL1
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -52,8 +64,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit {
#[cfg(test)]
mod tests {
use super::*;
use core::{cell::UnsafeCell, ops::Range};
use test_macros::kernel_test;
/// Check if KERNEL_TABLES is in .bss.
#[kernel_test]
fn kernel_tables_in_bss() {
let bss_range = bsp::memory::bss_range_inclusive();
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_exclusive: UnsafeCell<u64>;
}
let bss_range = unsafe {
Range {
start: __bss_start.get(),
end: __bss_end_exclusive.get(),
}
};
let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };
assert!(bss_range.contains(&kernel_tables_addr));

@ -46,14 +46,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -6,7 +6,7 @@
pub mod mmu;
use core::{cell::UnsafeCell, ops::RangeInclusive};
use core::cell::UnsafeCell;
//--------------------------------------------------------------------------------------------------
// Private Definitions
@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
extern "Rust" {
static __rx_start: UnsafeCell<()>;
static __rx_end_exclusive: UnsafeCell<()>;
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
@ -92,23 +89,3 @@ fn rx_start() -> usize {
fn rx_end_exclusive() -> usize {
unsafe { __rx_end_exclusive.get() as usize }
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -104,9 +104,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![allow(incomplete_features)]
@ -127,7 +125,6 @@
#![test_runner(crate::test_runner)]
mod panic_wait;
mod runtime_init;
mod synchronization;
pub mod bsp;
@ -152,6 +149,11 @@ pub fn version() -> &'static str {
)
}
#[cfg(not(test))]
extern "Rust" {
fn kernel_init() -> !;
}
//--------------------------------------------------------------------------------------------------
// Testing
//--------------------------------------------------------------------------------------------------
@ -171,7 +173,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) {
}
}
/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
/// The `kernel_init()` for unit tests.
#[cfg(test)]
#[no_mangle]
unsafe fn kernel_init() -> ! {

@ -5,65 +5,3 @@
//! Memory Management.
pub mod mmu;
use core::ops::RangeInclusive;
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Zero out an inclusive memory range.
///
/// # Safety
///
/// - `range.start` and `range.end` must be valid.
/// - `range.start` and `range.end` must be `T` aligned.
pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
where
T: From<u8>,
{
let mut ptr = *range.start();
let end_inclusive = *range.end();
while ptr <= end_inclusive {
core::ptr::write_volatile(ptr, T::from(0));
ptr = ptr.offset(1);
}
}
//--------------------------------------------------------------------------------------------------
// Testing
//--------------------------------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
use test_macros::kernel_test;
/// Check `zero_volatile()`.
#[kernel_test]
fn zero_volatile_works() {
let mut x: [usize; 3] = [10, 11, 12];
let x_range = x.as_mut_ptr_range();
let x_range_inclusive =
RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) });
unsafe { zero_volatile(x_range_inclusive) };
assert_eq!(x, [0, 0, 0]);
}
/// Check `bss` section layout.
#[kernel_test]
fn bss_section_is_sane() {
use crate::bsp::memory::bss_range_inclusive;
use core::mem;
let start = *bss_range_inclusive().start() as usize;
let end = *bss_range_inclusive().end() as usize;
assert_eq!(start % mem::size_of::<usize>(), 0);
assert_eq!(end % mem::size_of::<usize>(), 0);
assert!(end >= start);
}
}

@ -1,40 +0,0 @@
// SPDX-License-Identifier: MIT OR Apache-2.0
//
// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
//! Rust runtime initialization code.
use crate::{bsp, memory};
//--------------------------------------------------------------------------------------------------
// Private Code
//--------------------------------------------------------------------------------------------------
/// Zero out the .bss section.
///
/// # Safety
///
/// - Must only be called pre `kernel_init()`.
#[inline(always)]
unsafe fn zero_bss() {
memory::zero_volatile(bsp::memory::bss_range_inclusive());
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
/// init code.
///
/// # Safety
///
/// - Only a single core must be active and running this function.
pub unsafe fn runtime_init() -> ! {
extern "Rust" {
fn kernel_init() -> !;
}
zero_bss();
kernel_init()
}

@ -2104,7 +2104,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/exception.rs 13_exceptions_p
diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs
--- 12_integrated_testing/src/bsp/raspberrypi/memory.rs
+++ 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs
@@ -51,10 +51,12 @@
@@ -48,10 +48,12 @@
pub mod mmio {
use super::*;
@ -2121,7 +2121,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part
}
/// Physical devices.
@@ -65,6 +67,8 @@
@@ -62,6 +64,8 @@
pub const START: usize = 0xFE00_0000;
pub const GPIO_START: usize = START + GPIO_OFFSET;
pub const PL011_UART_START: usize = START + UART_OFFSET;
@ -2376,7 +2376,7 @@ diff -uNr 12_integrated_testing/src/exception/asynchronous.rs 13_exceptions_part
diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/src/lib.rs
--- 12_integrated_testing/src/lib.rs
+++ 13_exceptions_part2_peripheral_IRQs/src/lib.rs
@@ -110,6 +110,7 @@
@@ -108,6 +108,7 @@
#![allow(clippy::upper_case_acronyms)]
#![allow(incomplete_features)]
@ -2384,7 +2384,7 @@ diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/s
#![feature(const_fn_fn_ptr_basics)]
#![feature(const_generics)]
#![feature(const_panic)]
@@ -137,6 +138,7 @@
@@ -134,6 +135,7 @@
pub mod exception;
pub mod memory;
pub mod print;

@ -11,7 +11,6 @@
//!
//! crate::cpu::boot::arch_boot
use crate::runtime_init;
use cortex_a::{asm, regs::*};
// Assembly counterpart to this file.
@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
+ SPSR_EL2::M::EL1h,
);
// Second, let the link register point to runtime_init().
ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
// Second, let the link register point to kernel_init().
ELR_EL2.set(crate::kernel_init as *const () as u64);
// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
// are no plans to ever return to EL2, just re-use the same stack.
@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
///
/// # Safety
///
/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
#[no_mangle]
pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
// Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
asm::eret()
}

@ -33,17 +33,29 @@ _start:
// Only proceed if the core executes in EL2. Park it otherwise.
mrs x0, CurrentEL
cmp x0, _EL2
b.ne 1f
b.ne parking_loop
// Only proceed on the boot core. Park it otherwise.
mrs x1, MPIDR_EL1
and x1, x1, _core_id_mask
ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs
cmp x1, x2
b.ne 1f
b.ne parking_loop
// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
// If execution reaches here, it is the boot core.
// Initialize DRAM.
ADR_REL x0, __bss_start
ADR_REL x1, __bss_end_exclusive
bss_init_loop:
cmp x0, x1
b.eq prepare_rust
stp xzr, xzr, [x0], #16
b bss_init_loop
// Prepare the jump to Rust code.
prepare_rust:
// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
ADR_REL x0, __boot_core_stack_end_exclusive
mov sp, x0
@ -52,8 +64,9 @@ _start:
b _start_rust
// Infinitely wait for events (aka "park the core").
1: wfe
b 1b
parking_loop:
wfe
b parking_loop
.size _start, . - _start
.type _start, function

@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit {
#[cfg(test)]
mod tests {
use super::*;
use core::{cell::UnsafeCell, ops::Range};
use test_macros::kernel_test;
/// Check if KERNEL_TABLES is in .bss.
#[kernel_test]
fn kernel_tables_in_bss() {
let bss_range = bsp::memory::bss_range_inclusive();
extern "Rust" {
static __bss_start: UnsafeCell<u64>;
static __bss_end_exclusive: UnsafeCell<u64>;
}
let bss_range = unsafe {
Range {
start: __bss_start.get(),
end: __bss_end_exclusive.get(),
}
};
let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };
assert!(bss_range.contains(&kernel_tables_addr));

@ -46,14 +46,12 @@ SECTIONS
***********************************************************************************************/
.data : { *(.data*) } :segment_rw
/* Section is zeroed in u64 chunks, align start and end to 8 bytes */
.bss : ALIGN(8)
/* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
.bss : ALIGN(16)
{
__bss_start = .;
*(.bss*);
. = ALIGN(8);
. += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
__bss_end_inclusive = . - 8;
. = ALIGN(16);
__bss_end_exclusive = .;
} :NONE
}

@ -6,7 +6,7 @@
pub mod mmu;
use core::{cell::UnsafeCell, ops::RangeInclusive};
use core::cell::UnsafeCell;
//--------------------------------------------------------------------------------------------------
// Private Definitions
@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
extern "Rust" {
static __rx_start: UnsafeCell<()>;
static __rx_end_exclusive: UnsafeCell<()>;
static __bss_start: UnsafeCell<u64>;
static __bss_end_inclusive: UnsafeCell<u64>;
}
//--------------------------------------------------------------------------------------------------
@ -96,23 +93,3 @@ fn rx_start() -> usize {
fn rx_end_exclusive() -> usize {
unsafe { __rx_end_exclusive.get() as usize }
}
//--------------------------------------------------------------------------------------------------
// Public Code
//--------------------------------------------------------------------------------------------------
/// Return the inclusive range spanning the .bss section.
///
/// # Safety
///
/// - Values are provided by the linker script and must be trusted as-is.
/// - The linker-provided addresses must be u64 aligned.
pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
let range;
unsafe {
range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
}
assert!(!range.is_empty());
range
}

@ -104,9 +104,7 @@
//!
//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
//!
//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
#![allow(clippy::upper_case_acronyms)]
#![allow(incomplete_features)]
@ -128,7 +126,6 @@
#![test_runner(crate::test_runner)]
mod panic_wait;
mod runtime_init;
mod synchronization;
pub mod bsp;
@ -154,6 +151,11 @@ pub fn version() -> &'static str {
)
}
#[cfg(not(test))]
extern "Rust" {
fn kernel_init() -> !;
}
//--------------------------------------------------------------------------------------------------
// Testing
//--------------------------------------------------------------------------------------------------
@ -173,7 +175,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) {
}
}
/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
/// The `kernel_init()` for unit tests.
#[cfg(test)]
#[no_mangle]
unsafe fn kernel_init() -> ! {

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save