Init DRAM in assembly instead of Rust

See https://github.com/rust-embedded/cortex-m-rt/issues/300

02_runtime_init/README.md

@@ -2,8 +2,9 @@
 
 ## tl;dr
 
-- We extend `boot.s` to call into Rust code for the first time. There, we zero the [bss] section
+- We extend `boot.s` to call into Rust code for the first time. Before the jump
-  before execution is halted with a call to `panic()`.
+  to Rust happens, a bit of runtime init work is done.
+- The Rust code being called just halts execution with a call to `panic!()`.
 - Check out `make qemu` again to see the additional code run.
 
 ## Notable additions
@@ -12,11 +13,11 @@
      - New sections: `.rodata`, `.got`, `.data`, `.bss`.
      - A dedicated place for linking boot-time arguments that need to be read by `_start()`.
 - `_start()` in `_arch/__arch_name__/cpu/boot.s`:
-     1. Halt core if core != core0.
+     1. Halts core if core != core0.
-     1. Set up the `stack pointer`.
+     1. Initializes the `DRAM` by zeroing the [bss] section.
-     1. Jump to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`.
+     1. Sets up the `stack pointer`.
-- `runtime_init()` in `runtime_init.rs`:
+     1. Jumps to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`.
-     - Zeros the `.bss` section.
+- `_start_rust()`:
      - Calls `kernel_init()`, which calls `panic!()`, which eventually halts core0 as well.
 - The library now uses the [cortex-a] crate, which provides zero-overhead abstractions and wraps
   `unsafe` parts when dealing with the CPU's resources.
@@ -64,12 +65,8 @@ diff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile
 diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arch/aarch64/cpu/boot.rs
 --- 01_wait_forever/src/_arch/aarch64/cpu/boot.rs
 +++ 02_runtime_init/src/_arch/aarch64/cpu/boot.rs
-@@ -11,5 +11,23 @@
+@@ -13,3 +13,15 @@
- //!
- //! crate::cpu::boot::arch_boot
 
-+use crate::runtime_init;
-+
  // Assembly counterpart to this file.
  global_asm!(include_str!("boot.s"));
 +
@@ -80,13 +77,9 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arc
 +/// The Rust entry of the `kernel` binary.
 +///
 +/// The function is called from the assembly `_start` function.
-+///
-+/// # Safety
-+///
-+/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 +#[no_mangle]
 +pub unsafe fn _start_rust() -> ! {
-+    runtime_init::runtime_init()
++    crate::kernel_init()
 +}
 
 diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch/aarch64/cpu/boot.s
@@ -117,7 +110,7 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch
  // Public Code
  //--------------------------------------------------------------------------------------------------
  .section .text._start
-@@ -11,6 +29,22 @@
+@@ -11,9 +29,38 @@
  // fn _start()
  //------------------------------------------------------------------------------
  _start:
@@ -126,10 +119,22 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch
 +	and	x1, x1, _core_id_mask
 +	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 +	cmp	x1, x2
-+	b.ne	1f
++	b.ne	parking_loop
++
++	// If execution reaches here, it is the boot core.
++
++	// Initialize DRAM.
++	ADR_REL	x0, __bss_start
++	ADR_REL x1, __bss_end_exclusive
 +
-+	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
++bss_init_loop:
++	cmp	x0, x1
++	b.eq	prepare_rust
++	stp	xzr, xzr, [x0], #16
++	b	bss_init_loop
 +
++	// Prepare the jump to Rust code.
++prepare_rust:
 +	// Set the stack pointer.
 +	ADR_REL	x0, __boot_core_stack_end_exclusive
 +	mov	sp, x0
@@ -138,8 +143,14 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch
 +	b	_start_rust
 +
  	// Infinitely wait for events (aka "park the core").
- 1:	wfe
+-1:	wfe
- 	b	1b
+-	b	1b
++parking_loop:
++	wfe
++	b	parking_loop
+
+ .size	_start, . - _start
+ .type	_start, function
 
 diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs
 --- 01_wait_forever/src/_arch/aarch64/cpu.rs
@@ -194,7 +205,7 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/cpu.rs 02_runtime_init/src/bsp/ras
 diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld
 --- 01_wait_forever/src/bsp/raspberrypi/link.ld
 +++ 02_runtime_init/src/bsp/raspberrypi/link.ld
-@@ -11,17 +11,45 @@
+@@ -11,17 +11,43 @@
  PHDRS
  {
      segment_rx PT_LOAD FLAGS(5); /* 5 == RX */
@@ -230,70 +241,25 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra
 +    ***********************************************************************************************/
 +    .data : { *(.data*) } :segment_rw
 +
-+    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
++    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-+    .bss : ALIGN(8)
++    .bss : ALIGN(16)
 +    {
 +        __bss_start = .;
 +        *(.bss*);
-+        . = ALIGN(8);
++        . = ALIGN(16);
-+
++        __bss_end_exclusive = .;
-+        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-+        __bss_end_inclusive = . - 8;
 +    } :NONE
  }
 
-diff -uNr 01_wait_forever/src/bsp/raspberrypi/memory.rs 02_runtime_init/src/bsp/raspberrypi/memory.rs
---- 01_wait_forever/src/bsp/raspberrypi/memory.rs
-+++ 02_runtime_init/src/bsp/raspberrypi/memory.rs
-@@ -0,0 +1,37 @@
-+// SPDX-License-Identifier: MIT OR Apache-2.0
-+//
-+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-+
-+//! BSP Memory Management.
-+
-+use core::{cell::UnsafeCell, ops::RangeInclusive};
-+
-+//--------------------------------------------------------------------------------------------------
-+// Private Definitions
-+//--------------------------------------------------------------------------------------------------
-+
-+// Symbols from the linker script.
-+extern "Rust" {
-+    static __bss_start: UnsafeCell<u64>;
-+    static __bss_end_inclusive: UnsafeCell<u64>;
-+}
-+
-+//--------------------------------------------------------------------------------------------------
-+// Public Code
-+//--------------------------------------------------------------------------------------------------
-+
-+/// Return the inclusive range spanning the .bss section.
-+///
-+/// # Safety
-+///
-+/// - Values are provided by the linker script and must be trusted as-is.
-+/// - The linker-provided addresses must be u64 aligned.
-+pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-+    let range;
-+    unsafe {
-+        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-+    }
-+    assert!(!range.is_empty());
-+
-+    range
-+}
-
 diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs
 --- 01_wait_forever/src/bsp/raspberrypi.rs
 +++ 02_runtime_init/src/bsp/raspberrypi.rs
-@@ -4,4 +4,5 @@
+@@ -4,4 +4,4 @@
 
  //! Top-level BSP file for the Raspberry Pi 3 and 4.
 
 -// Coming soon.
 +pub mod cpu;
-+pub mod memory;
 
 diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs
 --- 01_wait_forever/src/cpu.rs
@@ -316,24 +282,19 @@ diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs
 diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs
 --- 01_wait_forever/src/main.rs
 +++ 02_runtime_init/src/main.rs
-@@ -102,14 +102,25 @@
+@@ -102,8 +102,8 @@
  //!
  //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
  //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
++//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-+//!
-+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 -#![feature(asm)]
  #![feature(global_asm)]
  #![no_main]
  #![no_std]
-
+@@ -112,4 +112,11 @@
- mod bsp;
  mod cpu;
-+mod memory;
  mod panic_wait;
-+mod runtime_init;
 
 -// Kernel code coming next tutorial.
 +/// Early init code.
@@ -345,41 +306,6 @@ diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs
 +    panic!()
 +}
 
-diff -uNr 01_wait_forever/src/memory.rs 02_runtime_init/src/memory.rs
---- 01_wait_forever/src/memory.rs
-+++ 02_runtime_init/src/memory.rs
-@@ -0,0 +1,30 @@
-+// SPDX-License-Identifier: MIT OR Apache-2.0
-+//
-+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-+
-+//! Memory Management.
-+
-+use core::ops::RangeInclusive;
-+
-+//--------------------------------------------------------------------------------------------------
-+// Public Code
-+//--------------------------------------------------------------------------------------------------
-+
-+/// Zero out an inclusive memory range.
-+///
-+/// # Safety
-+///
-+/// - `range.start` and `range.end` must be valid.
-+/// - `range.start` and `range.end` must be `T` aligned.
-+pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-+where
-+    T: From<u8>,
-+{
-+    let mut ptr = *range.start();
-+    let end_inclusive = *range.end();
-+
-+    while ptr <= end_inclusive {
-+        core::ptr::write_volatile(ptr, T::from(0));
-+        ptr = ptr.offset(1);
-+    }
-+}
-
 diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs
 --- 01_wait_forever/src/panic_wait.rs
 +++ 02_runtime_init/src/panic_wait.rs
@@ -396,46 +322,4 @@ diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs
 +    cpu::wait_forever()
  }
 
-diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.rs
---- 01_wait_forever/src/runtime_init.rs
-+++ 02_runtime_init/src/runtime_init.rs
-@@ -0,0 +1,37 @@
-+// SPDX-License-Identifier: MIT OR Apache-2.0
-+//
-+// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-+
-+//! Rust runtime initialization code.
-+
-+use crate::{bsp, memory};
-+
-+//--------------------------------------------------------------------------------------------------
-+// Private Code
-+//--------------------------------------------------------------------------------------------------
-+
-+/// Zero out the .bss section.
-+///
-+/// # Safety
-+///
-+/// - Must only be called pre `kernel_init()`.
-+#[inline(always)]
-+unsafe fn zero_bss() {
-+    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-+}
-+
-+//--------------------------------------------------------------------------------------------------
-+// Public Code
-+//--------------------------------------------------------------------------------------------------
-+
-+/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-+/// init code.
-+///
-+/// # Safety
-+///
-+/// - Only a single core must be active and running this function.
-+pub unsafe fn runtime_init() -> ! {
-+    zero_bss();
-+
-+    crate::kernel_init()
-+}
-
 ```

02_runtime_init/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

02_runtime_init/src/_arch/aarch64/cpu/boot.s

@@ -34,10 +34,22 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -46,8 +58,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

02_runtime_init/src/bsp/raspberrypi.rs

@@ -5,4 +5,3 @@
 //! Top-level BSP file for the Raspberry Pi 3 and 4.
 
 pub mod cpu;
-pub mod memory;

02_runtime_init/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

02_runtime_init/src/bsp/raspberrypi/memory.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! BSP Memory Management.
-
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

02_runtime_init/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![feature(global_asm)]
 #![no_main]
@@ -112,9 +110,7 @@
 
 mod bsp;
 mod cpu;
-mod memory;
 mod panic_wait;
-mod runtime_init;
 
 /// Early init code.
 ///

02_runtime_init/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

02_runtime_init/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

03_hacky_hello_world/README.md

@@ -117,13 +117,12 @@ diff -uNr 02_runtime_init/src/bsp/raspberrypi/console.rs 03_hacky_hello_world/sr
 diff -uNr 02_runtime_init/src/bsp/raspberrypi.rs 03_hacky_hello_world/src/bsp/raspberrypi.rs
 --- 02_runtime_init/src/bsp/raspberrypi.rs
 +++ 03_hacky_hello_world/src/bsp/raspberrypi.rs
-@@ -4,5 +4,6 @@
+@@ -4,4 +4,5 @@
 
  //! Top-level BSP file for the Raspberry Pi 3 and 4.
 
 +pub mod console;
  pub mod cpu;
- pub mod memory;
 
 diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs
 --- 02_runtime_init/src/console.rs
@@ -152,9 +151,9 @@ diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs
 diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs
 --- 02_runtime_init/src/main.rs
 +++ 03_hacky_hello_world/src/main.rs
-@@ -106,14 +106,18 @@
+@@ -104,13 +104,17 @@
- //!
+ //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
- //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+ //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
 
 +#![feature(format_args_nl)]
  #![feature(global_asm)]
@@ -165,13 +164,12 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs
  mod bsp;
 +mod console;
  mod cpu;
- mod memory;
  mod panic_wait;
 +mod print;
- mod runtime_init;
 
  /// Early init code.
-@@ -122,5 +126,7 @@
+ ///
+@@ -118,5 +122,7 @@
  ///
  /// - Only a single core must be active and running this function.
  unsafe fn kernel_init() -> ! {

03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s

@@ -34,10 +34,22 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -46,8 +58,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

03_hacky_hello_world/src/bsp/raspberrypi.rs

@@ -6,4 +6,3 @@
 
 pub mod console;
 pub mod cpu;
-pub mod memory;

03_hacky_hello_world/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

03_hacky_hello_world/src/bsp/raspberrypi/memory.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! BSP Memory Management.
-
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

03_hacky_hello_world/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![feature(format_args_nl)]
 #![feature(global_asm)]
@@ -115,10 +113,8 @@
 mod bsp;
 mod console;
 mod cpu;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 
 /// Early init code.
 ///

03_hacky_hello_world/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

03_hacky_hello_world/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

04_safe_globals/README.md

@@ -224,7 +224,7 @@ diff -uNr 03_hacky_hello_world/src/console.rs 04_safe_globals/src/console.rs
 diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs
 --- 03_hacky_hello_world/src/main.rs
 +++ 04_safe_globals/src/main.rs
-@@ -109,6 +109,7 @@
+@@ -107,6 +107,7 @@
  #![feature(format_args_nl)]
  #![feature(global_asm)]
  #![feature(panic_info_message)]
@@ -232,15 +232,15 @@ diff -uNr 03_hacky_hello_world/src/main.rs 04_safe_globals/src/main.rs
  #![no_main]
  #![no_std]
 
-@@ -119,6 +120,7 @@
+@@ -115,6 +116,7 @@
+ mod cpu;
  mod panic_wait;
  mod print;
- mod runtime_init;
 +mod synchronization;
 
  /// Early init code.
  ///
-@@ -126,7 +128,15 @@
+@@ -122,7 +124,15 @@
  ///
  /// - Only a single core must be active and running this function.
  unsafe fn kernel_init() -> ! {

04_safe_globals/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

04_safe_globals/src/_arch/aarch64/cpu/boot.s

@@ -34,10 +34,22 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -46,8 +58,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

04_safe_globals/src/bsp/raspberrypi.rs

@@ -6,4 +6,3 @@
 
 pub mod console;
 pub mod cpu;
-pub mod memory;

04_safe_globals/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

04_safe_globals/src/bsp/raspberrypi/memory.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! BSP Memory Management.
-
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

04_safe_globals/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![feature(format_args_nl)]
 #![feature(global_asm)]
@@ -116,10 +114,8 @@
 mod bsp;
 mod console;
 mod cpu;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 
 /// Early init code.

04_safe_globals/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

04_safe_globals/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

05_drivers_gpio_uart/README.md

@@ -1120,10 +1120,14 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/driver.rs 05_drivers_gpio_uart/src
 diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs
 --- 04_safe_globals/src/bsp/raspberrypi/memory.rs
 +++ 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs
-@@ -17,6 +17,38 @@
+@@ -0,0 +1,37 @@
- }
++// SPDX-License-Identifier: MIT OR Apache-2.0
-
++//
- //--------------------------------------------------------------------------------------------------
++// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
++
++//! BSP Memory Management.
++
++//--------------------------------------------------------------------------------------------------
 +// Public Definitions
 +//--------------------------------------------------------------------------------------------------
 +
@@ -1154,21 +1158,16 @@ diff -uNr 04_safe_globals/src/bsp/raspberrypi/memory.rs 05_drivers_gpio_uart/src
 +        pub const PL011_UART_START: usize = START + UART_OFFSET;
 +    }
 +}
-+
-+//--------------------------------------------------------------------------------------------------
- // Public Code
- //--------------------------------------------------------------------------------------------------
-
 
 diff -uNr 04_safe_globals/src/bsp/raspberrypi.rs 05_drivers_gpio_uart/src/bsp/raspberrypi.rs
 --- 04_safe_globals/src/bsp/raspberrypi.rs
 +++ 05_drivers_gpio_uart/src/bsp/raspberrypi.rs
-@@ -6,4 +6,33 @@
+@@ -6,3 +6,33 @@
 
  pub mod console;
  pub mod cpu;
 +pub mod driver;
- pub mod memory;
++pub mod memory;
 +
 +//--------------------------------------------------------------------------------------------------
 +// Global instances
@@ -1321,24 +1320,24 @@ diff -uNr 04_safe_globals/src/driver.rs 05_drivers_gpio_uart/src/driver.rs
 diff -uNr 04_safe_globals/src/main.rs 05_drivers_gpio_uart/src/main.rs
 --- 04_safe_globals/src/main.rs
 +++ 05_drivers_gpio_uart/src/main.rs
-@@ -106,6 +106,8 @@
+@@ -104,6 +104,8 @@
- //!
+ //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
- //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+ //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
 
 +#![allow(clippy::upper_case_acronyms)]
 +#![feature(const_fn_fn_ptr_basics)]
  #![feature(format_args_nl)]
  #![feature(global_asm)]
  #![feature(panic_info_message)]
-@@ -116,6 +118,7 @@
+@@ -114,6 +116,7 @@
  mod bsp;
  mod console;
  mod cpu;
 +mod driver;
- mod memory;
  mod panic_wait;
  mod print;
-@@ -127,16 +130,54 @@
+ mod synchronization;
+@@ -123,16 +126,54 @@
  /// # Safety
  ///
  /// - Only a single core must be active and running this function.

05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s

@@ -34,10 +34,22 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -46,8 +58,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs

@@ -4,18 +4,6 @@
 
 //! BSP Memory Management.
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
 //--------------------------------------------------------------------------------------------------
 // Public Definitions
 //--------------------------------------------------------------------------------------------------
@@ -47,23 +35,3 @@ pub(super) mod map {
         pub const PL011_UART_START: usize = START + UART_OFFSET;
     }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

05_drivers_gpio_uart/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![feature(const_fn_fn_ptr_basics)]
@@ -119,10 +117,8 @@ mod bsp;
 mod console;
 mod cpu;
 mod driver;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 
 /// Early init code.

05_drivers_gpio_uart/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

05_drivers_gpio_uart/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

06_uart_chainloader/README.md

@@ -232,26 +232,36 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/
  .equ _core_id_mask, 0b11
 
  //--------------------------------------------------------------------------------------------------
-@@ -34,20 +45,31 @@
+@@ -39,23 +50,35 @@
- 	and	x1, x1, _core_id_mask
+ 	// If execution reaches here, it is the boot core.
- 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
+
- 	cmp	x1, x2
+ 	// Initialize DRAM.
--	b.ne	1f
+-	ADR_REL	x0, __bss_start
-+	b.ne	2f
+-	ADR_REL x1, __bss_end_exclusive
-+
++	ADR_ABS	x0, __bss_start
-+	// If execution reaches here, it is the boot core.
++	ADR_ABS x1, __bss_end_exclusive
+
+ bss_init_loop:
+ 	cmp	x0, x1
+-	b.eq	prepare_rust
++	b.eq	relocate_binary
+ 	stp	xzr, xzr, [x0], #16
+ 	b	bss_init_loop
 
--	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
 +	// Next, relocate the binary.
++relocate_binary:
 +	ADR_REL	x0, __binary_nonzero_start         // The address the binary got loaded to.
 +	ADR_ABS	x1, __binary_nonzero_start         // The address the binary was linked to.
 +	ADR_ABS	x2, __binary_nonzero_end_exclusive
 +
-+1:	ldr	x3, [x0], #8
++copy_loop:
++	ldr	x3, [x0], #8
 +	str	x3, [x1], #8
 +	cmp	x1, x2
-+	b.lo	1b
++	b.lo	copy_loop
-
++
+ 	// Prepare the jump to Rust code.
+-prepare_rust:
  	// Set the stack pointer.
 -	ADR_REL	x0, __boot_core_stack_end_exclusive
 +	ADR_ABS	x0, __boot_core_stack_end_exclusive
@@ -264,13 +274,7 @@ diff -uNr 05_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 06_uart_chainloader/
 +	br	x1
 
  	// Infinitely wait for events (aka "park the core").
--1:	wfe
+ parking_loop:
--	b	1b
-+2:	wfe
-+	b	2b
-
- .size	_start, . - _start
- .type	_start, function
 
 diff -uNr 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 06_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs
 --- 05_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs
@@ -350,7 +354,7 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s
      .text :
      {
          KEEP(*(.text._start))
-@@ -42,8 +44,12 @@
+@@ -42,6 +44,10 @@
      ***********************************************************************************************/
      .data : { *(.data*) } :segment_rw
 
@@ -358,17 +362,14 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 06_uart_chainloader/s
 +    . = ALIGN(8);
 +    __binary_nonzero_end_exclusive = .;
 +
-     /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+     /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
--    .bss : ALIGN(8)
+     .bss : ALIGN(16)
-+    .bss :
      {
-         __bss_start = .;
-         *(.bss*);
 
 diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader/src/bsp/raspberrypi/memory.rs
 --- 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs
 +++ 06_uart_chainloader/src/bsp/raspberrypi/memory.rs
-@@ -23,9 +23,10 @@
+@@ -11,9 +11,10 @@
  /// The board's physical memory map.
  #[rustfmt::skip]
  pub(super) mod map {
@@ -381,34 +382,33 @@ diff -uNr 05_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 06_uart_chainloader
 
      /// Physical devices.
      #[cfg(feature = "bsp_rpi3")]
-@@ -52,7 +53,13 @@
+@@ -35,3 +36,13 @@
- // Public Code
+         pub const PL011_UART_START: usize = START + UART_OFFSET;
- //--------------------------------------------------------------------------------------------------
+     }
-
+ }
--/// Return the inclusive range spanning the .bss section.
++
++//--------------------------------------------------------------------------------------------------
++// Public Code
++//--------------------------------------------------------------------------------------------------
++
 +/// The address on which the Raspberry firmware loads every binary by default.
 +#[inline(always)]
 +pub fn board_default_load_addr() -> *const u64 {
 +    map::BOARD_DEFAULT_LOAD_ADDRESS as _
 +}
-+
-+/// Return the inclusive range spanning the relocated .bss section.
- ///
- /// # Safety
- ///
 
 diff -uNr 05_drivers_gpio_uart/src/main.rs 06_uart_chainloader/src/main.rs
 --- 05_drivers_gpio_uart/src/main.rs
 +++ 06_uart_chainloader/src/main.rs
-@@ -107,6 +107,7 @@
+@@ -105,6 +105,7 @@
- //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+ //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
 
  #![allow(clippy::upper_case_acronyms)]
 +#![feature(asm)]
  #![feature(const_fn_fn_ptr_basics)]
  #![feature(format_args_nl)]
  #![feature(global_asm)]
-@@ -146,38 +147,56 @@
+@@ -142,38 +143,56 @@
      kernel_main()
  }
 

06_uart_chainloader/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

06_uart_chainloader/src/_arch/aarch64/cpu/boot.s

@@ -45,20 +45,33 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	2f
+	b.ne	parking_loop
 
 	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_ABS	x0, __bss_start
+	ADR_ABS x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	relocate_binary
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
 	// Next, relocate the binary.
+relocate_binary:
 	ADR_REL	x0, __binary_nonzero_start         // The address the binary got loaded to.
 	ADR_ABS	x1, __binary_nonzero_start         // The address the binary was linked to.
 	ADR_ABS	x2, __binary_nonzero_end_exclusive
 
-1:	ldr	x3, [x0], #8
+copy_loop:
+	ldr	x3, [x0], #8
 	str	x3, [x1], #8
 	cmp	x1, x2
-	b.lo	1b
+	b.lo	copy_loop
 
+	// Prepare the jump to Rust code.
 	// Set the stack pointer.
 	ADR_ABS	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -68,8 +81,9 @@ _start:
 	br	x1
 
 	// Infinitely wait for events (aka "park the core").
-2:	wfe
+parking_loop:
-	b	2b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

06_uart_chainloader/src/bsp/raspberrypi/link.ld

@@ -48,14 +48,12 @@ SECTIONS
     . = ALIGN(8);
     __binary_nonzero_end_exclusive = .;
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss :
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

06_uart_chainloader/src/bsp/raspberrypi/memory.rs

@@ -4,18 +4,6 @@
 
 //! BSP Memory Management.
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
 //--------------------------------------------------------------------------------------------------
 // Public Definitions
 //--------------------------------------------------------------------------------------------------
@@ -58,19 +46,3 @@ pub(super) mod map {
 pub fn board_default_load_addr() -> *const u64 {
     map::BOARD_DEFAULT_LOAD_ADDRESS as _
 }
-
-/// Return the inclusive range spanning the relocated .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

06_uart_chainloader/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![feature(asm)]
@@ -120,10 +118,8 @@ mod bsp;
 mod console;
 mod cpu;
 mod driver;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 
 /// Early init code.

06_uart_chainloader/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

06_uart_chainloader/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

07_timestamps/README.md

@@ -154,26 +154,36 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar
  .equ _core_id_mask, 0b11
 
  //--------------------------------------------------------------------------------------------------
-@@ -45,31 +34,20 @@
+@@ -50,35 +39,23 @@
- 	and	x1, x1, _core_id_mask
+ 	// If execution reaches here, it is the boot core.
- 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
+
- 	cmp	x1, x2
+ 	// Initialize DRAM.
--	b.ne	2f
+-	ADR_ABS	x0, __bss_start
--
+-	ADR_ABS x1, __bss_end_exclusive
--	// If execution reaches here, it is the boot core.
++	ADR_REL	x0, __bss_start
-+	b.ne	1f
++	ADR_REL x1, __bss_end_exclusive
+
+ bss_init_loop:
+ 	cmp	x0, x1
+-	b.eq	relocate_binary
++	b.eq	prepare_rust
+ 	stp	xzr, xzr, [x0], #16
+ 	b	bss_init_loop
 
 -	// Next, relocate the binary.
+-relocate_binary:
 -	ADR_REL	x0, __binary_nonzero_start         // The address the binary got loaded to.
 -	ADR_ABS	x1, __binary_nonzero_start         // The address the binary was linked to.
 -	ADR_ABS	x2, __binary_nonzero_end_exclusive
 -
--1:	ldr	x3, [x0], #8
+-copy_loop:
+-	ldr	x3, [x0], #8
 -	str	x3, [x1], #8
 -	cmp	x1, x2
--	b.lo	1b
+-	b.lo	copy_loop
-+	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+-
-
+ 	// Prepare the jump to Rust code.
++prepare_rust:
  	// Set the stack pointer.
 -	ADR_ABS	x0, __boot_core_stack_end_exclusive
 +	ADR_REL	x0, __boot_core_stack_end_exclusive
@@ -186,13 +196,7 @@ diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu/boot.s 07_timestamps/src/_ar
 +	b	_start_rust
 
  	// Infinitely wait for events (aka "park the core").
--2:	wfe
+ parking_loop:
--	b	2b
-+1:	wfe
-+	b	1b
-
- .size	_start, . - _start
- .type	_start, function
 
 diff -uNr 06_uart_chainloader/src/_arch/aarch64/cpu.rs 07_timestamps/src/_arch/aarch64/cpu.rs
 --- 06_uart_chainloader/src/_arch/aarch64/cpu.rs
@@ -438,7 +442,7 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/
      .text :
      {
          KEEP(*(.text._start))
-@@ -44,12 +42,8 @@
+@@ -44,10 +42,6 @@
      ***********************************************************************************************/
      .data : { *(.data*) } :segment_rw
 
@@ -446,17 +450,14 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/link.ld 07_timestamps/src/bsp/
 -    . = ALIGN(8);
 -    __binary_nonzero_end_exclusive = .;
 -
-     /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+     /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
--    .bss :
+     .bss : ALIGN(16)
-+    .bss : ALIGN(8)
      {
-         __bss_start = .;
-         *(.bss*);
 
 diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bsp/raspberrypi/memory.rs
 --- 06_uart_chainloader/src/bsp/raspberrypi/memory.rs
 +++ 07_timestamps/src/bsp/raspberrypi/memory.rs
-@@ -23,10 +23,9 @@
+@@ -11,10 +11,9 @@
  /// The board's physical memory map.
  #[rustfmt::skip]
  pub(super) mod map {
@@ -469,21 +470,20 @@ diff -uNr 06_uart_chainloader/src/bsp/raspberrypi/memory.rs 07_timestamps/src/bs
 
      /// Physical devices.
      #[cfg(feature = "bsp_rpi3")]
-@@ -53,13 +52,7 @@
+@@ -36,13 +35,3 @@
- // Public Code
+         pub const PL011_UART_START: usize = START + UART_OFFSET;
- //--------------------------------------------------------------------------------------------------
+     }
-
+ }
+-
+-//--------------------------------------------------------------------------------------------------
+-// Public Code
+-//--------------------------------------------------------------------------------------------------
+-
 -/// The address on which the Raspberry firmware loads every binary by default.
 -#[inline(always)]
 -pub fn board_default_load_addr() -> *const u64 {
 -    map::BOARD_DEFAULT_LOAD_ADDRESS as _
 -}
--
--/// Return the inclusive range spanning the relocated .bss section.
-+/// Return the inclusive range spanning the .bss section.
- ///
- /// # Safety
- ///
 
 diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs
 --- 06_uart_chainloader/src/cpu.rs
@@ -499,23 +499,23 @@ diff -uNr 06_uart_chainloader/src/cpu.rs 07_timestamps/src/cpu.rs
 diff -uNr 06_uart_chainloader/src/main.rs 07_timestamps/src/main.rs
 --- 06_uart_chainloader/src/main.rs
 +++ 07_timestamps/src/main.rs
-@@ -107,7 +107,6 @@
+@@ -105,7 +105,6 @@
- //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+ //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
 
  #![allow(clippy::upper_case_acronyms)]
 -#![feature(asm)]
  #![feature(const_fn_fn_ptr_basics)]
  #![feature(format_args_nl)]
  #![feature(global_asm)]
-@@ -125,6 +124,7 @@
+@@ -121,6 +120,7 @@
+ mod panic_wait;
  mod print;
- mod runtime_init;
  mod synchronization;
 +mod time;
 
  /// Early init code.
  ///
-@@ -147,56 +147,38 @@
+@@ -143,56 +143,38 @@
      kernel_main()
  }
 

07_timestamps/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

07_timestamps/src/_arch/aarch64/cpu/boot.s

@@ -34,10 +34,22 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -46,8 +58,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

07_timestamps/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

07_timestamps/src/bsp/raspberrypi/memory.rs

@@ -4,18 +4,6 @@
 
 //! BSP Memory Management.
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
 //--------------------------------------------------------------------------------------------------
 // Public Definitions
 //--------------------------------------------------------------------------------------------------
@@ -47,23 +35,3 @@ pub(super) mod map {
         pub const PL011_UART_START: usize = START + UART_OFFSET;
     }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

07_timestamps/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![feature(const_fn_fn_ptr_basics)]
@@ -119,10 +117,8 @@ mod bsp;
 mod console;
 mod cpu;
 mod driver;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 mod time;
 

07_timestamps/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

07_timestamps/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs

@@ -11,8 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
-
 // Assembly counterpart to this file.
 global_asm!(include_str!("boot.s"));
 
@@ -23,11 +21,7 @@ global_asm!(include_str!("boot.s"));
 /// The Rust entry of the `kernel` binary.
 ///
 /// The function is called from the assembly `_start` function.
-///
-/// # Safety
-///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
 #[no_mangle]
 pub unsafe fn _start_rust() -> ! {
-    runtime_init::runtime_init()
+    crate::kernel_init()
 }

08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s

@@ -34,10 +34,22 @@ _start:
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -46,8 +58,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

08_hw_debug_JTAG/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

08_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs

@@ -4,18 +4,6 @@
 
 //! BSP Memory Management.
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
 //--------------------------------------------------------------------------------------------------
 // Public Definitions
 //--------------------------------------------------------------------------------------------------
@@ -47,23 +35,3 @@ pub(super) mod map {
         pub const PL011_UART_START: usize = START + UART_OFFSET;
     }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

08_hw_debug_JTAG/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![feature(const_fn_fn_ptr_basics)]
@@ -119,10 +117,8 @@ mod bsp;
 mod console;
 mod cpu;
 mod driver;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 mod time;
 

08_hw_debug_JTAG/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

08_hw_debug_JTAG/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

09_privilege_level/README.md

@@ -62,7 +62,7 @@ Afterwards, we continue with preparing the `EL2` -> `EL1` transition by calling
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }
 ```
@@ -125,19 +125,16 @@ SPSR_EL2.write(
         + SPSR_EL2::M::EL1h,
 );
 
-// Second, let the link register point to runtime_init().
+// Second, let the link register point to kernel_init().
-ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+ELR_EL2.set(crate::kernel_init as *const () as u64);
 
 // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
 // are no plans to ever return to EL2, just re-use the same stack.
 SP_EL1.set(phys_boot_core_stack_end_exclusive_addr);
 ```
 
-As you can see, we are populating `ELR_EL2` with the address of the [runtime_init()] function that
+As you can see, we are populating `ELR_EL2` with the address of the `kernel_init()` function that we
-we earlier used to call directly from the entrypoint. Finally, we set the stack pointer for
+earlier used to call directly from the entrypoint. Finally, we set the stack pointer for `SP_EL1`.
-`SP_EL1`.
-
-[runtime_init()]: src/runtime_init.rs
 
 You might have noticed that the stack's address was supplied as a function argument. As you might
 remember, in  `_start()` in `boot.s`, we are already setting up the stack for `EL2`. Since there
@@ -151,7 +148,7 @@ Lastly, back in `_start_rust()` a call to `ERET` is made:
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }
 ```
@@ -214,12 +211,12 @@ diff -uNr 08_hw_debug_JTAG/Cargo.toml 09_privilege_level/Cargo.toml
 diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/_arch/aarch64/cpu/boot.rs
 --- 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs
 +++ 09_privilege_level/src/_arch/aarch64/cpu/boot.rs
-@@ -12,11 +12,53 @@
+@@ -11,17 +11,67 @@
+ //!
  //! crate::cpu::boot::arch_boot
 
- use crate::runtime_init;
 +use cortex_a::{asm, regs::*};
-
++
  // Assembly counterpart to this file.
  global_asm!(include_str!("boot.s"));
 
@@ -256,8 +253,8 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/
 +            + SPSR_EL2::M::EL1h,
 +    );
 +
-+    // Second, let the link register point to runtime_init().
++    // Second, let the link register point to kernel_init().
-+    ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
++    ELR_EL2.set(crate::kernel_init as *const () as u64);
 +
 +    // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
 +    // are no plans to ever return to EL2, just re-use the same stack.
@@ -268,18 +265,20 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 09_privilege_level/src/
  // Public Code
  //--------------------------------------------------------------------------------------------------
 
-@@ -27,7 +69,11 @@
+ /// The Rust entry of the `kernel` binary.
- /// # Safety
  ///
- /// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+ /// The function is called from the assembly `_start` function.
-+/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
++///
++/// # Safety
++///
++/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
  #[no_mangle]
 -pub unsafe fn _start_rust() -> ! {
--    runtime_init::runtime_init()
+-    crate::kernel_init()
 +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
 +    prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 +
-+    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
++    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
 +    asm::eret()
  }
 
@@ -301,15 +300,15 @@ diff -uNr 08_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s 09_privilege_level/src/_
 +	// Only proceed if the core executes in EL2. Park it otherwise.
 +	mrs	x0, CurrentEL
 +	cmp	x0, _EL2
-+	b.ne	1f
++	b.ne	parking_loop
 +
  	// Only proceed on the boot core. Park it otherwise.
  	mrs	x1, MPIDR_EL1
  	and	x1, x1, _core_id_mask
-@@ -38,11 +44,11 @@
+@@ -50,11 +56,11 @@
-
- 	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
 
+ 	// Prepare the jump to Rust code.
+ prepare_rust:
 -	// Set the stack pointer.
 +	// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
  	ADR_REL	x0, __boot_core_stack_end_exclusive
@@ -499,15 +498,15 @@ diff -uNr 08_hw_debug_JTAG/src/exception.rs 09_privilege_level/src/exception.rs
 diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs
 --- 08_hw_debug_JTAG/src/main.rs
 +++ 09_privilege_level/src/main.rs
-@@ -119,6 +119,7 @@
+@@ -117,6 +117,7 @@
  mod console;
  mod cpu;
  mod driver;
 +mod exception;
- mod memory;
  mod panic_wait;
  mod print;
-@@ -149,6 +150,8 @@
+ mod synchronization;
+@@ -145,6 +146,8 @@
 
  /// The main function running after the early init.
  fn kernel_main() -> ! {
@@ -516,7 +515,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs
      use core::time::Duration;
      use driver::interface::DriverManager;
      use time::interface::TimeManager;
-@@ -160,6 +163,12 @@
+@@ -156,6 +159,12 @@
      );
      info!("Booting on: {}", bsp::board_name());
 
@@ -529,7 +528,7 @@ diff -uNr 08_hw_debug_JTAG/src/main.rs 09_privilege_level/src/main.rs
      info!(
          "Architectural timer resolution: {} ns",
          time::time_manager().resolution().as_nanos()
-@@ -174,11 +183,15 @@
+@@ -170,11 +179,15 @@
          info!("      {}. {}", i + 1, driver.compatible());
      }
 

09_privilege_level/src/_arch/aarch64/cpu/boot.rs

@@ -11,7 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
 use cortex_a::{asm, regs::*};
 
 // Assembly counterpart to this file.
@@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
             + SPSR_EL2::M::EL1h,
     );
 
-    // Second, let the link register point to runtime_init().
+    // Second, let the link register point to kernel_init().
-    ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+    ELR_EL2.set(crate::kernel_init as *const () as u64);
 
     // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
     // are no plans to ever return to EL2, just re-use the same stack.
@@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
 ///
 /// # Safety
 ///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
-/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
 #[no_mangle]
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }

09_privilege_level/src/_arch/aarch64/cpu/boot.s

@@ -33,17 +33,29 @@ _start:
 	// Only proceed if the core executes in EL2. Park it otherwise.
 	mrs	x0, CurrentEL
 	cmp	x0, _EL2
-	b.ne	1f
+	b.ne	parking_loop
 
 	// Only proceed on the boot core. Park it otherwise.
 	mrs	x1, MPIDR_EL1
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -52,8 +64,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

09_privilege_level/src/bsp/raspberrypi/link.ld

@@ -42,14 +42,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

09_privilege_level/src/bsp/raspberrypi/memory.rs

@@ -4,18 +4,6 @@
 
 //! BSP Memory Management.
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
-
-//--------------------------------------------------------------------------------------------------
-// Private Definitions
-//--------------------------------------------------------------------------------------------------
-
-// Symbols from the linker script.
-extern "Rust" {
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
-}
-
 //--------------------------------------------------------------------------------------------------
 // Public Definitions
 //--------------------------------------------------------------------------------------------------
@@ -47,23 +35,3 @@ pub(super) mod map {
         pub const PL011_UART_START: usize = START + UART_OFFSET;
     }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

09_privilege_level/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![feature(const_fn_fn_ptr_basics)]
@@ -120,10 +118,8 @@ mod console;
 mod cpu;
 mod driver;
 mod exception;
-mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 mod time;
 

09_privilege_level/src/memory.rs

@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Memory Management.
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

09_privilege_level/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

10_virtual_mem_part1_identity_mapping/README.md

@@ -939,26 +939,28 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 10_virtual_mem_pa
 diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs
 --- 09_privilege_level/src/bsp/raspberrypi/memory.rs
 +++ 10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs
-@@ -4,6 +4,8 @@
+@@ -4,6 +4,20 @@
 
  //! BSP Memory Management.
 
 +pub mod mmu;
 +
- use core::{cell::UnsafeCell, ops::RangeInclusive};
++use core::cell::UnsafeCell;
-
++
- //--------------------------------------------------------------------------------------------------
++//--------------------------------------------------------------------------------------------------
-@@ -12,6 +14,9 @@
++// Private Definitions
-
++//--------------------------------------------------------------------------------------------------
- // Symbols from the linker script.
++
- extern "Rust" {
++// Symbols from the linker script.
++extern "Rust" {
 +    static __rx_start: UnsafeCell<()>;
 +    static __rx_end_exclusive: UnsafeCell<()>;
++}
 +
-     static __bss_start: UnsafeCell<u64>;
+ //--------------------------------------------------------------------------------------------------
-     static __bss_end_inclusive: UnsafeCell<u64>;
+ // Public Definitions
- }
+ //--------------------------------------------------------------------------------------------------
-@@ -23,6 +28,20 @@
+@@ -11,6 +25,20 @@
  /// The board's physical memory map.
  #[rustfmt::skip]
  pub(super) mod map {
@@ -979,7 +981,7 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_
 
      pub const GPIO_OFFSET:         usize = 0x0020_0000;
      pub const UART_OFFSET:         usize = 0x0020_1000;
-@@ -35,6 +54,7 @@
+@@ -23,6 +51,7 @@
          pub const START:            usize =         0x3F00_0000;
          pub const GPIO_START:       usize = START + GPIO_OFFSET;
          pub const PL011_UART_START: usize = START + UART_OFFSET;
@@ -987,15 +989,15 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_
      }
 
      /// Physical devices.
-@@ -45,10 +65,35 @@
+@@ -33,5 +62,30 @@
          pub const START:            usize =         0xFE00_0000;
          pub const GPIO_START:       usize = START + GPIO_OFFSET;
          pub const PL011_UART_START: usize = START + UART_OFFSET;
 +        pub const END_INCLUSIVE:    usize =         0xFF84_FFFF;
      }
  }
-
++
- //--------------------------------------------------------------------------------------------------
++//--------------------------------------------------------------------------------------------------
 +// Private Code
 +//--------------------------------------------------------------------------------------------------
 +
@@ -1018,11 +1020,6 @@ diff -uNr 09_privilege_level/src/bsp/raspberrypi/memory.rs 10_virtual_mem_part1_
 +fn rx_end_exclusive() -> usize {
 +    unsafe { __rx_end_exclusive.get() as usize }
 +}
-+
-+//--------------------------------------------------------------------------------------------------
- // Public Code
- //--------------------------------------------------------------------------------------------------
-
 
 diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/src/bsp.rs
 --- 09_privilege_level/src/bsp.rs
@@ -1040,8 +1037,8 @@ diff -uNr 09_privilege_level/src/bsp.rs 10_virtual_mem_part1_identity_mapping/sr
 diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/src/main.rs
 --- 09_privilege_level/src/main.rs
 +++ 10_virtual_mem_part1_identity_mapping/src/main.rs
-@@ -107,7 +107,11 @@
+@@ -105,7 +105,11 @@
- //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
+ //! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
 
  #![allow(clippy::upper_case_acronyms)]
 +#![allow(incomplete_features)]
@@ -1052,7 +1049,15 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s
  #![feature(format_args_nl)]
  #![feature(global_asm)]
  #![feature(panic_info_message)]
-@@ -132,9 +136,17 @@
+@@ -118,6 +122,7 @@
+ mod cpu;
+ mod driver;
+ mod exception;
++mod memory;
+ mod panic_wait;
+ mod print;
+ mod synchronization;
+@@ -128,9 +133,17 @@
  /// # Safety
  ///
  /// - Only a single core must be active and running this function.
@@ -1071,7 +1076,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s
 
      for i in bsp::driver::driver_manager().all_device_drivers().iter() {
          if let Err(x) = i.init() {
-@@ -163,6 +175,9 @@
+@@ -159,6 +172,9 @@
      );
      info!("Booting on: {}", bsp::board_name());
 
@@ -1081,7 +1086,7 @@ diff -uNr 09_privilege_level/src/main.rs 10_virtual_mem_part1_identity_mapping/s
      let (_, privilege_level) = exception::current_privilege_level();
      info!("Current privilege level: {}", privilege_level);
 
-@@ -186,6 +201,13 @@
+@@ -182,6 +198,13 @@
      info!("Timer test, spinning for 1 second");
      time::time_manager().spin_for(Duration::from_secs(1));
 
@@ -1387,14 +1392,13 @@ diff -uNr 09_privilege_level/src/memory/mmu.rs 10_virtual_mem_part1_identity_map
 diff -uNr 09_privilege_level/src/memory.rs 10_virtual_mem_part1_identity_mapping/src/memory.rs
 --- 09_privilege_level/src/memory.rs
 +++ 10_virtual_mem_part1_identity_mapping/src/memory.rs
-@@ -4,6 +4,8 @@
+@@ -0,0 +1,7 @@
-
++// SPDX-License-Identifier: MIT OR Apache-2.0
- //! Memory Management.
++//
-
++// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-+pub mod mmu;
 +
- use core::ops::RangeInclusive;
++//! Memory Management.
-
++
- //--------------------------------------------------------------------------------------------------
++pub mod mmu;
 
 ```

10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs

@@ -11,7 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
 use cortex_a::{asm, regs::*};
 
 // Assembly counterpart to this file.
@@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
             + SPSR_EL2::M::EL1h,
     );
 
-    // Second, let the link register point to runtime_init().
+    // Second, let the link register point to kernel_init().
-    ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+    ELR_EL2.set(crate::kernel_init as *const () as u64);
 
     // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
     // are no plans to ever return to EL2, just re-use the same stack.
@@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
 ///
 /// # Safety
 ///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
-/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
 #[no_mangle]
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }

10_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s

@@ -33,17 +33,29 @@ _start:
 	// Only proceed if the core executes in EL2. Park it otherwise.
 	mrs	x0, CurrentEL
 	cmp	x0, _EL2
-	b.ne	1f
+	b.ne	parking_loop
 
 	// Only proceed on the boot core. Park it otherwise.
 	mrs	x1, MPIDR_EL1
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -52,8 +64,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld

@@ -46,14 +46,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

10_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs

@@ -6,7 +6,7 @@
 
 pub mod mmu;
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
+use core::cell::UnsafeCell;
 
 //--------------------------------------------------------------------------------------------------
 // Private Definitions
@@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
 extern "Rust" {
     static __rx_start: UnsafeCell<()>;
     static __rx_end_exclusive: UnsafeCell<()>;
-
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -92,23 +89,3 @@ fn rx_start() -> usize {
 fn rx_end_exclusive() -> usize {
     unsafe { __rx_end_exclusive.get() as usize }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

10_virtual_mem_part1_identity_mapping/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![allow(incomplete_features)]
@@ -127,7 +125,6 @@ mod exception;
 mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 mod time;
 

10_virtual_mem_part1_identity_mapping/src/memory.rs

@@ -5,28 +5,3 @@
 //! Memory Management.
 
 pub mod mmu;
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

10_virtual_mem_part1_identity_mapping/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

11_exceptions_part1_groundwork/README.md

@@ -967,7 +967,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/exception.rs 11_exceptions_p
 diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_groundwork/src/main.rs
 --- 10_virtual_mem_part1_identity_mapping/src/main.rs
 +++ 11_exceptions_part1_groundwork/src/main.rs
-@@ -144,6 +144,8 @@
+@@ -141,6 +141,8 @@
      use driver::interface::DriverManager;
      use memory::mmu::interface::MMU;
 
@@ -976,7 +976,7 @@ diff -uNr 10_virtual_mem_part1_identity_mapping/src/main.rs 11_exceptions_part1_
      if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() {
          panic!("MMU: {}", string);
      }
-@@ -201,13 +203,28 @@
+@@ -198,13 +200,28 @@
      info!("Timer test, spinning for 1 second");
      time::time_manager().spin_for(Duration::from_secs(1));
 

11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs

@@ -11,7 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
 use cortex_a::{asm, regs::*};
 
 // Assembly counterpart to this file.
@@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
             + SPSR_EL2::M::EL1h,
     );
 
-    // Second, let the link register point to runtime_init().
+    // Second, let the link register point to kernel_init().
-    ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+    ELR_EL2.set(crate::kernel_init as *const () as u64);
 
     // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
     // are no plans to ever return to EL2, just re-use the same stack.
@@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
 ///
 /// # Safety
 ///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
-/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
 #[no_mangle]
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }

11_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s

@@ -33,17 +33,29 @@ _start:
 	// Only proceed if the core executes in EL2. Park it otherwise.
 	mrs	x0, CurrentEL
 	cmp	x0, _EL2
-	b.ne	1f
+	b.ne	parking_loop
 
 	// Only proceed on the boot core. Park it otherwise.
 	mrs	x1, MPIDR_EL1
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -52,8 +64,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

11_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld

@@ -46,14 +46,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

11_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs

@@ -6,7 +6,7 @@
 
 pub mod mmu;
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
+use core::cell::UnsafeCell;
 
 //--------------------------------------------------------------------------------------------------
 // Private Definitions
@@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
 extern "Rust" {
     static __rx_start: UnsafeCell<()>;
     static __rx_end_exclusive: UnsafeCell<()>;
-
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -92,23 +89,3 @@ fn rx_start() -> usize {
 fn rx_end_exclusive() -> usize {
     unsafe { __rx_end_exclusive.get() as usize }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

11_exceptions_part1_groundwork/src/main.rs

@@ -102,9 +102,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![allow(incomplete_features)]
@@ -127,7 +125,6 @@ mod exception;
 mod memory;
 mod panic_wait;
 mod print;
-mod runtime_init;
 mod synchronization;
 mod time;
 

11_exceptions_part1_groundwork/src/memory.rs

@@ -5,28 +5,3 @@
 //! Memory Management.
 
 pub mod mmu;
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}

11_exceptions_part1_groundwork/src/runtime_init.rs

@@ -1,37 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    zero_bss();
-
-    crate::kernel_init()
-}

12_integrated_testing/README.md

@@ -215,25 +215,24 @@ kernel boot:
 
 | | Function  | File |
 | - | - | - |
-| 1. | `_start()` | `lib.rs` |
+| 1. | `_start()` | The library's `boot.s` |
-| 2. | (some more aarch64 code) | `lib.rs` |
+| 2. | (some more aarch64 code) | The library's `boot.rs` |
-| 3. | `runtime_init()` | `lib.rs` |
+| 3. | `kernel_init()` | `main.rs` |
-| 4. | `kernel_init()` | `main.rs` |
+| 4. | `kernel_main()` | `main.rs` |
-| 5. | `kernel_main()` | `main.rs` |
 
 A function named `main` is never called. Hence, the `main()` function generated by `cargo test`
-would be silently dropped, and therefore the tests would never be executed. As you can see,
+would be silently dropped, and therefore the tests would never be executed. As you can see, the
-`runtime_init()` is the last function residing in our carved-out `lib.rs`, and it calls into
+first function getting called in our carved-out `main.rs` is `kernel_init()`. So in order to get the
-`kernel_init()`. So in order to get the tests to execute, we add a test-environment version of
+tests to execute, we add a test-environment version of `kernel_init()` to `lib.rs` as well
-`kernel_init()` to `lib.rs` as well (conditional compilation ensures it is only present when the
+(conditional compilation ensures it is only present when the test flag is set), and call the `cargo
-test flag is set), and call the `cargo test` generated `main()` function from there.
+test` generated `main()` function from there.
 
 This is where `#![reexport_test_harness_main = "test_main"]` finally comes into picture. It declares
 the name of the generated main function so that we can manually call it. Here is the final
 implementation in `lib.rs`:
 
 ```rust
-/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
+/// The `kernel_init()` for unit tests.
 #[cfg(test)]
 #[no_mangle]
 unsafe fn kernel_init() -> ! {
@@ -1073,7 +1072,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu/translatio
 diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs
 --- 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs
 +++ 12_integrated_testing/src/_arch/aarch64/memory/mmu.rs
-@@ -162,3 +162,22 @@
+@@ -162,3 +162,33 @@
          SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable)
      }
  }
@@ -1085,12 +1084,23 @@ diff -uNr 11_exceptions_part1_groundwork/src/_arch/aarch64/memory/mmu.rs 12_inte
 +#[cfg(test)]
 +mod tests {
 +    use super::*;
++    use core::{cell::UnsafeCell, ops::Range};
 +    use test_macros::kernel_test;
 +
 +    /// Check if KERNEL_TABLES is in .bss.
 +    #[kernel_test]
 +    fn kernel_tables_in_bss() {
-+        let bss_range = bsp::memory::bss_range_inclusive();
++        extern "Rust" {
++            static __bss_start: UnsafeCell<u64>;
++            static __bss_end_exclusive: UnsafeCell<u64>;
++        }
++
++        let bss_range = unsafe {
++            Range {
++                start: __bss_start.get(),
++                end: __bss_end_exclusive.get(),
++            }
++        };
 +        let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };
 +
 +        assert!(bss_range.contains(&kernel_tables_addr));
@@ -1207,7 +1217,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/exception.rs 12_integrated_testing/
 diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/lib.rs
 --- 11_exceptions_part1_groundwork/src/lib.rs
 +++ 12_integrated_testing/src/lib.rs
-@@ -0,0 +1,184 @@
+@@ -0,0 +1,186 @@
 +// SPDX-License-Identifier: MIT OR Apache-2.0
 +//
 +// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
@@ -1314,9 +1324,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
 +//!
 +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 +//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
++//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-+//!
-+//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 +
 +#![allow(clippy::upper_case_acronyms)]
 +#![allow(incomplete_features)]
@@ -1337,7 +1345,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
 +#![test_runner(crate::test_runner)]
 +
 +mod panic_wait;
-+mod runtime_init;
 +mod synchronization;
 +
 +pub mod bsp;
@@ -1362,6 +1369,11 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
 +    )
 +}
 +
++#[cfg(not(test))]
++extern "Rust" {
++    fn kernel_init() -> !;
++}
++
 +//--------------------------------------------------------------------------------------------------
 +// Testing
 +//--------------------------------------------------------------------------------------------------
@@ -1381,7 +1393,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
 +    }
 +}
 +
-+/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
++/// The `kernel_init()` for unit tests.
 +#[cfg(test)]
 +#[no_mangle]
 +unsafe fn kernel_init() -> ! {
@@ -1396,7 +1408,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/lib.rs 12_integrated_testing/src/li
 diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/main.rs
 --- 11_exceptions_part1_groundwork/src/main.rs
 +++ 12_integrated_testing/src/main.rs
-@@ -6,130 +6,12 @@
+@@ -6,127 +6,12 @@
  #![doc(html_logo_url = "https://git.io/JeGIp")]
 
  //! The `kernel` binary.
@@ -1496,9 +1508,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
 -//!
 -//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 -//!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
--//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+-//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
--//!
--//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 -
 -#![allow(clippy::upper_case_acronyms)]
 -#![allow(incomplete_features)]
@@ -1522,14 +1532,13 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
 -mod memory;
 -mod panic_wait;
 -mod print;
--mod runtime_init;
 -mod synchronization;
 -mod time;
 +use libkernel::{bsp, console, driver, exception, info, memory, time};
 
  /// Early init code.
  ///
-@@ -140,6 +22,7 @@
+@@ -137,6 +22,7 @@
  ///     - MMU + Data caching must be activated at the earliest. Without it, any atomic operations,
  ///       e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ
  ///       NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs.
@@ -1537,7 +1546,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
  unsafe fn kernel_init() -> ! {
      use driver::interface::DriverManager;
      use memory::mmu::interface::MMU;
-@@ -166,15 +49,9 @@
+@@ -163,15 +49,9 @@
  fn kernel_main() -> ! {
      use bsp::console::console;
      use console::interface::All;
@@ -1554,7 +1563,7 @@ diff -uNr 11_exceptions_part1_groundwork/src/main.rs 12_integrated_testing/src/m
      info!("Booting on: {}", bsp::board_name());
 
      info!("MMU online. Special regions:");
-@@ -200,31 +77,6 @@
+@@ -197,31 +77,6 @@
          info!("      {}. {}", i + 1, driver.compatible());
      }
 
@@ -1609,51 +1618,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/memory/mmu.rs 12_integrated_testing
 +    }
  }
 
-diff -uNr 11_exceptions_part1_groundwork/src/memory.rs 12_integrated_testing/src/memory.rs
---- 11_exceptions_part1_groundwork/src/memory.rs
-+++ 12_integrated_testing/src/memory.rs
-@@ -30,3 +30,40 @@
-         ptr = ptr.offset(1);
-     }
- }
-+
-+//--------------------------------------------------------------------------------------------------
-+// Testing
-+//--------------------------------------------------------------------------------------------------
-+
-+#[cfg(test)]
-+mod tests {
-+    use super::*;
-+    use test_macros::kernel_test;
-+
-+    /// Check `zero_volatile()`.
-+    #[kernel_test]
-+    fn zero_volatile_works() {
-+        let mut x: [usize; 3] = [10, 11, 12];
-+        let x_range = x.as_mut_ptr_range();
-+        let x_range_inclusive =
-+            RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) });
-+
-+        unsafe { zero_volatile(x_range_inclusive) };
-+
-+        assert_eq!(x, [0, 0, 0]);
-+    }
-+
-+    /// Check `bss` section layout.
-+    #[kernel_test]
-+    fn bss_section_is_sane() {
-+        use crate::bsp::memory::bss_range_inclusive;
-+        use core::mem;
-+
-+        let start = *bss_range_inclusive().start() as usize;
-+        let end = *bss_range_inclusive().end() as usize;
-+
-+        assert_eq!(start modulo mem::size_of::<usize>(), 0);
-+        assert_eq!(end modulo mem::size_of::<usize>(), 0);
-+        assert!(end >= start);
-+    }
-+}
-
 diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing/src/panic_wait.rs
 --- 11_exceptions_part1_groundwork/src/panic_wait.rs
 +++ 12_integrated_testing/src/panic_wait.rs
@@ -1689,23 +1653,6 @@ diff -uNr 11_exceptions_part1_groundwork/src/panic_wait.rs 12_integrated_testing
 +    _panic_exit()
  }
 
-diff -uNr 11_exceptions_part1_groundwork/src/runtime_init.rs 12_integrated_testing/src/runtime_init.rs
---- 11_exceptions_part1_groundwork/src/runtime_init.rs
-+++ 12_integrated_testing/src/runtime_init.rs
-@@ -31,7 +31,10 @@
- ///
- /// - Only a single core must be active and running this function.
- pub unsafe fn runtime_init() -> ! {
--    zero_bss();
-+    extern "Rust" {
-+        fn kernel_init() -> !;
-+    }
-
--    crate::kernel_init()
-+    zero_bss();
-+    kernel_init()
- }
-
 diff -uNr 11_exceptions_part1_groundwork/test-macros/Cargo.toml 12_integrated_testing/test-macros/Cargo.toml
 --- 11_exceptions_part1_groundwork/test-macros/Cargo.toml
 +++ 12_integrated_testing/test-macros/Cargo.toml

12_integrated_testing/src/_arch/aarch64/cpu/boot.rs

@@ -11,7 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
 use cortex_a::{asm, regs::*};
 
 // Assembly counterpart to this file.
@@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
             + SPSR_EL2::M::EL1h,
     );
 
-    // Second, let the link register point to runtime_init().
+    // Second, let the link register point to kernel_init().
-    ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+    ELR_EL2.set(crate::kernel_init as *const () as u64);
 
     // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
     // are no plans to ever return to EL2, just re-use the same stack.
@@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
 ///
 /// # Safety
 ///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
-/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
 #[no_mangle]
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }

12_integrated_testing/src/_arch/aarch64/cpu/boot.s

@@ -33,17 +33,29 @@ _start:
 	// Only proceed if the core executes in EL2. Park it otherwise.
 	mrs	x0, CurrentEL
 	cmp	x0, _EL2
-	b.ne	1f
+	b.ne	parking_loop
 
 	// Only proceed on the boot core. Park it otherwise.
 	mrs	x1, MPIDR_EL1
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -52,8 +64,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

12_integrated_testing/src/_arch/aarch64/memory/mmu.rs

@@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use core::{cell::UnsafeCell, ops::Range};
     use test_macros::kernel_test;
 
     /// Check if KERNEL_TABLES is in .bss.
     #[kernel_test]
     fn kernel_tables_in_bss() {
-        let bss_range = bsp::memory::bss_range_inclusive();
+        extern "Rust" {
+            static __bss_start: UnsafeCell<u64>;
+            static __bss_end_exclusive: UnsafeCell<u64>;
+        }
+
+        let bss_range = unsafe {
+            Range {
+                start: __bss_start.get(),
+                end: __bss_end_exclusive.get(),
+            }
+        };
         let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };
 
         assert!(bss_range.contains(&kernel_tables_addr));

12_integrated_testing/src/bsp/raspberrypi/link.ld

@@ -46,14 +46,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

12_integrated_testing/src/bsp/raspberrypi/memory.rs

@@ -6,7 +6,7 @@
 
 pub mod mmu;
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
+use core::cell::UnsafeCell;
 
 //--------------------------------------------------------------------------------------------------
 // Private Definitions
@@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
 extern "Rust" {
     static __rx_start: UnsafeCell<()>;
     static __rx_end_exclusive: UnsafeCell<()>;
-
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -92,23 +89,3 @@ fn rx_start() -> usize {
 fn rx_end_exclusive() -> usize {
     unsafe { __rx_end_exclusive.get() as usize }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

12_integrated_testing/src/lib.rs

@@ -104,9 +104,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![allow(incomplete_features)]
@@ -127,7 +125,6 @@
 #![test_runner(crate::test_runner)]
 
 mod panic_wait;
-mod runtime_init;
 mod synchronization;
 
 pub mod bsp;
@@ -152,6 +149,11 @@ pub fn version() -> &'static str {
     )
 }
 
+#[cfg(not(test))]
+extern "Rust" {
+    fn kernel_init() -> !;
+}
+
 //--------------------------------------------------------------------------------------------------
 // Testing
 //--------------------------------------------------------------------------------------------------
@@ -171,7 +173,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) {
     }
 }
 
-/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
+/// The `kernel_init()` for unit tests.
 #[cfg(test)]
 #[no_mangle]
 unsafe fn kernel_init() -> ! {

12_integrated_testing/src/memory.rs

@@ -5,65 +5,3 @@
 //! Memory Management.
 
 pub mod mmu;
-
-use core::ops::RangeInclusive;
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out an inclusive memory range.
-///
-/// # Safety
-///
-/// - `range.start` and `range.end` must be valid.
-/// - `range.start` and `range.end` must be `T` aligned.
-pub unsafe fn zero_volatile<T>(range: RangeInclusive<*mut T>)
-where
-    T: From<u8>,
-{
-    let mut ptr = *range.start();
-    let end_inclusive = *range.end();
-
-    while ptr <= end_inclusive {
-        core::ptr::write_volatile(ptr, T::from(0));
-        ptr = ptr.offset(1);
-    }
-}
-
-//--------------------------------------------------------------------------------------------------
-// Testing
-//--------------------------------------------------------------------------------------------------
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use test_macros::kernel_test;
-
-    /// Check `zero_volatile()`.
-    #[kernel_test]
-    fn zero_volatile_works() {
-        let mut x: [usize; 3] = [10, 11, 12];
-        let x_range = x.as_mut_ptr_range();
-        let x_range_inclusive =
-            RangeInclusive::new(x_range.start, unsafe { x_range.end.offset(-1) });
-
-        unsafe { zero_volatile(x_range_inclusive) };
-
-        assert_eq!(x, [0, 0, 0]);
-    }
-
-    /// Check `bss` section layout.
-    #[kernel_test]
-    fn bss_section_is_sane() {
-        use crate::bsp::memory::bss_range_inclusive;
-        use core::mem;
-
-        let start = *bss_range_inclusive().start() as usize;
-        let end = *bss_range_inclusive().end() as usize;
-
-        assert_eq!(start % mem::size_of::<usize>(), 0);
-        assert_eq!(end % mem::size_of::<usize>(), 0);
-        assert!(end >= start);
-    }
-}

12_integrated_testing/src/runtime_init.rs

@@ -1,40 +0,0 @@
-// SPDX-License-Identifier: MIT OR Apache-2.0
-//
-// Copyright (c) 2018-2021 Andre Richter <andre.o.richter@gmail.com>
-
-//! Rust runtime initialization code.
-
-use crate::{bsp, memory};
-
-//--------------------------------------------------------------------------------------------------
-// Private Code
-//--------------------------------------------------------------------------------------------------
-
-/// Zero out the .bss section.
-///
-/// # Safety
-///
-/// - Must only be called pre `kernel_init()`.
-#[inline(always)]
-unsafe fn zero_bss() {
-    memory::zero_volatile(bsp::memory::bss_range_inclusive());
-}
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel
-/// init code.
-///
-/// # Safety
-///
-/// - Only a single core must be active and running this function.
-pub unsafe fn runtime_init() -> ! {
-    extern "Rust" {
-        fn kernel_init() -> !;
-    }
-
-    zero_bss();
-    kernel_init()
-}

13_exceptions_part2_peripheral_IRQs/README.md

@@ -2104,7 +2104,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/exception.rs 13_exceptions_p
 diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs
 --- 12_integrated_testing/src/bsp/raspberrypi/memory.rs
 +++ 13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs
-@@ -51,10 +51,12 @@
+@@ -48,10 +48,12 @@
      pub mod mmio {
          use super::*;
 
@@ -2121,7 +2121,7 @@ diff -uNr 12_integrated_testing/src/bsp/raspberrypi/memory.rs 13_exceptions_part
      }
 
      /// Physical devices.
-@@ -65,6 +67,8 @@
+@@ -62,6 +64,8 @@
          pub const START:            usize =         0xFE00_0000;
          pub const GPIO_START:       usize = START + GPIO_OFFSET;
          pub const PL011_UART_START: usize = START + UART_OFFSET;
@@ -2376,7 +2376,7 @@ diff -uNr 12_integrated_testing/src/exception/asynchronous.rs 13_exceptions_part
 diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/src/lib.rs
 --- 12_integrated_testing/src/lib.rs
 +++ 13_exceptions_part2_peripheral_IRQs/src/lib.rs
-@@ -110,6 +110,7 @@
+@@ -108,6 +108,7 @@
 
  #![allow(clippy::upper_case_acronyms)]
  #![allow(incomplete_features)]
@@ -2384,7 +2384,7 @@ diff -uNr 12_integrated_testing/src/lib.rs 13_exceptions_part2_peripheral_IRQs/s
  #![feature(const_fn_fn_ptr_basics)]
  #![feature(const_generics)]
  #![feature(const_panic)]
-@@ -137,6 +138,7 @@
+@@ -134,6 +135,7 @@
  pub mod exception;
  pub mod memory;
  pub mod print;

13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs

@@ -11,7 +11,6 @@
 //!
 //! crate::cpu::boot::arch_boot
 
-use crate::runtime_init;
 use cortex_a::{asm, regs::*};
 
 // Assembly counterpart to this file.
@@ -50,8 +49,8 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
             + SPSR_EL2::M::EL1h,
     );
 
-    // Second, let the link register point to runtime_init().
+    // Second, let the link register point to kernel_init().
-    ELR_EL2.set(runtime_init::runtime_init as *const () as u64);
+    ELR_EL2.set(crate::kernel_init as *const () as u64);
 
     // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there
     // are no plans to ever return to EL2, just re-use the same stack.
@@ -68,12 +67,11 @@ unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr:
 ///
 /// # Safety
 ///
-/// - The `bss` section is not initialized yet. The code must not use or reference it in any way.
+/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`.
-/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`.
 #[no_mangle]
 pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! {
     prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr);
 
-    // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1.
+    // Use `eret` to "return" to EL1. This results in execution of kernel_init() in EL1.
     asm::eret()
 }

13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s

@@ -33,17 +33,29 @@ _start:
 	// Only proceed if the core executes in EL2. Park it otherwise.
 	mrs	x0, CurrentEL
 	cmp	x0, _EL2
-	b.ne	1f
+	b.ne	parking_loop
 
 	// Only proceed on the boot core. Park it otherwise.
 	mrs	x1, MPIDR_EL1
 	and	x1, x1, _core_id_mask
 	ldr	x2, BOOT_CORE_ID      // provided by bsp/__board_name__/cpu.rs
 	cmp	x1, x2
-	b.ne	1f
+	b.ne	parking_loop
 
-	// If execution reaches here, it is the boot core. Now, prepare the jump to Rust code.
+	// If execution reaches here, it is the boot core.
 
+	// Initialize DRAM.
+	ADR_REL	x0, __bss_start
+	ADR_REL x1, __bss_end_exclusive
+
+bss_init_loop:
+	cmp	x0, x1
+	b.eq	prepare_rust
+	stp	xzr, xzr, [x0], #16
+	b	bss_init_loop
+
+	// Prepare the jump to Rust code.
+prepare_rust:
 	// Set the stack pointer. This ensures that any code in EL2 that needs the stack will work.
 	ADR_REL	x0, __boot_core_stack_end_exclusive
 	mov	sp, x0
@@ -52,8 +64,9 @@ _start:
 	b	_start_rust
 
 	// Infinitely wait for events (aka "park the core").
-1:	wfe
+parking_loop:
-	b	1b
+	wfe
+	b	parking_loop
 
 .size	_start, . - _start
 .type	_start, function

13_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs

@@ -170,12 +170,23 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use core::{cell::UnsafeCell, ops::Range};
     use test_macros::kernel_test;
 
     /// Check if KERNEL_TABLES is in .bss.
     #[kernel_test]
     fn kernel_tables_in_bss() {
-        let bss_range = bsp::memory::bss_range_inclusive();
+        extern "Rust" {
+            static __bss_start: UnsafeCell<u64>;
+            static __bss_end_exclusive: UnsafeCell<u64>;
+        }
+
+        let bss_range = unsafe {
+            Range {
+                start: __bss_start.get(),
+                end: __bss_end_exclusive.get(),
+            }
+        };
         let kernel_tables_addr = unsafe { &KERNEL_TABLES as *const _ as usize as *mut u64 };
 
         assert!(bss_range.contains(&kernel_tables_addr));

13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld

@@ -46,14 +46,12 @@ SECTIONS
     ***********************************************************************************************/
     .data : { *(.data*) } :segment_rw
 
-    /* Section is zeroed in u64 chunks, align start and end to 8 bytes */
+    /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */
-    .bss : ALIGN(8)
+    .bss : ALIGN(16)
     {
         __bss_start = .;
         *(.bss*);
-        . = ALIGN(8);
+        . = ALIGN(16);
-
+        __bss_end_exclusive = .;
-        . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */
-        __bss_end_inclusive = . - 8;
     } :NONE
 }

13_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs

@@ -6,7 +6,7 @@
 
 pub mod mmu;
 
-use core::{cell::UnsafeCell, ops::RangeInclusive};
+use core::cell::UnsafeCell;
 
 //--------------------------------------------------------------------------------------------------
 // Private Definitions
@@ -16,9 +16,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive};
 extern "Rust" {
     static __rx_start: UnsafeCell<()>;
     static __rx_end_exclusive: UnsafeCell<()>;
-
-    static __bss_start: UnsafeCell<u64>;
-    static __bss_end_inclusive: UnsafeCell<u64>;
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -96,23 +93,3 @@ fn rx_start() -> usize {
 fn rx_end_exclusive() -> usize {
     unsafe { __rx_end_exclusive.get() as usize }
 }
-
-//--------------------------------------------------------------------------------------------------
-// Public Code
-//--------------------------------------------------------------------------------------------------
-
-/// Return the inclusive range spanning the .bss section.
-///
-/// # Safety
-///
-/// - Values are provided by the linker script and must be trusted as-is.
-/// - The linker-provided addresses must be u64 aligned.
-pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> {
-    let range;
-    unsafe {
-        range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get());
-    }
-    assert!(!range.is_empty());
-
-    range
-}

13_exceptions_part2_peripheral_IRQs/src/lib.rs

@@ -104,9 +104,7 @@
 //!
 //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`.
 //!     - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`.
-//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`].
+//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`.
-//!
-//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html
 
 #![allow(clippy::upper_case_acronyms)]
 #![allow(incomplete_features)]
@@ -128,7 +126,6 @@
 #![test_runner(crate::test_runner)]
 
 mod panic_wait;
-mod runtime_init;
 mod synchronization;
 
 pub mod bsp;
@@ -154,6 +151,11 @@ pub fn version() -> &'static str {
     )
 }
 
+#[cfg(not(test))]
+extern "Rust" {
+    fn kernel_init() -> !;
+}
+
 //--------------------------------------------------------------------------------------------------
 // Testing
 //--------------------------------------------------------------------------------------------------
@@ -173,7 +175,7 @@ pub fn test_runner(tests: &[&test_types::UnitTest]) {
     }
 }
 
-/// The `kernel_init()` for unit tests. Called from `runtime_init()`.
+/// The `kernel_init()` for unit tests.
 #[cfg(test)]
 #[no_mangle]
 unsafe fn kernel_init() -> ! {