From 2432c0d283883ea87b5f2e70eed36971d74d44df Mon Sep 17 00:00:00 2001 From: Andre Richter Date: Sat, 20 Mar 2021 09:41:43 +0100 Subject: [PATCH] Revert to assembly boot code It is too risky to rely on the compiler to not insert any operations using the stack. Having a stack-setting call in Rust using the cortex-a crate as the first action in a Rust-only _start() function does not work if you're subsequently using the stack, because the compiler often inserts the operations to make room on the stack to prepare a function call BEFORE the call to set the stack, which crashes the boot process. Hence, keep on using a small piece of assembly boot code throughout. --- .editorconfig | 5 +- 01_wait_forever/Cargo.toml | 3 + 01_wait_forever/README.md | 2 +- 01_wait_forever/src/_arch/aarch64/cpu/boot.S | 11 - 01_wait_forever/src/_arch/aarch64/cpu/boot.rs | 4 +- 01_wait_forever/src/_arch/aarch64/cpu/boot.s | 20 + 01_wait_forever/src/bsp/raspberrypi/link.ld | 1 - 01_wait_forever/src/main.rs | 6 +- 02_runtime_init/Cargo.lock | 28 ++ 02_runtime_init/Cargo.toml | 4 + 02_runtime_init/README.md | 193 +++++++--- 02_runtime_init/src/_arch/aarch64/cpu.rs | 12 +- 02_runtime_init/src/_arch/aarch64/cpu/boot.S | 21 - 02_runtime_init/src/_arch/aarch64/cpu/boot.rs | 22 +- 02_runtime_init/src/_arch/aarch64/cpu/boot.s | 42 ++ 02_runtime_init/src/bsp/raspberrypi.rs | 1 + .../src/bsp/raspberrypi/cpu.rs | 4 +- 02_runtime_init/src/bsp/raspberrypi/link.ld | 16 +- 02_runtime_init/src/main.rs | 6 +- 02_runtime_init/src/runtime_init.rs | 1 - 03_hacky_hello_world/Cargo.lock | 28 ++ 03_hacky_hello_world/Cargo.toml | 5 + 03_hacky_hello_world/README.md | 29 +- 03_hacky_hello_world/src/_arch/aarch64/cpu.rs | 12 +- .../src/_arch/aarch64/cpu/boot.S | 21 - .../src/_arch/aarch64/cpu/boot.rs | 22 +- .../src/_arch/aarch64/cpu/boot.s | 42 ++ 03_hacky_hello_world/src/bsp/raspberrypi.rs | 1 + .../src/bsp/raspberrypi/cpu.rs | 14 +- .../src/bsp/raspberrypi/link.ld | 16 +- 03_hacky_hello_world/src/main.rs | 6 +- 03_hacky_hello_world/src/runtime_init.rs | 1 - .../.vscode/settings.json | 7 - 04_zero_overhead_abstraction/Cargo.lock | 34 -- 04_zero_overhead_abstraction/Cargo.toml | 24 -- 04_zero_overhead_abstraction/Makefile | 115 ------ 04_zero_overhead_abstraction/README.md | 301 --------------- 04_zero_overhead_abstraction/build.rs | 8 - .../src/_arch/aarch64/cpu.rs | 26 -- .../src/_arch/aarch64/cpu/boot.rs | 41 -- .../src/_arch/aarch64/cpu/smp.rs | 29 -- 04_zero_overhead_abstraction/src/bsp.rs | 11 - .../src/bsp/raspberrypi.rs | 9 - .../src/bsp/raspberrypi/console.rs | 47 --- .../src/bsp/raspberrypi/link.ld | 49 --- .../src/bsp/raspberrypi/memory.rs | 59 --- 04_zero_overhead_abstraction/src/console.rs | 19 - 04_zero_overhead_abstraction/src/cpu.rs | 18 - 04_zero_overhead_abstraction/src/cpu/boot.rs | 9 - 04_zero_overhead_abstraction/src/main.rs | 133 ------- 04_zero_overhead_abstraction/src/memory.rs | 30 -- .../src/panic_wait.rs | 19 - 04_zero_overhead_abstraction/src/print.rs | 38 -- .../src/runtime_init.rs | 37 -- 05_safe_globals/src/_arch/aarch64/cpu/boot.rs | 26 +- 05_safe_globals/src/_arch/aarch64/cpu/boot.s | 42 ++ 05_safe_globals/src/_arch/aarch64/cpu/smp.rs | 29 -- 05_safe_globals/src/bsp/raspberrypi/cpu.rs | 4 +- 05_safe_globals/src/bsp/raspberrypi/link.ld | 17 +- 05_safe_globals/src/bsp/raspberrypi/memory.rs | 22 -- 05_safe_globals/src/cpu.rs | 2 - 05_safe_globals/src/cpu/smp.rs | 14 - 05_safe_globals/src/main.rs | 6 +- 06_drivers_gpio_uart/README.md | 15 +- .../src/_arch/aarch64/cpu/boot.rs | 26 +- .../src/_arch/aarch64/cpu/boot.s | 42 ++ .../src/_arch/aarch64/cpu/smp.rs | 29 -- .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 17 +- .../src/bsp/raspberrypi/memory.rs | 22 -- 06_drivers_gpio_uart/src/cpu.rs | 2 - 06_drivers_gpio_uart/src/cpu/smp.rs | 14 - 06_drivers_gpio_uart/src/main.rs | 6 +- 07_uart_chainloader/Makefile | 4 +- 07_uart_chainloader/README.md | 358 +++++------------- 07_uart_chainloader/demo_payload_rpi3.img | Bin 6824 -> 6840 bytes 07_uart_chainloader/demo_payload_rpi4.img | Bin 6672 -> 6680 bytes 07_uart_chainloader/src/_arch/aarch64/cpu.rs | 16 - .../src/_arch/aarch64/cpu/boot.rs | 26 +- .../src/_arch/aarch64/cpu/boot.s | 53 +++ .../src/_arch/aarch64/cpu/smp.rs | 29 -- .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 30 +- .../src/bsp/raspberrypi/memory.rs | 30 +- 07_uart_chainloader/src/cpu.rs | 4 +- 07_uart_chainloader/src/cpu/smp.rs | 14 - 07_uart_chainloader/src/main.rs | 12 +- 07_uart_chainloader/src/relocate.rs | 49 --- 07_uart_chainloader/src/runtime_init.rs | 3 +- 08_timestamps/README.md | 290 ++++---------- 08_timestamps/src/_arch/aarch64/cpu/boot.rs | 26 +- 08_timestamps/src/_arch/aarch64/cpu/boot.s | 42 ++ 08_timestamps/src/_arch/aarch64/cpu/smp.rs | 29 -- 08_timestamps/src/bsp/raspberrypi/cpu.rs | 4 +- 08_timestamps/src/bsp/raspberrypi/link.ld | 17 +- 08_timestamps/src/bsp/raspberrypi/memory.rs | 22 -- 08_timestamps/src/cpu.rs | 2 - 08_timestamps/src/cpu/smp.rs | 14 - 08_timestamps/src/main.rs | 6 +- 09_hw_debug_JTAG/README.md | 16 +- .../src/_arch/aarch64/cpu/boot.rs | 26 +- 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s | 42 ++ 09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs | 29 -- 09_hw_debug_JTAG/src/bsp/raspberrypi/cpu.rs | 4 +- 09_hw_debug_JTAG/src/bsp/raspberrypi/link.ld | 17 +- .../src/bsp/raspberrypi/memory.rs | 22 -- 09_hw_debug_JTAG/src/cpu.rs | 2 - 09_hw_debug_JTAG/src/cpu/smp.rs | 14 - 09_hw_debug_JTAG/src/main.rs | 6 +- 10_privilege_level/README.md | 210 +++++----- .../src/_arch/aarch64/cpu/boot.rs | 48 +-- .../src/_arch/aarch64/cpu/boot.s | 48 +++ .../src/_arch/aarch64/cpu/smp.rs | 29 -- 10_privilege_level/src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 17 +- .../src/bsp/raspberrypi/memory.rs | 22 -- 10_privilege_level/src/cpu.rs | 2 - 10_privilege_level/src/cpu/smp.rs | 14 - 10_privilege_level/src/main.rs | 6 +- .../README.md | 49 ++- .../src/_arch/aarch64/cpu/boot.rs | 48 +-- .../src/_arch/aarch64/cpu/boot.s | 48 +++ .../src/_arch/aarch64/cpu/smp.rs | 29 -- .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 16 +- .../src/bsp/raspberrypi/memory.rs | 6 - .../src/cpu.rs | 2 - .../src/cpu/smp.rs | 14 - .../src/main.rs | 6 +- 12_exceptions_part1_groundwork/README.md | 291 +++++++------- .../src/_arch/aarch64/cpu/boot.rs | 48 +-- .../src/_arch/aarch64/cpu/boot.s | 48 +++ .../src/_arch/aarch64/cpu/smp.rs | 29 -- .../src/_arch/aarch64/exception.S | 138 ------- .../src/_arch/aarch64/exception.rs | 2 +- .../src/_arch/aarch64/exception.s | 148 ++++++++ .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 16 +- .../src/bsp/raspberrypi/memory.rs | 6 - 12_exceptions_part1_groundwork/src/cpu.rs | 2 - 12_exceptions_part1_groundwork/src/cpu/smp.rs | 14 - 12_exceptions_part1_groundwork/src/main.rs | 5 +- 13_integrated_testing/README.md | 26 +- .../src/_arch/aarch64/cpu/boot.rs | 48 +-- .../src/_arch/aarch64/cpu/boot.s | 48 +++ .../src/_arch/aarch64/cpu/smp.rs | 29 -- .../src/_arch/aarch64/exception.S | 138 ------- .../src/_arch/aarch64/exception.rs | 2 +- .../src/_arch/aarch64/exception.s | 148 ++++++++ .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 16 +- .../src/bsp/raspberrypi/memory.rs | 6 - 13_integrated_testing/src/cpu.rs | 2 - 13_integrated_testing/src/cpu/smp.rs | 14 - 13_integrated_testing/src/lib.rs | 5 +- .../tests/00_console_sanity.rs | 4 +- 14_exceptions_part2_peripheral_IRQs/README.md | 83 +++- .../src/_arch/aarch64/cpu/boot.rs | 48 +-- .../src/_arch/aarch64/cpu/boot.s | 48 +++ .../src/_arch/aarch64/exception.S | 138 ------- .../src/_arch/aarch64/exception.rs | 2 +- .../src/_arch/aarch64/exception.s | 148 ++++++++ .../src/_arch/aarch64/memory/mmu.rs | 2 +- .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 16 +- .../src/bsp/raspberrypi/memory.rs | 6 - .../src/lib.rs | 5 +- .../tests/00_console_sanity.rs | 4 +- 15_virtual_mem_part2_mmio_remap/README.md | 63 +-- .../src/_arch/aarch64/cpu/boot.rs | 48 +-- .../src/_arch/aarch64/cpu/boot.s | 48 +++ .../src/_arch/aarch64/exception.S | 138 ------- .../src/_arch/aarch64/exception.rs | 2 +- .../src/_arch/aarch64/exception.s | 148 ++++++++ .../src/bsp/raspberrypi/cpu.rs | 4 +- .../src/bsp/raspberrypi/link.ld | 19 +- .../src/bsp/raspberrypi/memory.rs | 8 - 15_virtual_mem_part2_mmio_remap/src/lib.rs | 5 +- .../tests/00_console_sanity.rs | 4 +- X1_JTAG_boot/jtag_boot_rpi3.img | Bin 8152 -> 8176 bytes X1_JTAG_boot/jtag_boot_rpi4.img | Bin 6840 -> 6848 bytes X1_JTAG_boot/src/_arch/aarch64/cpu/boot.rs | 26 +- X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s | 42 ++ X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs | 29 -- X1_JTAG_boot/src/bsp/raspberrypi/cpu.rs | 4 +- X1_JTAG_boot/src/bsp/raspberrypi/link.ld | 17 +- X1_JTAG_boot/src/bsp/raspberrypi/memory.rs | 22 -- X1_JTAG_boot/src/cpu.rs | 2 - X1_JTAG_boot/src/cpu/smp.rs | 14 - X1_JTAG_boot/src/main.rs | 6 +- rust-toolchain | 2 +- 191 files changed, 2586 insertions(+), 3695 deletions(-) delete mode 100644 01_wait_forever/src/_arch/aarch64/cpu/boot.S create mode 100644 01_wait_forever/src/_arch/aarch64/cpu/boot.s delete mode 100644 02_runtime_init/src/_arch/aarch64/cpu/boot.S create mode 100644 02_runtime_init/src/_arch/aarch64/cpu/boot.s rename {04_zero_overhead_abstraction => 02_runtime_init}/src/bsp/raspberrypi/cpu.rs (82%) delete mode 100644 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S create mode 100644 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s rename 04_zero_overhead_abstraction/src/cpu/smp.rs => 03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs (63%) delete mode 100644 04_zero_overhead_abstraction/.vscode/settings.json delete mode 100644 04_zero_overhead_abstraction/Cargo.lock delete mode 100644 04_zero_overhead_abstraction/Cargo.toml delete mode 100644 04_zero_overhead_abstraction/Makefile delete mode 100644 04_zero_overhead_abstraction/README.md delete mode 100644 04_zero_overhead_abstraction/build.rs delete mode 100644 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs delete mode 100644 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs delete mode 100644 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs delete mode 100644 04_zero_overhead_abstraction/src/bsp.rs delete mode 100644 04_zero_overhead_abstraction/src/bsp/raspberrypi.rs delete mode 100644 04_zero_overhead_abstraction/src/bsp/raspberrypi/console.rs delete mode 100644 04_zero_overhead_abstraction/src/bsp/raspberrypi/link.ld delete mode 100644 04_zero_overhead_abstraction/src/bsp/raspberrypi/memory.rs delete mode 100644 04_zero_overhead_abstraction/src/console.rs delete mode 100644 04_zero_overhead_abstraction/src/cpu.rs delete mode 100644 04_zero_overhead_abstraction/src/cpu/boot.rs delete mode 100644 04_zero_overhead_abstraction/src/main.rs delete mode 100644 04_zero_overhead_abstraction/src/memory.rs delete mode 100644 04_zero_overhead_abstraction/src/panic_wait.rs delete mode 100644 04_zero_overhead_abstraction/src/print.rs delete mode 100644 04_zero_overhead_abstraction/src/runtime_init.rs create mode 100644 05_safe_globals/src/_arch/aarch64/cpu/boot.s delete mode 100644 05_safe_globals/src/_arch/aarch64/cpu/smp.rs delete mode 100644 05_safe_globals/src/cpu/smp.rs create mode 100644 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s delete mode 100644 06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs delete mode 100644 06_drivers_gpio_uart/src/cpu/smp.rs create mode 100644 07_uart_chainloader/src/_arch/aarch64/cpu/boot.s delete mode 100644 07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs delete mode 100644 07_uart_chainloader/src/cpu/smp.rs delete mode 100644 07_uart_chainloader/src/relocate.rs create mode 100644 08_timestamps/src/_arch/aarch64/cpu/boot.s delete mode 100644 08_timestamps/src/_arch/aarch64/cpu/smp.rs delete mode 100644 08_timestamps/src/cpu/smp.rs create mode 100644 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s delete mode 100644 09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs delete mode 100644 09_hw_debug_JTAG/src/cpu/smp.rs create mode 100644 10_privilege_level/src/_arch/aarch64/cpu/boot.s delete mode 100644 10_privilege_level/src/_arch/aarch64/cpu/smp.rs delete mode 100644 10_privilege_level/src/cpu/smp.rs create mode 100644 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s delete mode 100644 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs delete mode 100644 11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs create mode 100644 12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s delete mode 100644 12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs delete mode 100644 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S create mode 100644 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.s delete mode 100644 12_exceptions_part1_groundwork/src/cpu/smp.rs create mode 100644 13_integrated_testing/src/_arch/aarch64/cpu/boot.s delete mode 100644 13_integrated_testing/src/_arch/aarch64/cpu/smp.rs delete mode 100644 13_integrated_testing/src/_arch/aarch64/exception.S create mode 100644 13_integrated_testing/src/_arch/aarch64/exception.s delete mode 100644 13_integrated_testing/src/cpu/smp.rs create mode 100644 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s delete mode 100644 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S create mode 100644 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.s create mode 100644 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s delete mode 100644 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.S create mode 100644 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.s create mode 100644 X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s delete mode 100644 X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs delete mode 100644 X1_JTAG_boot/src/cpu/smp.rs diff --git a/.editorconfig b/.editorconfig index 38ed8df1..dd8892ca 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,8 +23,9 @@ indent_size = 4 [*.rs] indent_size = 4 -[*.S] -indent_size = 4 +[*.s] +indent_style = tab +indent_size = 8 [*.sh] indent_size = 4 diff --git a/01_wait_forever/Cargo.toml b/01_wait_forever/Cargo.toml index 8a0187c4..a7580996 100644 --- a/01_wait_forever/Cargo.toml +++ b/01_wait_forever/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" authors = ["Andre Richter "] edition = "2018" +[profile.release] +lto = true + [features] default = [] bsp_rpi3 = [] diff --git a/01_wait_forever/README.md b/01_wait_forever/README.md index 66dc29d2..bf214b68 100644 --- a/01_wait_forever/README.md +++ b/01_wait_forever/README.md @@ -23,7 +23,7 @@ - Only `.text` section. - `main.rs`: Important [inner attributes]: - `#![no_std]`, `#![no_main]` -- `boot.S`: Assembly `_start()` function that executes `wfe` (Wait For Event), halting all cores +- `boot.s`: Assembly `_start()` function that executes `wfe` (Wait For Event), halting all cores that are executing `_start()`. - We (have to) define a `#[panic_handler]` function to make the compiler happy. - Make it `unimplemented!()` because it will be stripped out since it is not used. diff --git a/01_wait_forever/src/_arch/aarch64/cpu/boot.S b/01_wait_forever/src/_arch/aarch64/cpu/boot.S deleted file mode 100644 index 2de0c61c..00000000 --- a/01_wait_forever/src/_arch/aarch64/cpu/boot.S +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -.section ".text._start" - -.global _start - -_start: -1: wfe // Wait for event - b 1b // In case an event happened, jump back to 1 diff --git a/01_wait_forever/src/_arch/aarch64/cpu/boot.rs b/01_wait_forever/src/_arch/aarch64/cpu/boot.rs index c3c325d3..4bc6ed6d 100644 --- a/01_wait_forever/src/_arch/aarch64/cpu/boot.rs +++ b/01_wait_forever/src/_arch/aarch64/cpu/boot.rs @@ -11,5 +11,5 @@ //! //! crate::cpu::boot::arch_boot -// Assembly counterpart to this file. Includes function _start(). -global_asm!(include_str!("boot.S")); +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); diff --git a/01_wait_forever/src/_arch/aarch64/cpu/boot.s b/01_wait_forever/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..fb05382c --- /dev/null +++ b/01_wait_forever/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/01_wait_forever/src/bsp/raspberrypi/link.ld b/01_wait_forever/src/bsp/raspberrypi/link.ld index 7d649f88..60c393d6 100644 --- a/01_wait_forever/src/bsp/raspberrypi/link.ld +++ b/01_wait_forever/src/bsp/raspberrypi/link.ld @@ -23,6 +23,5 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) } :segment_rx } diff --git a/01_wait_forever/src/main.rs b/01_wait_forever/src/main.rs index 8cd4b144..b01a84cb 100644 --- a/01_wait_forever/src/main.rs +++ b/01_wait_forever/src/main.rs @@ -100,10 +100,8 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. -//! -//! [`cpu::boot::arch_boot::_start()`]: ../src/kernel/cpu/up/_arch/aarch64/cpu/boot.rs.html +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. #![feature(asm)] #![feature(global_asm)] diff --git a/02_runtime_init/Cargo.lock b/02_runtime_init/Cargo.lock index 3e9c3111..3b926a34 100644 --- a/02_runtime_init/Cargo.lock +++ b/02_runtime_init/Cargo.lock @@ -1,6 +1,34 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + +[[package]] +name = "cortex-a" +version = "5.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecefc30975eb87afc5a810d4b2305c0ec29e607ea97e51b2ecd80766e9268d28" +dependencies = [ + "register", +] + [[package]] name = "kernel" version = "0.1.0" +dependencies = [ + "cortex-a", +] +[[package]] +name = "register" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a247de29ab7cc8f5006cfe775c4a81c704f9914c5e2a79696862e643135433" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "tock-registers" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f521a79accce68c417c9c77ce22108056b626126da1932f7e2e9b5bbffee0cea" diff --git a/02_runtime_init/Cargo.toml b/02_runtime_init/Cargo.toml index a7580996..4a03b962 100644 --- a/02_runtime_init/Cargo.toml +++ b/02_runtime_init/Cargo.toml @@ -17,3 +17,7 @@ bsp_rpi4 = [] ##-------------------------------------------------------------------------------------------------- [dependencies] + +# Platform specific dependencies +[target.'cfg(target_arch = "aarch64")'.dependencies] +cortex-a = { version = "5.x.x" } diff --git a/02_runtime_init/README.md b/02_runtime_init/README.md index 5f5245d5..64e36440 100644 --- a/02_runtime_init/README.md +++ b/02_runtime_init/README.md @@ -2,22 +2,28 @@ ## tl;dr -- We extend `boot.S` to call into Rust code for the first time. There, we zero the [bss] section +- We extend `boot.s` to call into Rust code for the first time. There, we zero the [bss] section before execution is halted with a call to `panic()`. - Check out `make qemu` again to see the additional code run. ## Notable additions -- More sections in linker script: - - `.rodata`, `.got`, `.data`, `.bss` -- `_start()`: - - Halt core if core != `core0`. - - `core0` jumps to the `runtime_init()` Rust function. -- `runtime_init()` in `runtime_init.rs` +- More additions to the linker script: + - New sections: `.rodata`, `.got`, `.data`, `.bss`. + - A dedicated place for linking boot-time arguments that need to be read by `_start()`. +- `_start()` in `_arch/__arch_name__/cpu/boot.s`: + 1. Halt core if core != core0. + 1. Set up the `stack pointer`. + 1. Jump to the `_start_rust()` function, defined in `arch/__arch_name__/cpu/boot.rs`. +- `runtime_init()` in `runtime_init.rs`: - Zeros the `.bss` section. - - Calls `kernel_init()`, which calls `panic!()`, which eventually halts `core0` as well. + - Calls `kernel_init()`, which calls `panic!()`, which eventually halts core0 as well. +- The library now uses the [cortex-a] crate, which provides zero-overhead abstractions and wraps + `unsafe` parts when dealing with the CPU's resources. + - See it in action in `_arch/__arch_name__/cpu.rs`. [bss]: https://en.wikipedia.org/wiki/.bss +[cortex-a]: https://github.com/rust-embedded/cortex-a ## Diff to previous ```diff @@ -25,16 +31,14 @@ diff -uNr 01_wait_forever/Cargo.toml 02_runtime_init/Cargo.toml --- 01_wait_forever/Cargo.toml +++ 02_runtime_init/Cargo.toml -@@ -4,6 +4,9 @@ - authors = ["Andre Richter "] - edition = "2018" +@@ -17,3 +17,7 @@ + ##-------------------------------------------------------------------------------------------------- -+[profile.release] -+lto = true + [dependencies] + - [features] - default = [] - bsp_rpi3 = [] ++# Platform specific dependencies ++[target.'cfg(target_arch = "aarch64")'.dependencies] ++cortex-a = { version = "5.x.x" } diff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile --- 01_wait_forever/Makefile @@ -49,32 +53,78 @@ diff -uNr 01_wait_forever/Makefile 02_runtime_init/Makefile nm: $(KERNEL_ELF) -diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.S 02_runtime_init/src/_arch/aarch64/cpu/boot.S ---- 01_wait_forever/src/_arch/aarch64/cpu/boot.S -+++ 02_runtime_init/src/_arch/aarch64/cpu/boot.S -@@ -7,5 +7,15 @@ - .global _start +diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.rs 02_runtime_init/src/_arch/aarch64/cpu/boot.rs +--- 01_wait_forever/src/_arch/aarch64/cpu/boot.rs ++++ 02_runtime_init/src/_arch/aarch64/cpu/boot.rs +@@ -11,5 +11,23 @@ + //! + //! crate::cpu::boot::arch_boot + ++use crate::runtime_init; ++ + // Assembly counterpart to this file. + global_asm!(include_str!("boot.s")); ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// The Rust entry of the `kernel` binary. ++/// ++/// The function is called from the assembly `_start` function. ++/// ++/// # Safety ++/// ++/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. ++#[no_mangle] ++pub unsafe fn _start_rust() -> ! { ++ runtime_init::runtime_init() ++} +diff -uNr 01_wait_forever/src/_arch/aarch64/cpu/boot.s 02_runtime_init/src/_arch/aarch64/cpu/boot.s +--- 01_wait_forever/src/_arch/aarch64/cpu/boot.s ++++ 02_runtime_init/src/_arch/aarch64/cpu/boot.s +@@ -3,6 +3,12 @@ + // Copyright (c) 2021 Andre Richter + + //-------------------------------------------------------------------------------------------------- ++// Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++.equ _core_id_mask, 0b11 ++ ++//-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- + .section .text._start +@@ -11,6 +17,22 @@ + // fn _start() + //------------------------------------------------------------------------------ _start: --1: wfe // Wait for event -- b 1b // In case an event happened, jump back to 1 -+ mrs x1, mpidr_el1 // Read Multiprocessor Affinity Register -+ and x1, x1, #3 // Clear all bits except [1:0], which hold core id -+ cbz x1, 2f // Jump to label 2 if we are core 0 -+1: wfe // Wait for event -+ b 1b // In case an event happened, jump back to 1 -+2: // If we are here, we are core0 -+ ldr x1, =_start // Load address of function "_start()" -+ mov sp, x1 // Set start of stack to before our code, aka first -+ // address before "_start()" -+ bl runtime_init // Jump to the "runtime_init()" kernel function -+ b 1b // We should never reach here. But just in case, -+ // park this core aswell ++ // Only proceed on the boot core. Park it otherwise. ++ mrs x1, MPIDR_EL1 ++ and x1, x1, _core_id_mask ++ ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs ++ cmp x1, x2 ++ b.ne 1f ++ ++ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. ++ ++ // Set the stack pointer. ++ ldr x0, =__boot_core_stack_end_exclusive ++ mov sp, x0 ++ ++ // Jump to Rust code. ++ b _start_rust ++ + // Infinitely wait for events (aka "park the core"). + 1: wfe + b 1b diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aarch64/cpu.rs --- 01_wait_forever/src/_arch/aarch64/cpu.rs +++ 02_runtime_init/src/_arch/aarch64/cpu.rs -@@ -0,0 +1,30 @@ +@@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -88,6 +138,8 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aar +//! +//! crate::cpu::arch_cpu + ++use cortex_a::asm; ++ +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- @@ -95,21 +147,34 @@ diff -uNr 01_wait_forever/src/_arch/aarch64/cpu.rs 02_runtime_init/src/_arch/aar +/// Pause execution on the core. +#[inline(always)] +pub fn wait_forever() -> ! { -+ unsafe { -+ loop { -+ #[rustfmt::skip] -+ asm!( -+ "wfe", -+ options(nomem, nostack, preserves_flags) -+ ); -+ } ++ loop { ++ asm::wfe() + } +} +diff -uNr 01_wait_forever/src/bsp/raspberrypi/cpu.rs 02_runtime_init/src/bsp/raspberrypi/cpu.rs +--- 01_wait_forever/src/bsp/raspberrypi/cpu.rs ++++ 02_runtime_init/src/bsp/raspberrypi/cpu.rs +@@ -0,0 +1,14 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! BSP Processor code. ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Used by `arch` code to find the early boot core. ++#[no_mangle] ++#[link_section = ".text._start_arguments"] ++pub static BOOT_CORE_ID: u64 = 0; + diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/raspberrypi/link.ld --- 01_wait_forever/src/bsp/raspberrypi/link.ld +++ 02_runtime_init/src/bsp/raspberrypi/link.ld -@@ -11,6 +11,7 @@ +@@ -11,17 +11,52 @@ PHDRS { segment_rx PT_LOAD FLAGS(5); /* 5 == RX */ @@ -117,8 +182,13 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra } SECTIONS -@@ -18,11 +19,30 @@ + { . = __rpi_load_addr; ++ /* ^ */ ++ /* | stack */ ++ /* | growth */ ++ /* | direction */ ++ __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** - * Code @@ -127,7 +197,16 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/link.ld 02_runtime_init/src/bsp/ra .text : { KEEP(*(.text._start)) - *(.text*) ++ ++ /* Special constants (or statics in Rust speak) needed by _start(). ++ * ++ * They are placed in close proximity to _start() from where they will be read. This ensures ++ * that position-independent, PC-relative loads can be emitted. ++ */ ++ *(.text._start_arguments) ++ ++ *(.text._start_rust) /* The Rust entry point */ ++ *(.text*) /* Everything else */ } :segment_rx + + .rodata : ALIGN(8) { *(.rodata*) } :segment_rx @@ -195,11 +274,12 @@ diff -uNr 01_wait_forever/src/bsp/raspberrypi/memory.rs 02_runtime_init/src/bsp/ diff -uNr 01_wait_forever/src/bsp/raspberrypi.rs 02_runtime_init/src/bsp/raspberrypi.rs --- 01_wait_forever/src/bsp/raspberrypi.rs +++ 02_runtime_init/src/bsp/raspberrypi.rs -@@ -4,4 +4,4 @@ +@@ -4,4 +4,5 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. -// Coming soon. ++pub mod cpu; +pub mod memory; diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs @@ -223,18 +303,18 @@ diff -uNr 01_wait_forever/src/cpu.rs 02_runtime_init/src/cpu.rs diff -uNr 01_wait_forever/src/main.rs 02_runtime_init/src/main.rs --- 01_wait_forever/src/main.rs +++ 02_runtime_init/src/main.rs -@@ -102,8 +102,10 @@ +@@ -102,14 +102,25 @@ //! - //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. - //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. + //! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. + //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. - //! - //! [`cpu::boot::arch_boot::_start()`]: ../src/kernel/cpu/up/_arch/aarch64/cpu/boot.rs.html ++//! +//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html - #![feature(asm)] +-#![feature(asm)] #![feature(global_asm)] -@@ -112,6 +114,15 @@ + #![no_main] + #![no_std] mod bsp; mod cpu; @@ -306,7 +386,7 @@ diff -uNr 01_wait_forever/src/panic_wait.rs 02_runtime_init/src/panic_wait.rs diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.rs --- 01_wait_forever/src/runtime_init.rs +++ 02_runtime_init/src/runtime_init.rs -@@ -0,0 +1,38 @@ +@@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -339,7 +419,6 @@ diff -uNr 01_wait_forever/src/runtime_init.rs 02_runtime_init/src/runtime_init.r +/// # Safety +/// +/// - Only a single core must be active and running this function. -+#[no_mangle] +pub unsafe fn runtime_init() -> ! { + zero_bss(); + diff --git a/02_runtime_init/src/_arch/aarch64/cpu.rs b/02_runtime_init/src/_arch/aarch64/cpu.rs index 4fd7e313..5f97ea41 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu.rs +++ b/02_runtime_init/src/_arch/aarch64/cpu.rs @@ -11,6 +11,8 @@ //! //! crate::cpu::arch_cpu +use cortex_a::asm; + //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- @@ -18,13 +20,7 @@ /// Pause execution on the core. #[inline(always)] pub fn wait_forever() -> ! { - unsafe { - loop { - #[rustfmt::skip] - asm!( - "wfe", - options(nomem, nostack, preserves_flags) - ); - } + loop { + asm::wfe() } } diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.S b/02_runtime_init/src/_arch/aarch64/cpu/boot.S deleted file mode 100644 index 449cf502..00000000 --- a/02_runtime_init/src/_arch/aarch64/cpu/boot.S +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -.section ".text._start" - -.global _start - -_start: - mrs x1, mpidr_el1 // Read Multiprocessor Affinity Register - and x1, x1, #3 // Clear all bits except [1:0], which hold core id - cbz x1, 2f // Jump to label 2 if we are core 0 -1: wfe // Wait for event - b 1b // In case an event happened, jump back to 1 -2: // If we are here, we are core0 - ldr x1, =_start // Load address of function "_start()" - mov sp, x1 // Set start of stack to before our code, aka first - // address before "_start()" - bl runtime_init // Jump to the "runtime_init()" kernel function - b 1b // We should never reach here. But just in case, - // park this core aswell diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs index c3c325d3..c85bb94b 100644 --- a/02_runtime_init/src/_arch/aarch64/cpu/boot.rs +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.rs @@ -11,5 +11,23 @@ //! //! crate::cpu::boot::arch_boot -// Assembly counterpart to this file. Includes function _start(). -global_asm!(include_str!("boot.S")); +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The Rust entry of the `kernel` binary. +/// +/// The function is called from the assembly `_start` function. +/// +/// # Safety +/// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +#[no_mangle] +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() +} diff --git a/02_runtime_init/src/_arch/aarch64/cpu/boot.s b/02_runtime_init/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/02_runtime_init/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/02_runtime_init/src/bsp/raspberrypi.rs b/02_runtime_init/src/bsp/raspberrypi.rs index f9bd5b88..10535d7b 100644 --- a/02_runtime_init/src/bsp/raspberrypi.rs +++ b/02_runtime_init/src/bsp/raspberrypi.rs @@ -4,4 +4,5 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. +pub mod cpu; pub mod memory; diff --git a/04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs b/02_runtime_init/src/bsp/raspberrypi/cpu.rs similarity index 82% rename from 04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs rename to 02_runtime_init/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs +++ b/02_runtime_init/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/02_runtime_init/src/bsp/raspberrypi/link.ld b/02_runtime_init/src/bsp/raspberrypi/link.ld index acb7fe28..3b73b3c7 100644 --- a/02_runtime_init/src/bsp/raspberrypi/link.ld +++ b/02_runtime_init/src/bsp/raspberrypi/link.ld @@ -17,6 +17,11 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table @@ -24,7 +29,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/02_runtime_init/src/main.rs b/02_runtime_init/src/main.rs index fd9c36bc..faad6b09 100644 --- a/02_runtime_init/src/main.rs +++ b/02_runtime_init/src/main.rs @@ -100,14 +100,12 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: ../src/kernel/cpu/up/_arch/aarch64/cpu/boot.rs.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html -#![feature(asm)] #![feature(global_asm)] #![no_main] #![no_std] diff --git a/02_runtime_init/src/runtime_init.rs b/02_runtime_init/src/runtime_init.rs index 068dde84..ee094686 100644 --- a/02_runtime_init/src/runtime_init.rs +++ b/02_runtime_init/src/runtime_init.rs @@ -30,7 +30,6 @@ unsafe fn zero_bss() { /// # Safety /// /// - Only a single core must be active and running this function. -#[no_mangle] pub unsafe fn runtime_init() -> ! { zero_bss(); diff --git a/03_hacky_hello_world/Cargo.lock b/03_hacky_hello_world/Cargo.lock index 3e9c3111..3b926a34 100644 --- a/03_hacky_hello_world/Cargo.lock +++ b/03_hacky_hello_world/Cargo.lock @@ -1,6 +1,34 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + +[[package]] +name = "cortex-a" +version = "5.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecefc30975eb87afc5a810d4b2305c0ec29e607ea97e51b2ecd80766e9268d28" +dependencies = [ + "register", +] + [[package]] name = "kernel" version = "0.1.0" +dependencies = [ + "cortex-a", +] +[[package]] +name = "register" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a247de29ab7cc8f5006cfe775c4a81c704f9914c5e2a79696862e643135433" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "tock-registers" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f521a79accce68c417c9c77ce22108056b626126da1932f7e2e9b5bbffee0cea" diff --git a/03_hacky_hello_world/Cargo.toml b/03_hacky_hello_world/Cargo.toml index a7580996..5deb7010 100644 --- a/03_hacky_hello_world/Cargo.toml +++ b/03_hacky_hello_world/Cargo.toml @@ -17,3 +17,8 @@ bsp_rpi4 = [] ##-------------------------------------------------------------------------------------------------- [dependencies] + +# Platform specific dependencies +[target.'cfg(target_arch = "aarch64")'.dependencies] +cortex-a = { version = "5.x.x" } + diff --git a/03_hacky_hello_world/README.md b/03_hacky_hello_world/README.md index 753aa048..a002a619 100644 --- a/03_hacky_hello_world/README.md +++ b/03_hacky_hello_world/README.md @@ -4,8 +4,8 @@ - Introducing global `print!()` macros to enable "printf debugging" at the earliest. - To keep tutorial length reasonable, printing functions for now "abuse" a QEMU property that lets - us use the RPi's `UART` without setting it up properly. -- Using the real hardware `UART` is enabled step-by-step in following tutorials. + us use the Raspberry's `UART` without setting it up properly. +- Using the real hardware `UART` is enabled step-by-step in following tutorials. ## Notable additions @@ -28,6 +28,15 @@ Kernel panic: Stopping here. ## Diff to previous ```diff +diff -uNr 02_runtime_init/Cargo.toml 03_hacky_hello_world/Cargo.toml +--- 02_runtime_init/Cargo.toml ++++ 03_hacky_hello_world/Cargo.toml +@@ -21,3 +21,4 @@ + # Platform specific dependencies + [target.'cfg(target_arch = "aarch64")'.dependencies] + cortex-a = { version = "5.x.x" } ++ + diff -uNr 02_runtime_init/Makefile 03_hacky_hello_world/Makefile --- 02_runtime_init/Makefile +++ 03_hacky_hello_world/Makefile @@ -105,11 +114,12 @@ diff -uNr 02_runtime_init/src/bsp/raspberrypi/console.rs 03_hacky_hello_world/sr diff -uNr 02_runtime_init/src/bsp/raspberrypi.rs 03_hacky_hello_world/src/bsp/raspberrypi.rs --- 02_runtime_init/src/bsp/raspberrypi.rs +++ 03_hacky_hello_world/src/bsp/raspberrypi.rs -@@ -4,4 +4,5 @@ +@@ -4,5 +4,6 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. +pub mod console; + pub mod cpu; pub mod memory; diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs @@ -139,19 +149,10 @@ diff -uNr 02_runtime_init/src/console.rs 03_hacky_hello_world/src/console.rs diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs --- 02_runtime_init/src/main.rs +++ 03_hacky_hello_world/src/main.rs -@@ -101,21 +101,25 @@ - //! # Boot flow - //! - //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. --//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.S`. -+//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. - //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +@@ -106,14 +106,18 @@ //! --//! [`cpu::boot::arch_boot::_start()`]: ../src/kernel/cpu/up/_arch/aarch64/cpu/boot.rs.html -+//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html - #![feature(asm)] +#![feature(format_args_nl)] #![feature(global_asm)] +#![feature(panic_info_message)] @@ -167,7 +168,7 @@ diff -uNr 02_runtime_init/src/main.rs 03_hacky_hello_world/src/main.rs mod runtime_init; /// Early init code. -@@ -124,5 +128,7 @@ +@@ -122,5 +126,7 @@ /// /// - Only a single core must be active and running this function. unsafe fn kernel_init() -> ! { diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu.rs b/03_hacky_hello_world/src/_arch/aarch64/cpu.rs index 4fd7e313..5f97ea41 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu.rs +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu.rs @@ -11,6 +11,8 @@ //! //! crate::cpu::arch_cpu +use cortex_a::asm; + //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- @@ -18,13 +20,7 @@ /// Pause execution on the core. #[inline(always)] pub fn wait_forever() -> ! { - unsafe { - loop { - #[rustfmt::skip] - asm!( - "wfe", - options(nomem, nostack, preserves_flags) - ); - } + loop { + asm::wfe() } } diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S deleted file mode 100644 index 449cf502..00000000 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -.section ".text._start" - -.global _start - -_start: - mrs x1, mpidr_el1 // Read Multiprocessor Affinity Register - and x1, x1, #3 // Clear all bits except [1:0], which hold core id - cbz x1, 2f // Jump to label 2 if we are core 0 -1: wfe // Wait for event - b 1b // In case an event happened, jump back to 1 -2: // If we are here, we are core0 - ldr x1, =_start // Load address of function "_start()" - mov sp, x1 // Set start of stack to before our code, aka first - // address before "_start()" - bl runtime_init // Jump to the "runtime_init()" kernel function - b 1b // We should never reach here. But just in case, - // park this core aswell diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs index c3c325d3..c85bb94b 100644 --- a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs @@ -11,5 +11,23 @@ //! //! crate::cpu::boot::arch_boot -// Assembly counterpart to this file. Includes function _start(). -global_asm!(include_str!("boot.S")); +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The Rust entry of the `kernel` binary. +/// +/// The function is called from the assembly `_start` function. +/// +/// # Safety +/// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +#[no_mangle] +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() +} diff --git a/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/03_hacky_hello_world/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/03_hacky_hello_world/src/bsp/raspberrypi.rs b/03_hacky_hello_world/src/bsp/raspberrypi.rs index ac9d53e9..fdb7551a 100644 --- a/03_hacky_hello_world/src/bsp/raspberrypi.rs +++ b/03_hacky_hello_world/src/bsp/raspberrypi.rs @@ -5,4 +5,5 @@ //! Top-level BSP file for the Raspberry Pi 3 and 4. pub mod console; +pub mod cpu; pub mod memory; diff --git a/04_zero_overhead_abstraction/src/cpu/smp.rs b/03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs similarity index 63% rename from 04_zero_overhead_abstraction/src/cpu/smp.rs rename to 03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs index 38230ce1..d09ff956 100644 --- a/04_zero_overhead_abstraction/src/cpu/smp.rs +++ b/03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs @@ -2,13 +2,13 @@ // // Copyright (c) 2018-2021 Andre Richter -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; +//! BSP Processor code. //-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports +// Public Definitions //-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; + +/// Used by `arch` code to find the early boot core. +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/03_hacky_hello_world/src/bsp/raspberrypi/link.ld b/03_hacky_hello_world/src/bsp/raspberrypi/link.ld index acb7fe28..3b73b3c7 100644 --- a/03_hacky_hello_world/src/bsp/raspberrypi/link.ld +++ b/03_hacky_hello_world/src/bsp/raspberrypi/link.ld @@ -17,6 +17,11 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table @@ -24,7 +29,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/03_hacky_hello_world/src/main.rs b/03_hacky_hello_world/src/main.rs index 0815d5e2..bc290ddd 100644 --- a/03_hacky_hello_world/src/main.rs +++ b/03_hacky_hello_world/src/main.rs @@ -100,14 +100,12 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html -#![feature(asm)] #![feature(format_args_nl)] #![feature(global_asm)] #![feature(panic_info_message)] diff --git a/03_hacky_hello_world/src/runtime_init.rs b/03_hacky_hello_world/src/runtime_init.rs index 068dde84..ee094686 100644 --- a/03_hacky_hello_world/src/runtime_init.rs +++ b/03_hacky_hello_world/src/runtime_init.rs @@ -30,7 +30,6 @@ unsafe fn zero_bss() { /// # Safety /// /// - Only a single core must be active and running this function. -#[no_mangle] pub unsafe fn runtime_init() -> ! { zero_bss(); diff --git a/04_zero_overhead_abstraction/.vscode/settings.json b/04_zero_overhead_abstraction/.vscode/settings.json deleted file mode 100644 index a0d6a920..00000000 --- a/04_zero_overhead_abstraction/.vscode/settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "editor.formatOnSave": true, - "editor.rulers": [100], - "rust-analyzer.checkOnSave.overrideCommand": ["make", "check"], - "rust-analyzer.cargo.target": "aarch64-unknown-none-softfloat", - "rust-analyzer.cargo.features": ["bsp_rpi3"] -} diff --git a/04_zero_overhead_abstraction/Cargo.lock b/04_zero_overhead_abstraction/Cargo.lock deleted file mode 100644 index 3b926a34..00000000 --- a/04_zero_overhead_abstraction/Cargo.lock +++ /dev/null @@ -1,34 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "cortex-a" -version = "5.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecefc30975eb87afc5a810d4b2305c0ec29e607ea97e51b2ecd80766e9268d28" -dependencies = [ - "register", -] - -[[package]] -name = "kernel" -version = "0.1.0" -dependencies = [ - "cortex-a", -] - -[[package]] -name = "register" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a247de29ab7cc8f5006cfe775c4a81c704f9914c5e2a79696862e643135433" -dependencies = [ - "tock-registers", -] - -[[package]] -name = "tock-registers" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f521a79accce68c417c9c77ce22108056b626126da1932f7e2e9b5bbffee0cea" diff --git a/04_zero_overhead_abstraction/Cargo.toml b/04_zero_overhead_abstraction/Cargo.toml deleted file mode 100644 index 5deb7010..00000000 --- a/04_zero_overhead_abstraction/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "kernel" -version = "0.1.0" -authors = ["Andre Richter "] -edition = "2018" - -[profile.release] -lto = true - -[features] -default = [] -bsp_rpi3 = [] -bsp_rpi4 = [] - -##-------------------------------------------------------------------------------------------------- -## Dependencies -##-------------------------------------------------------------------------------------------------- - -[dependencies] - -# Platform specific dependencies -[target.'cfg(target_arch = "aarch64")'.dependencies] -cortex-a = { version = "5.x.x" } - diff --git a/04_zero_overhead_abstraction/Makefile b/04_zero_overhead_abstraction/Makefile deleted file mode 100644 index 2ed82a5f..00000000 --- a/04_zero_overhead_abstraction/Makefile +++ /dev/null @@ -1,115 +0,0 @@ -## SPDX-License-Identifier: MIT OR Apache-2.0 -## -## Copyright (c) 2018-2021 Andre Richter - -include ../utils/color.mk.in - -# Default to the RPi3 -BSP ?= rpi3 - -# BSP-specific arguments -ifeq ($(BSP),rpi3) - TARGET = aarch64-unknown-none-softfloat - KERNEL_BIN = kernel8.img - QEMU_BINARY = qemu-system-aarch64 - QEMU_MACHINE_TYPE = raspi3 - QEMU_RELEASE_ARGS = -serial stdio -display none - OBJDUMP_BINARY = aarch64-none-elf-objdump - NM_BINARY = aarch64-none-elf-nm - READELF_BINARY = aarch64-none-elf-readelf - LINKER_FILE = src/bsp/raspberrypi/link.ld - RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -else ifeq ($(BSP),rpi4) - TARGET = aarch64-unknown-none-softfloat - KERNEL_BIN = kernel8.img - QEMU_BINARY = qemu-system-aarch64 - QEMU_MACHINE_TYPE = - QEMU_RELEASE_ARGS = -serial stdio -display none - OBJDUMP_BINARY = aarch64-none-elf-objdump - NM_BINARY = aarch64-none-elf-nm - READELF_BINARY = aarch64-none-elf-readelf - LINKER_FILE = src/bsp/raspberrypi/link.ld - RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -endif - -# Export for build.rs -export LINKER_FILE - -QEMU_MISSING_STRING = "This board is not yet supported for QEMU." - -RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) -RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs - -FEATURES = --features bsp_$(BSP) -COMPILER_ARGS = --target=$(TARGET) \ - $(FEATURES) \ - --release - -RUSTC_CMD = cargo rustc $(COMPILER_ARGS) -DOC_CMD = cargo doc $(COMPILER_ARGS) -CLIPPY_CMD = cargo clippy $(COMPILER_ARGS) -CHECK_CMD = cargo check $(COMPILER_ARGS) -OBJCOPY_CMD = rust-objcopy \ - --strip-all \ - -O binary - -KERNEL_ELF = target/$(TARGET)/release/kernel - -DOCKER_IMAGE = rustembedded/osdev-utils -DOCKER_CMD = docker run --rm -v $(shell pwd):/work/tutorial -w /work/tutorial -DOCKER_CMD_INTERACT = $(DOCKER_CMD) -i -t - -DOCKER_QEMU = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE) -DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE) - -EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) - -.PHONY: all $(KERNEL_ELF) $(KERNEL_BIN) doc qemu clippy clean readelf objdump nm check - -all: $(KERNEL_BIN) - -$(KERNEL_ELF): - $(call colorecho, "\nCompiling kernel - $(BSP)") - @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(RUSTC_CMD) - -$(KERNEL_BIN): $(KERNEL_ELF) - @$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN) - -doc: - $(call colorecho, "\nGenerating docs") - @$(DOC_CMD) --document-private-items --open - -ifeq ($(QEMU_MACHINE_TYPE),) -qemu: - $(call colorecho, "\n$(QEMU_MISSING_STRING)") -else -qemu: $(KERNEL_BIN) - $(call colorecho, "\nLaunching QEMU") - @$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) -endif - -clippy: - @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) - -clean: - rm -rf target $(KERNEL_BIN) - -readelf: $(KERNEL_ELF) - $(call colorecho, "\nLaunching readelf") - @$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF) - -objdump: $(KERNEL_ELF) - $(call colorecho, "\nLaunching objdump") - @$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \ - --section .text \ - --section .rodata \ - --section .got \ - $(KERNEL_ELF) | rustfilt - -nm: $(KERNEL_ELF) - $(call colorecho, "\nLaunching nm") - @$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt - -# For rust-analyzer -check: - @RUSTFLAGS="$(RUSTFLAGS)" $(CHECK_CMD) --message-format=json diff --git a/04_zero_overhead_abstraction/README.md b/04_zero_overhead_abstraction/README.md deleted file mode 100644 index 69db5c72..00000000 --- a/04_zero_overhead_abstraction/README.md +++ /dev/null @@ -1,301 +0,0 @@ -# Tutorial 04 - Zero Overhead Abstraction - -## tl;dr - -- All hand-written assembly is replaced by Rust code from the [cortex-a] crate, which provides - zero-overhead abstractions and wraps the `unsafe` parts. - -[cortex-a]: https://github.com/rust-embedded/cortex-a - -## Diff to previous -```diff - -diff -uNr 03_hacky_hello_world/Cargo.toml 04_zero_overhead_abstraction/Cargo.toml ---- 03_hacky_hello_world/Cargo.toml -+++ 04_zero_overhead_abstraction/Cargo.toml -@@ -17,3 +17,8 @@ - ##-------------------------------------------------------------------------------------------------- - - [dependencies] -+ -+# Platform specific dependencies -+[target.'cfg(target_arch = "aarch64")'.dependencies] -+cortex-a = { version = "5.x.x" } -+ - -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs ---- 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.rs -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs -@@ -11,5 +11,31 @@ - //! - //! crate::cpu::boot::arch_boot - --// Assembly counterpart to this file. Includes function _start(). --global_asm!(include_str!("boot.S")); -+use crate::{bsp, cpu}; -+use cortex_a::regs::*; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// The entry of the `kernel` binary. -+/// -+/// The function must be named `_start`, because the linker is looking for this exact name. -+/// -+/// # Safety -+/// -+/// - Linker script must ensure to place this function where it is expected by the target machine. -+/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -+/// actually set (`SP.set()`). -+#[no_mangle] -+pub unsafe fn _start() -> ! { -+ use crate::runtime_init; -+ -+ if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { -+ SP.set(bsp::memory::boot_core_stack_end() as u64); -+ runtime_init::runtime_init() -+ } else { -+ // If not core0, infinitely wait for events. -+ cpu::wait_forever() -+ } -+} - -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.S ---- 03_hacky_hello_world/src/_arch/aarch64/cpu/boot.S -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.S -@@ -1,21 +0,0 @@ --// SPDX-License-Identifier: MIT OR Apache-2.0 --// --// Copyright (c) 2018-2021 Andre Richter -- --.section ".text._start" -- --.global _start -- --_start: -- mrs x1, mpidr_el1 // Read Multiprocessor Affinity Register -- and x1, x1, #3 // Clear all bits except [1:0], which hold core id -- cbz x1, 2f // Jump to label 2 if we are core 0 --1: wfe // Wait for event -- b 1b // In case an event happened, jump back to 1 --2: // If we are here, we are core0 -- ldr x1, =_start // Load address of function "_start()" -- mov sp, x1 // Set start of stack to before our code, aka first -- // address before "_start()" -- bl runtime_init // Jump to the "runtime_init()" kernel function -- b 1b // We should never reach here. But just in case, -- // park this core aswell - -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu/smp.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs ---- 03_hacky_hello_world/src/_arch/aarch64/cpu/smp.rs -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs -@@ -0,0 +1,29 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Architectural symmetric multiprocessing. -+//! -+//! # Orientation -+//! -+//! Since arch modules are imported into generic modules using the path attribute, the path of this -+//! file is: -+//! -+//! crate::cpu::smp::arch_smp -+ -+use cortex_a::regs::*; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Return the executing core's id. -+#[inline(always)] -+pub fn core_id() -> T -+where -+ T: From, -+{ -+ const CORE_MASK: u64 = 0b11; -+ -+ T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -+} - -diff -uNr 03_hacky_hello_world/src/_arch/aarch64/cpu.rs 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs ---- 03_hacky_hello_world/src/_arch/aarch64/cpu.rs -+++ 04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs -@@ -11,6 +11,8 @@ - //! - //! crate::cpu::arch_cpu - -+use cortex_a::asm; -+ - //-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- -@@ -18,13 +20,7 @@ - /// Pause execution on the core. - #[inline(always)] - pub fn wait_forever() -> ! { -- unsafe { -- loop { -- #[rustfmt::skip] -- asm!( -- "wfe", -- options(nomem, nostack, preserves_flags) -- ); -- } -+ loop { -+ asm::wfe() - } - } - -diff -uNr 03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs 04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs ---- 03_hacky_hello_world/src/bsp/raspberrypi/cpu.rs -+++ 04_zero_overhead_abstraction/src/bsp/raspberrypi/cpu.rs -@@ -0,0 +1,12 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! BSP Processor code. -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Definitions -+//-------------------------------------------------------------------------------------------------- -+ -+/// Used by `arch` code to find the early boot core. -+pub const BOOT_CORE_ID: u64 = 0; - -diff -uNr 03_hacky_hello_world/src/bsp/raspberrypi/link.ld 04_zero_overhead_abstraction/src/bsp/raspberrypi/link.ld ---- 03_hacky_hello_world/src/bsp/raspberrypi/link.ld -+++ 04_zero_overhead_abstraction/src/bsp/raspberrypi/link.ld -@@ -21,6 +21,7 @@ - /*********************************************************************************************** - * Code + RO Data + Global Offset Table - ***********************************************************************************************/ -+ __rx_start = .; - .text : - { - KEEP(*(.text._start)) - -diff -uNr 03_hacky_hello_world/src/bsp/raspberrypi/memory.rs 04_zero_overhead_abstraction/src/bsp/raspberrypi/memory.rs ---- 03_hacky_hello_world/src/bsp/raspberrypi/memory.rs -+++ 04_zero_overhead_abstraction/src/bsp/raspberrypi/memory.rs -@@ -12,14 +12,36 @@ - - // Symbols from the linker script. - extern "Rust" { -+ static __rx_start: UnsafeCell<()>; -+ - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; - } - - //-------------------------------------------------------------------------------------------------- -+// Private Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Start address of the Read+Execute (RX) range. -+/// -+/// # Safety -+/// -+/// - Value is provided by the linker script and must be trusted as-is. -+#[inline(always)] -+fn rx_start() -> usize { -+ unsafe { __rx_start.get() as usize } -+} -+ -+//-------------------------------------------------------------------------------------------------- - // Public Code - //-------------------------------------------------------------------------------------------------- - -+/// Exclusive end address of the boot core's stack. -+#[inline(always)] -+pub fn boot_core_stack_end() -> usize { -+ rx_start() -+} -+ - /// Return the inclusive range spanning the .bss section. - /// - /// # Safety - -diff -uNr 03_hacky_hello_world/src/bsp/raspberrypi.rs 04_zero_overhead_abstraction/src/bsp/raspberrypi.rs ---- 03_hacky_hello_world/src/bsp/raspberrypi.rs -+++ 04_zero_overhead_abstraction/src/bsp/raspberrypi.rs -@@ -5,4 +5,5 @@ - //! Top-level BSP file for the Raspberry Pi 3 and 4. - - pub mod console; -+pub mod cpu; - pub mod memory; - -diff -uNr 03_hacky_hello_world/src/cpu/smp.rs 04_zero_overhead_abstraction/src/cpu/smp.rs ---- 03_hacky_hello_world/src/cpu/smp.rs -+++ 04_zero_overhead_abstraction/src/cpu/smp.rs -@@ -0,0 +1,14 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Symmetric multiprocessing. -+ -+#[cfg(target_arch = "aarch64")] -+#[path = "../_arch/aarch64/cpu/smp.rs"] -+mod arch_smp; -+ -+//-------------------------------------------------------------------------------------------------- -+// Architectural Public Reexports -+//-------------------------------------------------------------------------------------------------- -+pub use arch_smp::core_id; - -diff -uNr 03_hacky_hello_world/src/cpu.rs 04_zero_overhead_abstraction/src/cpu.rs ---- 03_hacky_hello_world/src/cpu.rs -+++ 04_zero_overhead_abstraction/src/cpu.rs -@@ -10,6 +10,8 @@ - - mod boot; - -+pub mod smp; -+ - //-------------------------------------------------------------------------------------------------- - // Architectural Public Reexports - //-------------------------------------------------------------------------------------------------- - -diff -uNr 03_hacky_hello_world/src/main.rs 04_zero_overhead_abstraction/src/main.rs ---- 03_hacky_hello_world/src/main.rs -+++ 04_zero_overhead_abstraction/src/main.rs -@@ -107,9 +107,7 @@ - //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html - //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html - --#![feature(asm)] - #![feature(format_args_nl)] --#![feature(global_asm)] - #![feature(panic_info_message)] - #![no_main] - #![no_std] -@@ -128,7 +126,8 @@ - /// - /// - Only a single core must be active and running this function. - unsafe fn kernel_init() -> ! { -- println!("[0] Hello from Rust!"); -+ println!("[0] Hello from pure Rust!"); - -- panic!("Stopping here.") -+ println!("[1] Stopping here."); -+ cpu::wait_forever() - } - -diff -uNr 03_hacky_hello_world/src/runtime_init.rs 04_zero_overhead_abstraction/src/runtime_init.rs ---- 03_hacky_hello_world/src/runtime_init.rs -+++ 04_zero_overhead_abstraction/src/runtime_init.rs -@@ -30,7 +30,6 @@ - /// # Safety - /// - /// - Only a single core must be active and running this function. --#[no_mangle] - pub unsafe fn runtime_init() -> ! { - zero_bss(); - -``` diff --git a/04_zero_overhead_abstraction/build.rs b/04_zero_overhead_abstraction/build.rs deleted file mode 100644 index 3f0a2911..00000000 --- a/04_zero_overhead_abstraction/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -use std::env; - -fn main() { - let linker_file = env::var("LINKER_FILE").unwrap(); - - println!("cargo:rerun-if-changed={}", linker_file); - println!("cargo:rerun-if-changed=build.rs"); -} diff --git a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs deleted file mode 100644 index 5f97ea41..00000000 --- a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural processor code. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::arch_cpu - -use cortex_a::asm; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Pause execution on the core. -#[inline(always)] -pub fn wait_forever() -> ! { - loop { - asm::wfe() - } -} diff --git a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs deleted file mode 100644 index 549b5927..00000000 --- a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/boot.rs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2021 Andre Richter - -//! Architectural boot code. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::boot::arch_boot - -use crate::{bsp, cpu}; -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// The entry of the `kernel` binary. -/// -/// The function must be named `_start`, because the linker is looking for this exact name. -/// -/// # Safety -/// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). -#[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } -} diff --git a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs b/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/04_zero_overhead_abstraction/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/04_zero_overhead_abstraction/src/bsp.rs b/04_zero_overhead_abstraction/src/bsp.rs deleted file mode 100644 index 2b92251f..00000000 --- a/04_zero_overhead_abstraction/src/bsp.rs +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Conditional reexporting of Board Support Packages. - -#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] -mod raspberrypi; - -#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] -pub use raspberrypi::*; diff --git a/04_zero_overhead_abstraction/src/bsp/raspberrypi.rs b/04_zero_overhead_abstraction/src/bsp/raspberrypi.rs deleted file mode 100644 index fdb7551a..00000000 --- a/04_zero_overhead_abstraction/src/bsp/raspberrypi.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Top-level BSP file for the Raspberry Pi 3 and 4. - -pub mod console; -pub mod cpu; -pub mod memory; diff --git a/04_zero_overhead_abstraction/src/bsp/raspberrypi/console.rs b/04_zero_overhead_abstraction/src/bsp/raspberrypi/console.rs deleted file mode 100644 index fca7042e..00000000 --- a/04_zero_overhead_abstraction/src/bsp/raspberrypi/console.rs +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP console facilities. - -use crate::console; -use core::fmt; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -/// A mystical, magical device for generating QEMU output out of the void. -struct QEMUOutput; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are -/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`, -/// we get `write_fmt()` automatically. -/// -/// See [`src/print.rs`]. -/// -/// [`src/print.rs`]: ../../print/index.html -impl fmt::Write for QEMUOutput { - fn write_str(&mut self, s: &str) -> fmt::Result { - for c in s.chars() { - unsafe { - core::ptr::write_volatile(0x3F20_1000 as *mut u8, c as u8); - } - } - - Ok(()) - } -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return a reference to the console. -pub fn console() -> impl console::interface::Write { - QEMUOutput {} -} diff --git a/04_zero_overhead_abstraction/src/bsp/raspberrypi/link.ld b/04_zero_overhead_abstraction/src/bsp/raspberrypi/link.ld deleted file mode 100644 index 87e6a976..00000000 --- a/04_zero_overhead_abstraction/src/bsp/raspberrypi/link.ld +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: MIT OR Apache-2.0 - * - * Copyright (c) 2018-2021 Andre Richter - */ - -/* The address at which the the kernel binary will be loaded by the Raspberry's firmware */ -__rpi_load_addr = 0x80000; - -ENTRY(__rpi_load_addr) - -PHDRS -{ - segment_rx PT_LOAD FLAGS(5); /* 5 == RX */ - segment_rw PT_LOAD FLAGS(6); /* 6 == RW */ -} - -SECTIONS -{ - . = __rpi_load_addr; - - /*********************************************************************************************** - * Code + RO Data + Global Offset Table - ***********************************************************************************************/ - __rx_start = .; - .text : - { - KEEP(*(.text._start)) - *(.text*) - } :segment_rx - - .rodata : ALIGN(8) { *(.rodata*) } :segment_rx - .got : ALIGN(8) { *(.got) } :segment_rx - - /*********************************************************************************************** - * Data + BSS - ***********************************************************************************************/ - .data : { *(.data*) } :segment_rw - - /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) - { - __bss_start = .; - *(.bss*); - . = ALIGN(8); - - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; - } :NONE -} diff --git a/04_zero_overhead_abstraction/src/bsp/raspberrypi/memory.rs b/04_zero_overhead_abstraction/src/bsp/raspberrypi/memory.rs deleted file mode 100644 index bffe8eab..00000000 --- a/04_zero_overhead_abstraction/src/bsp/raspberrypi/memory.rs +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! BSP Memory Management. - -use core::{cell::UnsafeCell, ops::RangeInclusive}; - -//-------------------------------------------------------------------------------------------------- -// Private Definitions -//-------------------------------------------------------------------------------------------------- - -// Symbols from the linker script. -extern "Rust" { - static __rx_start: UnsafeCell<()>; - - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -} - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - -/// Return the inclusive range spanning the .bss section. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - } - assert!(!range.is_empty()); - - range -} diff --git a/04_zero_overhead_abstraction/src/console.rs b/04_zero_overhead_abstraction/src/console.rs deleted file mode 100644 index 27b79f7d..00000000 --- a/04_zero_overhead_abstraction/src/console.rs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! System console. - -//-------------------------------------------------------------------------------------------------- -// Public Definitions -//-------------------------------------------------------------------------------------------------- - -/// Console interfaces. -pub mod interface { - /// Console write functions. - /// - /// `core::fmt::Write` is exactly what we need for now. Re-export it here because - /// implementing `console::Write` gives a better hint to the reader about the - /// intention. - pub use core::fmt::Write; -} diff --git a/04_zero_overhead_abstraction/src/cpu.rs b/04_zero_overhead_abstraction/src/cpu.rs deleted file mode 100644 index 6f326b32..00000000 --- a/04_zero_overhead_abstraction/src/cpu.rs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2020-2021 Andre Richter - -//! Processor code. - -#[cfg(target_arch = "aarch64")] -#[path = "_arch/aarch64/cpu.rs"] -mod arch_cpu; - -mod boot; - -pub mod smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_cpu::wait_forever; diff --git a/04_zero_overhead_abstraction/src/cpu/boot.rs b/04_zero_overhead_abstraction/src/cpu/boot.rs deleted file mode 100644 index 1dc5c180..00000000 --- a/04_zero_overhead_abstraction/src/cpu/boot.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2021 Andre Richter - -//! Boot code. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/boot.rs"] -mod arch_boot; diff --git a/04_zero_overhead_abstraction/src/main.rs b/04_zero_overhead_abstraction/src/main.rs deleted file mode 100644 index 9dfc7d00..00000000 --- a/04_zero_overhead_abstraction/src/main.rs +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -// Rust embedded logo for `make doc`. -#![doc(html_logo_url = "https://git.io/JeGIp")] - -//! The `kernel` binary. -//! -//! # Code organization and architecture -//! -//! The code is divided into different *modules*, each representing a typical **subsystem** of the -//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example, -//! `src/memory.rs` contains code that is concerned with all things memory management. -//! -//! ## Visibility of processor architecture code -//! -//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target -//! processor architecture. For each supported processor architecture, there exists a subfolder in -//! `src/_arch`, for example, `src/_arch/aarch64`. -//! -//! The architecture folders mirror the subsystem modules laid out in `src`. For example, -//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go -//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in -//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic -//! module's name prefixed with `arch_`. -//! -//! For example, this is the top of `src/memory/mmu.rs`: -//! -//! ``` -//! #[cfg(target_arch = "aarch64")] -//! #[path = "../_arch/aarch64/memory/mmu.rs"] -//! mod arch_mmu; -//! ``` -//! -//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. -//! This way, each architecture specific module can provide its implementation of an item, while the -//! caller must not be concerned which architecture has been conditionally compiled. -//! -//! ## BSP code -//! -//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains -//! target board specific definitions and functions. These are things such as the board's memory map -//! or instances of drivers for devices that are featured on the respective board. -//! -//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the -//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is -//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. -//! -//! ## Kernel interfaces -//! -//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target -//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of -//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel` -//! code to play nicely with any of the two without much hassle. -//! -//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`, -//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined -//! in the respective subsystem module and help to enforce the idiom of *program to an interface, -//! not an implementation*. For example, there will be a common IRQ handling interface which the two -//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the -//! interface to the rest of the `kernel`. -//! -//! ``` -//! +-------------------+ -//! | Interface (Trait) | -//! | | -//! +--+-------------+--+ -//! ^ ^ -//! | | -//! | | -//! +----------+--+ +--+----------+ -//! | kernel code | | bsp code | -//! | | | arch code | -//! +-------------+ +-------------+ -//! ``` -//! -//! # Summary -//! -//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical -//! locations. Here is an example for the **memory** subsystem: -//! -//! - `src/memory.rs` and `src/memory/**/*` -//! - Common code that is agnostic of target processor architecture and `BSP` characteristics. -//! - Example: A function to zero a chunk of memory. -//! - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code. -//! - Example: An `MMU` interface that defines `MMU` function prototypes. -//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*` -//! - `BSP` specific code. -//! - Example: The board's memory map (physical addresses of DRAM and MMIO devices). -//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*` -//! - Processor architecture specific code. -//! - Example: Implementation of the `MMU` interface for the `__arch_name__` processor -//! architecture. -//! -//! From a namespace perspective, **memory** subsystem code lives in: -//! -//! - `crate::memory::*` -//! - `crate::bsp::memory::*` -//! -//! # Boot flow -//! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html - -#![feature(format_args_nl)] -#![feature(panic_info_message)] -#![no_main] -#![no_std] - -mod bsp; -mod console; -mod cpu; -mod memory; -mod panic_wait; -mod print; -mod runtime_init; - -/// Early init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -unsafe fn kernel_init() -> ! { - println!("[0] Hello from pure Rust!"); - - println!("[1] Stopping here."); - cpu::wait_forever() -} diff --git a/04_zero_overhead_abstraction/src/memory.rs b/04_zero_overhead_abstraction/src/memory.rs deleted file mode 100644 index 1f79e0c9..00000000 --- a/04_zero_overhead_abstraction/src/memory.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Memory Management. - -use core::ops::RangeInclusive; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out an inclusive memory range. -/// -/// # Safety -/// -/// - `range.start` and `range.end` must be valid. -/// - `range.start` and `range.end` must be `T` aligned. -pub unsafe fn zero_volatile(range: RangeInclusive<*mut T>) -where - T: From, -{ - let mut ptr = *range.start(); - let end_inclusive = *range.end(); - - while ptr <= end_inclusive { - core::ptr::write_volatile(ptr, T::from(0)); - ptr = ptr.offset(1); - } -} diff --git a/04_zero_overhead_abstraction/src/panic_wait.rs b/04_zero_overhead_abstraction/src/panic_wait.rs deleted file mode 100644 index 255eedee..00000000 --- a/04_zero_overhead_abstraction/src/panic_wait.rs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! A panic handler that infinitely waits. - -use crate::{cpu, println}; -use core::panic::PanicInfo; - -#[panic_handler] -fn panic(info: &PanicInfo) -> ! { - if let Some(args) = info.message() { - println!("\nKernel panic: {}", args); - } else { - println!("\nKernel panic!"); - } - - cpu::wait_forever() -} diff --git a/04_zero_overhead_abstraction/src/print.rs b/04_zero_overhead_abstraction/src/print.rs deleted file mode 100644 index 8ea5db24..00000000 --- a/04_zero_overhead_abstraction/src/print.rs +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Printing. - -use crate::{bsp, console}; -use core::fmt; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -#[doc(hidden)] -pub fn _print(args: fmt::Arguments) { - use console::interface::Write; - - bsp::console::console().write_fmt(args).unwrap(); -} - -/// Prints without a newline. -/// -/// Carbon copy from -#[macro_export] -macro_rules! print { - ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*))); -} - -/// Prints with a newline. -/// -/// Carbon copy from -#[macro_export] -macro_rules! println { - () => ($crate::print!("\n")); - ($($arg:tt)*) => ({ - $crate::print::_print(format_args_nl!($($arg)*)); - }) -} diff --git a/04_zero_overhead_abstraction/src/runtime_init.rs b/04_zero_overhead_abstraction/src/runtime_init.rs deleted file mode 100644 index ee094686..00000000 --- a/04_zero_overhead_abstraction/src/runtime_init.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Rust runtime initialization code. - -use crate::{bsp, memory}; - -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Zero out the .bss section. -/// -/// # Safety -/// -/// - Must only be called pre `kernel_init()`. -#[inline(always)] -unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::bss_range_inclusive()); -} - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel -/// init code. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -pub unsafe fn runtime_init() -> ! { - zero_bss(); - - crate::kernel_init() -} diff --git a/05_safe_globals/src/_arch/aarch64/cpu/boot.rs b/05_safe_globals/src/_arch/aarch64/cpu/boot.rs index 549b5927..c85bb94b 100644 --- a/05_safe_globals/src/_arch/aarch64/cpu/boot.rs +++ b/05_safe_globals/src/_arch/aarch64/cpu/boot.rs @@ -11,31 +11,23 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; -use cortex_a::regs::*; +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() } diff --git a/05_safe_globals/src/_arch/aarch64/cpu/boot.s b/05_safe_globals/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/05_safe_globals/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/05_safe_globals/src/_arch/aarch64/cpu/smp.rs b/05_safe_globals/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/05_safe_globals/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/05_safe_globals/src/bsp/raspberrypi/cpu.rs b/05_safe_globals/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/05_safe_globals/src/bsp/raspberrypi/cpu.rs +++ b/05_safe_globals/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/05_safe_globals/src/bsp/raspberrypi/link.ld b/05_safe_globals/src/bsp/raspberrypi/link.ld index 87e6a976..3b73b3c7 100644 --- a/05_safe_globals/src/bsp/raspberrypi/link.ld +++ b/05_safe_globals/src/bsp/raspberrypi/link.ld @@ -17,15 +17,28 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __rx_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/05_safe_globals/src/bsp/raspberrypi/memory.rs b/05_safe_globals/src/bsp/raspberrypi/memory.rs index bffe8eab..5c6b1ea5 100644 --- a/05_safe_globals/src/bsp/raspberrypi/memory.rs +++ b/05_safe_globals/src/bsp/raspberrypi/memory.rs @@ -12,36 +12,14 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; } -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/05_safe_globals/src/cpu.rs b/05_safe_globals/src/cpu.rs index 6f326b32..d8f78082 100644 --- a/05_safe_globals/src/cpu.rs +++ b/05_safe_globals/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/05_safe_globals/src/cpu/smp.rs b/05_safe_globals/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/05_safe_globals/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/05_safe_globals/src/main.rs b/05_safe_globals/src/main.rs index 83955815..579f90dd 100644 --- a/05_safe_globals/src/main.rs +++ b/05_safe_globals/src/main.rs @@ -100,14 +100,14 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/06_drivers_gpio_uart/README.md b/06_drivers_gpio_uart/README.md index 7dea308f..bd7c0208 100644 --- a/06_drivers_gpio_uart/README.md +++ b/06_drivers_gpio_uart/README.md @@ -18,7 +18,8 @@ - A `driver::interface::DeviceDriver` trait is added for abstracting `BSP` driver implementations from kernel code. - Drivers are stored in `src/bsp/device_driver`, and can be reused between `BSP`s. - - We introduce the `GPIO` driver, which pinmuxes the RPi's PL011 UART. + - We introduce the `GPIO` driver, which pinmuxes (that is, routing signals from inside the `SoC` + to actual HW pins) the RPi's PL011 UART. - Note how this driver differentiates between **RPi 3** and **RPi4**. Their HW is different, so we have to account for it in SW. - Most importantly, the `PL011Uart` driver: It implements the `console::interface::*` traits and @@ -1110,7 +1111,7 @@ diff -uNr 05_safe_globals/src/bsp/raspberrypi/driver.rs 06_drivers_gpio_uart/src diff -uNr 05_safe_globals/src/bsp/raspberrypi/memory.rs 06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs --- 05_safe_globals/src/bsp/raspberrypi/memory.rs +++ 06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs -@@ -19,6 +19,38 @@ +@@ -17,6 +17,38 @@ } //-------------------------------------------------------------------------------------------------- @@ -1146,7 +1147,7 @@ diff -uNr 05_safe_globals/src/bsp/raspberrypi/memory.rs 06_drivers_gpio_uart/src +} + +//-------------------------------------------------------------------------------------------------- - // Private Code + // Public Code //-------------------------------------------------------------------------------------------------- @@ -1249,7 +1250,7 @@ diff -uNr 05_safe_globals/src/console.rs 06_drivers_gpio_uart/src/console.rs diff -uNr 05_safe_globals/src/cpu.rs 06_drivers_gpio_uart/src/cpu.rs --- 05_safe_globals/src/cpu.rs +++ 06_drivers_gpio_uart/src/cpu.rs -@@ -15,4 +15,7 @@ +@@ -13,4 +13,7 @@ //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- @@ -1311,15 +1312,15 @@ diff -uNr 05_safe_globals/src/driver.rs 06_drivers_gpio_uart/src/driver.rs diff -uNr 05_safe_globals/src/main.rs 06_drivers_gpio_uart/src/main.rs --- 05_safe_globals/src/main.rs +++ 06_drivers_gpio_uart/src/main.rs -@@ -107,6 +107,8 @@ - //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html +@@ -106,6 +106,8 @@ + //! //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html +#![allow(clippy::clippy::upper_case_acronyms)] +#![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] + #![feature(global_asm)] #![feature(panic_info_message)] - #![feature(trait_alias)] @@ -116,6 +118,7 @@ mod bsp; mod console; diff --git a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs index 549b5927..c85bb94b 100644 --- a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs +++ b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs @@ -11,31 +11,23 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; -use cortex_a::regs::*; +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() } diff --git a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs b/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/06_drivers_gpio_uart/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/06_drivers_gpio_uart/src/bsp/raspberrypi/cpu.rs b/06_drivers_gpio_uart/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/06_drivers_gpio_uart/src/bsp/raspberrypi/cpu.rs +++ b/06_drivers_gpio_uart/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/06_drivers_gpio_uart/src/bsp/raspberrypi/link.ld b/06_drivers_gpio_uart/src/bsp/raspberrypi/link.ld index 87e6a976..3b73b3c7 100644 --- a/06_drivers_gpio_uart/src/bsp/raspberrypi/link.ld +++ b/06_drivers_gpio_uart/src/bsp/raspberrypi/link.ld @@ -17,15 +17,28 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __rx_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs b/06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs index 56a3306e..c6d65e36 100644 --- a/06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs +++ b/06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs @@ -12,8 +12,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; } @@ -50,30 +48,10 @@ pub(super) mod map { } } -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/06_drivers_gpio_uart/src/cpu.rs b/06_drivers_gpio_uart/src/cpu.rs index f0df4791..c5e956ea 100644 --- a/06_drivers_gpio_uart/src/cpu.rs +++ b/06_drivers_gpio_uart/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/06_drivers_gpio_uart/src/cpu/smp.rs b/06_drivers_gpio_uart/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/06_drivers_gpio_uart/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/06_drivers_gpio_uart/src/main.rs b/06_drivers_gpio_uart/src/main.rs index 627b5b59..0f3c493d 100644 --- a/06_drivers_gpio_uart/src/main.rs +++ b/06_drivers_gpio_uart/src/main.rs @@ -100,16 +100,16 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/07_uart_chainloader/Makefile b/07_uart_chainloader/Makefile index 4fc73f55..ce9d5a64 100644 --- a/07_uart_chainloader/Makefile +++ b/07_uart_chainloader/Makefile @@ -24,7 +24,7 @@ ifeq ($(BSP),rpi3) NM_BINARY = aarch64-none-elf-nm READELF_BINARY = aarch64-none-elf-readelf LINKER_FILE = src/bsp/raspberrypi/link.ld - RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C relocation-model=pic + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi3.img else ifeq ($(BSP),rpi4) TARGET = aarch64-unknown-none-softfloat @@ -36,7 +36,7 @@ else ifeq ($(BSP),rpi4) NM_BINARY = aarch64-none-elf-nm READELF_BINARY = aarch64-none-elf-readelf LINKER_FILE = src/bsp/raspberrypi/link.ld - RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -C relocation-model=pic + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi4.img endif diff --git a/07_uart_chainloader/README.md b/07_uart_chainloader/README.md index 923615b5..a60fe905 100644 --- a/07_uart_chainloader/README.md +++ b/07_uart_chainloader/README.md @@ -3,22 +3,34 @@ ## tl;dr - Running from an SD card was a nice experience, but it would be extremely tedious to do it for - every new binary. Let's write a [chainloader] using [position independent code]. + every new binary. So let's write a [chainloader]. - This will be the last binary you need to put on the SD card. Each following tutorial will provide a `chainboot` target in the `Makefile` that lets you conveniently load the kernel over `UART`. [chainloader]: https://en.wikipedia.org/wiki/Chain_loading -[position independent code]: https://en.wikipedia.org/wiki/Position-independent_code + ## Note -Please note that there is a lot of stuff going on in this tutorial that is very hard to grasp by -only looking at the source code changes. +Please note that there is stuff going on in this tutorial that is very hard to grasp by only looking +at the source code changes. + +The gist of it is that in `boot.s`, we are writing a piece of [position independent code] which +automatically determines where the firmware has loaded the binary (`0x8_0000`), and where it was +linked to (`0x200_0000`, see `link.ld`). The binary then copies itself from loaded to linked address +(aka "relocating" itself), and then jumps to the relocated version of `_start_rust()`. + +Since the chainloader has put itself "out of the way" now, it can now receive another kernel binary +from the `UART` and copy it to the standard load address of the RPi firmware at `0x8_0000`. Finally, +it jumps to `0x8_0000` and the newly loaded binary transparently executes as if it had been loaded +from SD card all along. Please bear with me until I find the time to write it all down here elaborately. For the time being, please see this tutorial as an enabler for a convenience feature that allows booting the following tutorials in a quick manner. +[position independent code]: https://en.wikipedia.org/wiki/Position-independent_code + ## Install and test it Our chainloader is called `MiniLoad` and is inspired by [raspbootin]. @@ -71,34 +83,36 @@ Minipush 1.0 [3] Echoing input now ``` -In this tutorial, a version of the kernel from the previous tutorial is loaded -for demo purposes. In subsequent tuts, it will be the working directory's -kernel. +In this tutorial, a version of the kernel from the previous tutorial is loaded for demo purposes. In +subsequent tutorials, it will be the working directory's kernel. ## Test it -The `Makefile` in this tutorial has an additional target, `qemuasm`, that lets -you nicely observe the jump from the loaded address (`0x80_XXX`) to the -relocated code at (`0x0200_0XXX`): +The `Makefile` in this tutorial has an additional target, `qemuasm`, that lets you nicely observe +how the kernel, after relocating itself, jumps the load address region (`0x80_XXX`) to the relocated +code at (`0x0200_0XXX`): ```console $ make qemuasm [...] +N: +0x00080030: 58000140 ldr x0, #0x80058 +0x00080034: 9100001f mov sp, x0 +0x00080038: 58000141 ldr x1, #0x80060 +0x0008003c: d61f0020 br x1 + +---------------- IN: -0x0008098c: b0000008 adrp x8, #0x81000 -0x00080990: b0000000 adrp x0, #0x81000 -0x00080994: 912a8000 add x0, x0, #0xaa0 -0x00080998: f9471908 ldr x8, [x8, #0xe30] -0x0008099c: d63f0100 blr x8 +0x02000070: 9400044c bl #0x20011a0 ---------------- IN: -0x02000b1c: b0000008 adrp x8, #0x2001000 -0x02000b20: b0000009 adrp x9, #0x2001000 -0x02000b24: f9475d08 ldr x8, [x8, #0xeb8] -0x02000b28: f9476129 ldr x9, [x9, #0xec0] -0x02000b2c: eb08013f cmp x9, x8 -0x02000b30: 540000c2 b.hs #0x2000b48 +0x020011a0: 90000008 adrp x8, #0x2001000 +0x020011a4: 90000009 adrp x9, #0x2001000 +0x020011a8: f9446508 ldr x8, [x8, #0x8c8] +0x020011ac: f9446929 ldr x9, [x9, #0x8d0] +0x020011b0: eb08013f cmp x9, x8 +0x020011b4: 54000109 b.ls #0x20011d4 [...] ``` @@ -110,22 +124,18 @@ Binary files 06_drivers_gpio_uart/demo_payload_rpi4.img and 07_uart_chainloader/ diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile --- 06_drivers_gpio_uart/Makefile +++ 07_uart_chainloader/Makefile -@@ -24,7 +24,8 @@ - NM_BINARY = aarch64-none-elf-nm +@@ -25,6 +25,7 @@ READELF_BINARY = aarch64-none-elf-readelf LINKER_FILE = src/bsp/raspberrypi/link.ld -- RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -+ RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C relocation-model=pic + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 + CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi3.img else ifeq ($(BSP),rpi4) TARGET = aarch64-unknown-none-softfloat KERNEL_BIN = kernel8.img -@@ -35,7 +36,8 @@ - NM_BINARY = aarch64-none-elf-nm +@@ -36,6 +37,7 @@ READELF_BINARY = aarch64-none-elf-readelf LINKER_FILE = src/bsp/raspberrypi/link.ld -- RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -+ RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -C relocation-model=pic + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 + CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi4.img endif @@ -166,47 +176,47 @@ diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile clippy: @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) -diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs ---- 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.rs -+++ 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs -@@ -29,11 +29,11 @@ - /// actually set (`SP.set()`). - #[no_mangle] - pub unsafe fn _start() -> ! { -- use crate::runtime_init; -+ use crate::relocate; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); -- runtime_init::runtime_init() -+ relocate::relocate_self() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - -diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs 07_uart_chainloader/src/_arch/aarch64/cpu.rs ---- 06_drivers_gpio_uart/src/_arch/aarch64/cpu.rs -+++ 07_uart_chainloader/src/_arch/aarch64/cpu.rs -@@ -35,3 +35,19 @@ - asm::wfe() - } - } +diff -uNr 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s 07_uart_chainloader/src/_arch/aarch64/cpu/boot.s +--- 06_drivers_gpio_uart/src/_arch/aarch64/cpu/boot.s ++++ 07_uart_chainloader/src/_arch/aarch64/cpu/boot.s +@@ -22,20 +22,31 @@ + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 +- b.ne 1f ++ b.ne 2f + +- // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. ++ // If execution reaches here, it is the boot core. + -+/// Branch to a raw integer value. -+/// -+/// # Safety -+/// -+/// - This is highly unsafe. Use with care. -+#[inline(always)] -+pub unsafe fn branch_to_raw_addr(addr: usize) -> ! { -+ asm!( -+ "blr {destination:x}", -+ destination = in(reg) addr, -+ options(nomem, nostack) -+ ); ++ // Next, relocate the binary. ++ adr x0, __binary_nonzero_start // The address the binary got loaded to. ++ ldr x1, =__binary_nonzero_start // The address the binary was linked to. ++ ldr x2, =__binary_nonzero_end_exclusive + -+ core::intrinsics::unreachable() -+} ++1: ldr x3, [x0], #8 ++ str x3, [x1], #8 ++ cmp x1, x2 ++ b.lo 1b + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + +- // Jump to Rust code. +- b _start_rust ++ // Jump to the relocated Rust code. ++ ldr x1, =_start_rust ++ br x1 + + // Infinitely wait for events (aka "park the core"). +-1: wfe +- b 1b ++2: wfe ++ b 2b + + .size _start, . - _start + .type _start, function diff -uNr 06_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 07_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs --- 06_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -268,58 +278,46 @@ diff -uNr 06_drivers_gpio_uart/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 0 diff -uNr 06_drivers_gpio_uart/src/bsp/raspberrypi/link.ld 07_uart_chainloader/src/bsp/raspberrypi/link.ld --- 06_drivers_gpio_uart/src/bsp/raspberrypi/link.ld +++ 07_uart_chainloader/src/bsp/raspberrypi/link.ld -@@ -16,12 +16,13 @@ +@@ -16,7 +16,8 @@ SECTIONS { - . = __rpi_load_addr; + /* Set the link address to 32 MiB */ + . = 0x2000000; - + /* ^ */ + /* | stack */ + /* | growth */ +@@ -26,6 +27,7 @@ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ -- __rx_start = .; -+ __binary_start = .; ++ __binary_nonzero_start = .; .text : { KEEP(*(.text._start)) -@@ -46,4 +47,10 @@ - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; - } :NONE -+ +@@ -49,8 +51,12 @@ + ***********************************************************************************************/ + .data : { *(.data*) } :segment_rw + + /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */ + . = ALIGN(8); -+ __binary_end_inclusive = . - 8; ++ __binary_nonzero_end_exclusive = .; + -+ __runtime_init_reloc = runtime_init; - } + /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ +- .bss : ALIGN(8) ++ .bss : + { + __bss_start = .; + *(.bss*); diff -uNr 06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 07_uart_chainloader/src/bsp/raspberrypi/memory.rs --- 06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs +++ 07_uart_chainloader/src/bsp/raspberrypi/memory.rs -@@ -12,10 +12,12 @@ - - // Symbols from the linker script. - extern "Rust" { -- static __rx_start: UnsafeCell<()>; -- -+ static __binary_start: UnsafeCell; - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -+ static __binary_end_inclusive: UnsafeCell; -+ -+ static __runtime_init_reloc: UnsafeCell; - } - - //-------------------------------------------------------------------------------------------------- -@@ -25,9 +27,12 @@ +@@ -23,9 +23,10 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { -+ pub const BOOT_CORE_STACK_END: usize = 0x8_0000; -+ + pub const BOARD_DEFAULT_LOAD_ADDRESS: usize = 0x8_0000; - pub const GPIO_OFFSET: usize = 0x0020_0000; @@ -329,111 +327,34 @@ diff -uNr 06_drivers_gpio_uart/src/bsp/raspberrypi/memory.rs 07_uart_chainloader /// Physical devices. #[cfg(feature = "bsp_rpi3")] -@@ -51,36 +56,44 @@ - } - - //-------------------------------------------------------------------------------------------------- --// Private Code -+// Public Code +@@ -52,7 +53,13 @@ + // Public Code //-------------------------------------------------------------------------------------------------- --/// Start address of the Read+Execute (RX) range. -+/// Exclusive end address of the boot core's stack. -+#[inline(always)] -+pub fn boot_core_stack_end() -> usize { -+ map::BOOT_CORE_STACK_END -+} -+ +-/// Return the inclusive range spanning the .bss section. +/// The address on which the Raspberry firmware loads every binary by default. +#[inline(always)] +pub fn board_default_load_addr() -> *const u64 { + map::BOARD_DEFAULT_LOAD_ADDRESS as _ +} + -+/// Return the inclusive range spanning the relocated kernel binary. - /// - /// # Safety - /// --/// - Value is provided by the linker script and must be trusted as-is. --#[inline(always)] --fn rx_start() -> usize { -- unsafe { __rx_start.get() as usize } -+/// - Values are provided by the linker script and must be trusted as-is. -+/// - The linker-provided addresses must be u64 aligned. -+pub fn relocated_binary_range_inclusive() -> RangeInclusive<*mut u64> { -+ unsafe { RangeInclusive::new(__binary_start.get(), __binary_end_inclusive.get()) } - } - --//-------------------------------------------------------------------------------------------------- --// Public Code --//-------------------------------------------------------------------------------------------------- -- --/// Exclusive end address of the boot core's stack. -+/// The relocated address of function `runtime_init()`. - #[inline(always)] --pub fn boot_core_stack_end() -> usize { -- rx_start() -+pub fn relocated_runtime_init_addr() -> *const u64 { -+ unsafe { __runtime_init_reloc.get() as _ } - } - --/// Return the inclusive range spanning the .bss section. +/// Return the inclusive range spanning the relocated .bss section. /// /// # Safety /// - /// - Values are provided by the linker script and must be trusted as-is. - /// - The linker-provided addresses must be u64 aligned. --pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { -+pub fn relocated_bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); - -diff -uNr 06_drivers_gpio_uart/src/cpu.rs 07_uart_chainloader/src/cpu.rs ---- 06_drivers_gpio_uart/src/cpu.rs -+++ 07_uart_chainloader/src/cpu.rs -@@ -15,7 +15,7 @@ - //-------------------------------------------------------------------------------------------------- - // Architectural Public Reexports - //-------------------------------------------------------------------------------------------------- --pub use arch_cpu::{nop, wait_forever}; -+pub use arch_cpu::{branch_to_raw_addr, nop, wait_forever}; - - #[cfg(feature = "bsp_rpi3")] - pub use arch_cpu::spin_for_cycles; diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs --- 06_drivers_gpio_uart/src/main.rs +++ 07_uart_chainloader/src/main.rs -@@ -102,13 +102,17 @@ - //! - //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. - //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. --//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -+//! 2. Once finished with architectural setup, the arch code calls [`relocate::relocate_self()`]. -+//! 3. Finally, [`runtime_init::runtime_init()`] is called. - //! - //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html -+//! [`relocate::relocate_self()`]: relocate/fn.relocate_self.html +@@ -107,6 +107,7 @@ //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] +#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] -+#![feature(core_intrinsics)] #![feature(format_args_nl)] - #![feature(panic_info_message)] - #![feature(trait_alias)] -@@ -122,6 +126,7 @@ - mod memory; - mod panic_wait; - mod print; -+mod relocate; - mod runtime_init; - mod synchronization; - -@@ -150,29 +155,49 @@ + #![feature(global_asm)] +@@ -150,29 +151,49 @@ fn kernel_main() -> ! { use bsp::console::console; use console::interface::All; @@ -504,81 +425,6 @@ diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs + kernel() } -diff -uNr 06_drivers_gpio_uart/src/relocate.rs 07_uart_chainloader/src/relocate.rs ---- 06_drivers_gpio_uart/src/relocate.rs -+++ 07_uart_chainloader/src/relocate.rs -@@ -0,0 +1,49 @@ -+// SPDX-License-Identifier: MIT OR Apache-2.0 -+// -+// Copyright (c) 2018-2021 Andre Richter -+ -+//! Relocation code. -+ -+use crate::{bsp, cpu}; -+ -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Relocates the own binary from `bsp::memory::board_default_load_addr()` to the `__binary_start` -+/// address from the linker script. -+/// -+/// # Safety -+/// -+/// - Only a single core must be active and running this function. -+/// - Function must not use the `bss` section. -+#[inline(never)] -+pub unsafe fn relocate_self() -> ! { -+ let range = bsp::memory::relocated_binary_range_inclusive(); -+ let mut relocated_binary_start_addr = *range.start(); -+ let relocated_binary_end_addr_inclusive = *range.end(); -+ -+ // The address of where the previous firmware loaded us. -+ let mut current_binary_start_addr = bsp::memory::board_default_load_addr(); -+ -+ // Copy the whole binary. -+ while relocated_binary_start_addr <= relocated_binary_end_addr_inclusive { -+ core::ptr::write_volatile( -+ relocated_binary_start_addr, -+ core::ptr::read_volatile(current_binary_start_addr), -+ ); -+ relocated_binary_start_addr = relocated_binary_start_addr.offset(1); -+ current_binary_start_addr = current_binary_start_addr.offset(1); -+ } -+ -+ // The following function calls realize an "absolute jump" to `runtime_init::runtime_init()` by -+ // forcing an indirection through the global offset table (GOT), so that execution continues -+ // from the relocated binary. -+ // -+ // Without the indirection through the assembly, the address of `runtime_init()` would be -+ // calculated as a relative offset from the current program counter, since we are compiling as -+ // `position independent code`. This would cause us to keep executing from the address to which -+ // the firmware loaded us, instead of the relocated position. -+ let relocated_runtime_init_addr = bsp::memory::relocated_runtime_init_addr() as usize; -+ cpu::branch_to_raw_addr(relocated_runtime_init_addr) -+} - -diff -uNr 06_drivers_gpio_uart/src/runtime_init.rs 07_uart_chainloader/src/runtime_init.rs ---- 06_drivers_gpio_uart/src/runtime_init.rs -+++ 07_uart_chainloader/src/runtime_init.rs -@@ -17,7 +17,7 @@ - /// - Must only be called pre `kernel_init()`. - #[inline(always)] - unsafe fn zero_bss() { -- memory::zero_volatile(bsp::memory::bss_range_inclusive()); -+ memory::zero_volatile(bsp::memory::relocated_bss_range_inclusive()); - } - - //-------------------------------------------------------------------------------------------------- -@@ -30,6 +30,7 @@ - /// # Safety - /// - /// - Only a single core must be active and running this function. -+#[no_mangle] - pub unsafe fn runtime_init() -> ! { - zero_bss(); - - diff -uNr 06_drivers_gpio_uart/update.sh 07_uart_chainloader/update.sh --- 06_drivers_gpio_uart/update.sh +++ 07_uart_chainloader/update.sh diff --git a/07_uart_chainloader/demo_payload_rpi3.img b/07_uart_chainloader/demo_payload_rpi3.img index 79eef6b4ca54773cbe15dd88f8b48e0ea9a8541a..91602b45b02de681e2e3f0706864f16396592fa9 100755 GIT binary patch delta 1220 zcmZ8fUrZE782@JXj@wlTw+G8|N3M5^oX25eO9|m$!Yv191;jrWv|Y6+qpNu;gr8B1G3Uch_C$WY)znW76}2+UDtsvRy3quEg97`Yzk=*q?L z7tn;WUOh9TH{lH)VZpX~eSk|ouA*tQj+Ni*o17foKEHgWMw@~a#{8Y#+3>Ko1-+5tAaoeZb? zN214%eAL<+J=vF`Zx#gCZ)d>iKb2RdK)UOm44w=A-<(%mOdC~R>B-59lk_&r_&UbF z@28U!N^&<-jIV{a5O( zkSk+}nR5(3XSm!!>8|X0{mf2{JgQfuE4<6ju~Q?{K}DRYcG*rM20J(@L{hUQ=|J8>p?Z+ZSrUEqI8aVhZ&n9gQ2*%!q^ou9g4pLP_ZV^ delta 1147 zcmZ8fOK1~O6uoaIV34YE?wVh0Bf~ zL{YTQC-_;2C`!|fh9H=Y)DH?ST!<~YOS|bu5vgvbNaLH#E2tOV+;`5o@80|Fyys1$ zP3kCYo(Kv0!T>-|f0&=h=ku}%@KkI`m>~rCEp90j&rvZ7MlwBMB4lLxxO#+ia*;oe z;-tvYiBP83u?L}e=CQDYTOY5%(X#(-e4*RL60V7uMYy?s+y5C7J5{*~l z$s2yG8sx4g=6YF)pS9IkXxh_=))2o`@7w6Xiw^W z8E%48o#;w%RW}*)HW%wP_`v-tJSZ5+3t{rzEBg`&vrf>nM}4J@bpm8vi$+pRkYGuB z!`6fujVH_)62H0-^Z>2N(FMAO2OUPTEvMm%-}zhCx6sp0wD!}tjF!RHX)La#xvhF6v$x&aZ%jy-JvU=Kj*3{^eDA;9&>A(s$ zM2^U@;e!D?%EQoeosjGrM>$_c59HCe&fePl2gp-1d8qf8woI2ic@ewhXm diff --git a/07_uart_chainloader/demo_payload_rpi4.img b/07_uart_chainloader/demo_payload_rpi4.img index 4ec2899812281f5c492516fa530527f937f5446b..b1060665b4af3eadced31cec72de42265985d110 100755 GIT binary patch delta 2148 zcmZ8iZ){Ul6hF7G-TKx-|In^oH(dW1O1mL{3Uq~NA8eQ~G|E^Ai66QJ3}I|y=NC@!(U^2In$eTfn7;3ln()P4g>6J1fbObu9RQ6p(u(6zzmPCF(aJt zga3yKA$3z}h3jUJDvAvX$dMjFj;RuXTs%9!gm#+8zg_yA-K7U5zk26QVC2l@Vj2`H)(Z)yN% zNwDUvM5@G){9_`PhYirV%_Jq-MWFlL;B2;oa|PnY4+-f3^rdvni*BjqQ}p+_eKF1> zUl!75b$^j_pr6OPWh5>OjnR@I<{u}!@=)malm5 zh?L+eklEz=tJwP@CRbY79P!DQu^GDTdo&>R+ z!M)ifwG5%}oZBbIQm73ioK}f*QjC-s#t>}wxeJ*C?YSc zdsgY>qH#6w(VPc=E!!=;X9$E?S!gFyqw9??5j>3^F+Fd1CJ>^L?pit%zLYCglUxxc z{Rt&qcwN#LneV!$FWf1*f4nn1es}cWYNv?|j4obug^-i%`r=i!={3{>khNl_QY#MT z-y7%<@iC8WC&{qaY$0-m9k;bQ94^dZ%*`M_-tv6=8TPxa(crIV)%NvK8l;mX(=ihU z6;O+OH=P*pvmuAFW#2F?r}zbsH3u5lUVD&qv7`3-z?lZh-?$qcW5JNhuE?g`J-I4} zLy`@2v9Ijm)ze{G6kK@|ffL0ehTMHc`Rjhaoz4vaol8Nkau#Ui6_R^D2o$@RJu$+b zax`Lw=N#L39!L1usRr7Pd1gKlawWx>Q@%Ilmh*EQDq0>KMr4`B0sL%UIVDv={H_2M zwcu{{r^B<3k1O7rBSsK0Y=m{(E0QfLDvHf4fj^!kQlUoNqBT=w8=Mq8A7oz;Gc&m)||3dt5UywF{E9->uD9!ZOU{EU=_S#6gY1-N;t&k{KGjHH?K~v zc>!J*v`-m*m)Lf%M|{(!Sx>XHH%`(l=f!#Fp4AJUF1-qe&c}BN|6bPZHr<|LtLqLp XuIbjLx^*+VP!|@xdgWert1kQ>Zelic delta 2218 zcmcIlZ){Ul6hHU9wtF38{j+xKI+p&Kvt5UR6zK+ud&n>_6DwU~Of-S2iTu&v<~Jp2 zw_zNDrq{cOL}QpnLi-KG&x!n(LV&wafk0TaG)lh^ys zpWiv>{?56tAFj`?PoIRYOFpxDV;4Yjr(eFL>w1#{a7y{Jx9lT8zcIm>AN;=-<(-%H z5typPNv;9ZvTlPy4@d*Uh?|Z=cNj8>b^uoo_y>XnlHmiEe#PuvS3;X&&m_Vwh?Zi| zy$$_ZxSy&H3=>EvxzUPG z#5CEO-y^BzwY)k)=66Yuw3^gzP=I#2!D)4X(>Ej!Y(YH_lEJ;!rFQk9ZLixK^-anH zBf?h1T|t|IHW$X5mwgdUCi&$UQbkX8I=s0W}v$*Q07Ia z7WAZsF)H!RIcz6Q8-10@l!862w8=S51}a%HiSLjqf^!h?yQ%g&&>7T!EWuc%1$!?O z%5jzWnOq4^#pfXa(qT??1)&WH^04DV{vZSpsAcG&h|jnFNdPfaS}mI*wgb zXFtDDeD(Z|;>Ld_Y>LxNq7%WDX9@X$U9Df+7+6PF0A%Yh|J)X!`^GoYBa_Xn)AkHe z*|4pR46_rqEsm)M%wfVt0>ueC)t6$6HlL(m}HVO{38a48h-&q@C zkO_|Y6fwt5@F@4UF7+BXez)epAL}7%q1L>$+=q9o*36!AZnC0u1f@DNd*8Vxtp%x! zxEIv`HQ|1+7%*a{%$dXrT|dgZXv8h|mLq1haM+Yd>~*`M5i=AN)L9{)uwI1%ADHUU zAj#Lc9v#@T%-Hc06UVoe~^H7Iv>CVdn5OIl3e)0M}~#L>r3@xtqMD8X;A6fJGVWj~e!+ z`={-vJ$IU~H&$h2&{$gs2lo%++ZCb5kvG=eem}DdjXmif`JW{&yYUMUZ?4MANejY` z6Kt##LdMHuth@cQ!oHk!RLBH^AtwQ@On`{mtq?*O<6+64$9QqG7&09_Y diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu.rs b/07_uart_chainloader/src/_arch/aarch64/cpu.rs index 893c7580..ade53a3c 100644 --- a/07_uart_chainloader/src/_arch/aarch64/cpu.rs +++ b/07_uart_chainloader/src/_arch/aarch64/cpu.rs @@ -35,19 +35,3 @@ pub fn wait_forever() -> ! { asm::wfe() } } - -/// Branch to a raw integer value. -/// -/// # Safety -/// -/// - This is highly unsafe. Use with care. -#[inline(always)] -pub unsafe fn branch_to_raw_addr(addr: usize) -> ! { - asm!( - "blr {destination:x}", - destination = in(reg) addr, - options(nomem, nostack) - ); - - core::intrinsics::unreachable() -} diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs b/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs index b343792f..c85bb94b 100644 --- a/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs +++ b/07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs @@ -11,31 +11,23 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; -use cortex_a::regs::*; +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] -pub unsafe fn _start() -> ! { - use crate::relocate; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - relocate::relocate_self() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() } diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu/boot.s b/07_uart_chainloader/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..4f45ebd1 --- /dev/null +++ b/07_uart_chainloader/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 2f + + // If execution reaches here, it is the boot core. + + // Next, relocate the binary. + adr x0, __binary_nonzero_start // The address the binary got loaded to. + ldr x1, =__binary_nonzero_start // The address the binary was linked to. + ldr x2, =__binary_nonzero_end_exclusive + +1: ldr x3, [x0], #8 + str x3, [x1], #8 + cmp x1, x2 + b.lo 1b + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to the relocated Rust code. + ldr x1, =_start_rust + br x1 + + // Infinitely wait for events (aka "park the core"). +2: wfe + b 2b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs b/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/07_uart_chainloader/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/07_uart_chainloader/src/bsp/raspberrypi/cpu.rs b/07_uart_chainloader/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/07_uart_chainloader/src/bsp/raspberrypi/cpu.rs +++ b/07_uart_chainloader/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/07_uart_chainloader/src/bsp/raspberrypi/link.ld b/07_uart_chainloader/src/bsp/raspberrypi/link.ld index 5642ed4e..f7523906 100644 --- a/07_uart_chainloader/src/bsp/raspberrypi/link.ld +++ b/07_uart_chainloader/src/bsp/raspberrypi/link.ld @@ -18,15 +18,29 @@ SECTIONS { /* Set the link address to 32 MiB */ . = 0x2000000; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __binary_start = .; + __binary_nonzero_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx @@ -37,8 +51,12 @@ SECTIONS ***********************************************************************************************/ .data : { *(.data*) } :segment_rw + /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */ + . = ALIGN(8); + __binary_nonzero_end_exclusive = .; + /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ - .bss : ALIGN(8) + .bss : { __bss_start = .; *(.bss*); @@ -47,10 +65,4 @@ SECTIONS . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ __bss_end_inclusive = . - 8; } :NONE - - /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */ - . = ALIGN(8); - __binary_end_inclusive = . - 8; - - __runtime_init_reloc = runtime_init; } diff --git a/07_uart_chainloader/src/bsp/raspberrypi/memory.rs b/07_uart_chainloader/src/bsp/raspberrypi/memory.rs index 5abfb9ef..9a66cc8a 100644 --- a/07_uart_chainloader/src/bsp/raspberrypi/memory.rs +++ b/07_uart_chainloader/src/bsp/raspberrypi/memory.rs @@ -12,12 +12,8 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __binary_start: UnsafeCell; static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; - static __binary_end_inclusive: UnsafeCell; - - static __runtime_init_reloc: UnsafeCell; } //-------------------------------------------------------------------------------------------------- @@ -27,8 +23,6 @@ extern "Rust" { /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { - pub const BOOT_CORE_STACK_END: usize = 0x8_0000; - pub const BOARD_DEFAULT_LOAD_ADDRESS: usize = 0x8_0000; pub const GPIO_OFFSET: usize = 0x0020_0000; @@ -59,41 +53,19 @@ pub(super) mod map { // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - map::BOOT_CORE_STACK_END -} - /// The address on which the Raspberry firmware loads every binary by default. #[inline(always)] pub fn board_default_load_addr() -> *const u64 { map::BOARD_DEFAULT_LOAD_ADDRESS as _ } -/// Return the inclusive range spanning the relocated kernel binary. -/// -/// # Safety -/// -/// - Values are provided by the linker script and must be trusted as-is. -/// - The linker-provided addresses must be u64 aligned. -pub fn relocated_binary_range_inclusive() -> RangeInclusive<*mut u64> { - unsafe { RangeInclusive::new(__binary_start.get(), __binary_end_inclusive.get()) } -} - -/// The relocated address of function `runtime_init()`. -#[inline(always)] -pub fn relocated_runtime_init_addr() -> *const u64 { - unsafe { __runtime_init_reloc.get() as _ } -} - /// Return the inclusive range spanning the relocated .bss section. /// /// # Safety /// /// - Values are provided by the linker script and must be trusted as-is. /// - The linker-provided addresses must be u64 aligned. -pub fn relocated_bss_range_inclusive() -> RangeInclusive<*mut u64> { +pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { let range; unsafe { range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); diff --git a/07_uart_chainloader/src/cpu.rs b/07_uart_chainloader/src/cpu.rs index 7de5c29c..c5e956ea 100644 --- a/07_uart_chainloader/src/cpu.rs +++ b/07_uart_chainloader/src/cpu.rs @@ -10,12 +10,10 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- -pub use arch_cpu::{branch_to_raw_addr, nop, wait_forever}; +pub use arch_cpu::{nop, wait_forever}; #[cfg(feature = "bsp_rpi3")] pub use arch_cpu::spin_for_cycles; diff --git a/07_uart_chainloader/src/cpu/smp.rs b/07_uart_chainloader/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/07_uart_chainloader/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/07_uart_chainloader/src/main.rs b/07_uart_chainloader/src/main.rs index 4fc6449a..1ac8140c 100644 --- a/07_uart_chainloader/src/main.rs +++ b/07_uart_chainloader/src/main.rs @@ -100,20 +100,17 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. -//! 2. Once finished with architectural setup, the arch code calls [`relocate::relocate_self()`]. -//! 3. Finally, [`runtime_init::runtime_init()`] is called. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html -//! [`relocate::relocate_self()`]: relocate/fn.relocate_self.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] #![feature(asm)] #![feature(const_fn_fn_ptr_basics)] -#![feature(core_intrinsics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] @@ -126,7 +123,6 @@ mod driver; mod memory; mod panic_wait; mod print; -mod relocate; mod runtime_init; mod synchronization; diff --git a/07_uart_chainloader/src/relocate.rs b/07_uart_chainloader/src/relocate.rs deleted file mode 100644 index 6ac6bcce..00000000 --- a/07_uart_chainloader/src/relocate.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Relocation code. - -use crate::{bsp, cpu}; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Relocates the own binary from `bsp::memory::board_default_load_addr()` to the `__binary_start` -/// address from the linker script. -/// -/// # Safety -/// -/// - Only a single core must be active and running this function. -/// - Function must not use the `bss` section. -#[inline(never)] -pub unsafe fn relocate_self() -> ! { - let range = bsp::memory::relocated_binary_range_inclusive(); - let mut relocated_binary_start_addr = *range.start(); - let relocated_binary_end_addr_inclusive = *range.end(); - - // The address of where the previous firmware loaded us. - let mut current_binary_start_addr = bsp::memory::board_default_load_addr(); - - // Copy the whole binary. - while relocated_binary_start_addr <= relocated_binary_end_addr_inclusive { - core::ptr::write_volatile( - relocated_binary_start_addr, - core::ptr::read_volatile(current_binary_start_addr), - ); - relocated_binary_start_addr = relocated_binary_start_addr.offset(1); - current_binary_start_addr = current_binary_start_addr.offset(1); - } - - // The following function calls realize an "absolute jump" to `runtime_init::runtime_init()` by - // forcing an indirection through the global offset table (GOT), so that execution continues - // from the relocated binary. - // - // Without the indirection through the assembly, the address of `runtime_init()` would be - // calculated as a relative offset from the current program counter, since we are compiling as - // `position independent code`. This would cause us to keep executing from the address to which - // the firmware loaded us, instead of the relocated position. - let relocated_runtime_init_addr = bsp::memory::relocated_runtime_init_addr() as usize; - cpu::branch_to_raw_addr(relocated_runtime_init_addr) -} diff --git a/07_uart_chainloader/src/runtime_init.rs b/07_uart_chainloader/src/runtime_init.rs index 5e9b7497..ee094686 100644 --- a/07_uart_chainloader/src/runtime_init.rs +++ b/07_uart_chainloader/src/runtime_init.rs @@ -17,7 +17,7 @@ use crate::{bsp, memory}; /// - Must only be called pre `kernel_init()`. #[inline(always)] unsafe fn zero_bss() { - memory::zero_volatile(bsp::memory::relocated_bss_range_inclusive()); + memory::zero_volatile(bsp::memory::bss_range_inclusive()); } //-------------------------------------------------------------------------------------------------- @@ -30,7 +30,6 @@ unsafe fn zero_bss() { /// # Safety /// /// - Only a single core must be active and running this function. -#[no_mangle] pub unsafe fn runtime_init() -> ! { zero_bss(); diff --git a/08_timestamps/README.md b/08_timestamps/README.md index eebdd20c..122308a6 100644 --- a/08_timestamps/README.md +++ b/08_timestamps/README.md @@ -49,23 +49,19 @@ Binary files 07_uart_chainloader/demo_payload_rpi4.img and 08_timestamps/demo_pa diff -uNr 07_uart_chainloader/Makefile 08_timestamps/Makefile --- 07_uart_chainloader/Makefile +++ 08_timestamps/Makefile -@@ -24,8 +24,7 @@ - NM_BINARY = aarch64-none-elf-nm +@@ -25,7 +25,6 @@ READELF_BINARY = aarch64-none-elf-readelf LINKER_FILE = src/bsp/raspberrypi/link.ld -- RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C relocation-model=pic + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 - CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi3.img -+ RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 else ifeq ($(BSP),rpi4) TARGET = aarch64-unknown-none-softfloat KERNEL_BIN = kernel8.img -@@ -36,8 +35,7 @@ - NM_BINARY = aarch64-none-elf-nm +@@ -37,7 +36,6 @@ READELF_BINARY = aarch64-none-elf-readelf LINKER_FILE = src/bsp/raspberrypi/link.ld -- RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -C relocation-model=pic + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 - CHAINBOOT_DEMO_PAYLOAD = demo_payload_rpi4.img -+ RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 endif # Export for build.rs @@ -97,23 +93,47 @@ diff -uNr 07_uart_chainloader/Makefile 08_timestamps/Makefile clippy: @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) -diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs 08_timestamps/src/_arch/aarch64/cpu/boot.rs ---- 07_uart_chainloader/src/_arch/aarch64/cpu/boot.rs -+++ 08_timestamps/src/_arch/aarch64/cpu/boot.rs -@@ -29,11 +29,11 @@ - /// actually set (`SP.set()`). - #[no_mangle] - pub unsafe fn _start() -> ! { -- use crate::relocate; -+ use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); -- relocate::relocate_self() -+ runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() +diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu/boot.s 08_timestamps/src/_arch/aarch64/cpu/boot.s +--- 07_uart_chainloader/src/_arch/aarch64/cpu/boot.s ++++ 08_timestamps/src/_arch/aarch64/cpu/boot.s +@@ -22,31 +22,20 @@ + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 +- b.ne 2f ++ b.ne 1f + +- // If execution reaches here, it is the boot core. +- +- // Next, relocate the binary. +- adr x0, __binary_nonzero_start // The address the binary got loaded to. +- ldr x1, =__binary_nonzero_start // The address the binary was linked to. +- ldr x2, =__binary_nonzero_end_exclusive +- +-1: ldr x3, [x0], #8 +- str x3, [x1], #8 +- cmp x1, x2 +- b.lo 1b ++ // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + +- // Jump to the relocated Rust code. +- ldr x1, =_start_rust +- br x1 ++ // Jump to Rust code. ++ b _start_rust + + // Infinitely wait for events (aka "park the core"). +-2: wfe +- b 2b ++1: wfe ++ b 1b + + .size _start, . - _start + .type _start, function diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/aarch64/cpu.rs --- 07_uart_chainloader/src/_arch/aarch64/cpu.rs @@ -134,26 +154,6 @@ diff -uNr 07_uart_chainloader/src/_arch/aarch64/cpu.rs 08_timestamps/src/_arch/a /// Pause execution on the core. #[inline(always)] pub fn wait_forever() -> ! { -@@ -35,19 +26,3 @@ - asm::wfe() - } - } -- --/// Branch to a raw integer value. --/// --/// # Safety --/// --/// - This is highly unsafe. Use with care. --#[inline(always)] --pub unsafe fn branch_to_raw_addr(addr: usize) -> ! { -- asm!( -- "blr {destination:x}", -- destination = in(reg) addr, -- options(nomem, nostack) -- ); -- -- core::intrinsics::unreachable() --} diff -uNr 07_uart_chainloader/src/_arch/aarch64/time.rs 08_timestamps/src/_arch/aarch64/time.rs --- 07_uart_chainloader/src/_arch/aarch64/time.rs @@ -361,58 +361,46 @@ diff -uNr 07_uart_chainloader/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 08 diff -uNr 07_uart_chainloader/src/bsp/raspberrypi/link.ld 08_timestamps/src/bsp/raspberrypi/link.ld --- 07_uart_chainloader/src/bsp/raspberrypi/link.ld +++ 08_timestamps/src/bsp/raspberrypi/link.ld -@@ -16,13 +16,12 @@ +@@ -16,8 +16,7 @@ SECTIONS { - /* Set the link address to 32 MiB */ - . = 0x2000000; + . = __rpi_load_addr; - + /* ^ */ + /* | stack */ + /* | growth */ +@@ -27,7 +26,6 @@ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ -- __binary_start = .; -+ __rx_start = .; +- __binary_nonzero_start = .; .text : { KEEP(*(.text._start)) -@@ -47,10 +46,4 @@ - . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ - __bss_end_inclusive = . - 8; - } :NONE -- +@@ -51,12 +49,8 @@ + ***********************************************************************************************/ + .data : { *(.data*) } :segment_rw + - /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */ - . = ALIGN(8); -- __binary_end_inclusive = . - 8; +- __binary_nonzero_end_exclusive = .; - -- __runtime_init_reloc = runtime_init; - } + /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ +- .bss : ++ .bss : ALIGN(8) + { + __bss_start = .; + *(.bss*); diff -uNr 07_uart_chainloader/src/bsp/raspberrypi/memory.rs 08_timestamps/src/bsp/raspberrypi/memory.rs --- 07_uart_chainloader/src/bsp/raspberrypi/memory.rs +++ 08_timestamps/src/bsp/raspberrypi/memory.rs -@@ -12,12 +12,10 @@ - - // Symbols from the linker script. - extern "Rust" { -- static __binary_start: UnsafeCell; -+ static __rx_start: UnsafeCell<()>; -+ - static __bss_start: UnsafeCell; - static __bss_end_inclusive: UnsafeCell; -- static __binary_end_inclusive: UnsafeCell; -- -- static __runtime_init_reloc: UnsafeCell; - } - - //-------------------------------------------------------------------------------------------------- -@@ -27,12 +25,9 @@ +@@ -23,10 +23,9 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { -- pub const BOOT_CORE_STACK_END: usize = 0x8_0000; -- - pub const BOARD_DEFAULT_LOAD_ADDRESS: usize = 0x8_0000; - pub const GPIO_OFFSET: usize = 0x0020_0000; @@ -422,114 +410,53 @@ diff -uNr 07_uart_chainloader/src/bsp/raspberrypi/memory.rs 08_timestamps/src/bs /// Physical devices. #[cfg(feature = "bsp_rpi3")] -@@ -56,44 +51,36 @@ - } - - //-------------------------------------------------------------------------------------------------- --// Public Code -+// Private Code +@@ -53,13 +52,7 @@ + // Public Code //-------------------------------------------------------------------------------------------------- --/// Exclusive end address of the boot core's stack. --#[inline(always)] --pub fn boot_core_stack_end() -> usize { -- map::BOOT_CORE_STACK_END --} -- -/// The address on which the Raspberry firmware loads every binary by default. -#[inline(always)] -pub fn board_default_load_addr() -> *const u64 { - map::BOARD_DEFAULT_LOAD_ADDRESS as _ -} - --/// Return the inclusive range spanning the relocated kernel binary. -+/// Start address of the Read+Execute (RX) range. - /// - /// # Safety - /// --/// - Values are provided by the linker script and must be trusted as-is. --/// - The linker-provided addresses must be u64 aligned. --pub fn relocated_binary_range_inclusive() -> RangeInclusive<*mut u64> { -- unsafe { RangeInclusive::new(__binary_start.get(), __binary_end_inclusive.get()) } -+/// - Value is provided by the linker script and must be trusted as-is. -+#[inline(always)] -+fn rx_start() -> usize { -+ unsafe { __rx_start.get() as usize } - } - --/// The relocated address of function `runtime_init()`. -+//-------------------------------------------------------------------------------------------------- -+// Public Code -+//-------------------------------------------------------------------------------------------------- -+ -+/// Exclusive end address of the boot core's stack. - #[inline(always)] --pub fn relocated_runtime_init_addr() -> *const u64 { -- unsafe { __runtime_init_reloc.get() as _ } -+pub fn boot_core_stack_end() -> usize { -+ rx_start() - } - -/// Return the inclusive range spanning the relocated .bss section. +/// Return the inclusive range spanning the .bss section. /// /// # Safety /// - /// - Values are provided by the linker script and must be trusted as-is. - /// - The linker-provided addresses must be u64 aligned. --pub fn relocated_bss_range_inclusive() -> RangeInclusive<*mut u64> { -+pub fn bss_range_inclusive() -> RangeInclusive<*mut u64> { - let range; - unsafe { - range = RangeInclusive::new(__bss_start.get(), __bss_end_inclusive.get()); diff -uNr 07_uart_chainloader/src/cpu.rs 08_timestamps/src/cpu.rs --- 07_uart_chainloader/src/cpu.rs +++ 08_timestamps/src/cpu.rs -@@ -15,7 +15,4 @@ - //-------------------------------------------------------------------------------------------------- +@@ -14,6 +14,3 @@ // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- --pub use arch_cpu::{branch_to_raw_addr, nop, wait_forever}; + pub use arch_cpu::{nop, wait_forever}; - -#[cfg(feature = "bsp_rpi3")] -pub use arch_cpu::spin_for_cycles; -+pub use arch_cpu::{nop, wait_forever}; diff -uNr 07_uart_chainloader/src/main.rs 08_timestamps/src/main.rs --- 07_uart_chainloader/src/main.rs +++ 08_timestamps/src/main.rs -@@ -102,17 +102,13 @@ - //! - //! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. - //! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. --//! 2. Once finished with architectural setup, the arch code calls [`relocate::relocate_self()`]. --//! 3. Finally, [`runtime_init::runtime_init()`] is called. -+//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. - //! - //! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html --//! [`relocate::relocate_self()`]: relocate/fn.relocate_self.html +@@ -107,7 +107,6 @@ //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] -#![feature(asm)] #![feature(const_fn_fn_ptr_basics)] --#![feature(core_intrinsics)] #![feature(format_args_nl)] - #![feature(panic_info_message)] - #![feature(trait_alias)] -@@ -126,9 +122,9 @@ - mod memory; - mod panic_wait; + #![feature(global_asm)] +@@ -125,6 +124,7 @@ mod print; --mod relocate; mod runtime_init; mod synchronization; +mod time; /// Early init code. /// -@@ -153,51 +149,31 @@ +@@ -149,51 +149,31 @@ /// The main function running after the early init. fn kernel_main() -> ! { @@ -680,81 +607,6 @@ diff -uNr 07_uart_chainloader/src/print.rs 08_timestamps/src/print.rs + }) +} -diff -uNr 07_uart_chainloader/src/relocate.rs 08_timestamps/src/relocate.rs ---- 07_uart_chainloader/src/relocate.rs -+++ 08_timestamps/src/relocate.rs -@@ -1,49 +0,0 @@ --// SPDX-License-Identifier: MIT OR Apache-2.0 --// --// Copyright (c) 2018-2021 Andre Richter -- --//! Relocation code. -- --use crate::{bsp, cpu}; -- --//-------------------------------------------------------------------------------------------------- --// Public Code --//-------------------------------------------------------------------------------------------------- -- --/// Relocates the own binary from `bsp::memory::board_default_load_addr()` to the `__binary_start` --/// address from the linker script. --/// --/// # Safety --/// --/// - Only a single core must be active and running this function. --/// - Function must not use the `bss` section. --#[inline(never)] --pub unsafe fn relocate_self() -> ! { -- let range = bsp::memory::relocated_binary_range_inclusive(); -- let mut relocated_binary_start_addr = *range.start(); -- let relocated_binary_end_addr_inclusive = *range.end(); -- -- // The address of where the previous firmware loaded us. -- let mut current_binary_start_addr = bsp::memory::board_default_load_addr(); -- -- // Copy the whole binary. -- while relocated_binary_start_addr <= relocated_binary_end_addr_inclusive { -- core::ptr::write_volatile( -- relocated_binary_start_addr, -- core::ptr::read_volatile(current_binary_start_addr), -- ); -- relocated_binary_start_addr = relocated_binary_start_addr.offset(1); -- current_binary_start_addr = current_binary_start_addr.offset(1); -- } -- -- // The following function calls realize an "absolute jump" to `runtime_init::runtime_init()` by -- // forcing an indirection through the global offset table (GOT), so that execution continues -- // from the relocated binary. -- // -- // Without the indirection through the assembly, the address of `runtime_init()` would be -- // calculated as a relative offset from the current program counter, since we are compiling as -- // `position independent code`. This would cause us to keep executing from the address to which -- // the firmware loaded us, instead of the relocated position. -- let relocated_runtime_init_addr = bsp::memory::relocated_runtime_init_addr() as usize; -- cpu::branch_to_raw_addr(relocated_runtime_init_addr) --} - -diff -uNr 07_uart_chainloader/src/runtime_init.rs 08_timestamps/src/runtime_init.rs ---- 07_uart_chainloader/src/runtime_init.rs -+++ 08_timestamps/src/runtime_init.rs -@@ -17,7 +17,7 @@ - /// - Must only be called pre `kernel_init()`. - #[inline(always)] - unsafe fn zero_bss() { -- memory::zero_volatile(bsp::memory::relocated_bss_range_inclusive()); -+ memory::zero_volatile(bsp::memory::bss_range_inclusive()); - } - - //-------------------------------------------------------------------------------------------------- -@@ -30,7 +30,6 @@ - /// # Safety - /// - /// - Only a single core must be active and running this function. --#[no_mangle] - pub unsafe fn runtime_init() -> ! { - zero_bss(); - - diff -uNr 07_uart_chainloader/src/time.rs 08_timestamps/src/time.rs --- 07_uart_chainloader/src/time.rs +++ 08_timestamps/src/time.rs diff --git a/08_timestamps/src/_arch/aarch64/cpu/boot.rs b/08_timestamps/src/_arch/aarch64/cpu/boot.rs index 549b5927..c85bb94b 100644 --- a/08_timestamps/src/_arch/aarch64/cpu/boot.rs +++ b/08_timestamps/src/_arch/aarch64/cpu/boot.rs @@ -11,31 +11,23 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; -use cortex_a::regs::*; +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() } diff --git a/08_timestamps/src/_arch/aarch64/cpu/boot.s b/08_timestamps/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/08_timestamps/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/08_timestamps/src/_arch/aarch64/cpu/smp.rs b/08_timestamps/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/08_timestamps/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/08_timestamps/src/bsp/raspberrypi/cpu.rs b/08_timestamps/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/08_timestamps/src/bsp/raspberrypi/cpu.rs +++ b/08_timestamps/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/08_timestamps/src/bsp/raspberrypi/link.ld b/08_timestamps/src/bsp/raspberrypi/link.ld index 87e6a976..3b73b3c7 100644 --- a/08_timestamps/src/bsp/raspberrypi/link.ld +++ b/08_timestamps/src/bsp/raspberrypi/link.ld @@ -17,15 +17,28 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __rx_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/08_timestamps/src/bsp/raspberrypi/memory.rs b/08_timestamps/src/bsp/raspberrypi/memory.rs index 56a3306e..c6d65e36 100644 --- a/08_timestamps/src/bsp/raspberrypi/memory.rs +++ b/08_timestamps/src/bsp/raspberrypi/memory.rs @@ -12,8 +12,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; } @@ -50,30 +48,10 @@ pub(super) mod map { } } -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/08_timestamps/src/cpu.rs b/08_timestamps/src/cpu.rs index 3834f183..7a095cb1 100644 --- a/08_timestamps/src/cpu.rs +++ b/08_timestamps/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/08_timestamps/src/cpu/smp.rs b/08_timestamps/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/08_timestamps/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/08_timestamps/src/main.rs b/08_timestamps/src/main.rs index 14e1d0ab..b7a74ea6 100644 --- a/08_timestamps/src/main.rs +++ b/08_timestamps/src/main.rs @@ -100,16 +100,16 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/09_hw_debug_JTAG/README.md b/09_hw_debug_JTAG/README.md index b13e1fa7..0c9fb497 100644 --- a/09_hw_debug_JTAG/README.md +++ b/09_hw_debug_JTAG/README.md @@ -54,6 +54,7 @@ infrastructure for JTAG debugging around it. We need to add another line to the `config.txt` file from the SD Card: ```toml +arm_64bit=1 init_uart_clock=48000000 enable_jtag_gpio=1 ``` @@ -234,15 +235,16 @@ $ make gdb [...] >>> target remote :3333 # Connect to OpenOCD, core0 >>> load # Load the kernel into the RPi's DRAM over JTAG. -Loading section .text, size 0x2340 lma 0x80000 -Loading section .rodata, size 0xc2d lma 0x82340 -Loading section .data, size 0x20 lma 0x82f70 -Start address 0x80000, load size 12173 -Transfer rate: 65 KB/sec, 4057 bytes/write. +Loading section .text, size 0x2454 lma 0x80000 +Loading section .rodata, size 0xa1d lma 0x82460 +Loading section .got, size 0x10 lma 0x82e80 +Loading section .data, size 0x20 lma 0x82e90 +Start address 0x0000000000080000, load size 11937 +Transfer rate: 63 KB/sec, 2984 bytes/write. >>> set $pc = 0x80000 # Set RPI's program counter to the start of the # kernel binary. ->>> break main.rs:70 -Breakpoint 1 at 0x80108: file src/main.rs, line 153. +>>> break main.rs:158 +Breakpoint 1 at 0x8025c: file src/main.rs, line 158. >>> cont >>> step # Single-step through the kernel >>> step diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs index 549b5927..c85bb94b 100644 --- a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs +++ b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs @@ -11,31 +11,23 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; -use cortex_a::regs::*; +use crate::runtime_init; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is -/// actually set (`SP.set()`). +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. #[no_mangle] -pub unsafe fn _start() -> ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() } diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs b/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/09_hw_debug_JTAG/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/09_hw_debug_JTAG/src/bsp/raspberrypi/cpu.rs b/09_hw_debug_JTAG/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/09_hw_debug_JTAG/src/bsp/raspberrypi/cpu.rs +++ b/09_hw_debug_JTAG/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/09_hw_debug_JTAG/src/bsp/raspberrypi/link.ld b/09_hw_debug_JTAG/src/bsp/raspberrypi/link.ld index 87e6a976..3b73b3c7 100644 --- a/09_hw_debug_JTAG/src/bsp/raspberrypi/link.ld +++ b/09_hw_debug_JTAG/src/bsp/raspberrypi/link.ld @@ -17,15 +17,28 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __rx_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/09_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs b/09_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs index 56a3306e..c6d65e36 100644 --- a/09_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs +++ b/09_hw_debug_JTAG/src/bsp/raspberrypi/memory.rs @@ -12,8 +12,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; } @@ -50,30 +48,10 @@ pub(super) mod map { } } -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/09_hw_debug_JTAG/src/cpu.rs b/09_hw_debug_JTAG/src/cpu.rs index 3834f183..7a095cb1 100644 --- a/09_hw_debug_JTAG/src/cpu.rs +++ b/09_hw_debug_JTAG/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/09_hw_debug_JTAG/src/cpu/smp.rs b/09_hw_debug_JTAG/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/09_hw_debug_JTAG/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/09_hw_debug_JTAG/src/main.rs b/09_hw_debug_JTAG/src/main.rs index 14e1d0ab..b7a74ea6 100644 --- a/09_hw_debug_JTAG/src/main.rs +++ b/09_hw_debug_JTAG/src/main.rs @@ -100,16 +100,16 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/10_privilege_level/README.md b/10_privilege_level/README.md index 2c435548..3e160e22 100644 --- a/10_privilege_level/README.md +++ b/10_privilege_level/README.md @@ -12,7 +12,6 @@ - [Checking for EL2 in the entrypoint](#checking-for-el2-in-the-entrypoint) - [Transition preparation](#transition-preparation) - [Returning from an exception that never happened](#returning-from-an-exception-that-never-happened) -- [Are we stackless?](#are-we-stackless) - [Test it](#test-it) - [Diff to previous](#diff-to-previous) @@ -39,32 +38,35 @@ ARMv8-A] before you continue. It gives a concise overview about the topic. ## Scope of this tutorial -By default, the Rpi will always start executing in `EL2`. Since we are writing a traditional +By default, the Raspberry will always start executing in `EL2`. Since we are writing a traditional `Kernel`, we have to transition into the more appropriate `EL1`. ## Checking for EL2 in the entrypoint First of all, we need to ensure that we actually execute in `EL2` before we can call respective code -to transition to `EL1`: +to transition to `EL1`. Therefore, we add a new checkt to the top of `boot.s`, which parks the CPU +core should it not be in `EL2`. + +``` +// Only proceed if the core executes in EL2. Park it otherwise. +mrs x0, CurrentEL +cmp x0, _EL2 +b.ne 1f +``` + +Afterwards, we continue with preparing the `EL2` -> `EL1` transition by calling +`prepare_el2_to_el1_transition()` in `boot.rs`: ```rust #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } ``` -If this is the case, we continue with preparing the `EL2` -> `EL1` transition in -`el2_to_el1_transition()`. - ## Transition preparation Since `EL2` is more privileged than `EL1`, it has control over various processor features and can @@ -104,8 +106,8 @@ This instruction will copy the contents of the [Saved Program Status Register - Program Status Register - EL1` and jump to the instruction address that is stored in the [Exception Link Register - EL2]. -This is basically the reverse of what is happening when an exception is taken. You'll learn about it -in an upcoming tutorial. +This is basically the reverse of what is happening when an exception is taken. You'll learn about +that in an upcoming tutorial. [Saved Program Status Register - EL2]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/regs/spsr_el2.rs.html [Exception Link Register - EL2]: https://docs.rs/cortex-a/5.1.2/src/cortex_a/regs/elr_el2.rs.html @@ -125,65 +127,39 @@ SPSR_EL2.write( // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + +// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there +// are no plans to ever return to EL2, just re-use the same stack. +SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); ``` As you can see, we are populating `ELR_EL2` with the address of the [runtime_init()] function that -we earlier used to call directly from the entrypoint. - -Finally, we set the stack pointer for `SP_EL1` and call `ERET`: +we earlier used to call directly from the entrypoint. Finally, we set the stack pointer for +`SP_EL1`. [runtime_init()]: src/runtime_init.rs -```rust -// Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. -SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - -// Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. -asm::eret() -``` +You might have noticed that the stack's address was supplied as a function argument. As you might +remember, in `_start()` in `boot.s`, we are already setting up the stack for `EL2`. Since there +are no plans to ever return to `EL2`, we can just re-use the same stack for `EL1`, so its address is +forwarded using function arguments. -## Are we stackless? +Lastly, back in `_start_rust()` a call to `ERET` is made: -We just wrote a big inline rust function, `el2_to_el1_transition()`, that is executed in a context -where we do not have a stack yet. We should double-check the generated machine code: +```rust +#[no_mangle] +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); -```console -$ make objdump -[...] -Disassembly of section .text: - -0000000000080000 <_start>: - 80000: d53800a8 mrs x8, mpidr_el1 - 80004: f240051f tst x8, #0x3 - 80008: 54000081 b.ne 80018 <_start+0x18> // b.any - 8000c: d5384248 mrs x8, currentel - 80010: f100211f cmp x8, #0x8 - 80014: 54000060 b.eq 80020 <_start+0x20> // b.none - 80018: d503205f wfe - 8001c: 17ffffff b 80018 <_start+0x18> - 80020: aa1f03e8 mov x8, xzr - 80024: 52800069 mov w9, #0x3 // #3 - 80028: d51ce109 msr cnthctl_el2, x9 - 8002c: d51ce068 msr cntvoff_el2, x8 - 80030: d0000008 adrp x8, 82000 - 80034: 52b0000a mov w10, #0x80000000 // #-2147483648 - 80038: 528078ab mov w11, #0x3c5 // #965 - 8003c: 52a0010c mov w12, #0x80000 // #524288 - 80040: d51c110a msr hcr_el2, x10 - 80044: d51c400b msr spsr_el2, x11 - 80048: 9114c108 add x8, x8, #0x530 - 8004c: d51c4028 msr elr_el2, x8 - 80050: d51c410c msr sp_el1, x12 - 80054: d69f03e0 eret + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} ``` -Looks good! Thanks zero-overhead abstractions in the [cortex-a] crate! :heart_eyes: - -[cortex-a]: https://github.com/rust-embedded/cortex-a - ## Test it -In `main.rs`, we additionally inspect if the mask bits in `SPSR_EL2` made it to `EL1` as well: +In `main.rs`, we print the `current privilege level` and additionally inspect if the mask bits in +`SPSR_EL2` made it to `EL1` as well: ```console $ make chainboot @@ -225,30 +201,27 @@ Minipush 1.0 diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 10_privilege_level/src/_arch/aarch64/cpu/boot.rs --- 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs +++ 10_privilege_level/src/_arch/aarch64/cpu/boot.rs -@@ -12,7 +12,55 @@ +@@ -12,11 +12,53 @@ //! crate::cpu::boot::arch_boot - use crate::{bsp, cpu}; --use cortex_a::regs::*; + use crate::runtime_init; +use cortex_a::{asm, regs::*}; -+ -+//-------------------------------------------------------------------------------------------------- + + // Assembly counterpart to this file. + global_asm!(include_str!("boot.s")); + + //-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + -+/// Transition from EL2 to EL1. ++/// Prepares the transition from EL2 to EL1. +/// +/// # Safety +/// ++/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - The HW state of EL1 must be prepared in a sound way. -+/// - Exception return from EL2 must must continue execution in EL1 with -+/// `runtime_init::runtime_init()`. -+/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -+/// a stack for EL2. +#[inline(always)] -+unsafe fn el2_to_el1_transition() -> ! { -+ use crate::runtime_init; -+ ++unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + @@ -273,38 +246,67 @@ diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.rs 10_privilege_level/src/ + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + -+ // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. -+ SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); -+ -+ // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. -+ asm::eret() ++ // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there ++ // are no plans to ever return to EL2, just re-use the same stack. ++ SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); +} - - //-------------------------------------------------------------------------------------------------- ++ ++//-------------------------------------------------------------------------------------------------- // Public Code -@@ -25,15 +73,15 @@ + //-------------------------------------------------------------------------------------------------- + +@@ -27,7 +69,11 @@ /// # Safety /// - /// - Linker script must ensure to place this function where it is expected by the target machine. --/// - We have to hope that the compiler omits any stack pointer usage before the stack pointer is --/// actually set (`SP.set()`). -+/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -+/// a stack for EL2. + /// - The `bss` section is not initialized yet. The code must not use or reference it in any way. ++/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] - pub unsafe fn _start() -> ! { -- use crate::runtime_init; -- -- if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { -- SP.set(bsp::memory::boot_core_stack_end() as u64); -- runtime_init::runtime_init() -+ // Expect the boot core to start in EL2. -+ if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) -+ && (CurrentEL.get() == CurrentEL::EL::EL2.value) -+ { -+ el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() +-pub unsafe fn _start_rust() -> ! { +- runtime_init::runtime_init() ++pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { ++ prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); ++ ++ // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. ++ asm::eret() + } + +diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s 10_privilege_level/src/_arch/aarch64/cpu/boot.s +--- 09_hw_debug_JTAG/src/_arch/aarch64/cpu/boot.s ++++ 10_privilege_level/src/_arch/aarch64/cpu/boot.s +@@ -6,6 +6,7 @@ + // Definitions + //-------------------------------------------------------------------------------------------------- + ++.equ _EL2, 0x8 + .equ _core_id_mask, 0b11 + + //-------------------------------------------------------------------------------------------------- +@@ -17,6 +18,11 @@ + // fn _start() + //------------------------------------------------------------------------------ + _start: ++ // Only proceed if the core executes in EL2. Park it otherwise. ++ mrs x0, CurrentEL ++ cmp x0, _EL2 ++ b.ne 1f ++ + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask +@@ -26,11 +32,11 @@ + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + +- // Set the stack pointer. ++ // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + +- // Jump to Rust code. ++ // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). diff -uNr 09_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs 10_privilege_level/src/_arch/aarch64/exception/asynchronous.rs --- 09_hw_debug_JTAG/src/_arch/aarch64/exception/asynchronous.rs diff --git a/10_privilege_level/src/_arch/aarch64/cpu/boot.rs b/10_privilege_level/src/_arch/aarch64/cpu/boot.rs index c2f5fe0b..02798fdc 100644 --- a/10_privilege_level/src/_arch/aarch64/cpu/boot.rs +++ b/10_privilege_level/src/_arch/aarch64/cpu/boot.rs @@ -11,26 +11,24 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; +use crate::runtime_init; use cortex_a::{asm, regs::*}; +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// Transition from EL2 to EL1. +/// Prepares the transition from EL2 to EL1. /// /// # Safety /// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. #[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - +unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -55,35 +53,27 @@ unsafe fn el2_to_el1_transition() -> ! { // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } diff --git a/10_privilege_level/src/_arch/aarch64/cpu/boot.s b/10_privilege_level/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..be81b20a --- /dev/null +++ b/10_privilege_level/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne 1f + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/10_privilege_level/src/_arch/aarch64/cpu/smp.rs b/10_privilege_level/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/10_privilege_level/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/10_privilege_level/src/bsp/raspberrypi/cpu.rs b/10_privilege_level/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/10_privilege_level/src/bsp/raspberrypi/cpu.rs +++ b/10_privilege_level/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/10_privilege_level/src/bsp/raspberrypi/link.ld b/10_privilege_level/src/bsp/raspberrypi/link.ld index 87e6a976..3b73b3c7 100644 --- a/10_privilege_level/src/bsp/raspberrypi/link.ld +++ b/10_privilege_level/src/bsp/raspberrypi/link.ld @@ -17,15 +17,28 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __rx_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/10_privilege_level/src/bsp/raspberrypi/memory.rs b/10_privilege_level/src/bsp/raspberrypi/memory.rs index 56a3306e..c6d65e36 100644 --- a/10_privilege_level/src/bsp/raspberrypi/memory.rs +++ b/10_privilege_level/src/bsp/raspberrypi/memory.rs @@ -12,8 +12,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; } @@ -50,30 +48,10 @@ pub(super) mod map { } } -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/10_privilege_level/src/cpu.rs b/10_privilege_level/src/cpu.rs index 3834f183..7a095cb1 100644 --- a/10_privilege_level/src/cpu.rs +++ b/10_privilege_level/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/10_privilege_level/src/cpu/smp.rs b/10_privilege_level/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/10_privilege_level/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/10_privilege_level/src/main.rs b/10_privilege_level/src/main.rs index 181a9aa5..0c3a0648 100644 --- a/10_privilege_level/src/main.rs +++ b/10_privilege_level/src/main.rs @@ -100,16 +100,16 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/11_virtual_mem_part1_identity_mapping/README.md b/11_virtual_mem_part1_identity_mapping/README.md index 8534ad88..b2313e1c 100644 --- a/11_virtual_mem_part1_identity_mapping/README.md +++ b/11_virtual_mem_part1_identity_mapping/README.md @@ -813,7 +813,15 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_mem_part diff -uNr 10_privilege_level/src/bsp/raspberrypi/link.ld 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld --- 10_privilege_level/src/bsp/raspberrypi/link.ld +++ 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld -@@ -31,6 +31,9 @@ +@@ -26,6 +26,7 @@ + /*********************************************************************************************** + * Code + RO Data + Global Offset Table + ***********************************************************************************************/ ++ __rx_start = .; + .text : + { + KEEP(*(.text._start)) +@@ -44,6 +45,9 @@ .rodata : ALIGN(8) { *(.rodata*) } :segment_rx .got : ALIGN(8) { *(.got) } :segment_rx @@ -927,15 +935,17 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_mem_part1_ use core::{cell::UnsafeCell, ops::RangeInclusive}; //-------------------------------------------------------------------------------------------------- -@@ -13,6 +15,7 @@ +@@ -12,6 +14,9 @@ + // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; ++ static __rx_start: UnsafeCell<()>; + static __rx_end_exclusive: UnsafeCell<()>; - ++ static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; -@@ -25,6 +28,20 @@ + } +@@ -23,6 +28,20 @@ /// The board's physical memory map. #[rustfmt::skip] pub(super) mod map { @@ -956,7 +966,7 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_mem_part1_ pub const GPIO_OFFSET: usize = 0x0020_0000; pub const UART_OFFSET: usize = 0x0020_1000; -@@ -37,6 +54,7 @@ +@@ -35,6 +54,7 @@ pub const START: usize = 0x3F00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; @@ -964,7 +974,7 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_mem_part1_ } /// Physical devices. -@@ -47,6 +65,7 @@ +@@ -45,10 +65,35 @@ pub const START: usize = 0xFE00_0000; pub const GPIO_START: usize = START + GPIO_OFFSET; pub const PL011_UART_START: usize = START + UART_OFFSET; @@ -972,10 +982,20 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_mem_part1_ } } -@@ -64,6 +83,16 @@ - unsafe { __rx_start.get() as usize } - } - + //-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Start address of the Read+Execute (RX) range. ++/// ++/// # Safety ++/// ++/// - Value is provided by the linker script and must be trusted as-is. ++#[inline(always)] ++fn rx_start() -> usize { ++ unsafe { __rx_start.get() as usize } ++} ++ +/// Exclusive end address of the Read+Execute (RX) range. +/// +/// # Safety @@ -986,10 +1006,11 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_mem_part1_ + unsafe { __rx_end_exclusive.get() as usize } +} + - //-------------------------------------------------------------------------------------------------- ++//-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- + diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_mem_part1_identity_mapping/src/bsp.rs --- 10_privilege_level/src/bsp.rs +++ 11_virtual_mem_part1_identity_mapping/src/bsp.rs @@ -1006,7 +1027,7 @@ diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_mem_part1_identity_mapping/sr diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/src/main.rs --- 10_privilege_level/src/main.rs +++ 11_virtual_mem_part1_identity_mapping/src/main.rs -@@ -108,7 +108,11 @@ +@@ -107,7 +107,11 @@ //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] @@ -1016,8 +1037,8 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_mem_part1_identity_mapping/s +#![feature(const_panic)] +#![feature(core_intrinsics)] #![feature(format_args_nl)] + #![feature(global_asm)] #![feature(panic_info_message)] - #![feature(trait_alias)] @@ -132,9 +136,17 @@ /// # Safety /// diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs index c2f5fe0b..02798fdc 100644 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.rs @@ -11,26 +11,24 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; +use crate::runtime_init; use cortex_a::{asm, regs::*}; +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// Transition from EL2 to EL1. +/// Prepares the transition from EL2 to EL1. /// /// # Safety /// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. #[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - +unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -55,35 +53,27 @@ unsafe fn el2_to_el1_transition() -> ! { // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..be81b20a --- /dev/null +++ b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne 1f + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs b/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld index 6c63ba10..854f5b38 100644 --- a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld +++ b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/link.ld @@ -17,6 +17,11 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table @@ -25,7 +30,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs index bb18dd20..81233775 100644 --- a/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs +++ b/11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory.rs @@ -97,12 +97,6 @@ fn rx_end_exclusive() -> usize { // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/11_virtual_mem_part1_identity_mapping/src/cpu.rs b/11_virtual_mem_part1_identity_mapping/src/cpu.rs index 3834f183..7a095cb1 100644 --- a/11_virtual_mem_part1_identity_mapping/src/cpu.rs +++ b/11_virtual_mem_part1_identity_mapping/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs b/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/11_virtual_mem_part1_identity_mapping/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/11_virtual_mem_part1_identity_mapping/src/main.rs b/11_virtual_mem_part1_identity_mapping/src/main.rs index 9037541d..8ce5d3d9 100644 --- a/11_virtual_mem_part1_identity_mapping/src/main.rs +++ b/11_virtual_mem_part1_identity_mapping/src/main.rs @@ -100,11 +100,10 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] @@ -114,6 +113,7 @@ #![feature(const_panic)] #![feature(core_intrinsics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/12_exceptions_part1_groundwork/README.md b/12_exceptions_part1_groundwork/README.md index 99838f3e..cc4b043f 100644 --- a/12_exceptions_part1_groundwork/README.md +++ b/12_exceptions_part1_groundwork/README.md @@ -177,47 +177,48 @@ handling code, restore the context, so that the processor can continue where it taking the exception. Context save and restore is one of the few places in system software where there is no way around -some hand-crafted assembly. Introducing `exception.S`: +some hand-crafted assembly. Introducing `exception.s`: ```asm /// Call the function provided by parameter `\handler` after saving the exception context. Provide /// the context as the first parameter to '\handler'. .macro CALL_WITH_CONTEXT handler - // Make room on the stack for the exception context. - sub sp, sp, #16 * 17 - - // Store all general purpose registers on the stack. - stp x0, x1, [sp, #16 * 0] - stp x2, x3, [sp, #16 * 1] - stp x4, x5, [sp, #16 * 2] - stp x6, x7, [sp, #16 * 3] - stp x8, x9, [sp, #16 * 4] - stp x10, x11, [sp, #16 * 5] - stp x12, x13, [sp, #16 * 6] - stp x14, x15, [sp, #16 * 7] - stp x16, x17, [sp, #16 * 8] - stp x18, x19, [sp, #16 * 9] - stp x20, x21, [sp, #16 * 10] - stp x22, x23, [sp, #16 * 11] - stp x24, x25, [sp, #16 * 12] - stp x26, x27, [sp, #16 * 13] - stp x28, x29, [sp, #16 * 14] - - // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). - mrs x1, ELR_EL1 - mrs x2, SPSR_EL1 - - stp lr, x1, [sp, #16 * 15] - str w2, [sp, #16 * 16] - - // x0 is the first argument for the function called through `\handler`. - mov x0, sp - - // Call `\handler`. - bl \handler - - // After returning from exception handling code, replay the saved context and return via `eret`. - b __exception_restore_context + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str x2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via + // `eret`. + b __exception_restore_context .endm ``` @@ -238,8 +239,6 @@ because `Rust` has no stable convention ([yet](https://github.com/rust-lang/rfcs Next, we craft the exception vector table: ```asm -.section .text - // Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. .align 11 @@ -254,23 +253,23 @@ __exception_vector_start: // // - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. .org 0x000 - CALL_WITH_CONTEXT current_el0_synchronous + CALL_WITH_CONTEXT current_el0_synchronous .org 0x080 - CALL_WITH_CONTEXT current_el0_irq + CALL_WITH_CONTEXT current_el0_irq .org 0x100 - FIQ_SUSPEND + FIQ_SUSPEND .org 0x180 - CALL_WITH_CONTEXT current_el0_serror + CALL_WITH_CONTEXT current_el0_serror // Current exception level with SP_ELx, x > 0. .org 0x200 - CALL_WITH_CONTEXT current_elx_synchronous + CALL_WITH_CONTEXT current_elx_synchronous .org 0x280 - CALL_WITH_CONTEXT current_elx_irq + CALL_WITH_CONTEXT current_elx_irq .org 0x300 - FIQ_SUSPEND + FIQ_SUSPEND .org 0x380 - CALL_WITH_CONTEXT current_elx_serror + CALL_WITH_CONTEXT current_elx_serror [...] ``` @@ -490,7 +489,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs 1 +use register::InMemoryRegister; + +// Assembly counterpart to this file. -+global_asm!(include_str!("exception.S")); ++global_asm!(include_str!("exception.s")); + +//-------------------------------------------------------------------------------------------------- +// Private Definitions @@ -731,64 +730,73 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.rs 1 + barrier::isb(barrier::SY); +} -diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.S 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S ---- 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.S -+++ 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S -@@ -0,0 +1,138 @@ +diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.s 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.s +--- 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.s ++++ 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.s +@@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + ++//-------------------------------------------------------------------------------------------------- ++// Definitions ++//-------------------------------------------------------------------------------------------------- ++ +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler -+ // Make room on the stack for the exception context. -+ sub sp, sp, #16 * 17 -+ -+ // Store all general purpose registers on the stack. -+ stp x0, x1, [sp, #16 * 0] -+ stp x2, x3, [sp, #16 * 1] -+ stp x4, x5, [sp, #16 * 2] -+ stp x6, x7, [sp, #16 * 3] -+ stp x8, x9, [sp, #16 * 4] -+ stp x10, x11, [sp, #16 * 5] -+ stp x12, x13, [sp, #16 * 6] -+ stp x14, x15, [sp, #16 * 7] -+ stp x16, x17, [sp, #16 * 8] -+ stp x18, x19, [sp, #16 * 9] -+ stp x20, x21, [sp, #16 * 10] -+ stp x22, x23, [sp, #16 * 11] -+ stp x24, x25, [sp, #16 * 12] -+ stp x26, x27, [sp, #16 * 13] -+ stp x28, x29, [sp, #16 * 14] -+ -+ // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). -+ mrs x1, ELR_EL1 -+ mrs x2, SPSR_EL1 -+ -+ stp lr, x1, [sp, #16 * 15] -+ str x2, [sp, #16 * 16] -+ -+ // x0 is the first argument for the function called through `\handler`. -+ mov x0, sp -+ -+ // Call `\handler`. -+ bl \handler -+ -+ // After returning from exception handling code, replay the saved context and return via `eret`. -+ b __exception_restore_context ++ // Make room on the stack for the exception context. ++ sub sp, sp, #16 * 17 ++ ++ // Store all general purpose registers on the stack. ++ stp x0, x1, [sp, #16 * 0] ++ stp x2, x3, [sp, #16 * 1] ++ stp x4, x5, [sp, #16 * 2] ++ stp x6, x7, [sp, #16 * 3] ++ stp x8, x9, [sp, #16 * 4] ++ stp x10, x11, [sp, #16 * 5] ++ stp x12, x13, [sp, #16 * 6] ++ stp x14, x15, [sp, #16 * 7] ++ stp x16, x17, [sp, #16 * 8] ++ stp x18, x19, [sp, #16 * 9] ++ stp x20, x21, [sp, #16 * 10] ++ stp x22, x23, [sp, #16 * 11] ++ stp x24, x25, [sp, #16 * 12] ++ stp x26, x27, [sp, #16 * 13] ++ stp x28, x29, [sp, #16 * 14] ++ ++ // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). ++ mrs x1, ELR_EL1 ++ mrs x2, SPSR_EL1 ++ ++ stp lr, x1, [sp, #16 * 15] ++ str x2, [sp, #16 * 16] ++ ++ // x0 is the first argument for the function called through `\handler`. ++ mov x0, sp ++ ++ // Call `\handler`. ++ bl \handler ++ ++ // After returning from exception handling code, replay the saved context and return via ++ // `eret`. ++ b __exception_restore_context +.endm + +.macro FIQ_SUSPEND -+1: wfe -+ b 1b ++1: wfe ++ b 1b +.endm + +//-------------------------------------------------------------------------------------------------- -+// The exception vector table. ++// Private Code +//-------------------------------------------------------------------------------------------------- +.section .text + ++//------------------------------------------------------------------------------ ++// The exception vector table. ++//------------------------------------------------------------------------------ ++ +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + @@ -803,76 +811,77 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/_arch/aarch64/exception.S 12 +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 -+ CALL_WITH_CONTEXT current_el0_synchronous ++ CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 -+ CALL_WITH_CONTEXT current_el0_irq ++ CALL_WITH_CONTEXT current_el0_irq +.org 0x100 -+ FIQ_SUSPEND ++ FIQ_SUSPEND +.org 0x180 -+ CALL_WITH_CONTEXT current_el0_serror ++ CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 -+ CALL_WITH_CONTEXT current_elx_synchronous ++ CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 -+ CALL_WITH_CONTEXT current_elx_irq ++ CALL_WITH_CONTEXT current_elx_irq +.org 0x300 -+ FIQ_SUSPEND ++ FIQ_SUSPEND +.org 0x380 -+ CALL_WITH_CONTEXT current_elx_serror ++ CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 -+ CALL_WITH_CONTEXT lower_aarch64_synchronous ++ CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 -+ CALL_WITH_CONTEXT lower_aarch64_irq ++ CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 -+ FIQ_SUSPEND ++ FIQ_SUSPEND +.org 0x580 -+ CALL_WITH_CONTEXT lower_aarch64_serror ++ CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 -+ CALL_WITH_CONTEXT lower_aarch32_synchronous ++ CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 -+ CALL_WITH_CONTEXT lower_aarch32_irq ++ CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 -+ FIQ_SUSPEND ++ FIQ_SUSPEND +.org 0x780 -+ CALL_WITH_CONTEXT lower_aarch32_serror ++ CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + -+//-------------------------------------------------------------------------------------------------- -+// Helper functions -+//-------------------------------------------------------------------------------------------------- -+.section .text -+ ++//------------------------------------------------------------------------------ ++// fn __exception_restore_context() ++//------------------------------------------------------------------------------ +__exception_restore_context: -+ ldr w19, [sp, #16 * 16] -+ ldp lr, x20, [sp, #16 * 15] -+ -+ msr SPSR_EL1, x19 -+ msr ELR_EL1, x20 -+ -+ ldp x0, x1, [sp, #16 * 0] -+ ldp x2, x3, [sp, #16 * 1] -+ ldp x4, x5, [sp, #16 * 2] -+ ldp x6, x7, [sp, #16 * 3] -+ ldp x8, x9, [sp, #16 * 4] -+ ldp x10, x11, [sp, #16 * 5] -+ ldp x12, x13, [sp, #16 * 6] -+ ldp x14, x15, [sp, #16 * 7] -+ ldp x16, x17, [sp, #16 * 8] -+ ldp x18, x19, [sp, #16 * 9] -+ ldp x20, x21, [sp, #16 * 10] -+ ldp x22, x23, [sp, #16 * 11] -+ ldp x24, x25, [sp, #16 * 12] -+ ldp x26, x27, [sp, #16 * 13] -+ ldp x28, x29, [sp, #16 * 14] -+ -+ add sp, sp, #16 * 17 -+ -+ eret ++ ldr w19, [sp, #16 * 16] ++ ldp lr, x20, [sp, #16 * 15] ++ ++ msr SPSR_EL1, x19 ++ msr ELR_EL1, x20 ++ ++ ldp x0, x1, [sp, #16 * 0] ++ ldp x2, x3, [sp, #16 * 1] ++ ldp x4, x5, [sp, #16 * 2] ++ ldp x6, x7, [sp, #16 * 3] ++ ldp x8, x9, [sp, #16 * 4] ++ ldp x10, x11, [sp, #16 * 5] ++ ldp x12, x13, [sp, #16 * 6] ++ ldp x14, x15, [sp, #16 * 7] ++ ldp x16, x17, [sp, #16 * 8] ++ ldp x18, x19, [sp, #16 * 9] ++ ldp x20, x21, [sp, #16 * 10] ++ ldp x22, x23, [sp, #16 * 11] ++ ldp x24, x25, [sp, #16 * 12] ++ ldp x26, x27, [sp, #16 * 13] ++ ldp x28, x29, [sp, #16 * 14] ++ ++ add sp, sp, #16 * 17 ++ ++ eret ++ ++.size __exception_restore_context, . - __exception_restore_context ++.type __exception_restore_context, function diff -uNr 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs --- 11_virtual_mem_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs @@ -945,15 +954,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/exception.rs 12_exceptions_p diff -uNr 11_virtual_mem_part1_identity_mapping/src/main.rs 12_exceptions_part1_groundwork/src/main.rs --- 11_virtual_mem_part1_identity_mapping/src/main.rs +++ 12_exceptions_part1_groundwork/src/main.rs -@@ -114,6 +114,7 @@ - #![feature(const_panic)] - #![feature(core_intrinsics)] - #![feature(format_args_nl)] -+#![feature(global_asm)] - #![feature(panic_info_message)] - #![feature(trait_alias)] - #![no_main] -@@ -144,6 +145,8 @@ +@@ -144,6 +144,8 @@ use driver::interface::DriverManager; use memory::mmu::interface::MMU; @@ -962,7 +963,7 @@ diff -uNr 11_virtual_mem_part1_identity_mapping/src/main.rs 12_exceptions_part1_ if let Err(string) = memory::mmu::mmu().enable_mmu_and_caching() { panic!("MMU: {}", string); } -@@ -196,13 +199,28 @@ +@@ -196,13 +198,28 @@ info!("Timer test, spinning for 1 second"); time::time_manager().spin_for(Duration::from_secs(1)); diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs index c2f5fe0b..02798fdc 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.rs @@ -11,26 +11,24 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; +use crate::runtime_init; use cortex_a::{asm, regs::*}; +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// Transition from EL2 to EL1. +/// Prepares the transition from EL2 to EL1. /// /// # Safety /// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. #[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - +unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -55,35 +53,27 @@ unsafe fn el2_to_el1_transition() -> ! { // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..be81b20a --- /dev/null +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne 1f + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S deleted file mode 100644 index fd7b1f93..00000000 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -/// Call the function provided by parameter `\handler` after saving the exception context. Provide -/// the context as the first parameter to '\handler'. -.macro CALL_WITH_CONTEXT handler - // Make room on the stack for the exception context. - sub sp, sp, #16 * 17 - - // Store all general purpose registers on the stack. - stp x0, x1, [sp, #16 * 0] - stp x2, x3, [sp, #16 * 1] - stp x4, x5, [sp, #16 * 2] - stp x6, x7, [sp, #16 * 3] - stp x8, x9, [sp, #16 * 4] - stp x10, x11, [sp, #16 * 5] - stp x12, x13, [sp, #16 * 6] - stp x14, x15, [sp, #16 * 7] - stp x16, x17, [sp, #16 * 8] - stp x18, x19, [sp, #16 * 9] - stp x20, x21, [sp, #16 * 10] - stp x22, x23, [sp, #16 * 11] - stp x24, x25, [sp, #16 * 12] - stp x26, x27, [sp, #16 * 13] - stp x28, x29, [sp, #16 * 14] - - // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). - mrs x1, ELR_EL1 - mrs x2, SPSR_EL1 - - stp lr, x1, [sp, #16 * 15] - str x2, [sp, #16 * 16] - - // x0 is the first argument for the function called through `\handler`. - mov x0, sp - - // Call `\handler`. - bl \handler - - // After returning from exception handling code, replay the saved context and return via `eret`. - b __exception_restore_context -.endm - -.macro FIQ_SUSPEND -1: wfe - b 1b -.endm - -//-------------------------------------------------------------------------------------------------- -// The exception vector table. -//-------------------------------------------------------------------------------------------------- -.section .text - -// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. -.align 11 - -// Export a symbol for the Rust code to use. -__exception_vector_start: - -// Current exception level with SP_EL0. -// -// .org sets the offset relative to section start. -// -// # Safety -// -// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. -.org 0x000 - CALL_WITH_CONTEXT current_el0_synchronous -.org 0x080 - CALL_WITH_CONTEXT current_el0_irq -.org 0x100 - FIQ_SUSPEND -.org 0x180 - CALL_WITH_CONTEXT current_el0_serror - -// Current exception level with SP_ELx, x > 0. -.org 0x200 - CALL_WITH_CONTEXT current_elx_synchronous -.org 0x280 - CALL_WITH_CONTEXT current_elx_irq -.org 0x300 - FIQ_SUSPEND -.org 0x380 - CALL_WITH_CONTEXT current_elx_serror - -// Lower exception level, AArch64 -.org 0x400 - CALL_WITH_CONTEXT lower_aarch64_synchronous -.org 0x480 - CALL_WITH_CONTEXT lower_aarch64_irq -.org 0x500 - FIQ_SUSPEND -.org 0x580 - CALL_WITH_CONTEXT lower_aarch64_serror - -// Lower exception level, AArch32 -.org 0x600 - CALL_WITH_CONTEXT lower_aarch32_synchronous -.org 0x680 - CALL_WITH_CONTEXT lower_aarch32_irq -.org 0x700 - FIQ_SUSPEND -.org 0x780 - CALL_WITH_CONTEXT lower_aarch32_serror -.org 0x800 - -//-------------------------------------------------------------------------------------------------- -// Helper functions -//-------------------------------------------------------------------------------------------------- -.section .text - -__exception_restore_context: - ldr w19, [sp, #16 * 16] - ldp lr, x20, [sp, #16 * 15] - - msr SPSR_EL1, x19 - msr ELR_EL1, x20 - - ldp x0, x1, [sp, #16 * 0] - ldp x2, x3, [sp, #16 * 1] - ldp x4, x5, [sp, #16 * 2] - ldp x6, x7, [sp, #16 * 3] - ldp x8, x9, [sp, #16 * 4] - ldp x10, x11, [sp, #16 * 5] - ldp x12, x13, [sp, #16 * 6] - ldp x14, x15, [sp, #16 * 7] - ldp x16, x17, [sp, #16 * 8] - ldp x18, x19, [sp, #16 * 9] - ldp x20, x21, [sp, #16 * 10] - ldp x22, x23, [sp, #16 * 11] - ldp x24, x25, [sp, #16 * 12] - ldp x26, x27, [sp, #16 * 13] - ldp x28, x29, [sp, #16 * 14] - - add sp, sp, #16 * 17 - - eret diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs index f08d0ce4..21f0c68c 100644 --- a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs @@ -16,7 +16,7 @@ use cortex_a::{asm, barrier, regs::*}; use register::InMemoryRegister; // Assembly counterpart to this file. -global_asm!(include_str!("exception.S")); +global_asm!(include_str!("exception.s")); //-------------------------------------------------------------------------------------------------- // Private Definitions diff --git a/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.s b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.s new file mode 100644 index 00000000..ffb1875c --- /dev/null +++ b/12_exceptions_part1_groundwork/src/_arch/aarch64/exception.s @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str x2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via + // `eret`. + b __exception_restore_context +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +.section .text + +//------------------------------------------------------------------------------ +// The exception vector table. +//------------------------------------------------------------------------------ + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + +//------------------------------------------------------------------------------ +// fn __exception_restore_context() +//------------------------------------------------------------------------------ +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 17 + + eret + +.size __exception_restore_context, . - __exception_restore_context +.type __exception_restore_context, function diff --git a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/cpu.rs b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/cpu.rs +++ b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld index 6c63ba10..854f5b38 100644 --- a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld +++ b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld @@ -17,6 +17,11 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table @@ -25,7 +30,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs index bb18dd20..81233775 100644 --- a/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs +++ b/12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory.rs @@ -97,12 +97,6 @@ fn rx_end_exclusive() -> usize { // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/12_exceptions_part1_groundwork/src/cpu.rs b/12_exceptions_part1_groundwork/src/cpu.rs index 3834f183..7a095cb1 100644 --- a/12_exceptions_part1_groundwork/src/cpu.rs +++ b/12_exceptions_part1_groundwork/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/12_exceptions_part1_groundwork/src/cpu/smp.rs b/12_exceptions_part1_groundwork/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/12_exceptions_part1_groundwork/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/12_exceptions_part1_groundwork/src/main.rs b/12_exceptions_part1_groundwork/src/main.rs index aed3404a..2af372ab 100644 --- a/12_exceptions_part1_groundwork/src/main.rs +++ b/12_exceptions_part1_groundwork/src/main.rs @@ -100,11 +100,10 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] diff --git a/13_integrated_testing/README.md b/13_integrated_testing/README.md index 65423fc4..046607ba 100644 --- a/13_integrated_testing/README.md +++ b/13_integrated_testing/README.md @@ -1157,7 +1157,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs 13_in diff -uNr 12_exceptions_part1_groundwork/src/cpu.rs 13_integrated_testing/src/cpu.rs --- 12_exceptions_part1_groundwork/src/cpu.rs +++ 13_integrated_testing/src/cpu.rs -@@ -16,3 +16,6 @@ +@@ -14,3 +14,6 @@ // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- pub use arch_cpu::{nop, wait_forever}; @@ -1194,7 +1194,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/exception.rs 13_integrated_testing/ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/lib.rs --- 12_exceptions_part1_groundwork/src/lib.rs +++ 13_integrated_testing/src/lib.rs -@@ -0,0 +1,172 @@ +@@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter @@ -1299,11 +1299,10 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li +//! +//! # Boot flow +//! -+//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -+//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. ++//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. ++//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. +//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. +//! -+//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html +//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html + +#![allow(clippy::clippy::upper_case_acronyms)] @@ -1371,7 +1370,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/lib.rs 13_integrated_testing/src/li diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/main.rs --- 12_exceptions_part1_groundwork/src/main.rs +++ 13_integrated_testing/src/main.rs -@@ -6,131 +6,12 @@ +@@ -6,130 +6,12 @@ #![doc(html_logo_url = "https://git.io/JeGIp")] //! The `kernel` binary. @@ -1469,11 +1468,10 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m -//! -//! # Boot flow -//! --//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. --//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +-//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +-//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. -//! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. -//! --//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html -//! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html - -#![allow(clippy::clippy::upper_case_acronyms)] @@ -1505,7 +1503,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m /// Early init code. /// -@@ -141,6 +22,7 @@ +@@ -140,6 +22,7 @@ /// - MMU + Data caching must be activated at the earliest. Without it, any atomic operations, /// e.g. the yet-to-be-introduced spinlocks in the device drivers (which currently employ /// NullLocks instead of spinlocks), will fail to work (properly) on the RPi SoCs. @@ -1513,7 +1511,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m unsafe fn kernel_init() -> ! { use driver::interface::DriverManager; use memory::mmu::interface::MMU; -@@ -167,9 +49,7 @@ +@@ -166,9 +49,7 @@ fn kernel_main() -> ! { use bsp::console::console; use console::interface::All; @@ -1523,7 +1521,7 @@ diff -uNr 12_exceptions_part1_groundwork/src/main.rs 13_integrated_testing/src/m info!("Booting on: {}", bsp::board_name()); -@@ -196,31 +76,6 @@ +@@ -195,31 +76,6 @@ info!(" {}. {}", i + 1, driver.compatible()); } @@ -1797,7 +1795,7 @@ diff -uNr 12_exceptions_part1_groundwork/tests/00_console_sanity.rs 13_integrate +#![no_main] +#![no_std] + -+use libkernel::{bsp, console, exception, print}; ++use libkernel::{bsp, console, cpu, exception, print}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { @@ -1820,7 +1818,7 @@ diff -uNr 12_exceptions_part1_groundwork/tests/00_console_sanity.rs 13_integrate + print!("{}", console().chars_read()); + + // The QEMU process running this test will be closed by the I/O test harness. -+ loop {} ++ cpu::wait_forever() +} diff -uNr 12_exceptions_part1_groundwork/tests/01_timer_sanity.rs 13_integrated_testing/tests/01_timer_sanity.rs diff --git a/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs b/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs index c2f5fe0b..02798fdc 100644 --- a/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs +++ b/13_integrated_testing/src/_arch/aarch64/cpu/boot.rs @@ -11,26 +11,24 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; +use crate::runtime_init; use cortex_a::{asm, regs::*}; +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// Transition from EL2 to EL1. +/// Prepares the transition from EL2 to EL1. /// /// # Safety /// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. #[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - +unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -55,35 +53,27 @@ unsafe fn el2_to_el1_transition() -> ! { // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } diff --git a/13_integrated_testing/src/_arch/aarch64/cpu/boot.s b/13_integrated_testing/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..be81b20a --- /dev/null +++ b/13_integrated_testing/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne 1f + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs b/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/13_integrated_testing/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/13_integrated_testing/src/_arch/aarch64/exception.S b/13_integrated_testing/src/_arch/aarch64/exception.S deleted file mode 100644 index fd7b1f93..00000000 --- a/13_integrated_testing/src/_arch/aarch64/exception.S +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -/// Call the function provided by parameter `\handler` after saving the exception context. Provide -/// the context as the first parameter to '\handler'. -.macro CALL_WITH_CONTEXT handler - // Make room on the stack for the exception context. - sub sp, sp, #16 * 17 - - // Store all general purpose registers on the stack. - stp x0, x1, [sp, #16 * 0] - stp x2, x3, [sp, #16 * 1] - stp x4, x5, [sp, #16 * 2] - stp x6, x7, [sp, #16 * 3] - stp x8, x9, [sp, #16 * 4] - stp x10, x11, [sp, #16 * 5] - stp x12, x13, [sp, #16 * 6] - stp x14, x15, [sp, #16 * 7] - stp x16, x17, [sp, #16 * 8] - stp x18, x19, [sp, #16 * 9] - stp x20, x21, [sp, #16 * 10] - stp x22, x23, [sp, #16 * 11] - stp x24, x25, [sp, #16 * 12] - stp x26, x27, [sp, #16 * 13] - stp x28, x29, [sp, #16 * 14] - - // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). - mrs x1, ELR_EL1 - mrs x2, SPSR_EL1 - - stp lr, x1, [sp, #16 * 15] - str x2, [sp, #16 * 16] - - // x0 is the first argument for the function called through `\handler`. - mov x0, sp - - // Call `\handler`. - bl \handler - - // After returning from exception handling code, replay the saved context and return via `eret`. - b __exception_restore_context -.endm - -.macro FIQ_SUSPEND -1: wfe - b 1b -.endm - -//-------------------------------------------------------------------------------------------------- -// The exception vector table. -//-------------------------------------------------------------------------------------------------- -.section .text - -// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. -.align 11 - -// Export a symbol for the Rust code to use. -__exception_vector_start: - -// Current exception level with SP_EL0. -// -// .org sets the offset relative to section start. -// -// # Safety -// -// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. -.org 0x000 - CALL_WITH_CONTEXT current_el0_synchronous -.org 0x080 - CALL_WITH_CONTEXT current_el0_irq -.org 0x100 - FIQ_SUSPEND -.org 0x180 - CALL_WITH_CONTEXT current_el0_serror - -// Current exception level with SP_ELx, x > 0. -.org 0x200 - CALL_WITH_CONTEXT current_elx_synchronous -.org 0x280 - CALL_WITH_CONTEXT current_elx_irq -.org 0x300 - FIQ_SUSPEND -.org 0x380 - CALL_WITH_CONTEXT current_elx_serror - -// Lower exception level, AArch64 -.org 0x400 - CALL_WITH_CONTEXT lower_aarch64_synchronous -.org 0x480 - CALL_WITH_CONTEXT lower_aarch64_irq -.org 0x500 - FIQ_SUSPEND -.org 0x580 - CALL_WITH_CONTEXT lower_aarch64_serror - -// Lower exception level, AArch32 -.org 0x600 - CALL_WITH_CONTEXT lower_aarch32_synchronous -.org 0x680 - CALL_WITH_CONTEXT lower_aarch32_irq -.org 0x700 - FIQ_SUSPEND -.org 0x780 - CALL_WITH_CONTEXT lower_aarch32_serror -.org 0x800 - -//-------------------------------------------------------------------------------------------------- -// Helper functions -//-------------------------------------------------------------------------------------------------- -.section .text - -__exception_restore_context: - ldr w19, [sp, #16 * 16] - ldp lr, x20, [sp, #16 * 15] - - msr SPSR_EL1, x19 - msr ELR_EL1, x20 - - ldp x0, x1, [sp, #16 * 0] - ldp x2, x3, [sp, #16 * 1] - ldp x4, x5, [sp, #16 * 2] - ldp x6, x7, [sp, #16 * 3] - ldp x8, x9, [sp, #16 * 4] - ldp x10, x11, [sp, #16 * 5] - ldp x12, x13, [sp, #16 * 6] - ldp x14, x15, [sp, #16 * 7] - ldp x16, x17, [sp, #16 * 8] - ldp x18, x19, [sp, #16 * 9] - ldp x20, x21, [sp, #16 * 10] - ldp x22, x23, [sp, #16 * 11] - ldp x24, x25, [sp, #16 * 12] - ldp x26, x27, [sp, #16 * 13] - ldp x28, x29, [sp, #16 * 14] - - add sp, sp, #16 * 17 - - eret diff --git a/13_integrated_testing/src/_arch/aarch64/exception.rs b/13_integrated_testing/src/_arch/aarch64/exception.rs index e67841f9..1ee0a198 100644 --- a/13_integrated_testing/src/_arch/aarch64/exception.rs +++ b/13_integrated_testing/src/_arch/aarch64/exception.rs @@ -16,7 +16,7 @@ use cortex_a::{barrier, regs::*}; use register::InMemoryRegister; // Assembly counterpart to this file. -global_asm!(include_str!("exception.S")); +global_asm!(include_str!("exception.s")); //-------------------------------------------------------------------------------------------------- // Private Definitions diff --git a/13_integrated_testing/src/_arch/aarch64/exception.s b/13_integrated_testing/src/_arch/aarch64/exception.s new file mode 100644 index 00000000..ffb1875c --- /dev/null +++ b/13_integrated_testing/src/_arch/aarch64/exception.s @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str x2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via + // `eret`. + b __exception_restore_context +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +.section .text + +//------------------------------------------------------------------------------ +// The exception vector table. +//------------------------------------------------------------------------------ + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + +//------------------------------------------------------------------------------ +// fn __exception_restore_context() +//------------------------------------------------------------------------------ +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 17 + + eret + +.size __exception_restore_context, . - __exception_restore_context +.type __exception_restore_context, function diff --git a/13_integrated_testing/src/bsp/raspberrypi/cpu.rs b/13_integrated_testing/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/13_integrated_testing/src/bsp/raspberrypi/cpu.rs +++ b/13_integrated_testing/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/13_integrated_testing/src/bsp/raspberrypi/link.ld b/13_integrated_testing/src/bsp/raspberrypi/link.ld index 6c63ba10..854f5b38 100644 --- a/13_integrated_testing/src/bsp/raspberrypi/link.ld +++ b/13_integrated_testing/src/bsp/raspberrypi/link.ld @@ -17,6 +17,11 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table @@ -25,7 +30,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/13_integrated_testing/src/bsp/raspberrypi/memory.rs b/13_integrated_testing/src/bsp/raspberrypi/memory.rs index bb18dd20..81233775 100644 --- a/13_integrated_testing/src/bsp/raspberrypi/memory.rs +++ b/13_integrated_testing/src/bsp/raspberrypi/memory.rs @@ -97,12 +97,6 @@ fn rx_end_exclusive() -> usize { // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/13_integrated_testing/src/cpu.rs b/13_integrated_testing/src/cpu.rs index 36b6a9d4..61c411fc 100644 --- a/13_integrated_testing/src/cpu.rs +++ b/13_integrated_testing/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/13_integrated_testing/src/cpu/smp.rs b/13_integrated_testing/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/13_integrated_testing/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/13_integrated_testing/src/lib.rs b/13_integrated_testing/src/lib.rs index b094daf3..745122f8 100644 --- a/13_integrated_testing/src/lib.rs +++ b/13_integrated_testing/src/lib.rs @@ -102,11 +102,10 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] diff --git a/13_integrated_testing/tests/00_console_sanity.rs b/13_integrated_testing/tests/00_console_sanity.rs index c3754aa9..ad7fd2bf 100644 --- a/13_integrated_testing/tests/00_console_sanity.rs +++ b/13_integrated_testing/tests/00_console_sanity.rs @@ -8,7 +8,7 @@ #![no_main] #![no_std] -use libkernel::{bsp, console, exception, print}; +use libkernel::{bsp, console, cpu, exception, print}; #[no_mangle] unsafe fn kernel_init() -> ! { @@ -31,5 +31,5 @@ unsafe fn kernel_init() -> ! { print!("{}", console().chars_read()); // The QEMU process running this test will be closed by the I/O test harness. - loop {} + cpu::wait_forever() } diff --git a/14_exceptions_part2_peripheral_IRQs/README.md b/14_exceptions_part2_peripheral_IRQs/README.md index 54e33071..b78da716 100644 --- a/14_exceptions_part2_peripheral_IRQs/README.md +++ b/14_exceptions_part2_peripheral_IRQs/README.md @@ -744,6 +744,40 @@ Minipush 1.0 ## Diff to previous ```diff +diff -uNr 13_integrated_testing/src/_arch/aarch64/cpu/smp.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs +--- 13_integrated_testing/src/_arch/aarch64/cpu/smp.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs +@@ -0,0 +1,29 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! Architectural symmetric multiprocessing. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::cpu::smp::arch_smp ++ ++use cortex_a::regs::*; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Return the executing core's id. ++#[inline(always)] ++pub fn core_id() -> T ++where ++ T: From, ++{ ++ const CORE_MASK: u64 = 0b11; ++ ++ T::from((MPIDR_EL1.get() & CORE_MASK) as u8) ++} + diff -uNr 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs --- 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs +++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs @@ -857,19 +891,6 @@ diff -uNr 13_integrated_testing/src/_arch/aarch64/exception.rs 14_exceptions_par #[no_mangle] -diff -uNr 13_integrated_testing/src/_arch/aarch64/memory/mmu.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs ---- 13_integrated_testing/src/_arch/aarch64/memory/mmu.rs -+++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs -@@ -149,7 +149,7 @@ - barrier::isb(barrier::SY); - - // Enable the MMU and turn on data and instruction caching. -- SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); -+ SCTLR_EL1.modify(SCTLR_EL1::M::Enable); - - // Force MMU init to complete before next instruction. - barrier::isb(barrier::SY); - diff -uNr 13_integrated_testing/src/bsp/device_driver/arm/gicv2/gicc.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs --- 13_integrated_testing/src/bsp/device_driver/arm/gicv2/gicc.rs +++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs @@ -2136,6 +2157,38 @@ diff -uNr 13_integrated_testing/src/bsp/raspberrypi.rs 14_exceptions_part2_perip //-------------------------------------------------------------------------------------------------- // Public Code +diff -uNr 13_integrated_testing/src/cpu/smp.rs 14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs +--- 13_integrated_testing/src/cpu/smp.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs +@@ -0,0 +1,14 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2018-2021 Andre Richter ++ ++//! Symmetric multiprocessing. ++ ++#[cfg(target_arch = "aarch64")] ++#[path = "../_arch/aarch64/cpu/smp.rs"] ++mod arch_smp; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++pub use arch_smp::core_id; + +diff -uNr 13_integrated_testing/src/cpu.rs 14_exceptions_part2_peripheral_IRQs/src/cpu.rs +--- 13_integrated_testing/src/cpu.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/cpu.rs +@@ -10,6 +10,8 @@ + + mod boot; + ++pub mod smp; ++ + //-------------------------------------------------------------------------------------------------- + // Architectural Public Reexports + //-------------------------------------------------------------------------------------------------- + diff -uNr 13_integrated_testing/src/driver.rs 14_exceptions_part2_peripheral_IRQs/src/driver.rs --- 13_integrated_testing/src/driver.rs +++ 14_exceptions_part2_peripheral_IRQs/src/driver.rs @@ -2309,7 +2362,7 @@ diff -uNr 13_integrated_testing/src/exception/asynchronous.rs 14_exceptions_part diff -uNr 13_integrated_testing/src/lib.rs 14_exceptions_part2_peripheral_IRQs/src/lib.rs --- 13_integrated_testing/src/lib.rs +++ 14_exceptions_part2_peripheral_IRQs/src/lib.rs -@@ -111,6 +111,7 @@ +@@ -110,6 +110,7 @@ #![allow(clippy::clippy::upper_case_acronyms)] #![allow(incomplete_features)] @@ -2317,7 +2370,7 @@ diff -uNr 13_integrated_testing/src/lib.rs 14_exceptions_part2_peripheral_IRQs/s #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -138,6 +139,7 @@ +@@ -137,6 +138,7 @@ pub mod exception; pub mod memory; pub mod print; diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs index c2f5fe0b..02798fdc 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs @@ -11,26 +11,24 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; +use crate::runtime_init; use cortex_a::{asm, regs::*}; +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// Transition from EL2 to EL1. +/// Prepares the transition from EL2 to EL1. /// /// # Safety /// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. #[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - +unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -55,35 +53,27 @@ unsafe fn el2_to_el1_transition() -> ! { // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..be81b20a --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne 1f + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S deleted file mode 100644 index fd7b1f93..00000000 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -/// Call the function provided by parameter `\handler` after saving the exception context. Provide -/// the context as the first parameter to '\handler'. -.macro CALL_WITH_CONTEXT handler - // Make room on the stack for the exception context. - sub sp, sp, #16 * 17 - - // Store all general purpose registers on the stack. - stp x0, x1, [sp, #16 * 0] - stp x2, x3, [sp, #16 * 1] - stp x4, x5, [sp, #16 * 2] - stp x6, x7, [sp, #16 * 3] - stp x8, x9, [sp, #16 * 4] - stp x10, x11, [sp, #16 * 5] - stp x12, x13, [sp, #16 * 6] - stp x14, x15, [sp, #16 * 7] - stp x16, x17, [sp, #16 * 8] - stp x18, x19, [sp, #16 * 9] - stp x20, x21, [sp, #16 * 10] - stp x22, x23, [sp, #16 * 11] - stp x24, x25, [sp, #16 * 12] - stp x26, x27, [sp, #16 * 13] - stp x28, x29, [sp, #16 * 14] - - // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). - mrs x1, ELR_EL1 - mrs x2, SPSR_EL1 - - stp lr, x1, [sp, #16 * 15] - str x2, [sp, #16 * 16] - - // x0 is the first argument for the function called through `\handler`. - mov x0, sp - - // Call `\handler`. - bl \handler - - // After returning from exception handling code, replay the saved context and return via `eret`. - b __exception_restore_context -.endm - -.macro FIQ_SUSPEND -1: wfe - b 1b -.endm - -//-------------------------------------------------------------------------------------------------- -// The exception vector table. -//-------------------------------------------------------------------------------------------------- -.section .text - -// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. -.align 11 - -// Export a symbol for the Rust code to use. -__exception_vector_start: - -// Current exception level with SP_EL0. -// -// .org sets the offset relative to section start. -// -// # Safety -// -// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. -.org 0x000 - CALL_WITH_CONTEXT current_el0_synchronous -.org 0x080 - CALL_WITH_CONTEXT current_el0_irq -.org 0x100 - FIQ_SUSPEND -.org 0x180 - CALL_WITH_CONTEXT current_el0_serror - -// Current exception level with SP_ELx, x > 0. -.org 0x200 - CALL_WITH_CONTEXT current_elx_synchronous -.org 0x280 - CALL_WITH_CONTEXT current_elx_irq -.org 0x300 - FIQ_SUSPEND -.org 0x380 - CALL_WITH_CONTEXT current_elx_serror - -// Lower exception level, AArch64 -.org 0x400 - CALL_WITH_CONTEXT lower_aarch64_synchronous -.org 0x480 - CALL_WITH_CONTEXT lower_aarch64_irq -.org 0x500 - FIQ_SUSPEND -.org 0x580 - CALL_WITH_CONTEXT lower_aarch64_serror - -// Lower exception level, AArch32 -.org 0x600 - CALL_WITH_CONTEXT lower_aarch32_synchronous -.org 0x680 - CALL_WITH_CONTEXT lower_aarch32_irq -.org 0x700 - FIQ_SUSPEND -.org 0x780 - CALL_WITH_CONTEXT lower_aarch32_serror -.org 0x800 - -//-------------------------------------------------------------------------------------------------- -// Helper functions -//-------------------------------------------------------------------------------------------------- -.section .text - -__exception_restore_context: - ldr w19, [sp, #16 * 16] - ldp lr, x20, [sp, #16 * 15] - - msr SPSR_EL1, x19 - msr ELR_EL1, x20 - - ldp x0, x1, [sp, #16 * 0] - ldp x2, x3, [sp, #16 * 1] - ldp x4, x5, [sp, #16 * 2] - ldp x6, x7, [sp, #16 * 3] - ldp x8, x9, [sp, #16 * 4] - ldp x10, x11, [sp, #16 * 5] - ldp x12, x13, [sp, #16 * 6] - ldp x14, x15, [sp, #16 * 7] - ldp x16, x17, [sp, #16 * 8] - ldp x18, x19, [sp, #16 * 9] - ldp x20, x21, [sp, #16 * 10] - ldp x22, x23, [sp, #16 * 11] - ldp x24, x25, [sp, #16 * 12] - ldp x26, x27, [sp, #16 * 13] - ldp x28, x29, [sp, #16 * 14] - - add sp, sp, #16 * 17 - - eret diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs index de4f1f18..5c44abb7 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs @@ -17,7 +17,7 @@ use cortex_a::{barrier, regs::*}; use register::InMemoryRegister; // Assembly counterpart to this file. -global_asm!(include_str!("exception.S")); +global_asm!(include_str!("exception.s")); //-------------------------------------------------------------------------------------------------- // Private Definitions diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.s b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.s new file mode 100644 index 00000000..ffb1875c --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.s @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str x2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via + // `eret`. + b __exception_restore_context +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +.section .text + +//------------------------------------------------------------------------------ +// The exception vector table. +//------------------------------------------------------------------------------ + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + +//------------------------------------------------------------------------------ +// fn __exception_restore_context() +//------------------------------------------------------------------------------ +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 17 + + eret + +.size __exception_restore_context, . - __exception_restore_context +.type __exception_restore_context, function diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs index cbf04a98..29e8125d 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs @@ -149,7 +149,7 @@ impl memory::mmu::interface::MMU for MemoryManagementUnit { barrier::isb(barrier::SY); // Enable the MMU and turn on data and instruction caching. - SCTLR_EL1.modify(SCTLR_EL1::M::Enable); + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); // Force MMU init to complete before next instruction. barrier::isb(barrier::SY); diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld index 6c63ba10..854f5b38 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld @@ -17,6 +17,11 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table @@ -25,7 +30,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs index d07f2dc3..8a58a17e 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs @@ -101,12 +101,6 @@ fn rx_end_exclusive() -> usize { // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/14_exceptions_part2_peripheral_IRQs/src/lib.rs b/14_exceptions_part2_peripheral_IRQs/src/lib.rs index 4ea416fd..c4e74460 100644 --- a/14_exceptions_part2_peripheral_IRQs/src/lib.rs +++ b/14_exceptions_part2_peripheral_IRQs/src/lib.rs @@ -102,11 +102,10 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] diff --git a/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs b/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs index c3754aa9..ad7fd2bf 100644 --- a/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs +++ b/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs @@ -8,7 +8,7 @@ #![no_main] #![no_std] -use libkernel::{bsp, console, exception, print}; +use libkernel::{bsp, console, cpu, exception, print}; #[no_mangle] unsafe fn kernel_init() -> ! { @@ -31,5 +31,5 @@ unsafe fn kernel_init() -> ! { print!("{}", console().chars_read()); // The QEMU process running this test will be closed by the I/O test harness. - loop {} + cpu::wait_forever() } diff --git a/15_virtual_mem_part2_mmio_remap/README.md b/15_virtual_mem_part2_mmio_remap/README.md index c512c51c..1745d4b1 100644 --- a/15_virtual_mem_part2_mmio_remap/README.md +++ b/15_virtual_mem_part2_mmio_remap/README.md @@ -369,19 +369,6 @@ Minipush 1.0 ## Diff to previous ```diff -diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs ---- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/boot.rs -+++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs -@@ -56,7 +56,7 @@ - ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. -- SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); -+ SP_EL1.set(bsp::memory::phys_boot_core_stack_end().into_usize() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() - diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs --- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs +++ 15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs @@ -805,15 +792,6 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15 self.configure_translation_control(); -@@ -149,7 +140,7 @@ - barrier::isb(barrier::SY); - - // Enable the MMU and turn on data and instruction caching. -- SCTLR_EL1.modify(SCTLR_EL1::M::Enable); -+ SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); - - // Force MMU init to complete before next instruction. - barrier::isb(barrier::SY); @@ -162,22 +153,3 @@ SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) } @@ -1478,7 +1456,19 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs 15_v diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld --- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld +++ 15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld -@@ -37,6 +37,7 @@ +@@ -17,11 +17,6 @@ + SECTIONS + { + . = __rpi_load_addr; +- /* ^ */ +- /* | stack */ +- /* | growth */ +- /* | direction */ +- __boot_core_stack_end_exclusive = .; /* | */ + + /*********************************************************************************************** + * Code + RO Data + Global Offset Table +@@ -51,6 +46,7 @@ /*********************************************************************************************** * Data + BSS ***********************************************************************************************/ @@ -1486,7 +1476,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 15_vir .data : { *(.data*) } :segment_rw /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ -@@ -49,4 +50,21 @@ +@@ -63,4 +59,23 @@ . += 8; /* Fill for the bss == 0 case, so that __bss_start <= __bss_end_inclusive holds */ __bss_end_inclusive = . - 8; } :NONE @@ -1504,9 +1494,11 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 15_vir + /*********************************************************************************************** + * Boot Core Stack + ***********************************************************************************************/ -+ __boot_core_stack_start = .; -+ . += 512K; -+ __boot_core_stack_end_exclusive = .; ++ __boot_core_stack_start = .; /* ^ */ ++ /* | stack */ ++ . += 512K; /* | growth */ ++ /* | direction */ ++ __boot_core_stack_end_exclusive = .; /* | */ } diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs @@ -2002,19 +1994,6 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 15_v } //-------------------------------------------------------------------------------------------------- -@@ -103,8 +193,10 @@ - - /// Exclusive end address of the boot core's stack. - #[inline(always)] --pub fn boot_core_stack_end() -> usize { -- rx_start() -+pub fn phys_boot_core_stack_end() -> Address { -+ // The binary is still identity mapped, so we don't need to convert here. -+ let end = virt_boot_core_stack_start().into_usize() + boot_core_stack_size(); -+ Address::new(end) - } - - /// Return the inclusive range spanning the .bss section. diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs 15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi.rs --- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs @@ -2138,7 +2117,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/driver.rs 15_virtual_mem_part2 diff -uNr 14_exceptions_part2_peripheral_IRQs/src/lib.rs 15_virtual_mem_part2_mmio_remap/src/lib.rs --- 14_exceptions_part2_peripheral_IRQs/src/lib.rs +++ 15_virtual_mem_part2_mmio_remap/src/lib.rs -@@ -112,6 +112,8 @@ +@@ -111,6 +111,8 @@ #![allow(clippy::clippy::upper_case_acronyms)] #![allow(incomplete_features)] #![feature(asm)] @@ -2147,7 +2126,7 @@ diff -uNr 14_exceptions_part2_peripheral_IRQs/src/lib.rs 15_virtual_mem_part2_mm #![feature(const_fn_fn_ptr_basics)] #![feature(const_generics)] #![feature(const_panic)] -@@ -133,6 +135,7 @@ +@@ -132,6 +134,7 @@ mod synchronization; pub mod bsp; diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs index 27696514..02798fdc 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.rs @@ -11,26 +11,24 @@ //! //! crate::cpu::boot::arch_boot -use crate::{bsp, cpu}; +use crate::runtime_init; use cortex_a::{asm, regs::*}; +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + //-------------------------------------------------------------------------------------------------- // Private Code //-------------------------------------------------------------------------------------------------- -/// Transition from EL2 to EL1. +/// Prepares the transition from EL2 to EL1. /// /// # Safety /// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. /// - The HW state of EL1 must be prepared in a sound way. -/// - Exception return from EL2 must must continue execution in EL1 with -/// `runtime_init::runtime_init()`. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. #[inline(always)] -unsafe fn el2_to_el1_transition() -> ! { - use crate::runtime_init; - +unsafe fn prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr: u64) { // Enable timer counter registers for EL1. CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); @@ -55,35 +53,27 @@ unsafe fn el2_to_el1_transition() -> ! { // Second, let the link register point to runtime_init(). ELR_EL2.set(runtime_init::runtime_init as *const () as u64); - // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. - SP_EL1.set(bsp::memory::phys_boot_core_stack_end().into_usize() as u64); - - // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. - asm::eret() + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(phys_boot_core_stack_end_exclusive_addr); } //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// The entry of the `kernel` binary. +/// The Rust entry of the `kernel` binary. /// -/// The function must be named `_start`, because the linker is looking for this exact name. +/// The function is called from the assembly `_start` function. /// /// # Safety /// -/// - Linker script must ensure to place this function where it is expected by the target machine. -/// - We have to hope that the compiler omits any stack pointer usage, because we are not setting up -/// a stack for EL2. +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - Exception return from EL2 must must continue execution in EL1 with `runtime_init()`. #[no_mangle] -pub unsafe fn _start() -> ! { - // Expect the boot core to start in EL2. - if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) - && (CurrentEL.get() == CurrentEL::EL::EL2.value) - { - el2_to_el1_transition() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe extern "C" fn _start_rust(phys_boot_core_stack_end_exclusive_addr: u64) -> ! { + prepare_el2_to_el1_transition(phys_boot_core_stack_end_exclusive_addr); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() } diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..be81b20a --- /dev/null +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne 1f + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. This ensures that any code in EL2 that needs the stack will work. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. x0 holds the function argument provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.S b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.S deleted file mode 100644 index fd7b1f93..00000000 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.S +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -/// Call the function provided by parameter `\handler` after saving the exception context. Provide -/// the context as the first parameter to '\handler'. -.macro CALL_WITH_CONTEXT handler - // Make room on the stack for the exception context. - sub sp, sp, #16 * 17 - - // Store all general purpose registers on the stack. - stp x0, x1, [sp, #16 * 0] - stp x2, x3, [sp, #16 * 1] - stp x4, x5, [sp, #16 * 2] - stp x6, x7, [sp, #16 * 3] - stp x8, x9, [sp, #16 * 4] - stp x10, x11, [sp, #16 * 5] - stp x12, x13, [sp, #16 * 6] - stp x14, x15, [sp, #16 * 7] - stp x16, x17, [sp, #16 * 8] - stp x18, x19, [sp, #16 * 9] - stp x20, x21, [sp, #16 * 10] - stp x22, x23, [sp, #16 * 11] - stp x24, x25, [sp, #16 * 12] - stp x26, x27, [sp, #16 * 13] - stp x28, x29, [sp, #16 * 14] - - // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). - mrs x1, ELR_EL1 - mrs x2, SPSR_EL1 - - stp lr, x1, [sp, #16 * 15] - str x2, [sp, #16 * 16] - - // x0 is the first argument for the function called through `\handler`. - mov x0, sp - - // Call `\handler`. - bl \handler - - // After returning from exception handling code, replay the saved context and return via `eret`. - b __exception_restore_context -.endm - -.macro FIQ_SUSPEND -1: wfe - b 1b -.endm - -//-------------------------------------------------------------------------------------------------- -// The exception vector table. -//-------------------------------------------------------------------------------------------------- -.section .text - -// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. -.align 11 - -// Export a symbol for the Rust code to use. -__exception_vector_start: - -// Current exception level with SP_EL0. -// -// .org sets the offset relative to section start. -// -// # Safety -// -// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. -.org 0x000 - CALL_WITH_CONTEXT current_el0_synchronous -.org 0x080 - CALL_WITH_CONTEXT current_el0_irq -.org 0x100 - FIQ_SUSPEND -.org 0x180 - CALL_WITH_CONTEXT current_el0_serror - -// Current exception level with SP_ELx, x > 0. -.org 0x200 - CALL_WITH_CONTEXT current_elx_synchronous -.org 0x280 - CALL_WITH_CONTEXT current_elx_irq -.org 0x300 - FIQ_SUSPEND -.org 0x380 - CALL_WITH_CONTEXT current_elx_serror - -// Lower exception level, AArch64 -.org 0x400 - CALL_WITH_CONTEXT lower_aarch64_synchronous -.org 0x480 - CALL_WITH_CONTEXT lower_aarch64_irq -.org 0x500 - FIQ_SUSPEND -.org 0x580 - CALL_WITH_CONTEXT lower_aarch64_serror - -// Lower exception level, AArch32 -.org 0x600 - CALL_WITH_CONTEXT lower_aarch32_synchronous -.org 0x680 - CALL_WITH_CONTEXT lower_aarch32_irq -.org 0x700 - FIQ_SUSPEND -.org 0x780 - CALL_WITH_CONTEXT lower_aarch32_serror -.org 0x800 - -//-------------------------------------------------------------------------------------------------- -// Helper functions -//-------------------------------------------------------------------------------------------------- -.section .text - -__exception_restore_context: - ldr w19, [sp, #16 * 16] - ldp lr, x20, [sp, #16 * 15] - - msr SPSR_EL1, x19 - msr ELR_EL1, x20 - - ldp x0, x1, [sp, #16 * 0] - ldp x2, x3, [sp, #16 * 1] - ldp x4, x5, [sp, #16 * 2] - ldp x6, x7, [sp, #16 * 3] - ldp x8, x9, [sp, #16 * 4] - ldp x10, x11, [sp, #16 * 5] - ldp x12, x13, [sp, #16 * 6] - ldp x14, x15, [sp, #16 * 7] - ldp x16, x17, [sp, #16 * 8] - ldp x18, x19, [sp, #16 * 9] - ldp x20, x21, [sp, #16 * 10] - ldp x22, x23, [sp, #16 * 11] - ldp x24, x25, [sp, #16 * 12] - ldp x26, x27, [sp, #16 * 13] - ldp x28, x29, [sp, #16 * 14] - - add sp, sp, #16 * 17 - - eret diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs index b9338734..4a2c8de9 100644 --- a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.rs @@ -21,7 +21,7 @@ use cortex_a::{barrier, regs::*}; use register::InMemoryRegister; // Assembly counterpart to this file. -global_asm!(include_str!("exception.S")); +global_asm!(include_str!("exception.s")); //-------------------------------------------------------------------------------------------------- // Private Definitions diff --git a/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.s b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.s new file mode 100644 index 00000000..ffb1875c --- /dev/null +++ b/15_virtual_mem_part2_mmio_remap/src/_arch/aarch64/exception.s @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str x2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via + // `eret`. + b __exception_restore_context +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +.section .text + +//------------------------------------------------------------------------------ +// The exception vector table. +//------------------------------------------------------------------------------ + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + +//------------------------------------------------------------------------------ +// fn __exception_restore_context() +//------------------------------------------------------------------------------ +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 17 + + eret + +.size __exception_restore_context, . - __exception_restore_context +.type __exception_restore_context, function diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld index f2b5d05d..7fddf3ec 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/link.ld @@ -25,7 +25,16 @@ SECTIONS .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx @@ -64,7 +73,9 @@ SECTIONS /*********************************************************************************************** * Boot Core Stack ***********************************************************************************************/ - __boot_core_stack_start = .; - . += 512K; - __boot_core_stack_end_exclusive = .; + __boot_core_stack_start = .; /* ^ */ + /* | stack */ + . += 512K; /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ } diff --git a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs index 19545c00..7f571e08 100644 --- a/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs +++ b/15_virtual_mem_part2_mmio_remap/src/bsp/raspberrypi/memory.rs @@ -191,14 +191,6 @@ fn phys_addr_space_end() -> Address { // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn phys_boot_core_stack_end() -> Address { - // The binary is still identity mapped, so we don't need to convert here. - let end = virt_boot_core_stack_start().into_usize() + boot_core_stack_size(); - Address::new(end) -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/15_virtual_mem_part2_mmio_remap/src/lib.rs b/15_virtual_mem_part2_mmio_remap/src/lib.rs index 30f684af..3346134a 100644 --- a/15_virtual_mem_part2_mmio_remap/src/lib.rs +++ b/15_virtual_mem_part2_mmio_remap/src/lib.rs @@ -102,11 +102,10 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] diff --git a/15_virtual_mem_part2_mmio_remap/tests/00_console_sanity.rs b/15_virtual_mem_part2_mmio_remap/tests/00_console_sanity.rs index c3754aa9..ad7fd2bf 100644 --- a/15_virtual_mem_part2_mmio_remap/tests/00_console_sanity.rs +++ b/15_virtual_mem_part2_mmio_remap/tests/00_console_sanity.rs @@ -8,7 +8,7 @@ #![no_main] #![no_std] -use libkernel::{bsp, console, exception, print}; +use libkernel::{bsp, console, cpu, exception, print}; #[no_mangle] unsafe fn kernel_init() -> ! { @@ -31,5 +31,5 @@ unsafe fn kernel_init() -> ! { print!("{}", console().chars_read()); // The QEMU process running this test will be closed by the I/O test harness. - loop {} + cpu::wait_forever() } diff --git a/X1_JTAG_boot/jtag_boot_rpi3.img b/X1_JTAG_boot/jtag_boot_rpi3.img index d45cc1ff0028c6d32ecd176a828fc440da668d97..8eed7dba0e7d56b27a14d14f981f77b599d9b6c3 100755 GIT binary patch delta 1590 zcmb7EUrbw782?UhYkM~k3jM=??zX~g3uY6_27^H_-6Ct9uvLs?QvpfX9}*+Phb@uN z8XuU!gyWe*At8mAd2vKZFBQ+?jD=`IOUA$XV6&9#Ilb3}?L|Gwmvg`G z{Jwv`@0@$ryY4+W0X26VTJb#}f;KN`R>A?q2b zq+mCMf{PwFICCV0G-65At0_YoC?!4ITPRFcxjHmY#&a%{IqjEw!?oDmSc;dO*QK}~ zNpTMFAuU|8gZ-yDuKj26i_ni9OEa5s^Dmo`ukMqNbq7#5x}-a$N6~0Q_I+eN;lVor zhO+=ZO&$i^bGr@?Es|$)UPSZJ@to60)8Qg|WB2}vTaTgje_JrKBEGWoiFHp#Qru8gH1!nKYKuVs9A*+6P%PzJgMuIZr@~}^pdH(a&zw!c=Z!bO6-FaPXj&MKRuTFDoN%EWt~sqtLg5AY-$~x zHWUTYeGAi$a*U{+JPomTD0Q1x^pb<7^W;}%hiY+`-Ge)Iq2SvQ(wE=0BjmykE)?{E zoBWn<8Avvzc%2yYn@t{G2eC6$7v6`MPnJj6HxC~K`41u1pz7>Qp!!zJ{6~ymCUlC! ztV+XTQ_B5EQ_71BD?EZ$I@@6l6$sViPAyGoZ@|AkkVn{T7@lFt8>T&)jtVknYA&Nq zC+Ya&`o|>Jj$^J0$LU0tom%NAZ7b>2Ng21sDw1#R-lMYT;=dWqnCVbfDU0{YK|1&O z#b6+7vhP+e88-`b0N2f4%lrWxpv)zTAQkxR@uCDx#aPO>RBq>_#bRlEh%^sg;t($1 zQtFmD&Fw7@r2cHtOLf!+dYHXLi(?g-e!tSUOryF_kQAN;rwp zZ=la$c|a#EeGS-|mPh7NauuVWqoSn(tR=Bdxd(G~nBitAj*Wuh5DEpMqaL$y#yMeX zkqY~+2M|-EY1wKh6(!2URGc%=Lw?G#C0lO*%+h|R+$OQcr-cpP3^fz7+{zr;yB5hz zfz|!GMtS8&)Yo34>b}Fz*^eK4x}w7VLSvBLm07Y;U`5vmzw3KrWcR~=hO;EGs~WWt zM`3lvUq#9pZ7s7-saSetGV4*_{p4by;QPN`57aEVYSf*@(PUvWa{M4D+0Pd%`(0I! zsCuSnmG34;q{p=D$1^KhTo>5%;B0`aG}inoRwoH@WY= zbIj?!)kE*!MnZUQuk3%>hFC`n|%6uIsXckgJa6uCzi4%V8~U$4PVsNi2G8 zAWbcas`*2f8fuAE!&|~h>VsTs8*PWhg7fgB_2XxJtt_}y&DQ-lRLxFRO(3k(O3r$T z_uGWf|6N3y`X-WmZ!_`g&CNvXBlz65lltZ}wnKK>IsbD0Z`8R~&+-VF*oH-I z*v3u|!r_9KX#4zP!BJ|_{7@|oIYWi4Tf8-$i?Ka4%pU0a!v_@Jl@vM%H>6rDDJ3ab zE#wuwLSKjBBEvCWL`hiqd1$F?Wzyc^%mFXdt2F&i}h^K z7K^^&g)_xR_L2be39)ELEWk3q>yw5Ynch#-Vmx5n8jBwLxhpN&BJpl#NrPx3al=;5 zk$8u$PiB4OLR6e0@!eS)c9$rjm9BK~PFFfi9UCGgeFRUk;3Q-h8^jYkF79GK@k|Dp z;2k(q;d#HOMrDH>25eRD**r(#SLvd);qUpF=JwzfH}8mKvbWIrJ2lEFgX{ zo!v&g@1u10YKc0$cq-bn?b66s6c(Kds{Xw*>}uc1dQh&Uo1}X_e8Qc?RE9KluKLl( zggDI0GfEe5T5*!UN!P&c3Y67mOu>veN@G#7(87#f(gbC84X5$13zp?f14%#XwX2O7 zPweFSS!80ATk^~0k##H~7^-4$)3rVS6b3j@(XsO+{w8d>_P7#N>{vO88{Csk=IsZU z+Mr*qpaT!rSl&dNp}o8* zwCXl}cqiG>#$&Nlvg4mvC#dC0$NxqBz_S$cQ#ai6x$b?`x1gBW4|>cs!R;8gvz(=v z%k)n0H2^anSsD<`-Pibvv6XV}ABE=PWr!! z@GQ(!+{-g0@7mwpOGpSCxTk`94#Je; Pmk7VsRqkDdTZ;cr4O^>0 diff --git a/X1_JTAG_boot/jtag_boot_rpi4.img b/X1_JTAG_boot/jtag_boot_rpi4.img index 080ff5e6dc2f097df2754fccfcc5f3acd0ef3fbb..cbdd1b808832df622e2be50a4bb24e5437756355 100755 GIT binary patch delta 1876 zcmZ9MUu;uV9LIm>-WGb-!f4mKb{(*G14_#f<`uR|LT|f~xB-cjDi|NODr#U`p!i16 z+Xal#j-=Ov&fo(bJV=`nS;i6<|ImRiOEd-*eKTXm1X~CZ60;Jw>+jrq`6DN3PVf1B ze}B&Bch328OKFQT3LQ5B7Wtz%f&Ee7DnkItJiz`Y0Cg#VyVuX(&@{~hV1p+|$pWYS z;Qyt>_x`GN!gU+SRjn4LFh=?u#<(d#Bm|j6$O@B(F-{cxY0Qxf9bdAyW<@3uc7qgn z!>2sUgynXKbfb8Nhsgs>EX(P0C>Bu2#2bz~_J&)RNFj7e9c+B$^Vc?r97I2XgUOfC zFLd0c2e+e-gZXtiq*nCPlMSxa{VJI2Hu{yL5cvQUbQa)pSpv122w67ovVw|K+Bd8O zSDAMd9Yn5582l^|DtQi)JFRl0M*y{71Xt?{aBW20l_5TT0&TsfX16GJTtxez=#y9+ zWtvZ)H|?`52HKgws9@j<-xMzMLh;F9R~NKo*6EX2ayM%3n3^X8o6}e7Fi+!!X}|leZV`Sj%RtML}47Slb4R90}M!9Yi2Io#5(4n4P#D1)-9` zxz#Oq45MvK^eIv=l4PWrv4nFNjXh|52DjV)RKb;v5bIN(mG6f8~4IA7)+1F zKo1~9?6j_75X-3PM~GR3c!<=5dl6y=kE5M*Jf=8+6u(D`hmc|oLhM0cH3*RvmYUa+ z3>u?>m*x_b=xhA@Bt^^oHZnxd)P6$0wtVwUYKF3t1BsKB7_0^^1#KiNpbRPlu6 z;;>!~@R>yXUu}dv5V-#w^sP~OoL`3))q?ZMObebIqNc2wgk3Xq_`KwVJv*vlt2~vp zWD`Qe=)mG^`j!{Cu;9@e#41)7x6+-$AEs1glPc5ST)zBk1IrHIFOU3jBns2kr#- zoiO-4iXZyq5%0xh+x-x@h<@9Cpe`aTB$04($$Xp|4&_enqDxTM z?O<>7uMih2@_i)8R*hHKE#0RNyPBytTTAoq&yu-vqPFBU{@-gTnXY{!ZNI;ldHIXI z%<(_iC;D#UTHDhad?canKSYge{sJB*s=vui9PJbf*oH(;r_7jTqjY*~NMI zS(bmh{QIM<&ukatK3at?AH>=7<~9;%i{_?Lb!-TZ4WVxqO?aEo;m-qkdF3zpk^hvOYc7>M(V}7|^|8<#(x*^^RHg)-gySHj zeksZ}z&SdnN~9Cql;C`^u(g$VSlnv!_HuXtT&?(isY}}r430XVe7wD*!}(M&gsd#F ztkp)YvhS^5j|lY|w&Vfz%3lF4?3B8OW@gUM;4fa&*J}E^ar?DwulBJHp|0eP%~V&V zwnrw^b$2=go3#Dvx@s?ZO=##9ZGTbQtK|iHttHg3COul2b%(Jv`5a%uy`)JG%huV% z{pyl7&$jB^ ! { - use crate::runtime_init; - - if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { - SP.set(bsp::memory::boot_core_stack_end() as u64); - runtime_init::runtime_init() - } else { - // If not core0, infinitely wait for events. - cpu::wait_forever() - } +pub unsafe fn _start_rust() -> ! { + runtime_init::runtime_init() } diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..ad4a2689 --- /dev/null +++ b/X1_JTAG_boot/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne 1f + + // If execution reaches here, it is the boot core. Now, prepare the jump to Rust code. + + // Set the stack pointer. + ldr x0, =__boot_core_stack_end_exclusive + mov sp, x0 + + // Jump to Rust code. + b _start_rust + + // Infinitely wait for events (aka "park the core"). +1: wfe + b 1b + +.size _start, . - _start +.type _start, function +.global _start diff --git a/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs b/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs deleted file mode 100644 index b9fdd0f7..00000000 --- a/X1_JTAG_boot/src/_arch/aarch64/cpu/smp.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Architectural symmetric multiprocessing. -//! -//! # Orientation -//! -//! Since arch modules are imported into generic modules using the path attribute, the path of this -//! file is: -//! -//! crate::cpu::smp::arch_smp - -use cortex_a::regs::*; - -//-------------------------------------------------------------------------------------------------- -// Public Code -//-------------------------------------------------------------------------------------------------- - -/// Return the executing core's id. -#[inline(always)] -pub fn core_id() -> T -where - T: From, -{ - const CORE_MASK: u64 = 0b11; - - T::from((MPIDR_EL1.get() & CORE_MASK) as u8) -} diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/cpu.rs b/X1_JTAG_boot/src/bsp/raspberrypi/cpu.rs index 16ab927d..d09ff956 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/cpu.rs +++ b/X1_JTAG_boot/src/bsp/raspberrypi/cpu.rs @@ -9,4 +9,6 @@ //-------------------------------------------------------------------------------------------------- /// Used by `arch` code to find the early boot core. -pub const BOOT_CORE_ID: u64 = 0; +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/link.ld b/X1_JTAG_boot/src/bsp/raspberrypi/link.ld index 87e6a976..3b73b3c7 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/link.ld +++ b/X1_JTAG_boot/src/bsp/raspberrypi/link.ld @@ -17,15 +17,28 @@ PHDRS SECTIONS { . = __rpi_load_addr; + /* ^ */ + /* | stack */ + /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ /*********************************************************************************************** * Code + RO Data + Global Offset Table ***********************************************************************************************/ - __rx_start = .; .text : { KEEP(*(.text._start)) - *(.text*) + + /* Special constants (or statics in Rust speak) needed by _start(). + * + * They are placed in close proximity to _start() from where they will be read. This ensures + * that position-independent, PC-relative loads can be emitted. + */ + *(.text._start_arguments) + + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ } :segment_rx .rodata : ALIGN(8) { *(.rodata*) } :segment_rx diff --git a/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs b/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs index 56a3306e..c6d65e36 100644 --- a/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs +++ b/X1_JTAG_boot/src/bsp/raspberrypi/memory.rs @@ -12,8 +12,6 @@ use core::{cell::UnsafeCell, ops::RangeInclusive}; // Symbols from the linker script. extern "Rust" { - static __rx_start: UnsafeCell<()>; - static __bss_start: UnsafeCell; static __bss_end_inclusive: UnsafeCell; } @@ -50,30 +48,10 @@ pub(super) mod map { } } -//-------------------------------------------------------------------------------------------------- -// Private Code -//-------------------------------------------------------------------------------------------------- - -/// Start address of the Read+Execute (RX) range. -/// -/// # Safety -/// -/// - Value is provided by the linker script and must be trusted as-is. -#[inline(always)] -fn rx_start() -> usize { - unsafe { __rx_start.get() as usize } -} - //-------------------------------------------------------------------------------------------------- // Public Code //-------------------------------------------------------------------------------------------------- -/// Exclusive end address of the boot core's stack. -#[inline(always)] -pub fn boot_core_stack_end() -> usize { - rx_start() -} - /// Return the inclusive range spanning the .bss section. /// /// # Safety diff --git a/X1_JTAG_boot/src/cpu.rs b/X1_JTAG_boot/src/cpu.rs index 3834f183..7a095cb1 100644 --- a/X1_JTAG_boot/src/cpu.rs +++ b/X1_JTAG_boot/src/cpu.rs @@ -10,8 +10,6 @@ mod arch_cpu; mod boot; -pub mod smp; - //-------------------------------------------------------------------------------------------------- // Architectural Public Reexports //-------------------------------------------------------------------------------------------------- diff --git a/X1_JTAG_boot/src/cpu/smp.rs b/X1_JTAG_boot/src/cpu/smp.rs deleted file mode 100644 index 38230ce1..00000000 --- a/X1_JTAG_boot/src/cpu/smp.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 -// -// Copyright (c) 2018-2021 Andre Richter - -//! Symmetric multiprocessing. - -#[cfg(target_arch = "aarch64")] -#[path = "../_arch/aarch64/cpu/smp.rs"] -mod arch_smp; - -//-------------------------------------------------------------------------------------------------- -// Architectural Public Reexports -//-------------------------------------------------------------------------------------------------- -pub use arch_smp::core_id; diff --git a/X1_JTAG_boot/src/main.rs b/X1_JTAG_boot/src/main.rs index aeff7a4d..19c864c4 100644 --- a/X1_JTAG_boot/src/main.rs +++ b/X1_JTAG_boot/src/main.rs @@ -100,16 +100,16 @@ //! //! # Boot flow //! -//! 1. The kernel's entry point is the function [`cpu::boot::arch_boot::_start()`]. -//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.rs`. +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. //! 2. Once finished with architectural setup, the arch code calls [`runtime_init::runtime_init()`]. //! -//! [`cpu::boot::arch_boot::_start()`]: cpu/boot/arch_boot/fn._start.html //! [`runtime_init::runtime_init()`]: runtime_init/fn.runtime_init.html #![allow(clippy::clippy::upper_case_acronyms)] #![feature(const_fn_fn_ptr_basics)] #![feature(format_args_nl)] +#![feature(global_asm)] #![feature(panic_info_message)] #![feature(trait_alias)] #![no_main] diff --git a/rust-toolchain b/rust-toolchain index 1b5ba1c0..e0e7993d 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2021-03-06" +channel = "nightly-2021-03-18" components = ["llvm-tools-preview"] targets = ["aarch64-unknown-none-softfloat"]