diff --git a/14_exceptions_part2_peripheral_IRQs/.cargo/config b/14_exceptions_part2_peripheral_IRQs/.cargo/config new file mode 100644 index 00000000..e3476485 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/.cargo/config @@ -0,0 +1,2 @@ +[target.'cfg(target_os = "none")'] +runner = "target/kernel_test_runner.sh" diff --git a/14_exceptions_part2_peripheral_IRQs/.vscode/settings.json b/14_exceptions_part2_peripheral_IRQs/.vscode/settings.json new file mode 100644 index 00000000..c520a512 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/.vscode/settings.json @@ -0,0 +1,13 @@ +{ + "editor.formatOnSave": true, + "editor.rulers": [ + 100 + ], + "rust-analyzer.cargoFeatures.features": [ + "bsp_rpi3" + ], + "rust-analyzer.cargo-watch.allTargets": false, + "rust-analyzer.cargo-watch.arguments": [ + "--target=aarch64-unknown-none-softfloat" + ] +} diff --git a/14_exceptions_part2_peripheral_IRQs/Cargo.lock b/14_exceptions_part2_peripheral_IRQs/Cargo.lock new file mode 100644 index 00000000..114c0206 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/Cargo.lock @@ -0,0 +1,175 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "array-init" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bit_field" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cast" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cortex-a" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "register 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel" +version = "0.1.0" +dependencies = [ + "cortex-a 2.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "qemu-exit 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "test-macros 0.1.0", + "test-types 0.1.0", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "qemu-exit" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "x86_64 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "register" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tock-registers 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "test-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "test-types 0.1.0", +] + +[[package]] +name = "test-types" +version = "0.1.0" + +[[package]] +name = "tock-registers" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ux" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "x86_64" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "array-init 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cast 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum array-init 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" +"checksum bit_field 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum cast 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +"checksum cortex-a 2.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd807a12241940332c7c9f6aeaf3f7539d4213853d32445d17c97f3e7eebe118" +"checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" +"checksum qemu-exit 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6eef8966dfa42074458b801f4ca70c0a070a84a500022584cc11d7a3c1fdb105" +"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum register 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e10c2ba523bc81270ecb9b6ca6704f42cd1efec89fddbfe0e520ad0e218f401" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" +"checksum tock-registers 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "70323afdb8082186c0986da0e10f6e4ed103d681c921c00597e98d9806dac20f" +"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum ux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "88dfeb711b61ce620c0cb6fd9f8e3e678622f0c971da2a63c4b3e25e88ed012f" +"checksum x86_64 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1f27d9168654aee1b0c1b73746caeb4aa33248f8b8c8f6e100e697fcc2a794b2" diff --git a/14_exceptions_part2_peripheral_IRQs/Cargo.toml b/14_exceptions_part2_peripheral_IRQs/Cargo.toml new file mode 100644 index 00000000..93c4d204 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "kernel" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" + +[package.metadata.cargo-xbuild] +sysroot_path = "../xbuild_sysroot" + +# The features section is used to select the target board. +[features] +default = [] +bsp_rpi3 = ["cortex-a", "register"] +bsp_rpi4 = ["cortex-a", "register"] +qemu-quirks = [] + +[dependencies] +qemu-exit = "0.1.x" +test-types = { path = "test-types" } + +# Optional dependencies +cortex-a = { version = "2.9.x", optional = true } +register = { version = "0.5.x", features=["no_std_unit_tests"], optional = true } + +##-------------------------------------------------------------------------------------------------- +## Testing +##-------------------------------------------------------------------------------------------------- + +[dev-dependencies] +test-macros = { path = "test-macros" } + +# Unit tests are done in the library part of the kernel. +[lib] +name = "libkernel" +test = true + +# Disable unit tests for the kernel binary. +[[bin]] +name = "kernel" +test = false + +# List of tests without harness. +[[test]] +name = "00_console_sanity" +harness = false + +[[test]] +name = "02_exception_sync_page_fault" +harness = false diff --git a/14_exceptions_part2_peripheral_IRQs/Makefile b/14_exceptions_part2_peripheral_IRQs/Makefile new file mode 100644 index 00000000..a006eb87 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/Makefile @@ -0,0 +1,166 @@ +## SPDX-License-Identifier: MIT OR Apache-2.0 +## +## Copyright (c) 2018-2020 Andre Richter + +# Default to the RPi3 +ifndef BSP + BSP = rpi3 +endif + +# Default to /dev/ttyUSB0 +ifndef DEV_SERIAL + DEV_SERIAL = /dev/ttyUSB0 +endif + +# BSP-specific arguments +ifeq ($(BSP),rpi3) + TARGET = aarch64-unknown-none-softfloat + OUTPUT = kernel8.img + QEMU_BINARY = qemu-system-aarch64 + QEMU_MACHINE_TYPE = raspi3 + QEMU_RELEASE_ARGS = -serial stdio -display none + QEMU_TEST_ARGS = $(QEMU_RELEASE_ARGS) -semihosting + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg + JTAG_BOOT_IMAGE = jtag_boot_rpi3.img + LINKER_FILE = src/bsp/raspberrypi/link.ld + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 +else ifeq ($(BSP),rpi4) + TARGET = aarch64-unknown-none-softfloat + OUTPUT = kernel8.img + # QEMU_BINARY = qemu-system-aarch64 + # QEMU_MACHINE_TYPE = + # QEMU_RELEASE_ARGS = -serial stdio -display none + # QEMU_TEST_ARGS = $(QEMU_RELEASE_ARGS) -semihosting + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg + JTAG_BOOT_IMAGE = jtag_boot_rpi4.img + LINKER_FILE = src/bsp/raspberrypi/link.ld + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 +endif + +# Testing-specific arguments +ifdef TEST + ifeq ($(TEST),unit) + TEST_ARG = --lib + else + TEST_ARG = --test $(TEST) + endif +endif + +QEMU_MISSING_STRING = "This board is not yet supported for QEMU." + +RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) +RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs +FEATURES = bsp_$(BSP) + +SOURCES = $(wildcard **/*.rs) $(wildcard **/*.S) $(wildcard **/*.ld) + +X_CMD_ARGS = --target=$(TARGET) \ + --features $(FEATURES) \ + --release +XRUSTC_CMD = cargo xrustc $(X_CMD_ARGS) +XTEST_CMD = cargo xtest $(X_CMD_ARGS) + +CARGO_OUTPUT = target/$(TARGET)/release/kernel + +OBJCOPY_CMD = cargo objcopy \ + -- \ + --strip-all \ + -O binary + +DOCKER_IMAGE = rustembedded/osdev-utils +DOCKER_CMD_TEST = docker run -i --rm +DOCKER_CMD_USER = $(DOCKER_CMD_TEST) -t +DOCKER_ARG_DIR_TUT = -v $(shell pwd):/work -w /work +DOCKER_ARG_DIR_UTILS = -v $(shell pwd)/../utils:/utils +DOCKER_ARG_DIR_JTAG = -v $(shell pwd)/../X1_JTAG_boot:/jtag +DOCKER_ARG_TTY = --privileged -v /dev:/dev +DOCKER_ARG_NET = --network host +DOCKER_EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) +DOCKER_EXEC_MINIPUSH = ruby /utils/minipush.rb + +.PHONY: all doc qemu chainboot jtagboot openocd gdb gdb-opt0 clippy clean readelf objdump nm test + +all: clean $(OUTPUT) + +$(CARGO_OUTPUT): $(SOURCES) + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(XRUSTC_CMD) + +$(OUTPUT): $(CARGO_OUTPUT) + cp $< . + $(OBJCOPY_CMD) $< $(OUTPUT) + +doc: + cargo xdoc --target=$(TARGET) --features bsp_$(BSP) --document-private-items --open + +qemu: FEATURES += --features qemu-quirks +ifeq ($(QEMU_MACHINE_TYPE),) +qemu: + @echo $(QEMU_MISSING_STRING) + +test: + @echo $(QEMU_MISSING_STRING) +else +qemu: all + @$(DOCKER_CMD_USER) $(DOCKER_ARG_DIR_TUT) $(DOCKER_IMAGE) \ + $(DOCKER_EXEC_QEMU) $(QEMU_RELEASE_ARGS) \ + -kernel $(OUTPUT) + +define KERNEL_TEST_RUNNER + #!/usr/bin/env bash + + $(OBJCOPY_CMD) $$1 $$1.img + TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g') + $(DOCKER_CMD_TEST) $(DOCKER_ARG_DIR_TUT) $(DOCKER_IMAGE) \ + ruby tests/runner.rb $(DOCKER_EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY +endef + +export KERNEL_TEST_RUNNER +test: FEATURES += --features qemu-quirks +test: $(SOURCES) + @mkdir -p target + @echo "$$KERNEL_TEST_RUNNER" > target/kernel_test_runner.sh + @chmod +x target/kernel_test_runner.sh + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(XTEST_CMD) $(TEST_ARG) +endif + +chainboot: all + @$(DOCKER_CMD_USER) $(DOCKER_ARG_DIR_TUT) $(DOCKER_ARG_DIR_UTILS) $(DOCKER_ARG_TTY) \ + $(DOCKER_IMAGE) $(DOCKER_EXEC_MINIPUSH) $(DEV_SERIAL) \ + $(OUTPUT) + +jtagboot: + @$(DOCKER_CMD_USER) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_ARG_DIR_UTILS) $(DOCKER_ARG_TTY) \ + $(DOCKER_IMAGE) $(DOCKER_EXEC_MINIPUSH) $(DEV_SERIAL) \ + /jtag/$(JTAG_BOOT_IMAGE) + +openocd: + @$(DOCKER_CMD_USER) $(DOCKER_ARG_TTY) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) \ + openocd $(OPENOCD_ARG) + +define gen_gdb + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC) $1" $(XRUSTC_CMD) + cp $(CARGO_OUTPUT) kernel_for_jtag + @$(DOCKER_CMD_USER) $(DOCKER_ARG_DIR_TUT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) \ + gdb-multiarch -q kernel_for_jtag +endef + +gdb: clean $(SOURCES) + $(call gen_gdb,-C debuginfo=2) + +gdb-opt0: clean $(SOURCES) + $(call gen_gdb,-C debuginfo=2 -C opt-level=0) + +clippy: + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" cargo xclippy --target=$(TARGET) --features bsp_$(BSP) + +clean: + rm -rf target + +readelf: + readelf -a kernel + +objdump: + cargo objdump --target $(TARGET) -- -disassemble -no-show-raw-insn -print-imm-hex kernel + +nm: + cargo nm --target $(TARGET) -- -print-size kernel | sort diff --git a/14_exceptions_part2_peripheral_IRQs/README.md b/14_exceptions_part2_peripheral_IRQs/README.md new file mode 100644 index 00000000..03003fd3 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/README.md @@ -0,0 +1,2940 @@ +# Tutorial 14 - Exceptions Part 2: Peripheral IRQs + +## tl;dr + +- We write `device drivers` for the two interrupt controllers on the **Raspberry Pi 3** (Broadcom + custom controller) and **Pi 4** (ARM Generic Interrupt Controller v2, `GICv2`). +- Modularity is ensured by interfacing everything through a trait named `IRQManager`. +- Handling for our first peripheral IRQ is implemented: The `UART`'s receive IRQ - one IRQ per + received character. + + + +## Table of Contents + +- [Introduction](#introduction) +- [Different Controllers: A Usecase for Abstraction](#different-controllers-a-usecase-for-abstraction) +- [New Challenges: Reentrancy](#new-challenges-reentrancy) +- [Implementation](#implementation) + * [The Kernel's Interfaces for Interrupt Handling](#the-kernels-interfaces-for-interrupt-handling) + + [Uniquely Identifying an IRQ](#uniquely-identifying-an-irq) + - [The BCM IRQ Number Scheme](#the-bcm-irq-number-scheme) + - [The GICv2 IRQ Number Scheme](#the-gicv2-irq-number-scheme) + + [Registering IRQ Handlers](#registering-irq-handlers) + + [Handling Pending IRQs](#handling-pending-irqs) + * [Reentrancy: What to protect?](#reentrancy-what-to-protect) + * [The Interrupt Controller Device Drivers](#the-interrupt-controller-device-drivers) + + [The BCM Driver (Pi 3)](#the-bcm-driver-pi-3) + - [Peripheral Controller Register Access](#peripheral-controller-register-access) + - [The IRQ Handler Table](#the-irq-handler-table) + + [The GICv2 Driver (Pi 4)](#the-gicv2-driver-pi-4) + - [GICC Details](#gicc-details) + - [GICD Details](#gicd-details) +- [UART hack](#uart-hack) +- [Test it](#test-it) +- [Diff to previous](#diff-to-previous) + +## Introduction + +In [tutorial 12], we laid the groundwork for exception handling from the processor architecture +side. Handler stubs for the different exception types were set up, and a first glimpse at exception +handling was presented by causing a `synchronous` exception by means of a `page fault`. + +[tutorial 12]: ../12_exceptions_part1_groundwork + +In this tutorial, we will add a first level of support for one of the three types of `asynchronous` +exceptions that are defined for `AArch64`: `IRQs`. The overall goal for this tutorial is to get rid +of the busy-loop at the end of our current `kernel_main()` function, which actively polls the +`UART` for newly received characters. Instead, we will let the processor idle and wait for the +`UART`'s RX IRQ, which indicates that a new character was received. A respective `IRQ` service +routine, provided by the `UART` driver, will run in response to the `IRQ` and print the character. + +## Different Controllers: A Usecase for Abstraction + +One very exciting aspect of this tutorial is that the `Pi 3` and the `Pi 4` feature completely +different interrupt controllers. This is also a first in all of the tutorial series. Until now, both +Raspberrys did not need differentiation with respect to their devices. + +The `Pi 3` has a very simple custom controller made by Broadcom (BCM), the manufacturer of the Pi's +`System-on-Chip`. The `Pi 4` features an implementation of `ARM`'s Generic Interrupt Controller +version 2 (`GICv2`). Since ARM's GIC controllers are the prevalent interrupt controllers in ARM +application procesors, it is very beneficial to finally have it on the Raspberry Pi. It will enable +people to learn about one of the most common building blocks in ARM-based embedded computing. + +This also means that we can finally make full use of all the infrastructure for abstraction that we +prepared already. We will design an `IRQManager` interface trait and implement it in both controller +drivers. The generic part of our `kernel` code will only be exposed to this trait (compare to the +diagram in the [tl;dr] section). This common idiom of *program to an interface, not an +implementation* enables a clean abstraction and makes the code modular and pluggable. + +[tl;dr]: #tldr + +## New Challenges: Reentrancy + +Enabling interrupts also poses new challenges with respect to protecting certain code sections in +the kernel from being [re-entered]. Please read the linked article for background on that topic. + +[re-entered]: https://en.wikipedia.org/wiki/Reentrancy_(computing) + +Our `kernel` is still running on a single core. For this reason, we are still using our `NullLock` +pseudo-locks for `Critical Sections` or `shared resources`, instead of real `Spinlocks`. Hence, +interrupt handling at this point in time does not put us at risk of running into one of those +dreaded `deadlocks`, which is one of several side-effects that reentrancy can cause. For example, a +`deadlock` because of interrupts can happen happen when the executing CPU core has locked a +`Spinlock` at the beginning of a function, an IRQ happens, and the IRQ service routine is trying to +execute the same function. Since the lock is already locked, the core would spin forever waiting for +it to be released. + +There is no straight-forward way to tell if a function is `reentrantcy`-safe or not. It usually +needs careful manual checking to conclude. Even though it might be technically safe to `re-enter` a +function, sometimes you don't want that to happen for functional reasons. For example, printing of a +string should not be interrupted by a an interrupt service routine that starts printing another +string, so that the output mixes. In the course of this tutorial, we will check and see where we +want to protect against `reentrancy`. + +## Implementation + +Okay, let's start. The following sections cover the the implementation in a top-down fashion, +starting with the trait that interfaces all the `kernel` components to each other. + +### The Kernel's Interfaces for Interrupt Handling + +First, we design the `IRQManager` trait that interrupt controller drivers must implement. The +minimal set of functionality that we need for starters is: + +1. Registering an IRQ `handler` for a given IRQ `number`. +2. Enabling an IRQ (from the controller side). +3. Handling pending IRQs. +4. Printing the list of registered IRQ handlers. + +The trait is defined as `exception::asynchronous::interface::IRQManager`: + +```rust +pub trait IRQManager { + /// The IRQ number type depends on the implementation. + type IRQNumberType; + + /// Register a handler. + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: super::IRQDescriptor, + ) -> Result<(), &'static str>; + + /// Enable an interrupt in the controller. + fn enable(&self, irq_number: Self::IRQNumberType); + + /// Handle pending interrupts. + /// + /// This function is called directly from the CPU's IRQ exception vector. On AArch64, + /// this means that the respective CPU core has disabled exception handling. + /// This function can therefore not be preempted and runs start to finish. + /// + /// Takes an IRQContext token to ensure it can only be called from IRQ context. + #[allow(clippy::trivially_copy_pass_by_ref)] + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &super::IRQContext<'irq_context>, + ); + + /// Print list of registered handlers. + fn print_handler(&self); +} +``` + +#### Uniquely Identifying an IRQ + +The first member of the trait is the [associated type] `IRQNumberType`. The following explains why +we make it customizable for the implementor and do not define the type as a plain integer right +away. + +Interrupts can generally be characterizied with the following properties: + +[associated type]: https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#specifying-placeholder-types-in-trait-definitions-with-associated-types + +1. Software-generated vs hardware-generated. +2. Private vs shared. + +Different interrupt controllers take different approaches at categorizing and numbering IRQs that +have one or the other property. Often times, this leads to situations where a plain integer does not +suffice to uniquely identify an IRQ, and makes it necessary to encode additional information in the +used type. Letting the respective interrupt controller driver define `IRQManager::IRQNumberType` +itself addresses this issue. The rest of the `BSP` must then conditionally use this type. + +##### The BCM IRQ Number Scheme + +The `BCM` controller of the `Raspberry Pi 3`, for example, is composed of two functional parts: A +**local** controller and a **peripheral** controller. The BCM's **local controller** handles all +`private` IRQs, which means private SW-generated IRQs and IRQs of private HW devices. An example for +the latter would be the `ARMv8` timer. Each CPU core has its own private instance of it. The BCM's +**peripheral controller** handles all IRQs of `non-private` HW devices such as the `UART` (if those +IRQs can be declared as `shared` according to our taxonomy above is a different discussion, because +the BCM controller allows these HW interrupts to be routed to _only one CPU core at a time_). + +The IRQ numbers of the BCM **local controller** range from `0..11`. The numbers of the **peripheral +controller** range from `0..63`. This demonstrates why a primitive integer type would not be +sufficient to uniquely encode the IRQs, because their ranges overlap. In the driver for the `BCM` +controller, we therefore define the associated type as follows: + +```rust +pub type LocalIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>; +pub type PeripheralIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>; + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +#[derive(Copy, Clone)] +pub enum IRQNumber { + Local(LocalIRQ), + Peripheral(PeripheralIRQ), +} +``` + +The type `exception::asynchronous::IRQNumber` is a newtype around an `usize` that uses a [const +generic] to ensure that the value of the encapsulated IRQ number is in the allowed range (e.g. +`0..MAX_LOCAL_IRQ_NUMBER` for `LocalIRQ`, with `MAX_LOCAL_IRQ_NUMBER == 11`). + +[const generic]: https://github.com/rust-lang/rfcs/blob/master/text/2000-const-generics.md + +##### The GICv2 IRQ Number Scheme + +The `GICv2` in the `Raspberry Pi 4`, on the other hand, uses a different scheme. IRQ numbers `0..31` +are for `private` IRQs. Those are further subdivided into `SW-generated` (SGIs, `0..15`) and +`HW-generated` (PPIs, Private Peripheral Interrupts, `16..31`). Numbers `32..1019` are for `shared +hardware-generated` interrupts (SPI, Shared Peripheral Interrupts). + +There are no overlaps, so this scheme enables us to actually have a plain integer as a unique +identifier for the IRQs. We define the type as follows: + +```rust +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +pub type IRQNumber = exception::asynchronous::IRQNumber<{ GICv2::MAX_IRQ_NUMBER }>; +``` + +#### Registering IRQ Handlers + +To enable the controller driver to manage interrupt handling, it must know where to find respective +handlers, and it must know how to call them. For the latter, we define an `IRQHandler` trait in +`exception::asynchronous` that must be implemented by any SW entity that wants to handle IRQs: + +```rust +/// Implemented by types that handle IRQs. +pub trait IRQHandler { + /// Called when the corresponding interrupt is asserted. + fn handle(&self) -> Result<(), &'static str>; +} +``` + +The `PL011Uart` driver gets the honors for being our first driver to ever implement this trait. In +this tutorial, only receive (RX) interrupts are enabled, and there will be one RX IRQ fired for each +character that is received. In the handler, our standard scheme of echoing any received characters +back to the host is used: + +```rust +impl exception::asynchronous::interface::IRQHandler for PL011Uart { + fn handle(&self) -> Result<(), &'static str> { + let mut r = &self.inner; + r.lock(|inner| { + // Echo any received characters. + loop { + match inner.read_char_converting(false) { + None => break, + Some(c) => inner.write_char(c), + } + } + }); + + Ok(()) + } +} +``` + +Registering and enabling handlers in the interrupt controller is supposed to be done by the +respective drivers themselves. Therefore, we added a new function to the standard device driver +trait in `driver::interface::DeviceDriver` that must be implemented if IRQ handling is supported: + +```rust +/// Called by the kernel to register and enable the device's IRQ handlers, if any. +/// +/// Rust's type system will prevent a call to this function unless the calling instance +/// itself has static lifetime. +fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + Ok(()) +} +``` + +Here is the implementation for the `PL011Uart`: + +```rust +fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + use bsp::exception::asynchronous::irq_manager; + use exception::asynchronous::{interface::IRQManager, IRQDescriptor}; + + let descriptor = IRQDescriptor { + name: "BCM PL011 UART", + handler: self, + }; + + irq_manager().register_handler(self.irq_number, descriptor)?; + irq_manager().enable(self.irq_number); + + Ok(()) +} +``` + +The `bsp::exception::asynchronous::irq_manager()` function used here returns a reference to an +implementor of the `IRQManager` trait. Since the implementation is supposed to be done by the +platform's interrupt controller, this call will redirect to the `kernel`'s instance of either the +driver for the `BCM` controller (`Raspberry Pi 3`) or the driver for the `GICv2` (`Pi 4`). We will +look into the implementation of the `register_handler()` function from the driver's perspective +later. The gist here is that the calls on `irq_manager()` will make the platform's interrupt +controller aware that the `UART` driver (i) wants to handle its interrupt and (ii) which function it +provides to do so. + +Also note how `irq_number` is a member of the `PL011Uart` struct and not hardcoded. The reason is +that the `UART` driver code is agnostic about the **IRQ numbers** that are associated to it. This is +vendor-supplied information and as such typically part of the Board Support Package (`BSP`). It can +vary from `BSP` to `BSP`, same like the board's memory map, which provides the `UART`'s MMIO +register addresses. Therefore, we extend the instantiation of the `UART` driver accordingly, so that +the `BSP` now additionally provides the IRQ number as an argument: + +```rust +static PL011_UART: device_driver::PL011Uart = unsafe { + device_driver::PL011Uart::new( + memory::map::mmio::PL011_UART_BASE, + exception::asynchronous::irq_map::PL011_UART, + ) +}; +``` + +With all this in place, we can finally let drivers register and enable their IRQ handlers with the +interrupt controller, and unmask IRQ reception on the boot CPU core during the kernel init phase in +`main.rs`. After unmasking, IRQ handling is live: + +```rust +// Let device drivers register and enable their handlers with the interrupt controller. +for i in bsp::driver::driver_manager().all_device_drivers() { + if let Err(msg) = i.register_and_enable_irq_handler() { + warn!("Error registering IRQ handler: {}", msg); + } +} + +// Unmask interrupts on the boot CPU core. +exception::asynchronous::local_irq_unmask(); +``` + +#### Handling Pending IRQs + +Now that interrupts can happen, the `kernel` needs a way of requesting the interrupt controller +driver to handle pending interrupts. Therefore, implementors of the trait `IRQManager` must also +supply the following function: + +```rust +fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &super::IRQContext<'irq_context>, +); +``` + +An important aspect of this function signature is that we want to ensure that IRQ handling is only +possible from IRQ context. Part of the reason is that this invariant allows us to make some implicit +assumptions (which might depend on the target architecture, though). For example, as we have learned +in [tutorial 12], in `AArch64`, _"all kinds of exceptions are turned off upon taking an exception, +so that by default, exception handlers can not get interrupted themselves"_ (note that an IRQ is an +exception). This is a useful property that relieves us from explicitly protecting IRQ handling from +being interrupted itself. Another reason would be that calling IRQ handling functions from arbitrary +execution contexts just doesn't make a lot of sense. + +[tutorial 12]: ../12_exceptions_part1_groundwork/ + +So in order to ensure that this function is only being called from IRQ context, we borrow a +technique that I first saw in the [Rust embedded WG]'s [bare-metal crate]. It uses Rust's type +system to create a "token" that is only valid for the duration of the IRQ context. We create it +directly at the top of the IRQ vector function in `_arch/aarch64/exception.rs`, and pass it on to +the the implementation of the trait's handling function: + +[Rust embedded WG]: https://github.com/rust-embedded/bare-metal +[bare-metal crate]: https://github.com/rust-embedded/bare-metal/blob/master/src/lib.rs#L20 + +```rust +#[no_mangle] +unsafe extern "C" fn current_elx_irq(_e: &mut ExceptionContext) { + use exception::asynchronous::interface::IRQManager; + + let token = &exception::asynchronous::IRQContext::new(); + bsp::exception::asynchronous::irq_manager().handle_pending_irqs(token); +} +``` + +By requiring the caller of the function `handle_pending_irqs()` to provide this `IRQContext` token, +we can prevent that the same function is accidentally being called from somewhere else. It is +evident, though, that for this to work, it is the _user's responsibility_ to only ever create this +token from within an IRQ context. If you want to circumvent this on purpose, you can do it. + +### Reentrancy: What to protect? + +Now that interrupt handling is live, we need to think about `reentrancy`. At [the beginning of this +tutorial], we mused about the need to protect certain functions from being re-entered, and that it +is not straight-forward to identify all the places that need protection. + +[the beginning of this tutorial]: #new-challenges-reentrancy + +In this tutorial, we will keep this part short nonetheless by taking a better-safe-than-sorry +approach. In the past, we already made efforts to prepare parts of `shared resources` (e.g. global +device driver instances) to be protected against parallel access. We did so by wrapping them into +`NullLocks`, which we will upgrade to real `Spinlocks` once we boot secondary CPU cores. + +We can hook on that previous work and reason that anything that we wanted protected against parallel +access so far, we also want it protected against reentrancy now. Therefore, we upgrade all +`NullLocks` to `IRQSafeNullocks`: + +```rust +impl interface::Mutex for &IRQSafeNullLock { + type Data = T; + + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + // In a real lock, there would be code encapsulating this line that ensures that this + // mutable reference will ever only be given out once at a time. + let data = unsafe { &mut *self.data.get() }; + + // Execute the closure while IRQs are masked. + exception::asynchronous::exec_with_irq_masked(|| f(data)) + } +} +``` + +The new part is that the call to `f(data)` is executed as a closure in +`exception::asynchronous::exec_with_irq_masked()`. Inside that function, IRQs on the executing CPU +core are masked before the `f(data)` is being executed, and restored afterwards: + +```rust +/// Executes the provided closure while IRQs are masked on the executing core. +/// +/// While the function temporarily changes the HW state of the executing core, it restores it to the +/// previous state before returning, so this is deemed safe. +#[inline(always)] +pub fn exec_with_irq_masked(f: impl FnOnce() -> T) -> T { + let ret: T; + + unsafe { + let saved = local_irq_mask_save(); + ret = f(); + local_irq_restore(saved); + } + + ret +} +``` + +The helper functions used here are defined in `src/_arch/aarch64/exception/asynchronous.rs`. + +### The Interrupt Controller Device Drivers + +The previous sections explained how the `kernel` uses the `IRQManager` trait. Now, let's have a look +at the driver-side of it in the Raspberry Pi `BSP`. We start with the Broadcom interrupt controller +featured in the `Pi 3`. + +#### The BCM Driver (Pi 3) + +As mentioned earlier, the `BCM` driver consists of two subcomponents, a **local** and a +**peripheral** controller. The local controller owns a bunch of configuration registers, among +others, the `routing` configuration for peripheral IRQs such as those from the `UART`. Peripheral +IRQs can be routed to _one core only_. In our case, we leave the default unchanged, which means +everything is routed to the boot CPU core. The image below depicts the `struct diagram` of the +driver implementation. + + + +We have a top-level driver, which implements the `IRQManager` trait. _Only the top-level driver_ is +exposed to the rest of the `kernel`. The top-level itself has two members, representing the local +and the peripheral controller, respectively, which implement the `IRQManager` trait as well. This +design allows for easy forwarding of function calls from the top-level driver to one of the +subcontrollers. + +For this tutorial, we leave out implementation of the local controller, because we will only be +concerned with the peripheral `UART` IRQ. + +##### Peripheral Controller Register Access + +When writing a device driver for a kernel with exception handling and multi-core support, it is +always important to analyze what parts of the driver will need protection against reentrancy (we +talked about this earlier in this tutorial) and/or parallel execution of other driver parts. If a +driver function needs to follow a vendor-defined sequence of multiple register operations that +include `write operations`, this is usually a good hint that protection might be needed. But that is +only one of many examples. + +For the driver implementation in this tutorial, we are following a simple rule: Register read access +is deemed always safe. Write access is guarded by an `IRQSafeNullLock`, which means that we are safe +against `reentrancy` issues, and also in the future when the kernel will be running on multiple +cores, we can easily upgrade to a real spinlock, which serializes register write operations from +different CPU cores. + +In fact, for this tutorial, we probably would not have needed any protection yet, because all the +driver does is read from the `PENDING_*` registers for the `handle_pending_irqs()` implementation, +and writing to the `ENABLE_*` registers for the `enable()` implementation. However, the chosen +architecture will have us set up for future extensions, when more complex register manipulation +sequences might be needed. + +Since nothing complex is happening in the implementation, it is not covered in detail here. Please +refer to [the source of the **peripheral** controller] to check it out. + +[the source of the **peripheral** controller]: src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs + +##### The IRQ Handler Table + +Calls to `register_handler()` result in the driver inserting the provided handler reference in a +specific table (the handler reference is a member of `IRQDescriptor`): + +```rust +type HandlerTable = + [Option; InterruptController::NUM_PERIPHERAL_IRQS]; +``` + +One of the requirements for safe operation of the `kernel` is that those handlers are not +registered, removed or exchanged in the middle of an IRQ handling situation. This, again, is a +multi-core scenario where one core might look up a handler entry while another core is modifying the +same in parallel. + +While we want to allow drivers to take the decision of registering or not registering a handler at +runtime, there is no need to allow it for the _whole_ runtime of the kernel. It is fine to restrict +this option to the kernel `init phase`, at which only a single boot core runs and IRQs are masked. + +We introduce the so called `InitStateLock` for cases like that. From an API-perspective, it is a +special variant of a `Read/Write exclusion synchronization primitive`. RWLocks in the Rust standard +library [are characterized] as allowing _"a number of readers or at most one writer at any point in +time"_. For the `InitStateLock`, we only implement the `read()` and `write()` functions: + +[are characterized]: https://doc.rust-lang.org/std/sync/struct.RwLock.html + +```rust +impl interface::ReadWriteEx for &InitStateLock { + type Data = T; + + fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + assert!( + state::state_manager().state() == state::State::Init, + "InitStateLock::write called after kernel init phase" + ); + assert!( + !exception::asynchronous::is_local_irq_masked(), + "InitStateLock::write called with IRQs unmasked" + ); + + let data = unsafe { &mut *self.data.get() }; + + f(data) + } + + fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R { + let data = unsafe { &*self.data.get() }; + + f(data) + } +} +``` + +The `write()` function is guarded by two `assertions`. One ensures that IRQs are masked, the other +checks the `state::state_manager()` if the kernel is still in the init phase. The `State Manager` is +new since this tutorial, and implemented in `src/state.rs`. It provides atomic state transition and +reporting functions that are called when the kernel enters a new phase. In the current kernel, the +only call is happening before the transition from `kernel_init()` to `kernel_main()`: + +```rust +// Announce conclusion of the kernel_init() phase. +state::state_manager().transition_to_single_core_main(); +``` + +#### The GICv2 Driver (Pi 4) + +As we learned earlier, the ARM `GICv2` in the `Raspberry Pi 4` features a continuous interrupt +number range: +- IRQ numbers `0..31` represent IRQs that are private (aka local) to the respective processor core. +- IRQ numbers `32..1019` are for shared IRQs. + +The `GIC` has a so-called `Distributor`, the `GICD`, and a `CPU Interface`, the `GICC`. The `GICD`, +among other things, is used to enable IRQs and route them to one or more CPU cores. The `GICC` is +used by CPU cores to check which IRQs are pending, and to acknowledge them once they were handled. +There is one dedicated `GICC` for _each CPU core_. + +One neat thing about the `GICv2` is that any MMIO registers that are associated to core-private IRQs +are `banked`. That means that different CPU cores can assert the same MMIO address, but they will +end up accessing a core-private copy of the referenced register. This makes it very comfortable to +program the `GIC`, because this hardware design ensures that each core only ever gets access to its +own resources. Preventing one core to accidentally or willfully fiddle with the IRQ state of another +core must therefore not be enforced in software. + +In summary, this means that any registers in the `GICD` that deal with the core-private IRQ range +are banked. Since there is one `GICC` per CPU core, the whole thing is banked. This allows us to +design the following `struct diagram` for our driver implementation: + + + +The top-level struct is composed of a `GICD`, a `GICC` and a `HandlerTable`. The latter is +implemented identically as in the `Pi 3`. + +##### GICC Details + +Since the `GICC` is banked wholly, the top-level driver can directly forward any requests to it, +without worrying about concurrency issues for now. Note that this only works as long as the `GICC` +implementation is only accessing the banked `GICC` registers, and does not save any state in member +variables that are stored in `DRAM`. The two main duties of the `GICC` struct are to read the `IAR` +(Interrupt Acknowledge) register, which returns the number of the highest-priority pending IRQ, and +writing to the `EOIR` (End Of Interrupt) register, which tells the hardware that handling of an +interrupt is now concluded. + +##### GICD Details + +The `GICD` hardware block differentiates between `shared` and `banked` registers. As with the +`GICC`, we don't have to protect the banked registers against concurrent access. The shared +registers are wrapped into an `IRQSafeNullLock` again. The important parts of the `GICD` for this +tutorial are the `ITARGETSR[256]` and `ISENABLER[32]` register arrays. + +Each `ITARGETSR` is subdivided into four _bytes_. Each byte represents one IRQ, and stores a bitmask +that encodes all the `GICCs` to which the respective IRQ is forwarded. For example, +`ITARGETSR[0].byte0` would represent IRQ number 0, and `ITARGETSR[0].byte3` IRQ number 3. In the +`ISENABLER`, each _bit_ represents an IRQ. For example, `ISENABLER[0].bit3` is IRQ number 3. + +In summary, this means that `ITARGETSR[0..7]` and `ISENABLER[0]` represent the first 32 IRQs (the +banked ones), and as such, we split the register block into `shared` and `banked` parts accordingly +in `gicd.rs`: + +```rust +register_structs! { + #[allow(non_snake_case)] + SharedRegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => TYPER: ReadOnly), + (0x008 => _reserved1), + (0x104 => ISENABLER: [ReadWrite; 31]), + (0x108 => _reserved2), + (0x820 => ITARGETSR: [ReadWrite; 248]), + (0xBFC => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + BankedRegisterBlock { + (0x000 => _reserved1), + (0x100 => ISENABLER: ReadWrite), + (0x104 => _reserved2), + (0x800 => ITARGETSR: [ReadOnly; 8]), + (0xBFC => @END), + } +} +``` + +As with the implementation of the BCM interrupt controller driver, we won't cover the remaining +parts in exhaustive detail. For that, please refer to [this folder] folder which contains all the +sources. + +[this folder]: src/bsp/device_driver/arm + +## UART hack + +In the introduction, we stated that we want to set up the system such that the `UART` fires one IRQ +per received character. As it turns out, this usually common mode of operation is not natively +supported by the Raspberry's `PL011 UART` when receive FIFOs are activated. We can only set a +relative FIFO fill level at which the first IRQ is fired. The lowest fill level is `1/8` of the RX +FIFO size, which translates to `2 characters` for the `Pi 3` and `4 characters` for the `Pi 4`. + +To circumvent this constraint, we added a hack to the `UART` driver that "abuses" debug facilities +of the controller to always pre-fill the RX FIFOs to one character short of the `1/8` fill level. +This enusres that the next real character received from the connected user will fire an IRQ. + +To make things even more convoluted, `QEMU` does not emulate the fill-level-based IRQ generation, +but generates an IRQ on every received character. In the end, this whole situation adds a bit of +conditional-compilation mumbo-jumbo to the `UART` driver. But hey, it works! + +## Test it + +When you load the kernel, any keystroke results in echoing back the character by way of IRQ +handling. There is no more polling done at the end of `kernel_main()`, just waiting for events such +as IRQs: + +```rust +fn kernel_main() -> ! { + + // omitted for brevity + + info!("Echoing input now"); + cpu::wait_forever(); +} +``` + +Raspberry Pi 3: + +```console +$ make chainboot +[...] +Minipush 1.0 + +[MP] ⏳ Waiting for /dev/ttyUSB0 +[MP] ✅ Connected + __ __ _ _ _ _ +| \/ (_)_ _ (_) | ___ __ _ __| | +| |\/| | | ' \| | |__/ _ \/ _` / _` | +|_| |_|_|_||_|_|____\___/\__,_\__,_| + + Raspberry Pi 3 + +[ML] Requesting binary +[MP] ⏩ Pushing 66 KiB ========================================🦀 100% 33 KiB/s Time: 00:00:02 +[ML] Loaded! Executing the payload now + +[ 3.134937] Booting on: Raspberry Pi 3 +[ 3.136023] MMU online. Special regions: +[ 3.137934] 0x00080000 - 0x0008ffff | 64 KiB | C RO PX | Kernel code and RO data +[ 3.142017] 0x3f000000 - 0x4000ffff | 16 MiB | Dev RW PXN | Device MMIO +[ 3.145579] Current privilege level: EL1 +[ 3.147490] Exception handling state: +[ 3.149271] Debug: Masked +[ 3.150835] SError: Masked +[ 3.152398] IRQ: Unmasked +[ 3.154049] FIQ: Masked +[ 3.155613] Architectural timer resolution: 52 ns +[ 3.157915] Drivers loaded: +[ 3.159262] 1. BCM GPIO +[ 3.160695] 2. BCM PL011 UART +[ 3.162389] 3. BCM Interrupt Controller +[ 3.164518] Registered IRQ handlers: +[ 3.166255] Peripheral handler: +[ 3.168038] 57. BCM PL011 UART +[ 3.170078] Echoing input now +``` + +Raspberry Pi 4: + +```console +$ BSP=rpi4 make chainboot +[...] +Minipush 1.0 + +[MP] ⏳ Waiting for /dev/ttyUSB0 +[MP] ✅ Connected + __ __ _ _ _ _ +| \/ (_)_ _ (_) | ___ __ _ __| | +| |\/| | | ' \| | |__/ _ \/ _` / _` | +|_| |_|_|_||_|_|____\___/\__,_\__,_| + + Raspberry Pi 4 + +[ML] Requesting binary +[MP] ⏩ Pushing 73 KiB ========================================🦀 100% 24 KiB/s Time: 00:00:03 +[ML] Loaded! Executing the payload now + +[ 3.413865] Booting on: Raspberry Pi 4 +[ 3.414255] MMU online. Special regions: +[ 3.416166] 0x00080000 - 0x0008ffff | 64 KiB | C RO PX | Kernel code and RO data +[ 3.420249] 0xfe000000 - 0xff84ffff | 24 MiB | Dev RW PXN | Device MMIO +[ 3.423811] Current privilege level: EL1 +[ 3.425722] Exception handling state: +[ 3.427503] Debug: Masked +[ 3.429067] SError: Masked +[ 3.430630] IRQ: Unmasked +[ 3.432281] FIQ: Masked +[ 3.433845] Architectural timer resolution: 18 ns +[ 3.436147] Drivers loaded: +[ 3.437494] 1. BCM GPIO +[ 3.438927] 2. BCM PL011 UART +[ 3.440621] 3. GICv2 (ARM Generic Interrupt Controller v2) +[ 3.443575] Registered IRQ handlers: +[ 3.445312] Peripheral handler: +[ 3.447096] 153. BCM PL011 UART +[ 3.449136] Echoing input now +``` + +## Diff to previous +```diff + +diff -uNr 13_integrated_testing/Cargo.toml 14_exceptions_part2_peripheral_IRQs/Cargo.toml +--- 13_integrated_testing/Cargo.toml ++++ 14_exceptions_part2_peripheral_IRQs/Cargo.toml +@@ -12,6 +12,7 @@ + default = [] + bsp_rpi3 = ["cortex-a", "register"] + bsp_rpi4 = ["cortex-a", "register"] ++qemu-quirks = [] + + [dependencies] + qemu-exit = "0.1.x" + +diff -uNr 13_integrated_testing/Makefile 14_exceptions_part2_peripheral_IRQs/Makefile +--- 13_integrated_testing/Makefile ++++ 14_exceptions_part2_peripheral_IRQs/Makefile +@@ -50,11 +50,12 @@ + + RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) + RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs ++FEATURES = bsp_$(BSP) + + SOURCES = $(wildcard **/*.rs) $(wildcard **/*.S) $(wildcard **/*.ld) + + X_CMD_ARGS = --target=$(TARGET) \ +- --features bsp_$(BSP) \ ++ --features $(FEATURES) \ + --release + XRUSTC_CMD = cargo xrustc $(X_CMD_ARGS) + XTEST_CMD = cargo xtest $(X_CMD_ARGS) +@@ -91,6 +92,7 @@ + doc: + cargo xdoc --target=$(TARGET) --features bsp_$(BSP) --document-private-items --open + ++qemu: FEATURES += --features qemu-quirks + ifeq ($(QEMU_MACHINE_TYPE),) + qemu: + @echo $(QEMU_MISSING_STRING) +@@ -113,6 +115,7 @@ + endef + + export KERNEL_TEST_RUNNER ++test: FEATURES += --features qemu-quirks + test: $(SOURCES) + @mkdir -p target + @echo "$$KERNEL_TEST_RUNNER" > target/kernel_test_runner.sh + +diff -uNr 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs +--- 13_integrated_testing/src/_arch/aarch64/exception/asynchronous.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs +@@ -10,6 +10,10 @@ + // Private Definitions + //-------------------------------------------------------------------------------------------------- + ++mod daif_bits { ++ pub const IRQ: u8 = 0b0010; ++} ++ + trait DaifField { + fn daif_field() -> register::Field; + } +@@ -55,6 +59,71 @@ + // Public Code + //-------------------------------------------------------------------------------------------------- + ++/// Returns whether IRQs are masked on the executing core. ++pub fn is_local_irq_masked() -> bool { ++ !is_masked::() ++} ++ ++/// Unmask IRQs on the executing core. ++/// ++/// It is not needed to place an explicit instruction synchronization barrier after the `msr`. ++/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3: ++/// ++/// "Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional ++/// synchronization." ++/// ++/// # Safety ++/// ++/// - Changes the HW state of the executing core. ++#[inline(always)] ++pub unsafe fn local_irq_unmask() { ++ asm!("msr DAIFClr, $0" ++ : // outputs ++ : "i"(daif_bits::IRQ) // inputs ++ : // clobbers ++ : "volatile" // options ++ ); ++} ++ ++/// Mask IRQs on the executing core. ++/// ++/// # Safety ++/// ++/// - Changes the HW state of the executing core. ++#[inline(always)] ++pub unsafe fn local_irq_mask() { ++ asm!("msr DAIFSet, $0" ++ : // outputs ++ : "i"(daif_bits::IRQ) // inputs ++ : // clobbers ++ : "volatile" // options ++ ); ++} ++ ++/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF). ++/// ++/// # Safety ++/// ++/// - Changes the HW state of the executing core. ++#[inline(always)] ++pub unsafe fn local_irq_mask_save() -> u32 { ++ let saved = DAIF.get(); ++ local_irq_mask(); ++ ++ saved ++} ++ ++/// Restore the interrupt mask bits (DAIF) using the callee's argument. ++/// ++/// # Safety ++/// ++/// - Changes the HW state of the executing core. ++/// - No sanity checks on the input. ++#[inline(always)] ++pub unsafe fn local_irq_restore(saved: u32) { ++ DAIF.set(saved); ++} ++ + /// Print the AArch64 exceptions status. + #[rustfmt::skip] + pub fn print_state() { + +diff -uNr 13_integrated_testing/src/_arch/aarch64/exception.rs 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs +--- 13_integrated_testing/src/_arch/aarch64/exception.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs +@@ -4,6 +4,7 @@ + + //! Architectural synchronous and asynchronous exception handling. + ++use crate::{bsp, exception}; + use core::fmt; + use cortex_a::{barrier, regs::*}; + use register::InMemoryRegister; +@@ -84,8 +85,11 @@ + } + + #[no_mangle] +-unsafe extern "C" fn current_elx_irq(e: &mut ExceptionContext) { +- default_exception_handler(e); ++unsafe extern "C" fn current_elx_irq(_e: &mut ExceptionContext) { ++ use exception::asynchronous::interface::IRQManager; ++ ++ let token = &exception::asynchronous::IRQContext::new(); ++ bsp::exception::asynchronous::irq_manager().handle_pending_irqs(token); + } + + #[no_mangle] + +diff -uNr 13_integrated_testing/src/bsp/device_driver/arm/gicv2/gicc.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs +--- 13_integrated_testing/src/bsp/device_driver/arm/gicv2/gicc.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs +@@ -0,0 +1,146 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! GICC Driver - GIC CPU interface. ++ ++use crate::exception; ++use core::ops; ++use register::{mmio::*, register_bitfields, register_structs}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++register_bitfields! { ++ u32, ++ ++ /// CPU Interface Control Register ++ CTLR [ ++ Enable OFFSET(0) NUMBITS(1) [] ++ ], ++ ++ /// Interrupt Priority Mask Register ++ PMR [ ++ Priority OFFSET(0) NUMBITS(8) [] ++ ], ++ ++ /// Interrupt Acknowledge Register ++ IAR [ ++ InterruptID OFFSET(0) NUMBITS(10) [] ++ ], ++ ++ /// End of Interrupt Register ++ EOIR [ ++ EOIINTID OFFSET(0) NUMBITS(10) [] ++ ] ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++register_structs! { ++ #[allow(non_snake_case)] ++ pub RegisterBlock { ++ (0x000 => CTLR: ReadWrite), ++ (0x004 => PMR: ReadWrite), ++ (0x008 => _reserved1), ++ (0x00C => IAR: ReadWrite), ++ (0x010 => EOIR: ReadWrite), ++ (0x014 => @END), ++ } ++} ++ ++/// Representation of the GIC CPU interface. ++pub struct GICC { ++ base_addr: usize, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl ops::Deref for GICC { ++ type Target = RegisterBlock; ++ ++ fn deref(&self) -> &Self::Target { ++ unsafe { &*self.ptr() } ++ } ++} ++ ++impl GICC { ++ /// Create an instance. ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide the correct `base_addr`. ++ pub const unsafe fn new(base_addr: usize) -> Self { ++ Self { base_addr } ++ } ++ ++ /// Return a pointer to the associated MMIO register block. ++ fn ptr(&self) -> *const RegisterBlock { ++ self.base_addr as *const _ ++ } ++ ++ /// Accept interrupts of any priority. ++ /// ++ /// Quoting the GICv2 Architecture Specification: ++ /// ++ /// "Writing 255 to the GICC_PMR always sets it to the largest supported priority field ++ /// value." ++ /// ++ /// # Safety ++ /// ++ /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead ++ /// of `&mut self`. ++ pub fn priority_accept_all(&self) { ++ self.PMR.write(PMR::Priority.val(255)); // Comment in arch spec. ++ } ++ ++ /// Enable the interface - start accepting IRQs. ++ /// ++ /// # Safety ++ /// ++ /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead ++ /// of `&mut self`. ++ pub fn enable(&self) { ++ self.CTLR.write(CTLR::Enable::SET); ++ } ++ ++ /// Extract the number of the highest-priority pending IRQ. ++ /// ++ /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. ++ /// ++ /// # Safety ++ /// ++ /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead ++ /// of `&mut self`. ++ #[allow(clippy::trivially_copy_pass_by_ref)] ++ pub fn get_pending_number<'irq_context>( ++ &self, ++ _ic: &exception::asynchronous::IRQContext<'irq_context>, ++ ) -> usize { ++ self.IAR.read(IAR::InterruptID) as usize ++ } ++ ++ /// Complete handling of the currently active IRQ. ++ /// ++ /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. ++ /// ++ /// To be called after `get_pending_number()`. ++ /// ++ /// # Safety ++ /// ++ /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead ++ /// of `&mut self`. ++ #[allow(clippy::trivially_copy_pass_by_ref)] ++ pub fn mark_comleted<'irq_context>( ++ &self, ++ irq_number: u32, ++ _ic: &exception::asynchronous::IRQContext<'irq_context>, ++ ) { ++ self.EOIR.write(EOIR::EOIINTID.val(irq_number)); ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver/arm/gicv2/gicd.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs +--- 13_integrated_testing/src/bsp/device_driver/arm/gicv2/gicd.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs +@@ -0,0 +1,197 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! GICD Driver - GIC Distributor. ++//! ++//! # Glossary ++//! - SPI - Shared Peripheral Interrupt. ++ ++use crate::{ ++ bsp::device_driver::common::MMIODerefWrapper, state, synchronization, ++ synchronization::IRQSafeNullLock, ++}; ++use register::{mmio::*, register_bitfields, register_structs}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++register_bitfields! { ++ u32, ++ ++ /// Distributor Control Register ++ CTLR [ ++ Enable OFFSET(0) NUMBITS(1) [] ++ ], ++ ++ /// Interrupt Controller Type Register ++ TYPER [ ++ ITLinesNumber OFFSET(0) NUMBITS(5) [] ++ ], ++ ++ /// Interrupt Processor Targets Registers ++ ITARGETSR [ ++ Offset3 OFFSET(24) NUMBITS(8) [], ++ Offset2 OFFSET(16) NUMBITS(8) [], ++ Offset1 OFFSET(8) NUMBITS(8) [], ++ Offset0 OFFSET(0) NUMBITS(8) [] ++ ] ++} ++ ++register_structs! { ++ #[allow(non_snake_case)] ++ SharedRegisterBlock { ++ (0x000 => CTLR: ReadWrite), ++ (0x004 => TYPER: ReadOnly), ++ (0x008 => _reserved1), ++ (0x104 => ISENABLER: [ReadWrite; 31]), ++ (0x108 => _reserved2), ++ (0x820 => ITARGETSR: [ReadWrite; 248]), ++ (0xBFC => @END), ++ } ++} ++ ++register_structs! { ++ #[allow(non_snake_case)] ++ BankedRegisterBlock { ++ (0x000 => _reserved1), ++ (0x100 => ISENABLER: ReadWrite), ++ (0x104 => _reserved2), ++ (0x800 => ITARGETSR: [ReadOnly; 8]), ++ (0xBFC => @END), ++ } ++} ++ ++/// Abstraction for the non-banked parts of the associated MMIO registers. ++type SharedRegs = MMIODerefWrapper; ++ ++/// Abstraction for the banked parts of the associated MMIO registers. ++type BankedRegs = MMIODerefWrapper; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Representation of the GIC Distributor. ++pub struct GICD { ++ /// Access to shared registers is guarded with a lock. ++ shared_regs: IRQSafeNullLock, ++ ++ /// Access to banked registers is unguarded. ++ banked_regs: BankedRegs, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl SharedRegs { ++ /// Return the number of IRQs that this HW implements. ++ #[inline(always)] ++ fn num_irqs(&mut self) -> usize { ++ // Query number of implemented IRQs. ++ // ++ // Refer to GICv2 Architecture Specification, Section 4.3.2. ++ ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 ++ } ++ ++ /// Return a slice of the implemented ITARGETSR. ++ #[inline(always)] ++ fn implemented_itargets_slice(&mut self) -> &[ReadWrite] { ++ assert!(self.num_irqs() >= 36); ++ ++ // Calculate the max index of the shared ITARGETSR array. ++ // ++ // The first 32 IRQs are private, so not included in `shared_regs`. Each ITARGETS ++ // register has four entries, so shift right by two. Subtract one because we start ++ // counting at zero. ++ let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1; ++ ++ // Rust automatically inserts slice range sanity check, i.e. max >= min. ++ &self.ITARGETSR[0..spi_itargetsr_max_index] ++ } ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++use synchronization::interface::Mutex; ++ ++impl GICD { ++ /// Create an instance. ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide the correct `base_addr`. ++ pub const unsafe fn new(base_addr: usize) -> Self { ++ Self { ++ shared_regs: IRQSafeNullLock::new(SharedRegs::new(base_addr)), ++ banked_regs: BankedRegs::new(base_addr), ++ } ++ } ++ ++ /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask. ++ /// ++ /// Quoting the GICv2 Architecture Specification: ++ /// ++ /// "GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that ++ /// corresponds only to the processor reading the register." ++ fn local_gic_target_mask(&self) -> u32 { ++ self.banked_regs.ITARGETSR[0].read(ITARGETSR::Offset0) ++ } ++ ++ /// Route all SPIs to the boot core and enable the distributor. ++ pub fn boot_core_init(&self) { ++ assert!( ++ state::state_manager().state() == state::State::Init, ++ "Only allowed during kernel init phase" ++ ); ++ ++ // Target all SPIs to the boot core only. ++ let mask = self.local_gic_target_mask(); ++ ++ let mut r = &self.shared_regs; ++ r.lock(|regs| { ++ for i in regs.implemented_itargets_slice().iter() { ++ i.write( ++ ITARGETSR::Offset3.val(mask) ++ + ITARGETSR::Offset2.val(mask) ++ + ITARGETSR::Offset1.val(mask) ++ + ITARGETSR::Offset0.val(mask), ++ ); ++ } ++ ++ regs.CTLR.write(CTLR::Enable::SET); ++ }); ++ } ++ ++ /// Enable an interrupt. ++ pub fn enable(&self, irq_num: super::IRQNumber) { ++ let irq_num = irq_num.get(); ++ ++ // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5 ++ // (division by 32) and arrive at the index for the respective ISENABLER[i]. ++ let enable_reg_index = irq_num >> 5; ++ let enable_bit: u32 = 1u32 << (irq_num modulo 32); ++ ++ // Check if we are handling a private or shared IRQ. ++ match irq_num { ++ // Private. ++ 0..=31 => { ++ let enable_reg = &self.banked_regs.ISENABLER; ++ enable_reg.set(enable_reg.get() | enable_bit); ++ } ++ // Shared. ++ _ => { ++ let enable_reg_index_shared = enable_reg_index - 1; ++ ++ let mut r = &self.shared_regs; ++ r.lock(|regs| { ++ let enable_reg = ®s.ISENABLER[enable_reg_index_shared]; ++ enable_reg.set(enable_reg.get() | enable_bit); ++ }); ++ } ++ } ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver/arm/gicv2.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs +--- 13_integrated_testing/src/bsp/device_driver/arm/gicv2.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs +@@ -0,0 +1,222 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! GICv2 Driver - ARM Generic Interrupt Controller v2. ++//! ++//! The following is a collection of excerpts with useful information from ++//! - `Programmer's Guide for ARMv8-A` ++//! - `ARM Generic Interrupt Controller Architecture Specification` ++//! ++//! # Programmer's Guide - 10.6.1 Configuration ++//! ++//! The GIC is accessed as a memory-mapped peripheral. ++//! ++//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core ++//! uses the same address to access its own private CPU interface. ++//! ++//! It is not possible for a core to access the CPU interface of another core. ++//! ++//! # Architecture Specification - 10.6.2 Initialization ++//! ++//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized ++//! after reset before it can deliver interrupts to the core. ++//! ++//! In the Distributor, software must configure the priority, target, security and enable individual ++//! interrupts. The Distributor must subsequently be enabled through its control register ++//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption ++//! settings. ++//! ++//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This ++//! prepares the GIC to deliver interrupts to the core. ++//! ++//! Before interrupts are expected in the core, software prepares the core to take interrupts by ++//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in ++//! PSTATE, and setting the routing controls. ++//! ++//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor. ++//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface. ++//! Individual interrupts can also be disabled (or enabled) in the distributor. ++//! ++//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must ++//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the ++//! core's priority mask. ++//! ++//! # Architecture Specification - 1.4.2 Interrupt types ++//! ++//! - Peripheral interrupt ++//! - Private Peripheral Interrupt (PPI) ++//! - This is a peripheral interrupt that is specific to a single processor. ++//! - Shared Peripheral Interrupt (SPI) ++//! - This is a peripheral interrupt that the Distributor can route to any of a specified ++//! combination of processors. ++//! ++//! - Software-generated interrupt (SGI) ++//! - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The ++//! system uses SGIs for interprocessor communication. ++//! - An SGI has edge-triggered properties. The software triggering of the interrupt is ++//! equivalent to the edge transition of the interrupt request signal. ++//! - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt ++//! Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR, ++//! identifies the processor that requested the interrupt. ++//! ++//! # Architecture Specification - 2.2.1 Interrupt IDs ++//! ++//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020 ++//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by ++//! the Distributor. ++//! ++//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows: ++//! - Interrupt numbers 32..1019 are used for SPIs. ++//! - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These ++//! interrupts are banked in the Distributor. ++//! - A banked interrupt is one where the Distributor can have multiple interrupts with the ++//! same ID. A banked interrupt is identified uniquely by its ID number and its associated ++//! CPU interface number. Of the banked interrupt IDs: ++//! - 00..15 SGIs ++//! - 16..31 PPIs ++ ++mod gicc; ++mod gicd; ++ ++use crate::{bsp, cpu, driver, exception, synchronization, synchronization::InitStateLock}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++type HandlerTable = [Option; GICv2::NUM_IRQS]; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. ++pub type IRQNumber = exception::asynchronous::IRQNumber<{ GICv2::MAX_IRQ_NUMBER }>; ++ ++/// Representation of the GIC. ++pub struct GICv2 { ++ /// The Distributor. ++ gicd: gicd::GICD, ++ ++ /// The CPU Interface. ++ gicc: gicc::GICC, ++ ++ /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. ++ handler_table: InitStateLock, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl GICv2 { ++ const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space. ++ const NUM_IRQS: usize = Self::MAX_IRQ_NUMBER + 1; ++ ++ /// Create an instance. ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide the correct `base_addr`. ++ pub const unsafe fn new(gicd_base_addr: usize, gicc_base_addr: usize) -> Self { ++ Self { ++ gicd: gicd::GICD::new(gicd_base_addr), ++ gicc: gicc::GICC::new(gicc_base_addr), ++ handler_table: InitStateLock::new([None; Self::NUM_IRQS]), ++ } ++ } ++} ++ ++//------------------------------------------------------------------------------ ++// OS Interface Code ++//------------------------------------------------------------------------------ ++use synchronization::interface::ReadWriteEx; ++ ++impl driver::interface::DeviceDriver for GICv2 { ++ fn compatible(&self) -> &str { ++ "GICv2 (ARM Generic Interrupt Controller v2)" ++ } ++ ++ fn init(&self) -> Result<(), ()> { ++ if cpu::smp::core_id::() == bsp::cpu::BOOT_CORE_ID { ++ self.gicd.boot_core_init(); ++ } ++ ++ self.gicc.priority_accept_all(); ++ self.gicc.enable(); ++ ++ Ok(()) ++ } ++} ++ ++impl exception::asynchronous::interface::IRQManager for GICv2 { ++ type IRQNumberType = IRQNumber; ++ ++ fn register_handler( ++ &self, ++ irq_number: Self::IRQNumberType, ++ descriptor: exception::asynchronous::IRQDescriptor, ++ ) -> Result<(), &'static str> { ++ let mut r = &self.handler_table; ++ r.write(|table| { ++ let irq_number = irq_number.get(); ++ ++ if table[irq_number].is_some() { ++ return Err("IRQ handler already registered"); ++ } ++ ++ table[irq_number] = Some(descriptor); ++ ++ Ok(()) ++ }) ++ } ++ ++ fn enable(&self, irq_number: Self::IRQNumberType) { ++ self.gicd.enable(irq_number); ++ } ++ ++ fn handle_pending_irqs<'irq_context>( ++ &'irq_context self, ++ ic: &exception::asynchronous::IRQContext<'irq_context>, ++ ) { ++ // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register ++ // (IAR). ++ let irq_number = self.gicc.get_pending_number(ic); ++ ++ // Guard against spurious interrupts. ++ if irq_number > GICv2::MAX_IRQ_NUMBER { ++ return; ++ } ++ ++ // Call the IRQ handler. Panic if there is none. ++ let mut r = &self.handler_table; ++ r.read(|table| { ++ match table[irq_number] { ++ None => panic!("No handler registered for IRQ {}", irq_number), ++ Some(descriptor) => { ++ // Call the IRQ handler. Panics on failure. ++ descriptor.handler.handle().expect("Error handling IRQ"); ++ } ++ } ++ }); ++ ++ // Signal completion of handling. ++ self.gicc.mark_comleted(irq_number as u32, ic); ++ } ++ ++ fn print_handler(&self) { ++ use crate::info; ++ ++ info!(" Peripheral handler:"); ++ ++ let mut r = &self.handler_table; ++ r.read(|table| { ++ for (i, opt) in table.iter().skip(32).enumerate() { ++ if let Some(handler) = opt { ++ info!(" {: >3}. {}", i + 32, handler.name); ++ } ++ } ++ }); ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver/arm.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm.rs +--- 13_integrated_testing/src/bsp/device_driver/arm.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm.rs +@@ -0,0 +1,11 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! ARM driver top level. ++ ++#[cfg(feature = "bsp_rpi4")] ++pub mod gicv2; ++ ++#[cfg(feature = "bsp_rpi4")] ++pub use gicv2::*; + +diff -uNr 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs +--- 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs +@@ -4,8 +4,10 @@ + + //! GPIO Driver. + +-use crate::{cpu, driver, synchronization, synchronization::NullLock}; +-use core::ops; ++use crate::{ ++ bsp::device_driver::common::MMIODerefWrapper, cpu, driver, synchronization, ++ synchronization::IRQSafeNullLock, ++}; + use register::{mmio::*, register_bitfields, register_structs}; + + //-------------------------------------------------------------------------------------------------- +@@ -70,9 +72,8 @@ + } + } + +-struct GPIOInner { +- base_addr: usize, +-} ++/// Abstraction for the associated MMIO registers. ++type Regs = MMIODerefWrapper; + + //-------------------------------------------------------------------------------------------------- + // Public Definitions +@@ -80,30 +81,7 @@ + + /// Representation of the GPIO HW. + pub struct GPIO { +- inner: NullLock, +-} +- +-//-------------------------------------------------------------------------------------------------- +-// Private Code +-//-------------------------------------------------------------------------------------------------- +- +-impl ops::Deref for GPIOInner { +- type Target = RegisterBlock; +- +- fn deref(&self) -> &Self::Target { +- unsafe { &*self.ptr() } +- } +-} +- +-impl GPIOInner { +- const fn new(base_addr: usize) -> Self { +- Self { base_addr } +- } +- +- /// Return a pointer to the associated MMIO register block. +- fn ptr(&self) -> *const RegisterBlock { +- self.base_addr as *const _ +- } ++ inner: IRQSafeNullLock, + } + + //-------------------------------------------------------------------------------------------------- +@@ -118,7 +96,7 @@ + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { +- inner: NullLock::new(GPIOInner::new(base_addr)), ++ inner: IRQSafeNullLock::new(Regs::new(base_addr)), + } + } + + +diff -uNr 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs +--- 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs +@@ -0,0 +1,167 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! Peripheral Interrupt regsler Driver. ++ ++use super::{InterruptController, PendingIRQs, PeripheralIRQ}; ++use crate::{ ++ bsp::device_driver::common::MMIODerefWrapper, ++ exception, synchronization, ++ synchronization::{IRQSafeNullLock, InitStateLock}, ++}; ++use register::{mmio::*, register_structs}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++register_structs! { ++ #[allow(non_snake_case)] ++ WORegisterBlock { ++ (0x00 => _reserved1), ++ (0x10 => ENABLE_1: WriteOnly), ++ (0x14 => ENABLE_2: WriteOnly), ++ (0x24 => @END), ++ } ++} ++ ++register_structs! { ++ #[allow(non_snake_case)] ++ RORegisterBlock { ++ (0x00 => _reserved1), ++ (0x04 => PENDING_1: ReadOnly), ++ (0x08 => PENDING_2: ReadOnly), ++ (0x0c => @END), ++ } ++} ++ ++/// Abstraction for the WriteOnly parts of the associated MMIO registers. ++type WriteOnlyRegs = MMIODerefWrapper; ++ ++/// Abstraction for the ReadOnly parts of the associated MMIO registers. ++type ReadOnlyRegs = MMIODerefWrapper; ++ ++type HandlerTable = ++ [Option; InterruptController::NUM_PERIPHERAL_IRQS]; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Representation of the peripheral interrupt regsler. ++pub struct PeripheralIC { ++ /// Access to write registers is guarded with a lock. ++ wo_regs: IRQSafeNullLock, ++ ++ /// Register read access is unguarded. ++ ro_regs: ReadOnlyRegs, ++ ++ /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. ++ handler_table: InitStateLock, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl PeripheralIC { ++ /// Create an instance. ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide the correct `base_addr`. ++ pub const unsafe fn new(base_addr: usize) -> Self { ++ Self { ++ wo_regs: IRQSafeNullLock::new(WriteOnlyRegs::new(base_addr)), ++ ro_regs: ReadOnlyRegs::new(base_addr), ++ handler_table: InitStateLock::new([None; InterruptController::NUM_PERIPHERAL_IRQS]), ++ } ++ } ++ ++ /// Query the list of pending IRQs. ++ fn get_pending(&self) -> PendingIRQs { ++ let pending_mask: u64 = (u64::from(self.ro_regs.PENDING_2.get()) << 32) ++ | u64::from(self.ro_regs.PENDING_1.get()); ++ ++ PendingIRQs::new(pending_mask) ++ } ++} ++ ++//------------------------------------------------------------------------------ ++// OS Interface Code ++//------------------------------------------------------------------------------ ++use synchronization::interface::{Mutex, ReadWriteEx}; ++ ++impl exception::asynchronous::interface::IRQManager for PeripheralIC { ++ type IRQNumberType = PeripheralIRQ; ++ ++ fn register_handler( ++ &self, ++ irq: Self::IRQNumberType, ++ descriptor: exception::asynchronous::IRQDescriptor, ++ ) -> Result<(), &'static str> { ++ let mut r = &self.handler_table; ++ r.write(|table| { ++ let irq_number = irq.get(); ++ ++ if table[irq_number].is_some() { ++ return Err("IRQ handler already registered"); ++ } ++ ++ table[irq_number] = Some(descriptor); ++ ++ Ok(()) ++ }) ++ } ++ ++ fn enable(&self, irq: Self::IRQNumberType) { ++ let mut r = &self.wo_regs; ++ r.lock(|regs| { ++ let enable_reg = if irq.get() <= 31 { ++ ®s.ENABLE_1 ++ } else { ++ ®s.ENABLE_2 ++ }; ++ ++ let enable_bit: u32 = 1 << (irq.get() modulo 32); ++ ++ // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable ++ // bits are unaffected. So we don't need read and OR'ing here. ++ enable_reg.set(enable_bit); ++ }); ++ } ++ ++ fn handle_pending_irqs<'irq_context>( ++ &'irq_context self, ++ _ic: &exception::asynchronous::IRQContext<'irq_context>, ++ ) { ++ let mut r = &self.handler_table; ++ r.read(|table| { ++ for irq_number in self.get_pending() { ++ match table[irq_number] { ++ None => panic!("No handler registered for IRQ {}", irq_number), ++ Some(descriptor) => { ++ // Call the IRQ handler. Panics on failure. ++ descriptor.handler.handle().expect("Error handling IRQ"); ++ } ++ } ++ } ++ }) ++ } ++ ++ fn print_handler(&self) { ++ use crate::info; ++ ++ info!(" Peripheral handler:"); ++ ++ let mut r = &self.handler_table; ++ r.read(|table| { ++ for (i, opt) in table.iter().enumerate() { ++ if let Some(handler) = opt { ++ info!(" {: >3}. {}", i, handler.name); ++ } ++ } ++ }); ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs +--- 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs +@@ -0,0 +1,131 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! Interrupt Controller Driver. ++ ++mod peripheral_ic; ++ ++use crate::{driver, exception}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Wrapper struct for a bitmask indicating pending IRQ numbers. ++struct PendingIRQs { ++ bitmask: u64, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++pub type LocalIRQ = ++ exception::asynchronous::IRQNumber<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>; ++pub type PeripheralIRQ = ++ exception::asynchronous::IRQNumber<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>; ++ ++/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. ++#[derive(Copy, Clone)] ++pub enum IRQNumber { ++ Local(LocalIRQ), ++ Peripheral(PeripheralIRQ), ++} ++ ++/// Representation of the Interrupt Controller. ++pub struct InterruptController { ++ periph: peripheral_ic::PeripheralIC, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl PendingIRQs { ++ pub fn new(bitmask: u64) -> Self { ++ Self { bitmask } ++ } ++} ++ ++impl Iterator for PendingIRQs { ++ type Item = usize; ++ ++ fn next(&mut self) -> Option { ++ use core::intrinsics::cttz; ++ ++ let next = cttz(self.bitmask); ++ if next == 64 { ++ return None; ++ } ++ ++ self.bitmask &= !(1 << next); ++ ++ Some(next as usize) ++ } ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl InterruptController { ++ const MAX_LOCAL_IRQ_NUMBER: usize = 11; ++ const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63; ++ const NUM_PERIPHERAL_IRQS: usize = Self::MAX_PERIPHERAL_IRQ_NUMBER + 1; ++ ++ /// Create an instance. ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide the correct `base_addr`. ++ pub const unsafe fn new(_local_base_addr: usize, periph_base_addr: usize) -> Self { ++ Self { ++ periph: peripheral_ic::PeripheralIC::new(periph_base_addr), ++ } ++ } ++} ++ ++//------------------------------------------------------------------------------ ++// OS Interface Code ++//------------------------------------------------------------------------------ ++ ++impl driver::interface::DeviceDriver for InterruptController { ++ fn compatible(&self) -> &str { ++ "BCM Interrupt Controller" ++ } ++} ++ ++impl exception::asynchronous::interface::IRQManager for InterruptController { ++ type IRQNumberType = IRQNumber; ++ ++ fn register_handler( ++ &self, ++ irq: Self::IRQNumberType, ++ descriptor: exception::asynchronous::IRQDescriptor, ++ ) -> Result<(), &'static str> { ++ match irq { ++ IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), ++ IRQNumber::Peripheral(pirq) => self.periph.register_handler(pirq, descriptor), ++ } ++ } ++ ++ fn enable(&self, irq: Self::IRQNumberType) { ++ match irq { ++ IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), ++ IRQNumber::Peripheral(pirq) => self.periph.enable(pirq), ++ } ++ } ++ ++ fn handle_pending_irqs<'irq_context>( ++ &'irq_context self, ++ ic: &exception::asynchronous::IRQContext<'irq_context>, ++ ) { ++ // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet. ++ self.periph.handle_pending_irqs(ic) ++ } ++ ++ fn print_handler(&self) { ++ self.periph.print_handler(); ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs +--- 13_integrated_testing/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs +@@ -3,8 +3,32 @@ + // Copyright (c) 2018-2020 Andre Richter + + //! PL011 UART driver. +- +-use crate::{console, cpu, driver, synchronization, synchronization::NullLock}; ++//! ++//! # FIFO fill level IRQ hack ++//! ++//! For learning purposes, we want the UART to raise an IRQ on _every_ received character. ++//! Unfortunately, this rather common mode of operation is not supported by the PL011 when operating ++//! in FIFO mode. It is only possible to set a fill level fraction on which the IRQ is triggered. ++//! The lowest fill level is 1/8. ++//! ++//! On the RPi3, the RX FIFO is 16 chars deep, so the IRQ would trigger after 2 chars have been ++//! received. On the RPi4, the FIFO seems to be 32 chars deep, because experiments showed that the ++//! RX IRQ triggers after receiving 4 chars. ++//! ++//! Fortunately, the PL011 has a test mode which allows to push characters into the FIFOs. We make ++//! use of this testing facilities to employ a little hack that pushes (fill-level - 1) chars into ++//! the RX FIFO by default. This way, we get an IRQ for the first received char that arrives from ++//! external. ++//! ++//! To make things even more complicated, QEMU is not honoring the fill-level dependent IRQ ++//! generation. Instead, QEMU creates an IRQ on every received char. ++//! ++//! We use conditional compilation to differentiate between the three modes of operation (RPi3, ++//! RPI4, QEMU) respectively. ++ ++use crate::{ ++ bsp, console, cpu, driver, exception, synchronization, synchronization::IRQSafeNullLock, ++}; + use core::{fmt, ops}; + use register::{mmio::*, register_bitfields, register_structs}; + +@@ -106,10 +130,45 @@ + ] + ], + ++ /// Interrupt FIFO Level Select Register ++ IFLS [ ++ /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as ++ /// follows. ++ RXIFLSEL OFFSET(3) NUMBITS(5) [ ++ OneEigth = 0b000, ++ OneQuarter = 0b001, ++ OneHalf = 0b010, ++ ThreeQuarters = 0b011, ++ SevenEights = 0b100 ++ ] ++ ], ++ ++ /// Interrupt Mask Set Clear Register ++ IMSC [ ++ /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt. On ++ /// a write of 1, the mask of the interrupt is set. A write of 0 clears the mask. ++ RXIM OFFSET(4) NUMBITS(1) [ ++ Disabled = 0, ++ Enabled = 1 ++ ] ++ ], ++ + /// Interrupt Clear Register + ICR [ + /// Meta field for all pending interrupts + ALL OFFSET(0) NUMBITS(11) [] ++ ], ++ ++ /// Test Control Register ++ ITCR [ ++ /// Test FIFO enable. When this bit it 1, a write to the Test Data Register, UART_DR writes ++ /// data into the receive FIFO, and reads from the UART_DR register reads data out of the ++ /// transmit FIFO. When this bit is 0, data cannot be read directly from the transmit FIFO ++ /// or written directly to the receive FIFO (normal operation). ++ ITCR1 OFFSET(1) NUMBITS(1) [ ++ Disabled = 0, ++ Enabled = 1 ++ ] + ] + } + +@@ -128,9 +187,15 @@ + (0x28 => FBRD: WriteOnly), + (0x2c => LCRH: WriteOnly), + (0x30 => CR: WriteOnly), +- (0x34 => _reserved3), ++ (0x34 => IFLS: ReadWrite), ++ (0x38 => IMSC: ReadWrite), ++ (0x3C => _reserved3), + (0x44 => ICR: WriteOnly), +- (0x48 => @END), ++ (0x48 => _reserved4), ++ (0x80 => ITCR: ReadWrite), ++ (0x84 => _reserved5), ++ (0x8c => TDR: ReadWrite), ++ (0x90 => @END), + } + } + +@@ -145,7 +210,8 @@ + + /// Representation of the UART. + pub struct PL011Uart { +- inner: NullLock, ++ inner: IRQSafeNullLock, ++ irq_number: bsp::device_driver::IRQNumber, + } + + //-------------------------------------------------------------------------------------------------- +@@ -199,6 +265,14 @@ + .write(LCRH::WLEN::EightBit + LCRH::FEN::FifosEnabled); // 8N1 + Fifo on + self.CR + .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled); ++ ++ // Trigger the RX interrupt at 1/8 of the FIFO fill level (this is the lowest possible) and ++ // enable RX interrupts. ++ self.IFLS.write(IFLS::RXIFLSEL::OneEigth); ++ self.IMSC.write(IMSC::RXIM::Enabled); ++ ++ #[cfg(not(feature = "qemu-quirks"))] ++ self.fill_hack_push(); + } + + /// Return a pointer to the register block. +@@ -218,6 +292,70 @@ + + self.chars_written += 1; + } ++ ++ /// Retrieve a character. ++ fn read_char_converting(&mut self, blocking: bool) -> Option { ++ #[cfg(not(feature = "qemu-quirks"))] ++ self.fill_hack_pop(); ++ ++ // If blocking, spin while RX FIFO empty is set, else return None. ++ while self.FR.matches_all(FR::RXFE::SET) { ++ if !blocking { ++ #[cfg(not(feature = "qemu-quirks"))] ++ self.fill_hack_push(); ++ ++ return None; ++ } ++ ++ cpu::nop(); ++ } ++ ++ // Read one character. ++ let mut ret = self.DR.get() as u8 as char; ++ ++ // Convert carrige return to newline. ++ if ret == '\r' { ++ ret = '\n' ++ } ++ ++ // Update statistics. ++ self.chars_read += 1; ++ ++ #[cfg(not(feature = "qemu-quirks"))] ++ self.fill_hack_push(); ++ ++ Some(ret) ++ } ++ ++ /// Push characters into the receive FIFO. ++ /// ++ /// See top level comments why this is needed. ++ #[cfg(not(feature = "qemu-quirks"))] ++ fn fill_hack_push(&mut self) { ++ self.ITCR.write(ITCR::ITCR1::Enabled); ++ ++ #[cfg(feature = "bsp_rpi4")] ++ { ++ self.TDR.set(b'X' as u32); ++ self.TDR.set(b'Y' as u32); ++ } ++ self.TDR.set(b'Z' as u32); ++ ++ self.ITCR.write(ITCR::ITCR1::Disabled); ++ } ++ ++ /// Pop characters from the receive FIFO. ++ /// ++ /// See top level comments why this is needed. ++ #[cfg(not(feature = "qemu-quirks"))] ++ fn fill_hack_pop(&mut self) { ++ #[cfg(feature = "bsp_rpi4")] ++ { ++ self.DR.get(); ++ self.DR.get(); ++ } ++ self.DR.get(); ++ } + } + + /// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are +@@ -243,9 +381,10 @@ + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. +- pub const unsafe fn new(base_addr: usize) -> Self { ++ pub const unsafe fn new(base_addr: usize, irq_number: bsp::device_driver::IRQNumber) -> Self { + Self { +- inner: NullLock::new(PL011UartInner::new(base_addr)), ++ inner: IRQSafeNullLock::new(PL011UartInner::new(base_addr)), ++ irq_number, + } + } + } +@@ -266,6 +405,21 @@ + + Ok(()) + } ++ ++ fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { ++ use bsp::exception::asynchronous::irq_manager; ++ use exception::asynchronous::{interface::IRQManager, IRQDescriptor}; ++ ++ let descriptor = IRQDescriptor { ++ name: "BCM PL011 UART", ++ handler: self, ++ }; ++ ++ irq_manager().register_handler(self.irq_number, descriptor)?; ++ irq_manager().enable(self.irq_number); ++ ++ Ok(()) ++ } + } + + impl console::interface::Write for PL011Uart { +@@ -297,25 +451,7 @@ + impl console::interface::Read for PL011Uart { + fn read_char(&self) -> char { + let mut r = &self.inner; +- r.lock(|inner| { +- // Spin while RX FIFO empty is set. +- while inner.FR.matches_all(FR::RXFE::SET) { +- cpu::nop(); +- } +- +- // Read one character. +- let mut ret = inner.DR.get() as u8 as char; +- +- // Convert carrige return to newline. +- if ret == '\r' { +- ret = '\n' +- } +- +- // Update statistics. +- inner.chars_read += 1; +- +- ret +- }) ++ r.lock(|inner| inner.read_char_converting(true).unwrap()) + } + + fn clear(&self) { +@@ -340,3 +476,20 @@ + r.lock(|inner| inner.chars_read) + } + } ++ ++impl exception::asynchronous::interface::IRQHandler for PL011Uart { ++ fn handle(&self) -> Result<(), &'static str> { ++ let mut r = &self.inner; ++ r.lock(|inner| { ++ // Echo any received characters. ++ loop { ++ match inner.read_char_converting(false) { ++ None => break, ++ Some(c) => inner.write_char(c), ++ } ++ } ++ }); ++ ++ Ok(()) ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver/bcm.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm.rs +--- 13_integrated_testing/src/bsp/device_driver/bcm.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm.rs +@@ -5,7 +5,11 @@ + //! BCM driver top level. + + mod bcm2xxx_gpio; ++#[cfg(feature = "bsp_rpi3")] ++mod bcm2xxx_interrupt_controller; + mod bcm2xxx_pl011_uart; + + pub use bcm2xxx_gpio::*; ++#[cfg(feature = "bsp_rpi3")] ++pub use bcm2xxx_interrupt_controller::*; + pub use bcm2xxx_pl011_uart::*; + +diff -uNr 13_integrated_testing/src/bsp/device_driver/common.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/common.rs +--- 13_integrated_testing/src/bsp/device_driver/common.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/common.rs +@@ -0,0 +1,35 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! Common device driver code. ++ ++use core::{marker::PhantomData, ops}; ++ ++pub struct MMIODerefWrapper { ++ base_addr: usize, ++ phantom: PhantomData, ++} ++ ++impl MMIODerefWrapper { ++ /// Create an instance. ++ pub const unsafe fn new(base_addr: usize) -> Self { ++ Self { ++ base_addr, ++ phantom: PhantomData, ++ } ++ } ++ ++ /// Return a pointer to the associated MMIO register block. ++ fn ptr(&self) -> *const T { ++ self.base_addr as *const _ ++ } ++} ++ ++impl ops::Deref for MMIODerefWrapper { ++ type Target = T; ++ ++ fn deref(&self) -> &Self::Target { ++ unsafe { &*self.ptr() } ++ } ++} + +diff -uNr 13_integrated_testing/src/bsp/device_driver.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver.rs +--- 13_integrated_testing/src/bsp/device_driver.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver.rs +@@ -4,8 +4,13 @@ + + //! Device driver. + ++#[cfg(feature = "bsp_rpi4")] ++mod arm; + #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] + mod bcm; ++mod common; + ++#[cfg(feature = "bsp_rpi4")] ++pub use arm::*; + #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] + pub use bcm::*; + +diff -uNr 13_integrated_testing/src/bsp/raspberrypi/cpu.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs +--- 13_integrated_testing/src/bsp/raspberrypi/cpu.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs +@@ -13,3 +13,6 @@ + + /// The early boot core's stack address. + pub const BOOT_CORE_STACK_START: u64 = 0x80_000; ++ ++/// The number of processor cores. ++pub const NUM_CORES: usize = 4; + +diff -uNr 13_integrated_testing/src/bsp/raspberrypi/driver.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs +--- 13_integrated_testing/src/bsp/raspberrypi/driver.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs +@@ -12,7 +12,7 @@ + + /// Device Driver Manager type. + pub struct BSPDriverManager { +- device_drivers: [&'static (dyn DeviceDriver + Sync); 2], ++ device_drivers: [&'static (dyn DeviceDriver + Sync); 3], + } + + //-------------------------------------------------------------------------------------------------- +@@ -20,7 +20,11 @@ + //-------------------------------------------------------------------------------------------------- + + static BSP_DRIVER_MANAGER: BSPDriverManager = BSPDriverManager { +- device_drivers: [&super::GPIO, &super::PL011_UART], ++ device_drivers: [ ++ &super::GPIO, ++ &super::PL011_UART, ++ &super::INTERRUPT_CONTROLLER, ++ ], + }; + + //-------------------------------------------------------------------------------------------------- + +diff -uNr 13_integrated_testing/src/bsp/raspberrypi/exception/asynchronous.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception/asynchronous.rs +--- 13_integrated_testing/src/bsp/raspberrypi/exception/asynchronous.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception/asynchronous.rs +@@ -0,0 +1,36 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! BSP asynchronous exception handling. ++ ++use crate::{bsp, exception}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++#[cfg(feature = "bsp_rpi3")] ++pub(in crate::bsp) mod irq_map { ++ use super::bsp::device_driver::{IRQNumber, PeripheralIRQ}; ++ ++ pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57)); ++} ++ ++#[cfg(feature = "bsp_rpi4")] ++pub(in crate::bsp) mod irq_map { ++ use super::bsp::device_driver::IRQNumber; ++ ++ pub const PL011_UART: IRQNumber = IRQNumber::new(153); ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Return a reference to the IRQ manager. ++pub fn irq_manager() -> &'static impl exception::asynchronous::interface::IRQManager< ++ IRQNumberType = bsp::device_driver::IRQNumber, ++> { ++ &super::super::INTERRUPT_CONTROLLER ++} + +diff -uNr 13_integrated_testing/src/bsp/raspberrypi/exception.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception.rs +--- 13_integrated_testing/src/bsp/raspberrypi/exception.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception.rs +@@ -0,0 +1,7 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! BSP synchronous and asynchronous exception handling. ++ ++pub mod asynchronous; + +diff -uNr 13_integrated_testing/src/bsp/raspberrypi/memory.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs +--- 13_integrated_testing/src/bsp/raspberrypi/memory.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs +@@ -24,8 +24,10 @@ + use super::*; + + pub const BASE: usize = 0x3F00_0000; ++ pub const PERIPHERAL_INTERRUPT_CONTROLLER_BASE: usize = BASE + 0x0000_B200; + pub const GPIO_BASE: usize = BASE + GPIO_OFFSET; + pub const PL011_UART_BASE: usize = BASE + UART_OFFSET; ++ pub const LOCAL_INTERRUPT_CONTROLLER_BASE: usize = 0x4000_0000; + pub const END_INCLUSIVE: usize = 0x4000_FFFF; + } + +@@ -37,6 +39,8 @@ + pub const BASE: usize = 0xFE00_0000; + pub const GPIO_BASE: usize = BASE + GPIO_OFFSET; + pub const PL011_UART_BASE: usize = BASE + UART_OFFSET; ++ pub const GICD_BASE: usize = 0xFF84_1000; ++ pub const GICC_BASE: usize = 0xFF84_2000; + pub const END_INCLUSIVE: usize = 0xFF84_FFFF; + } + } + +diff -uNr 13_integrated_testing/src/bsp/raspberrypi.rs 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs +--- 13_integrated_testing/src/bsp/raspberrypi.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs +@@ -7,6 +7,7 @@ + pub mod console; + pub mod cpu; + pub mod driver; ++pub mod exception; + pub mod memory; + + //-------------------------------------------------------------------------------------------------- +@@ -17,8 +18,25 @@ + static GPIO: device_driver::GPIO = + unsafe { device_driver::GPIO::new(memory::map::mmio::GPIO_BASE) }; + +-static PL011_UART: device_driver::PL011Uart = +- unsafe { device_driver::PL011Uart::new(memory::map::mmio::PL011_UART_BASE) }; ++static PL011_UART: device_driver::PL011Uart = unsafe { ++ device_driver::PL011Uart::new( ++ memory::map::mmio::PL011_UART_BASE, ++ exception::asynchronous::irq_map::PL011_UART, ++ ) ++}; ++ ++#[cfg(feature = "bsp_rpi3")] ++static INTERRUPT_CONTROLLER: device_driver::InterruptController = unsafe { ++ device_driver::InterruptController::new( ++ memory::map::mmio::LOCAL_INTERRUPT_CONTROLLER_BASE, ++ memory::map::mmio::PERIPHERAL_INTERRUPT_CONTROLLER_BASE, ++ ) ++}; ++ ++#[cfg(feature = "bsp_rpi4")] ++static INTERRUPT_CONTROLLER: device_driver::GICv2 = unsafe { ++ device_driver::GICv2::new(memory::map::mmio::GICD_BASE, memory::map::mmio::GICC_BASE) ++}; + + //-------------------------------------------------------------------------------------------------- + // Public Code + +diff -uNr 13_integrated_testing/src/driver.rs 14_exceptions_part2_peripheral_IRQs/src/driver.rs +--- 13_integrated_testing/src/driver.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/driver.rs +@@ -20,6 +20,14 @@ + fn init(&self) -> Result<(), ()> { + Ok(()) + } ++ ++ /// Called by the kernel to register and enable the device's IRQ handlers, if any. ++ /// ++ /// Rust's type system will prevent a call to this function unless the calling instance ++ /// itself has static lifetime. ++ fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { ++ Ok(()) ++ } + } + + /// Device driver management functions. + +diff -uNr 13_integrated_testing/src/exception/asynchronous.rs 14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs +--- 13_integrated_testing/src/exception/asynchronous.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs +@@ -8,3 +8,138 @@ + #[path = "../_arch/aarch64/exception/asynchronous.rs"] + mod arch_exception_async; + pub use arch_exception_async::*; ++ ++use core::{fmt, marker::PhantomData}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Asynchronous exception handling interfaces. ++pub mod interface { ++ ++ /// Implemented by types that handle IRQs. ++ pub trait IRQHandler { ++ /// Called when the corresponding interrupt is asserted. ++ fn handle(&self) -> Result<(), &'static str>; ++ } ++ ++ /// IRQ management functions. ++ /// ++ /// The `BSP` is supposed to supply one global instance. Typically implemented by the ++ /// platform's interrupt controller. ++ pub trait IRQManager { ++ /// The IRQ number type depends on the implementation. ++ type IRQNumberType; ++ ++ /// Register a handler. ++ fn register_handler( ++ &self, ++ irq_number: Self::IRQNumberType, ++ descriptor: super::IRQDescriptor, ++ ) -> Result<(), &'static str>; ++ ++ /// Enable an interrupt in the controller. ++ fn enable(&self, irq_number: Self::IRQNumberType); ++ ++ /// Handle pending interrupts. ++ /// ++ /// This function is called directly from the CPU's IRQ exception vector. On AArch64, ++ /// this means that the respective CPU core has disabled exception handling. ++ /// This function can therefore not be preempted and runs start to finish. ++ /// ++ /// Takes an IRQContext token to ensure it can only be called from IRQ context. ++ #[allow(clippy::trivially_copy_pass_by_ref)] ++ fn handle_pending_irqs<'irq_context>( ++ &'irq_context self, ++ ic: &super::IRQContext<'irq_context>, ++ ); ++ ++ /// Print list of registered handlers. ++ fn print_handler(&self); ++ } ++} ++ ++/// Interrupt descriptor. ++#[derive(Copy, Clone)] ++pub struct IRQDescriptor { ++ /// Descriptive name. ++ pub name: &'static str, ++ ++ /// Reference to handler trait object. ++ pub handler: &'static (dyn interface::IRQHandler + Sync), ++} ++ ++/// IRQContext token. ++/// ++/// An instance of this type indicates that the local core is currently executing in IRQ ++/// context, aka executing an interrupt vector or subcalls of it. ++/// ++/// Concept and implementation derived from the `CriticalSection` introduced in ++/// https://github.com/rust-embedded/bare-metal ++#[derive(Clone, Copy)] ++pub struct IRQContext<'irq_context> { ++ _0: PhantomData<&'irq_context ()>, ++} ++ ++/// A wrapper type for IRQ numbers with integrated range sanity check. ++#[derive(Copy, Clone)] ++pub struct IRQNumber(usize); ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl<'irq_context> IRQContext<'irq_context> { ++ /// Creates an IRQContext token. ++ /// ++ /// # Safety ++ /// ++ /// - This must only be called when the current core is in an interrupt context and will not ++ /// live beyond the end of it. That is, creation is allowed in interrupt vector functions. For ++ /// example, in the ARMv8-A case, in `extern "C" fn current_elx_irq()`. ++ /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code ++ /// must not be able to influence the lifetime picked for this type, since that might cause it ++ /// to be inferred to `'static`. ++ #[inline(always)] ++ pub unsafe fn new() -> Self { ++ IRQContext { _0: PhantomData } ++ } ++} ++ ++impl IRQNumber<{ MAX_INCLUSIVE }> { ++ /// Creates a new instance if number <= MAX_INCLUSIVE. ++ pub const fn new(number: usize) -> Self { ++ assert!(number <= MAX_INCLUSIVE); ++ ++ Self { 0: number } ++ } ++ ++ /// Return the wrapped number. ++ pub fn get(self) -> usize { ++ self.0 ++ } ++} ++ ++impl fmt::Display for IRQNumber<{ MAX_INCLUSIVE }> { ++ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { ++ write!(f, "{}", self.0) ++ } ++} ++ ++/// Executes the provided closure while IRQs are masked on the executing core. ++/// ++/// While the function temporarily changes the HW state of the executing core, it restores it to the ++/// previous state before returning, so this is deemed safe. ++#[inline(always)] ++pub fn exec_with_irq_masked(f: impl FnOnce() -> T) -> T { ++ let ret: T; ++ ++ unsafe { ++ let saved = local_irq_mask_save(); ++ ret = f(); ++ local_irq_restore(saved); ++ } ++ ++ ret ++} + +diff -uNr 13_integrated_testing/src/lib.rs 14_exceptions_part2_peripheral_IRQs/src/lib.rs +--- 13_integrated_testing/src/lib.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/lib.rs +@@ -13,12 +13,17 @@ + //! + //! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. + //! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. ++//! - [`bsp::exception::asynchronous::irq_manager()`] - Returns a reference to the kernel's [IRQ ++//! Handling interface]. + //! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. ++//! - [`state::state_manager()`] - Returns a reference to the kernel's [state management] instance. + //! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. + //! + //! [console interface]: ../libkernel/console/interface/index.html + //! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html ++//! [IRQ Handling interface]: ../libkernel/exception/asynchronous/interface/trait.IRQManager.html + //! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html ++//! [state management]: ../libkernel/state/struct.StateManager.html + //! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html + //! + //! # Code organization and architecture +@@ -107,7 +112,12 @@ + //! - `crate::bsp::memory::*` + + #![allow(incomplete_features)] ++#![feature(asm)] ++#![feature(const_fn)] + #![feature(const_generics)] ++#![feature(const_if_match)] ++#![feature(const_panic)] ++#![feature(core_intrinsics)] + #![feature(custom_inner_attributes)] + #![feature(format_args_nl)] + #![feature(global_asm)] +@@ -137,6 +147,7 @@ + pub mod exception; + pub mod memory; + pub mod print; ++pub mod state; + pub mod time; + + //-------------------------------------------------------------------------------------------------- + +diff -uNr 13_integrated_testing/src/main.rs 14_exceptions_part2_peripheral_IRQs/src/main.rs +--- 13_integrated_testing/src/main.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/main.rs +@@ -11,7 +11,7 @@ + #![no_main] + #![no_std] + +-use libkernel::{bsp, console, driver, exception, info, memory, time}; ++use libkernel::{bsp, cpu, driver, exception, info, memory, state, time, warn}; + + /// Early init code. + /// +@@ -21,8 +21,8 @@ + /// - The init calls in this function must appear in the correct order: + /// - Virtual memory must be activated before the device drivers. + /// - Without it, any atomic operations, e.g. the yet-to-be-introduced spinlocks in the device +-/// drivers (which currently employ NullLocks instead of spinlocks), will fail to work on +-/// the RPi SoCs. ++/// drivers (which currently employ IRQSafeNullLocks instead of spinlocks), will fail to ++/// work on the RPi SoCs. + #[no_mangle] + unsafe fn kernel_init() -> ! { + use driver::interface::DriverManager; +@@ -42,14 +42,27 @@ + bsp::driver::driver_manager().post_device_driver_init(); + // println! is usable from here on. + ++ // Let device drivers register and enable their handlers with the interrupt controller. ++ for i in bsp::driver::driver_manager().all_device_drivers() { ++ if let Err(msg) = i.register_and_enable_irq_handler() { ++ warn!("Error registering IRQ handler: {}", msg); ++ } ++ } ++ ++ // Unmask interrupts on the boot CPU core. ++ exception::asynchronous::local_irq_unmask(); ++ ++ // Announce conclusion of the kernel_init() phase. ++ state::state_manager().transition_to_single_core_main(); ++ + // Transition from unsafe to safe. + kernel_main() + } + + /// The main function running after the early init. + fn kernel_main() -> ! { +- use console::interface::All; + use driver::interface::DriverManager; ++ use exception::asynchronous::interface::IRQManager; + + info!("Booting on: {}", bsp::board_name()); + +@@ -76,9 +89,9 @@ + info!(" {}. {}", i + 1, driver.compatible()); + } + ++ info!("Registered IRQ handlers:"); ++ bsp::exception::asynchronous::irq_manager().print_handler(); ++ + info!("Echoing input now"); +- loop { +- let c = bsp::console::console().read_char(); +- bsp::console::console().write_char(c); +- } ++ cpu::wait_forever(); + } + +diff -uNr 13_integrated_testing/src/state.rs 14_exceptions_part2_peripheral_IRQs/src/state.rs +--- 13_integrated_testing/src/state.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/state.rs +@@ -0,0 +1,83 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! State information about the kernel itself. ++ ++use core::sync::atomic::{AtomicU8, Ordering}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Different stages in the kernel execution. ++#[derive(Copy, Clone, Eq, PartialEq)] ++pub enum State { ++ /// The kernel starts booting in this state. ++ Init, ++ ++ /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of ++ /// `kernel_init()`, after all init calls are done). ++ SingleCoreMain, ++ ++ /// The kernel transitions to this state when it boots the secondary cores, aka switches ++ /// exectution mode to symmetric multiprocessing (SMP). ++ MultiCoreMain, ++} ++ ++/// Maintains the kernel state and state transitions. ++pub struct StateManager(AtomicU8); ++ ++//-------------------------------------------------------------------------------------------------- ++// Global instances ++//-------------------------------------------------------------------------------------------------- ++ ++static STATE_MANAGER: StateManager = StateManager::new(); ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Return a reference to the global StateManager. ++pub fn state_manager() -> &'static StateManager { ++ &STATE_MANAGER ++} ++ ++impl StateManager { ++ const INIT: u8 = 0; ++ const SINGLE_CORE_MAIN: u8 = 1; ++ const MULTI_CORE_MAIN: u8 = 2; ++ ++ /// Create a new instance. ++ pub const fn new() -> Self { ++ Self(AtomicU8::new(Self::INIT)) ++ } ++ ++ /// Return the current state. ++ pub fn state(&self) -> State { ++ let state = self.0.load(Ordering::Acquire); ++ ++ match state { ++ Self::INIT => State::Init, ++ Self::SINGLE_CORE_MAIN => State::SingleCoreMain, ++ Self::MULTI_CORE_MAIN => State::MultiCoreMain, ++ _ => panic!("Invalid KERNEL_STATE"), ++ } ++ } ++ ++ /// Transition from Init to SingleCoreMain. ++ pub fn transition_to_single_core_main(&self) { ++ if self ++ .0 ++ .compare_exchange( ++ Self::INIT, ++ Self::SINGLE_CORE_MAIN, ++ Ordering::Acquire, ++ Ordering::Relaxed, ++ ) ++ .is_err() ++ { ++ panic!("transition_to_single_core_main() called while state != Init"); ++ } ++ } ++} + +diff -uNr 13_integrated_testing/src/synchronization.rs 14_exceptions_part2_peripheral_IRQs/src/synchronization.rs +--- 13_integrated_testing/src/synchronization.rs ++++ 14_exceptions_part2_peripheral_IRQs/src/synchronization.rs +@@ -42,6 +42,21 @@ + /// Creates a critical section and grants temporary mutable access to the encapsulated data. + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + } ++ ++ /// A reader-writer exclusion type. ++ /// ++ /// The implementing object allows either a number of readers or at most one writer at any point ++ /// in time. ++ pub trait ReadWriteEx { ++ /// The type of encapsulated data. ++ type Data; ++ ++ /// Grants temporary mutable access to the encapsulated data. ++ fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; ++ ++ /// Grants temporary immutable access to the encapsulated data. ++ fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R; ++ } + } + + /// A pseudo-lock for teaching purposes. +@@ -52,10 +67,17 @@ + /// other cores to the contained data. This part is preserved for later lessons. + /// + /// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is +-/// executing single-threaded, aka only running on a single core with interrupts disabled. ++/// executing on a single core. + /// + /// [interior mutability]: https://doc.rust-lang.org/std/cell/index.html +-pub struct NullLock { ++pub struct IRQSafeNullLock { ++ data: UnsafeCell, ++} ++ ++/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards. ++/// ++/// Intended to encapsulate data that is populated during kernel init when no concurrency exists. ++pub struct InitStateLock { + data: UnsafeCell, + } + +@@ -63,10 +85,20 @@ + // Public Code + //-------------------------------------------------------------------------------------------------- + +-unsafe impl Sync for NullLock {} ++unsafe impl Sync for IRQSafeNullLock {} ++ ++impl IRQSafeNullLock { ++ /// Wraps `data` into a new `IRQSafeNullLock`. ++ pub const fn new(data: T) -> Self { ++ Self { ++ data: UnsafeCell::new(data), ++ } ++ } ++} ++ ++unsafe impl Sync for InitStateLock {} + +-impl NullLock { +- /// Wraps `data` into a new `NullLock`. ++impl InitStateLock { + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), +@@ -77,8 +109,9 @@ + //------------------------------------------------------------------------------ + // OS Interface Code + //------------------------------------------------------------------------------ ++use crate::{exception, state}; + +-impl interface::Mutex for &NullLock { ++impl interface::Mutex for &IRQSafeNullLock { + type Data = T; + + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { +@@ -86,6 +119,32 @@ + // mutable reference will ever only be given out once at a time. + let data = unsafe { &mut *self.data.get() }; + ++ // Execute the closure while IRQs are masked. ++ exception::asynchronous::exec_with_irq_masked(|| f(data)) ++ } ++} ++ ++impl interface::ReadWriteEx for &InitStateLock { ++ type Data = T; ++ ++ fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { ++ assert!( ++ state::state_manager().state() == state::State::Init, ++ "InitStateLock::write called after kernel init phase" ++ ); ++ assert!( ++ !exception::asynchronous::is_local_irq_masked(), ++ "InitStateLock::write called with IRQs unmasked" ++ ); ++ ++ let data = unsafe { &mut *self.data.get() }; ++ ++ f(data) ++ } ++ ++ fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R { ++ let data = unsafe { &*self.data.get() }; ++ + f(data) + } + } + +diff -uNr 13_integrated_testing/tests/03_exception_irq_sanity.rs 14_exceptions_part2_peripheral_IRQs/tests/03_exception_irq_sanity.rs +--- 13_integrated_testing/tests/03_exception_irq_sanity.rs ++++ 14_exceptions_part2_peripheral_IRQs/tests/03_exception_irq_sanity.rs +@@ -0,0 +1,68 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! IRQ handling sanity tests. ++ ++#![feature(custom_test_frameworks)] ++#![no_main] ++#![no_std] ++#![reexport_test_harness_main = "test_main"] ++#![test_runner(libkernel::test_runner)] ++ ++mod panic_exit_failure; ++ ++use libkernel::{bsp, cpu, exception}; ++use test_macros::kernel_test; ++ ++#[no_mangle] ++unsafe fn kernel_init() -> ! { ++ bsp::console::qemu_bring_up_console(); ++ ++ exception::handling_init(); ++ exception::asynchronous::local_irq_unmask(); ++ ++ test_main(); ++ ++ cpu::qemu_exit_success() ++} ++ ++/// Check that IRQ masking works. ++#[kernel_test] ++fn local_irq_mask_works() { ++ // Precondition: IRQs are unmasked. ++ assert!(exception::asynchronous::is_local_irq_masked()); ++ ++ unsafe { exception::asynchronous::local_irq_mask() }; ++ assert!(!exception::asynchronous::is_local_irq_masked()); ++ ++ // Restore earlier state. ++ unsafe { exception::asynchronous::local_irq_unmask() }; ++} ++ ++/// Check that IRQ unmasking works. ++#[kernel_test] ++fn local_irq_unmask_works() { ++ // Precondition: IRQs are masked. ++ unsafe { exception::asynchronous::local_irq_mask() }; ++ assert!(!exception::asynchronous::is_local_irq_masked()); ++ ++ unsafe { exception::asynchronous::local_irq_unmask() }; ++ assert!(exception::asynchronous::is_local_irq_masked()); ++} ++ ++/// Check that IRQ mask save is saving "something". ++#[kernel_test] ++fn local_irq_mask_save_works() { ++ // Precondition: IRQs are unmasked. ++ assert!(exception::asynchronous::is_local_irq_masked()); ++ ++ let first = unsafe { exception::asynchronous::local_irq_mask_save() }; ++ assert!(!exception::asynchronous::is_local_irq_masked()); ++ ++ let second = unsafe { exception::asynchronous::local_irq_mask_save() }; ++ assert_ne!(first, second); ++ ++ unsafe { exception::asynchronous::local_irq_restore(first) }; ++ assert!(exception::asynchronous::is_local_irq_masked()); ++} + +``` diff --git a/14_exceptions_part2_peripheral_IRQs/kernel b/14_exceptions_part2_peripheral_IRQs/kernel new file mode 100755 index 00000000..14dcfa76 Binary files /dev/null and b/14_exceptions_part2_peripheral_IRQs/kernel differ diff --git a/14_exceptions_part2_peripheral_IRQs/kernel8.img b/14_exceptions_part2_peripheral_IRQs/kernel8.img new file mode 100755 index 00000000..9bc0ea3b Binary files /dev/null and b/14_exceptions_part2_peripheral_IRQs/kernel8.img differ diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs new file mode 100644 index 00000000..7839f142 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural processor code. + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Boot Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function at `0x80_000`. +#[naked] +#[no_mangle] +pub unsafe extern "C" fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + wait_forever() + } +} + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::cpu::BOOT_CORE_STACK_START); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +pub use asm::nop; + +/// Spin for `n` cycles. +#[inline(always)] +pub fn spin_for_cycles(n: usize) { + for _ in 0..n { + asm::nop(); + } +} + +/// Pause execution on the core. +#[inline(always)] +pub fn wait_forever() -> ! { + loop { + asm::wfe() + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// Make the host QEMU binary execute `exit(1)`. +pub fn qemu_exit_failure() -> ! { + qemu_exit::aarch64::exit_failure() +} + +/// Make the host QEMU binary execute `exit(0)`. +pub fn qemu_exit_success() -> ! { + qemu_exit::aarch64::exit_success() +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs new file mode 100644 index 00000000..8429e1d2 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu/smp.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural symmetric multiprocessing. + +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return the executing core's id. +#[inline(always)] +pub fn core_id() -> T +where + T: From, +{ + const CORE_MASK: u64 = 0b11; + + T::from((MPIDR_EL1.get() & CORE_MASK) as u8) +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S new file mode 100644 index 00000000..70817be4 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.S @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str w2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via `eret`. + b __exception_restore_context +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// The exception vector table. +//-------------------------------------------------------------------------------------------------- +.section .exception_vectors, "ax", @progbits + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + +//-------------------------------------------------------------------------------------------------- +// Helper functions +//-------------------------------------------------------------------------------------------------- +.section .text + +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 17 + + eret diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs new file mode 100644 index 00000000..396618bf --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural synchronous and asynchronous exception handling. + +use crate::{bsp, exception}; +use core::fmt; +use cortex_a::{barrier, regs::*}; +use register::InMemoryRegister; + +// Assembly counterpart to this file. +global_asm!(include_str!("exception.S")); + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Wrapper struct for memory copy of SPSR_EL1. +#[repr(transparent)] +struct SpsrEL1(InMemoryRegister); + +/// The exception context as it is stored on the stack on exception entry. +#[repr(C)] +struct ExceptionContext { + /// General Purpose Registers. + gpr: [u64; 30], + + /// The link register, aka x30. + lr: u64, + + /// Exception link register. The program counter at the time the exception happened. + elr_el1: u64, + + /// Saved program status. + spsr_el1: SpsrEL1, +} + +/// Wrapper struct for pretty printing ESR_EL1. +struct EsrEL1; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Print verbose information about the exception and the panic. +fn default_exception_handler(e: &ExceptionContext) { + panic!( + "\n\nCPU Exception!\n\ + FAR_EL1: {:#018x}\n\ + {}\n\ + {}", + FAR_EL1.get(), + EsrEL1 {}, + e + ); +} + +//------------------------------------------------------------------------------ +// Current, EL0 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn current_el0_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_el0_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_el0_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Current, ELx +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_elx_irq(_e: &mut ExceptionContext) { + use exception::asynchronous::interface::IRQManager; + + let token = &exception::asynchronous::IRQContext::new(); + bsp::exception::asynchronous::irq_manager().handle_pending_irqs(token); +} + +#[no_mangle] +unsafe extern "C" fn current_elx_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch64 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch32 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Pretty printing +//------------------------------------------------------------------------------ + +/// Human readable ESR_EL1. +#[rustfmt::skip] +impl fmt::Display for EsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let esr_el1 = ESR_EL1.extract(); + + // Raw print of whole register. + writeln!(f, "ESR_EL1: {:#010x}", esr_el1.get())?; + + // Raw print of exception class. + write!(f, " Exception Class (EC) : {:#x}", esr_el1.read(ESR_EL1::EC))?; + + // Exception class, translation. + let ec_translation = match esr_el1.read_as_enum(ESR_EL1::EC) { + Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => "Data Abort, current EL", + _ => "N/A", + }; + writeln!(f, " - {}", ec_translation)?; + + // Raw print of instruction specific syndrome. + write!(f, " Instr Specific Syndrome (ISS): {:#x}", esr_el1.read(ESR_EL1::ISS))?; + + Ok(()) + } +} + +/// Human readable SPSR_EL1. +#[rustfmt::skip] +impl fmt::Display for SpsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Raw value. + writeln!(f, "SPSR_EL1: {:#010x}", self.0.get())?; + + let to_flag_str = |x| -> _ { + if x { "Set" } else { "Not set" } + }; + + writeln!(f, " Flags:")?; + writeln!(f, " Negative (N): {}", to_flag_str(self.0.is_set(SPSR_EL1::N)))?; + writeln!(f, " Zero (Z): {}", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?; + writeln!(f, " Carry (C): {}", to_flag_str(self.0.is_set(SPSR_EL1::C)))?; + writeln!(f, " Overflow (V): {}", to_flag_str(self.0.is_set(SPSR_EL1::V)))?; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + writeln!(f, " Exception handling state:")?; + writeln!(f, " Debug (D): {}", to_mask_str(self.0.is_set(SPSR_EL1::D)))?; + writeln!(f, " SError (A): {}", to_mask_str(self.0.is_set(SPSR_EL1::A)))?; + writeln!(f, " IRQ (I): {}", to_mask_str(self.0.is_set(SPSR_EL1::I)))?; + writeln!(f, " FIQ (F): {}", to_mask_str(self.0.is_set(SPSR_EL1::F)))?; + + write!(f, " Illegal Execution State (IL): {}", + to_flag_str(self.0.is_set(SPSR_EL1::IL)) + )?; + + Ok(()) + } +} + +/// Human readable print of the exception context. +impl fmt::Display for ExceptionContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "ELR_EL1: {:#018x}", self.elr_el1)?; + writeln!(f, "{}", self.spsr_el1)?; + writeln!(f)?; + writeln!(f, "General purpose register:")?; + + #[rustfmt::skip] + let alternating = |x| -> _ { + if x % 2 == 0 { " " } else { "\n" } + }; + + // Print two registers per line. + for (i, reg) in self.gpr.iter().enumerate() { + write!(f, " x{: <2}: {: >#018x}{}", i, reg, alternating(i))?; + } + write!(f, " lr : {:#018x}", self.lr)?; + + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::exception::PrivilegeLevel; + +/// The processing element's current privilege level. +pub fn current_privilege_level() -> (PrivilegeLevel, &'static str) { + let el = CurrentEL.read_as_enum(CurrentEL::EL); + match el { + Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, "EL2"), + Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, "EL1"), + Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, "EL0"), + _ => (PrivilegeLevel::Unknown, "Unknown"), + } +} + +/// Init exception handling by setting the exception vector base address register. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must +/// adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference +/// Manual. +pub unsafe fn handling_init() { + // Provided by exception.S. + extern "C" { + static mut __exception_vector_start: u64; + } + let addr: u64 = &__exception_vector_start as *const _ as u64; + + VBAR_EL1.set(addr); + + // Force VBAR update to complete before next instruction. + barrier::isb(barrier::SY); +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs new file mode 100644 index 00000000..e2887289 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/exception/asynchronous.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural asynchronous exception handling. + +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +mod daif_bits { + pub const IRQ: u8 = 0b0010; +} + +trait DaifField { + fn daif_field() -> register::Field; +} + +struct Debug; +struct SError; +struct IRQ; +struct FIQ; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl DaifField for Debug { + fn daif_field() -> register::Field { + DAIF::D + } +} + +impl DaifField for SError { + fn daif_field() -> register::Field { + DAIF::A + } +} + +impl DaifField for IRQ { + fn daif_field() -> register::Field { + DAIF::I + } +} + +impl DaifField for FIQ { + fn daif_field() -> register::Field { + DAIF::F + } +} + +fn is_masked() -> bool { + DAIF.is_set(T::daif_field()) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Returns whether IRQs are masked on the executing core. +pub fn is_local_irq_masked() -> bool { + !is_masked::() +} + +/// Unmask IRQs on the executing core. +/// +/// It is not needed to place an explicit instruction synchronization barrier after the `msr`. +/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3: +/// +/// "Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional +/// synchronization." +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_unmask() { + asm!("msr DAIFClr, $0" + : // outputs + : "i"(daif_bits::IRQ) // inputs + : // clobbers + : "volatile" // options + ); +} + +/// Mask IRQs on the executing core. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_mask() { + asm!("msr DAIFSet, $0" + : // outputs + : "i"(daif_bits::IRQ) // inputs + : // clobbers + : "volatile" // options + ); +} + +/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF). +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_mask_save() -> u32 { + let saved = DAIF.get(); + local_irq_mask(); + + saved +} + +/// Restore the interrupt mask bits (DAIF) using the callee's argument. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +/// - No sanity checks on the input. +#[inline(always)] +pub unsafe fn local_irq_restore(saved: u32) { + DAIF.set(saved); +} + +/// Print the AArch64 exceptions status. +#[rustfmt::skip] +pub fn print_state() { + use crate::info; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + info!(" Debug: {}", to_mask_str(is_masked::())); + info!(" SError: {}", to_mask_str(is_masked::())); + info!(" IRQ: {}", to_mask_str(is_masked::())); + info!(" FIQ: {}", to_mask_str(is_masked::())); +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs new file mode 100644 index 00000000..e9c9d038 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Memory Management Unit Driver. +//! +//! Static page tables, compiled on boot; Everything 64 KiB granule. + +use super::{AccessPermissions, AttributeFields, MemAttributes}; +use crate::{bsp, memory}; +use core::convert; +use cortex_a::{barrier, regs::*}; +use register::register_bitfields; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D4-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next page table. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D4-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next page table (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) +const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(transparent)] +struct TableDescriptor(u64); + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(transparent)] +struct PageDescriptor(u64); + +/// Big monolithic struct for storing the page tables. Individual levels must be 64 KiB aligned, +/// hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +struct PageTables { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; N], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; N], +} + +/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. +const ENTRIES_512_MIB: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; + +/// The page tables. +/// +/// # Safety +/// +/// - Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: PageTables<{ ENTRIES_512_MIB }> = PageTables { + lvl3: [[PageDescriptor(0); 8192]; ENTRIES_512_MIB], + lvl2: [TableDescriptor(0); ENTRIES_512_MIB], +}; + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Memory Management Unit type. +pub struct MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static MMU: MemoryManagementUnit = MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl BaseAddr for [T; N] { + fn base_addr_u64(&self) -> u64 { + self as *const T as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const T as usize + } +} + +impl convert::From for TableDescriptor { + fn from(next_lvl_table_addr: usize) -> Self { + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; + let val = (STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64)) + .value; + + TableDescriptor(val) + } +} + +/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + desc + } +} + +impl PageDescriptor { + fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { + let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; + let val = (STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64)) + .value; + + Self(val) + } +} + +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc + + // Attribute 0 - Device. + + MAIR_EL1::Attr0_HIGH::Device + + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, + ); +} + +/// Iterates over all static page table entries and fills them at once. +/// +/// # Safety +/// +/// - Modifies a `static mut`. Ensure it only happens from here. +unsafe fn populate_pt_entries() -> Result<(), &'static str> { + for (l2_nr, l2_entry) in TABLES.lvl2.iter_mut().enumerate() { + *l2_entry = TABLES.lvl3[l2_nr].base_addr_usize().into(); + + for (l3_nr, l3_entry) in TABLES.lvl3[l2_nr].iter_mut().enumerate() { + let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); + + let (output_addr, attribute_fields) = + bsp::memory::mmu::virt_mem_layout().get_virt_addr_properties(virt_addr)?; + + *l3_entry = PageDescriptor::new(output_addr, attribute_fields); + } + } + + Ok(()) +} + +/// Configure various settings of stage 1 of the EL1 translation regime. +fn configure_translation_control() { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(32), // TTBR0 spans 4 GiB total. + ); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the MMU. +pub fn mmu() -> &'static impl memory::mmu::interface::MMU { + &MMU +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl memory::mmu::interface::MMU for MemoryManagementUnit { + unsafe fn init(&self) -> Result<(), &'static str> { + // Fail early if translation granule is not supported. Both RPis support it, though. + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { + return Err("64 KiB translation granule not supported"); + } + + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Populate page tables. + populate_pt_entries()?; + + // Set the "Translation Table Base Register". + TTBR0_EL1.set_baddr(TABLES.lvl2.base_addr_u64()); + + configure_translation_control(); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction. + barrier::isb(barrier::SY); + + Ok(()) + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs new file mode 100644 index 00000000..fb01ced1 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/time.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural timer primitives. + +use crate::{time, warn}; +use core::time::Duration; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +const NS_PER_S: u64 = 1_000_000_000; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// ARMv8 Generic Timer. +pub struct GenericTimer; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static TIME_MANAGER: GenericTimer = GenericTimer; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the time manager. +pub fn time_manager() -> &'static impl time::interface::TimeManager { + &TIME_MANAGER +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl time::interface::TimeManager for GenericTimer { + fn resolution(&self) -> Duration { + Duration::from_nanos(NS_PER_S / (CNTFRQ_EL0.get() as u64)) + } + + fn uptime(&self) -> Duration { + let frq: u64 = CNTFRQ_EL0.get() as u64; + let current_count: u64 = CNTPCT_EL0.get() * NS_PER_S; + + Duration::from_nanos(current_count / frq) + } + + fn spin_for(&self, duration: Duration) { + // Instantly return on zero. + if duration.as_nanos() == 0 { + return; + } + + // Calculate the register compare value. + let frq = CNTFRQ_EL0.get() as u64; + let x = match frq.checked_mul(duration.as_nanos() as u64) { + None => { + warn!("Spin duration too long, skipping"); + return; + } + Some(val) => val, + }; + let tval = x / NS_PER_S; + + // Check if it is within supported bounds. + let warn: Option<&str> = if tval == 0 { + Some("smaller") + } else if tval > u32::max_value().into() { + Some("bigger") + } else { + None + }; + + if let Some(w) = warn { + warn!( + "Spin duration {} than architecturally supported, skipping", + w + ); + return; + } + + // Set the compare value register. + CNTP_TVAL_EL0.set(tval as u32); + + // Kick off the counting. // Disable timer interrupt. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); + + // ISTATUS will be '1' when cval ticks have passed. Busy-check it. + while !CNTP_CTL_EL0.matches_all(CNTP_CTL_EL0::ISTATUS::SET) {} + + // Disable counting again. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::CLEAR); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp.rs new file mode 100644 index 00000000..3dbbba8c --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Conditional re-exporting of Board Support Packages. + +mod device_driver; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod raspberrypi; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use raspberrypi::*; + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Ensure the kernel's virtual memory layout is free of overlaps. + #[kernel_test] + fn virt_mem_layout_has_no_overlaps() { + let layout = memory::mmu::virt_mem_layout().inner(); + + for (i, first) in layout.iter().enumerate() { + for second in layout.iter().skip(i + 1) { + let first_range = first.virtual_range; + let second_range = second.virtual_range; + + assert!(!first_range().contains(second_range().start())); + assert!(!first_range().contains(second_range().end())); + assert!(!second_range().contains(first_range().start())); + assert!(!second_range().contains(first_range().end())); + } + } + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver.rs new file mode 100644 index 00000000..3fe1fa55 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Device driver. + +#[cfg(feature = "bsp_rpi4")] +mod arm; +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod bcm; +mod common; + +#[cfg(feature = "bsp_rpi4")] +pub use arm::*; +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use bcm::*; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm.rs new file mode 100644 index 00000000..9e42fa49 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! ARM driver top level. + +#[cfg(feature = "bsp_rpi4")] +pub mod gicv2; + +#[cfg(feature = "bsp_rpi4")] +pub use gicv2::*; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs new file mode 100644 index 00000000..4501bec7 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! GICv2 Driver - ARM Generic Interrupt Controller v2. +//! +//! The following is a collection of excerpts with useful information from +//! - `Programmer's Guide for ARMv8-A` +//! - `ARM Generic Interrupt Controller Architecture Specification` +//! +//! # Programmer's Guide - 10.6.1 Configuration +//! +//! The GIC is accessed as a memory-mapped peripheral. +//! +//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core +//! uses the same address to access its own private CPU interface. +//! +//! It is not possible for a core to access the CPU interface of another core. +//! +//! # Architecture Specification - 10.6.2 Initialization +//! +//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized +//! after reset before it can deliver interrupts to the core. +//! +//! In the Distributor, software must configure the priority, target, security and enable individual +//! interrupts. The Distributor must subsequently be enabled through its control register +//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption +//! settings. +//! +//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This +//! prepares the GIC to deliver interrupts to the core. +//! +//! Before interrupts are expected in the core, software prepares the core to take interrupts by +//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in +//! PSTATE, and setting the routing controls. +//! +//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor. +//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface. +//! Individual interrupts can also be disabled (or enabled) in the distributor. +//! +//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must +//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the +//! core's priority mask. +//! +//! # Architecture Specification - 1.4.2 Interrupt types +//! +//! - Peripheral interrupt +//! - Private Peripheral Interrupt (PPI) +//! - This is a peripheral interrupt that is specific to a single processor. +//! - Shared Peripheral Interrupt (SPI) +//! - This is a peripheral interrupt that the Distributor can route to any of a specified +//! combination of processors. +//! +//! - Software-generated interrupt (SGI) +//! - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The +//! system uses SGIs for interprocessor communication. +//! - An SGI has edge-triggered properties. The software triggering of the interrupt is +//! equivalent to the edge transition of the interrupt request signal. +//! - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt +//! Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR, +//! identifies the processor that requested the interrupt. +//! +//! # Architecture Specification - 2.2.1 Interrupt IDs +//! +//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020 +//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by +//! the Distributor. +//! +//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows: +//! - Interrupt numbers 32..1019 are used for SPIs. +//! - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These +//! interrupts are banked in the Distributor. +//! - A banked interrupt is one where the Distributor can have multiple interrupts with the +//! same ID. A banked interrupt is identified uniquely by its ID number and its associated +//! CPU interface number. Of the banked interrupt IDs: +//! - 00..15 SGIs +//! - 16..31 PPIs + +mod gicc; +mod gicd; + +use crate::{bsp, cpu, driver, exception, synchronization, synchronization::InitStateLock}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +type HandlerTable = [Option; GICv2::NUM_IRQS]; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +pub type IRQNumber = exception::asynchronous::IRQNumber<{ GICv2::MAX_IRQ_NUMBER }>; + +/// Representation of the GIC. +pub struct GICv2 { + /// The Distributor. + gicd: gicd::GICD, + + /// The CPU Interface. + gicc: gicc::GICC, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl GICv2 { + const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space. + const NUM_IRQS: usize = Self::MAX_IRQ_NUMBER + 1; + + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(gicd_base_addr: usize, gicc_base_addr: usize) -> Self { + Self { + gicd: gicd::GICD::new(gicd_base_addr), + gicc: gicc::GICC::new(gicc_base_addr), + handler_table: InitStateLock::new([None; Self::NUM_IRQS]), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::ReadWriteEx; + +impl driver::interface::DeviceDriver for GICv2 { + fn compatible(&self) -> &str { + "GICv2 (ARM Generic Interrupt Controller v2)" + } + + fn init(&self) -> Result<(), ()> { + if cpu::smp::core_id::() == bsp::cpu::BOOT_CORE_ID { + self.gicd.boot_core_init(); + } + + self.gicc.priority_accept_all(); + self.gicc.enable(); + + Ok(()) + } +} + +impl exception::asynchronous::interface::IRQManager for GICv2 { + type IRQNumberType = IRQNumber; + + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + let mut r = &self.handler_table; + r.write(|table| { + let irq_number = irq_number.get(); + + if table[irq_number].is_some() { + return Err("IRQ handler already registered"); + } + + table[irq_number] = Some(descriptor); + + Ok(()) + }) + } + + fn enable(&self, irq_number: Self::IRQNumberType) { + self.gicd.enable(irq_number); + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register + // (IAR). + let irq_number = self.gicc.get_pending_number(ic); + + // Guard against spurious interrupts. + if irq_number > GICv2::MAX_IRQ_NUMBER { + return; + } + + // Call the IRQ handler. Panic if there is none. + let mut r = &self.handler_table; + r.read(|table| { + match table[irq_number] { + None => panic!("No handler registered for IRQ {}", irq_number), + Some(descriptor) => { + // Call the IRQ handler. Panics on failure. + descriptor.handler.handle().expect("Error handling IRQ"); + } + } + }); + + // Signal completion of handling. + self.gicc.mark_comleted(irq_number as u32, ic); + } + + fn print_handler(&self) { + use crate::info; + + info!(" Peripheral handler:"); + + let mut r = &self.handler_table; + r.read(|table| { + for (i, opt) in table.iter().skip(32).enumerate() { + if let Some(handler) = opt { + info!(" {: >3}. {}", i + 32, handler.name); + } + } + }); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs new file mode 100644 index 00000000..eaa60363 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! GICC Driver - GIC CPU interface. + +use crate::exception; +use core::ops; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_bitfields! { + u32, + + /// CPU Interface Control Register + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + + /// Interrupt Priority Mask Register + PMR [ + Priority OFFSET(0) NUMBITS(8) [] + ], + + /// Interrupt Acknowledge Register + IAR [ + InterruptID OFFSET(0) NUMBITS(10) [] + ], + + /// End of Interrupt Register + EOIR [ + EOIINTID OFFSET(0) NUMBITS(10) [] + ] +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +register_structs! { + #[allow(non_snake_case)] + pub RegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => PMR: ReadWrite), + (0x008 => _reserved1), + (0x00C => IAR: ReadWrite), + (0x010 => EOIR: ReadWrite), + (0x014 => @END), + } +} + +/// Representation of the GIC CPU interface. +pub struct GICC { + base_addr: usize, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl ops::Deref for GICC { + type Target = RegisterBlock; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.ptr() } + } +} + +impl GICC { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { base_addr } + } + + /// Return a pointer to the associated MMIO register block. + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ + } + + /// Accept interrupts of any priority. + /// + /// Quoting the GICv2 Architecture Specification: + /// + /// "Writing 255 to the GICC_PMR always sets it to the largest supported priority field + /// value." + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn priority_accept_all(&self) { + self.PMR.write(PMR::Priority.val(255)); // Comment in arch spec. + } + + /// Enable the interface - start accepting IRQs. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn enable(&self) { + self.CTLR.write(CTLR::Enable::SET); + } + + /// Extract the number of the highest-priority pending IRQ. + /// + /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + #[allow(clippy::trivially_copy_pass_by_ref)] + pub fn get_pending_number<'irq_context>( + &self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) -> usize { + self.IAR.read(IAR::InterruptID) as usize + } + + /// Complete handling of the currently active IRQ. + /// + /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. + /// + /// To be called after `get_pending_number()`. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + #[allow(clippy::trivially_copy_pass_by_ref)] + pub fn mark_comleted<'irq_context>( + &self, + irq_number: u32, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + self.EOIR.write(EOIR::EOIINTID.val(irq_number)); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs new file mode 100644 index 00000000..25a6bdda --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! GICD Driver - GIC Distributor. +//! +//! # Glossary +//! - SPI - Shared Peripheral Interrupt. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, state, synchronization, + synchronization::IRQSafeNullLock, +}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_bitfields! { + u32, + + /// Distributor Control Register + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + + /// Interrupt Controller Type Register + TYPER [ + ITLinesNumber OFFSET(0) NUMBITS(5) [] + ], + + /// Interrupt Processor Targets Registers + ITARGETSR [ + Offset3 OFFSET(24) NUMBITS(8) [], + Offset2 OFFSET(16) NUMBITS(8) [], + Offset1 OFFSET(8) NUMBITS(8) [], + Offset0 OFFSET(0) NUMBITS(8) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + SharedRegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => TYPER: ReadOnly), + (0x008 => _reserved1), + (0x104 => ISENABLER: [ReadWrite; 31]), + (0x108 => _reserved2), + (0x820 => ITARGETSR: [ReadWrite; 248]), + (0xBFC => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + BankedRegisterBlock { + (0x000 => _reserved1), + (0x100 => ISENABLER: ReadWrite), + (0x104 => _reserved2), + (0x800 => ITARGETSR: [ReadOnly; 8]), + (0xBFC => @END), + } +} + +/// Abstraction for the non-banked parts of the associated MMIO registers. +type SharedRegs = MMIODerefWrapper; + +/// Abstraction for the banked parts of the associated MMIO registers. +type BankedRegs = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the GIC Distributor. +pub struct GICD { + /// Access to shared registers is guarded with a lock. + shared_regs: IRQSafeNullLock, + + /// Access to banked registers is unguarded. + banked_regs: BankedRegs, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl SharedRegs { + /// Return the number of IRQs that this HW implements. + #[inline(always)] + fn num_irqs(&mut self) -> usize { + // Query number of implemented IRQs. + // + // Refer to GICv2 Architecture Specification, Section 4.3.2. + ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 + } + + /// Return a slice of the implemented ITARGETSR. + #[inline(always)] + fn implemented_itargets_slice(&mut self) -> &[ReadWrite] { + assert!(self.num_irqs() >= 36); + + // Calculate the max index of the shared ITARGETSR array. + // + // The first 32 IRQs are private, so not included in `shared_regs`. Each ITARGETS + // register has four entries, so shift right by two. Subtract one because we start + // counting at zero. + let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1; + + // Rust automatically inserts slice range sanity check, i.e. max >= min. + &self.ITARGETSR[0..spi_itargetsr_max_index] + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use synchronization::interface::Mutex; + +impl GICD { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { + shared_regs: IRQSafeNullLock::new(SharedRegs::new(base_addr)), + banked_regs: BankedRegs::new(base_addr), + } + } + + /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask. + /// + /// Quoting the GICv2 Architecture Specification: + /// + /// "GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that + /// corresponds only to the processor reading the register." + fn local_gic_target_mask(&self) -> u32 { + self.banked_regs.ITARGETSR[0].read(ITARGETSR::Offset0) + } + + /// Route all SPIs to the boot core and enable the distributor. + pub fn boot_core_init(&self) { + assert!( + state::state_manager().state() == state::State::Init, + "Only allowed during kernel init phase" + ); + + // Target all SPIs to the boot core only. + let mask = self.local_gic_target_mask(); + + let mut r = &self.shared_regs; + r.lock(|regs| { + for i in regs.implemented_itargets_slice().iter() { + i.write( + ITARGETSR::Offset3.val(mask) + + ITARGETSR::Offset2.val(mask) + + ITARGETSR::Offset1.val(mask) + + ITARGETSR::Offset0.val(mask), + ); + } + + regs.CTLR.write(CTLR::Enable::SET); + }); + } + + /// Enable an interrupt. + pub fn enable(&self, irq_num: super::IRQNumber) { + let irq_num = irq_num.get(); + + // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5 + // (division by 32) and arrive at the index for the respective ISENABLER[i]. + let enable_reg_index = irq_num >> 5; + let enable_bit: u32 = 1u32 << (irq_num % 32); + + // Check if we are handling a private or shared IRQ. + match irq_num { + // Private. + 0..=31 => { + let enable_reg = &self.banked_regs.ISENABLER; + enable_reg.set(enable_reg.get() | enable_bit); + } + // Shared. + _ => { + let enable_reg_index_shared = enable_reg_index - 1; + + let mut r = &self.shared_regs; + r.lock(|regs| { + let enable_reg = ®s.ISENABLER[enable_reg_index_shared]; + enable_reg.set(enable_reg.get() | enable_bit); + }); + } + } + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm.rs new file mode 100644 index 00000000..5e4c9e70 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BCM driver top level. + +mod bcm2xxx_gpio; +#[cfg(feature = "bsp_rpi3")] +mod bcm2xxx_interrupt_controller; +mod bcm2xxx_pl011_uart; + +pub use bcm2xxx_gpio::*; +#[cfg(feature = "bsp_rpi3")] +pub use bcm2xxx_interrupt_controller::*; +pub use bcm2xxx_pl011_uart::*; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs new file mode 100644 index 00000000..0e7384f3 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! GPIO Driver. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, cpu, driver, synchronization, + synchronization::IRQSafeNullLock, +}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// GPIO registers. +// +// Descriptions taken from +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +register_bitfields! { + u32, + + /// GPIO Function Select 1 + GPFSEL1 [ + /// Pin 15 + FSEL15 OFFSET(15) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART RX + + ], + + /// Pin 14 + FSEL14 OFFSET(12) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART TX + ] + ], + + /// GPIO Pull-up/down Clock Register 0 + GPPUDCLK0 [ + /// Pin 15 + PUDCLK15 OFFSET(15) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ], + + /// Pin 14 + PUDCLK14 OFFSET(14) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ] + ] +} + +register_structs! { + #[allow(non_snake_case)] + RegisterBlock { + (0x00 => GPFSEL0: ReadWrite), + (0x04 => GPFSEL1: ReadWrite), + (0x08 => GPFSEL2: ReadWrite), + (0x0C => GPFSEL3: ReadWrite), + (0x10 => GPFSEL4: ReadWrite), + (0x14 => GPFSEL5: ReadWrite), + (0x18 => _reserved1), + (0x94 => GPPUD: ReadWrite), + (0x98 => GPPUDCLK0: ReadWrite), + (0x9C => GPPUDCLK1: ReadWrite), + (0xA0 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Regs = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the GPIO HW. +pub struct GPIO { + inner: IRQSafeNullLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl GPIO { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { + inner: IRQSafeNullLock::new(Regs::new(base_addr)), + } + } + + /// Map PL011 UART as standard output. + /// + /// TX to pin 14 + /// RX to pin 15 + pub fn map_pl011_uart(&self) { + let mut r = &self.inner; + r.lock(|inner| { + // Map to pins. + inner + .GPFSEL1 + .modify(GPFSEL1::FSEL14::AltFunc0 + GPFSEL1::FSEL15::AltFunc0); + + // Enable pins 14 and 15. + inner.GPPUD.set(0); + cpu::spin_for_cycles(150); + + inner + .GPPUDCLK0 + .write(GPPUDCLK0::PUDCLK14::AssertClock + GPPUDCLK0::PUDCLK15::AssertClock); + cpu::spin_for_cycles(150); + + inner.GPPUDCLK0.set(0); + }) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::Mutex; + +impl driver::interface::DeviceDriver for GPIO { + fn compatible(&self) -> &str { + "BCM GPIO" + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs new file mode 100644 index 00000000..9b2f00a3 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Interrupt Controller Driver. + +mod peripheral_ic; + +use crate::{driver, exception}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Wrapper struct for a bitmask indicating pending IRQ numbers. +struct PendingIRQs { + bitmask: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub type LocalIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>; +pub type PeripheralIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>; + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +#[derive(Copy, Clone)] +pub enum IRQNumber { + Local(LocalIRQ), + Peripheral(PeripheralIRQ), +} + +/// Representation of the Interrupt Controller. +pub struct InterruptController { + periph: peripheral_ic::PeripheralIC, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl PendingIRQs { + pub fn new(bitmask: u64) -> Self { + Self { bitmask } + } +} + +impl Iterator for PendingIRQs { + type Item = usize; + + fn next(&mut self) -> Option { + use core::intrinsics::cttz; + + let next = cttz(self.bitmask); + if next == 64 { + return None; + } + + self.bitmask &= !(1 << next); + + Some(next as usize) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl InterruptController { + const MAX_LOCAL_IRQ_NUMBER: usize = 11; + const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63; + const NUM_PERIPHERAL_IRQS: usize = Self::MAX_PERIPHERAL_IRQ_NUMBER + 1; + + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(_local_base_addr: usize, periph_base_addr: usize) -> Self { + Self { + periph: peripheral_ic::PeripheralIC::new(periph_base_addr), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl driver::interface::DeviceDriver for InterruptController { + fn compatible(&self) -> &str { + "BCM Interrupt Controller" + } +} + +impl exception::asynchronous::interface::IRQManager for InterruptController { + type IRQNumberType = IRQNumber; + + fn register_handler( + &self, + irq: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + match irq { + IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), + IRQNumber::Peripheral(pirq) => self.periph.register_handler(pirq, descriptor), + } + } + + fn enable(&self, irq: Self::IRQNumberType) { + match irq { + IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), + IRQNumber::Peripheral(pirq) => self.periph.enable(pirq), + } + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet. + self.periph.handle_pending_irqs(ic) + } + + fn print_handler(&self) { + self.periph.print_handler(); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs new file mode 100644 index 00000000..bd640216 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Peripheral Interrupt regsler Driver. + +use super::{InterruptController, PendingIRQs, PeripheralIRQ}; +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, + exception, synchronization, + synchronization::{IRQSafeNullLock, InitStateLock}, +}; +use register::{mmio::*, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_structs! { + #[allow(non_snake_case)] + WORegisterBlock { + (0x00 => _reserved1), + (0x10 => ENABLE_1: WriteOnly), + (0x14 => ENABLE_2: WriteOnly), + (0x24 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + RORegisterBlock { + (0x00 => _reserved1), + (0x04 => PENDING_1: ReadOnly), + (0x08 => PENDING_2: ReadOnly), + (0x0c => @END), + } +} + +/// Abstraction for the WriteOnly parts of the associated MMIO registers. +type WriteOnlyRegs = MMIODerefWrapper; + +/// Abstraction for the ReadOnly parts of the associated MMIO registers. +type ReadOnlyRegs = MMIODerefWrapper; + +type HandlerTable = + [Option; InterruptController::NUM_PERIPHERAL_IRQS]; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the peripheral interrupt regsler. +pub struct PeripheralIC { + /// Access to write registers is guarded with a lock. + wo_regs: IRQSafeNullLock, + + /// Register read access is unguarded. + ro_regs: ReadOnlyRegs, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl PeripheralIC { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { + wo_regs: IRQSafeNullLock::new(WriteOnlyRegs::new(base_addr)), + ro_regs: ReadOnlyRegs::new(base_addr), + handler_table: InitStateLock::new([None; InterruptController::NUM_PERIPHERAL_IRQS]), + } + } + + /// Query the list of pending IRQs. + fn get_pending(&self) -> PendingIRQs { + let pending_mask: u64 = (u64::from(self.ro_regs.PENDING_2.get()) << 32) + | u64::from(self.ro_regs.PENDING_1.get()); + + PendingIRQs::new(pending_mask) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::{Mutex, ReadWriteEx}; + +impl exception::asynchronous::interface::IRQManager for PeripheralIC { + type IRQNumberType = PeripheralIRQ; + + fn register_handler( + &self, + irq: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + let mut r = &self.handler_table; + r.write(|table| { + let irq_number = irq.get(); + + if table[irq_number].is_some() { + return Err("IRQ handler already registered"); + } + + table[irq_number] = Some(descriptor); + + Ok(()) + }) + } + + fn enable(&self, irq: Self::IRQNumberType) { + let mut r = &self.wo_regs; + r.lock(|regs| { + let enable_reg = if irq.get() <= 31 { + ®s.ENABLE_1 + } else { + ®s.ENABLE_2 + }; + + let enable_bit: u32 = 1 << (irq.get() % 32); + + // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable + // bits are unaffected. So we don't need read and OR'ing here. + enable_reg.set(enable_bit); + }); + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + let mut r = &self.handler_table; + r.read(|table| { + for irq_number in self.get_pending() { + match table[irq_number] { + None => panic!("No handler registered for IRQ {}", irq_number), + Some(descriptor) => { + // Call the IRQ handler. Panics on failure. + descriptor.handler.handle().expect("Error handling IRQ"); + } + } + } + }) + } + + fn print_handler(&self) { + use crate::info; + + info!(" Peripheral handler:"); + + let mut r = &self.handler_table; + r.read(|table| { + for (i, opt) in table.iter().enumerate() { + if let Some(handler) = opt { + info!(" {: >3}. {}", i, handler.name); + } + } + }); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs new file mode 100644 index 00000000..84f45e48 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! PL011 UART driver. +//! +//! # FIFO fill level IRQ hack +//! +//! For learning purposes, we want the UART to raise an IRQ on _every_ received character. +//! Unfortunately, this rather common mode of operation is not supported by the PL011 when operating +//! in FIFO mode. It is only possible to set a fill level fraction on which the IRQ is triggered. +//! The lowest fill level is 1/8. +//! +//! On the RPi3, the RX FIFO is 16 chars deep, so the IRQ would trigger after 2 chars have been +//! received. On the RPi4, the FIFO seems to be 32 chars deep, because experiments showed that the +//! RX IRQ triggers after receiving 4 chars. +//! +//! Fortunately, the PL011 has a test mode which allows to push characters into the FIFOs. We make +//! use of this testing facilities to employ a little hack that pushes (fill-level - 1) chars into +//! the RX FIFO by default. This way, we get an IRQ for the first received char that arrives from +//! external. +//! +//! To make things even more complicated, QEMU is not honoring the fill-level dependent IRQ +//! generation. Instead, QEMU creates an IRQ on every received char. +//! +//! We use conditional compilation to differentiate between the three modes of operation (RPi3, +//! RPI4, QEMU) respectively. + +use crate::{ + bsp, console, cpu, driver, exception, synchronization, synchronization::IRQSafeNullLock, +}; +use core::{fmt, ops}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// PL011 UART registers. +// +// Descriptions taken from +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +register_bitfields! { + u32, + + /// Flag Register + FR [ + /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// Line Control Register, UARTLCR_ LCRH. + /// + /// If the FIFO is disabled, this bit is set when the transmit holding register is empty. If + /// the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty. This bit does + /// not indicate if there is data in the transmit shift register. + TXFE OFFSET(7) NUMBITS(1) [], + + /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the + /// UARTLCR_ LCRH Register. + /// + /// If the FIFO is disabled, this bit is set when the transmit holding register is full. If + /// the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full. + TXFF OFFSET(5) NUMBITS(1) [], + + /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// UARTLCR_H Register. + /// + /// If the FIFO is disabled, this bit is set when the receive holding register is empty. If + /// the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty. + RXFE OFFSET(4) NUMBITS(1) [] + ], + + /// Integer Baud rate divisor + IBRD [ + /// Integer Baud rate divisor + IBRD OFFSET(0) NUMBITS(16) [] + ], + + /// Fractional Baud rate divisor + FBRD [ + /// Fractional Baud rate divisor + FBRD OFFSET(0) NUMBITS(6) [] + ], + + /// Line Control register + LCRH [ + /// Word length. These bits indicate the number of data bits transmitted or received in a + /// frame. + WLEN OFFSET(5) NUMBITS(2) [ + FiveBit = 0b00, + SixBit = 0b01, + SevenBit = 0b10, + EightBit = 0b11 + ], + + /// Enable FIFOs: + /// + /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding + /// registers + /// + /// 1 = transmit and receive FIFO buffers are enabled (FIFO mode). + FEN OFFSET(4) NUMBITS(1) [ + FifosDisabled = 0, + FifosEnabled = 1 + ] + ], + + /// Control Register + CR [ + /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled. + /// Data reception occurs for UART signals. When the UART is disabled in the middle of + /// reception, it completes the current character before stopping. + RXE OFFSET(9) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled. + /// Data transmission occurs for UART signals. When the UART is disabled in the middle of + /// transmission, it completes the current character before stopping. + TXE OFFSET(8) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// UART enable + UARTEN OFFSET(0) NUMBITS(1) [ + /// If the UART is disabled in the middle of transmission or reception, it completes the + /// current character before stopping. + Disabled = 0, + Enabled = 1 + ] + ], + + /// Interrupt FIFO Level Select Register + IFLS [ + /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as + /// follows. + RXIFLSEL OFFSET(3) NUMBITS(5) [ + OneEigth = 0b000, + OneQuarter = 0b001, + OneHalf = 0b010, + ThreeQuarters = 0b011, + SevenEights = 0b100 + ] + ], + + /// Interrupt Mask Set Clear Register + IMSC [ + /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt. On + /// a write of 1, the mask of the interrupt is set. A write of 0 clears the mask. + RXIM OFFSET(4) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ] + ], + + /// Interrupt Clear Register + ICR [ + /// Meta field for all pending interrupts + ALL OFFSET(0) NUMBITS(11) [] + ], + + /// Test Control Register + ITCR [ + /// Test FIFO enable. When this bit it 1, a write to the Test Data Register, UART_DR writes + /// data into the receive FIFO, and reads from the UART_DR register reads data out of the + /// transmit FIFO. When this bit is 0, data cannot be read directly from the transmit FIFO + /// or written directly to the receive FIFO (normal operation). + ITCR1 OFFSET(1) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ] + ] +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +register_structs! { + #[allow(non_snake_case)] + pub RegisterBlock { + (0x00 => DR: ReadWrite), + (0x04 => _reserved1), + (0x18 => FR: ReadOnly), + (0x1c => _reserved2), + (0x24 => IBRD: WriteOnly), + (0x28 => FBRD: WriteOnly), + (0x2c => LCRH: WriteOnly), + (0x30 => CR: WriteOnly), + (0x34 => IFLS: ReadWrite), + (0x38 => IMSC: ReadWrite), + (0x3C => _reserved3), + (0x44 => ICR: WriteOnly), + (0x48 => _reserved4), + (0x80 => ITCR: ReadWrite), + (0x84 => _reserved5), + (0x8c => TDR: ReadWrite), + (0x90 => @END), + } +} + +pub struct PL011UartInner { + base_addr: usize, + chars_written: usize, + chars_read: usize, +} + +// Export the inner struct so that BSPs can use it for the panic handler. +pub use PL011UartInner as PanicUart; + +/// Representation of the UART. +pub struct PL011Uart { + inner: IRQSafeNullLock, + irq_number: bsp::device_driver::IRQNumber, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Deref to RegisterBlock. +/// +/// Allows writing +/// ``` +/// self.DR.read() +/// ``` +/// instead of something along the lines of +/// ``` +/// unsafe { (*PL011UartInner::ptr()).DR.read() } +/// ``` +impl ops::Deref for PL011UartInner { + type Target = RegisterBlock; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.ptr() } + } +} + +impl PL011UartInner { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { + base_addr, + chars_written: 0, + chars_read: 0, + } + } + + /// Set up baud rate and characteristics. + /// + /// Results in 8N1 and 230400 baud (if the clk has been previously set to 48 MHz by the + /// firmware). + pub fn init(&mut self) { + // Turn it off temporarily. + self.CR.set(0); + + self.ICR.write(ICR::ALL::CLEAR); + self.IBRD.write(IBRD::IBRD.val(13)); + self.FBRD.write(FBRD::FBRD.val(2)); + self.LCRH + .write(LCRH::WLEN::EightBit + LCRH::FEN::FifosEnabled); // 8N1 + Fifo on + self.CR + .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled); + + // Trigger the RX interrupt at 1/8 of the FIFO fill level (this is the lowest possible) and + // enable RX interrupts. + self.IFLS.write(IFLS::RXIFLSEL::OneEigth); + self.IMSC.write(IMSC::RXIM::Enabled); + + #[cfg(not(feature = "qemu-quirks"))] + self.fill_hack_push(); + } + + /// Return a pointer to the register block. + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ + } + + /// Send a character. + fn write_char(&mut self, c: char) { + // Spin while TX FIFO full is set, waiting for an empty slot. + while self.FR.matches_all(FR::TXFF::SET) { + cpu::nop(); + } + + // Write the character to the buffer. + self.DR.set(c as u32); + + self.chars_written += 1; + } + + /// Retrieve a character. + fn read_char_converting(&mut self, blocking: bool) -> Option { + #[cfg(not(feature = "qemu-quirks"))] + self.fill_hack_pop(); + + // If blocking, spin while RX FIFO empty is set, else return None. + while self.FR.matches_all(FR::RXFE::SET) { + if !blocking { + #[cfg(not(feature = "qemu-quirks"))] + self.fill_hack_push(); + + return None; + } + + cpu::nop(); + } + + // Read one character. + let mut ret = self.DR.get() as u8 as char; + + // Convert carrige return to newline. + if ret == '\r' { + ret = '\n' + } + + // Update statistics. + self.chars_read += 1; + + #[cfg(not(feature = "qemu-quirks"))] + self.fill_hack_push(); + + Some(ret) + } + + /// Push characters into the receive FIFO. + /// + /// See top level comments why this is needed. + #[cfg(not(feature = "qemu-quirks"))] + fn fill_hack_push(&mut self) { + self.ITCR.write(ITCR::ITCR1::Enabled); + + #[cfg(feature = "bsp_rpi4")] + { + self.TDR.set(b'X' as u32); + self.TDR.set(b'Y' as u32); + } + self.TDR.set(b'Z' as u32); + + self.ITCR.write(ITCR::ITCR1::Disabled); + } + + /// Pop characters from the receive FIFO. + /// + /// See top level comments why this is needed. + #[cfg(not(feature = "qemu-quirks"))] + fn fill_hack_pop(&mut self) { + #[cfg(feature = "bsp_rpi4")] + { + self.DR.get(); + self.DR.get(); + } + self.DR.get(); + } +} + +/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are +/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`, +/// we get `write_fmt()` automatically. +/// +/// The function takes an `&mut self`, so it must be implemented for the inner struct. +/// +/// See [`src/print.rs`]. +/// +/// [`src/print.rs`]: ../../print/index.html +impl fmt::Write for PL011UartInner { + fn write_str(&mut self, s: &str) -> fmt::Result { + for c in s.chars() { + self.write_char(c); + } + + Ok(()) + } +} + +impl PL011Uart { + /// # Safety + /// + /// - The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize, irq_number: bsp::device_driver::IRQNumber) -> Self { + Self { + inner: IRQSafeNullLock::new(PL011UartInner::new(base_addr)), + irq_number, + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::Mutex; + +impl driver::interface::DeviceDriver for PL011Uart { + fn compatible(&self) -> &str { + "BCM PL011 UART" + } + + fn init(&self) -> Result<(), ()> { + let mut r = &self.inner; + r.lock(|inner| inner.init()); + + Ok(()) + } + + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + use bsp::exception::asynchronous::irq_manager; + use exception::asynchronous::{interface::IRQManager, IRQDescriptor}; + + let descriptor = IRQDescriptor { + name: "BCM PL011 UART", + handler: self, + }; + + irq_manager().register_handler(self.irq_number, descriptor)?; + irq_manager().enable(self.irq_number); + + Ok(()) + } +} + +impl console::interface::Write for PL011Uart { + /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to + /// serialize access. + fn write_char(&self, c: char) { + let mut r = &self.inner; + r.lock(|inner| inner.write_char(c)); + } + + fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result { + // Fully qualified syntax for the call to `core::fmt::Write::write:fmt()` to increase + // readability. + let mut r = &self.inner; + r.lock(|inner| fmt::Write::write_fmt(inner, args)) + } + + fn flush(&self) { + // Spin until TX FIFO empty is set. + let mut r = &self.inner; + r.lock(|inner| { + while !inner.FR.matches_all(FR::TXFE::SET) { + cpu::nop(); + } + }); + } +} + +impl console::interface::Read for PL011Uart { + fn read_char(&self) -> char { + let mut r = &self.inner; + r.lock(|inner| inner.read_char_converting(true).unwrap()) + } + + fn clear(&self) { + let mut r = &self.inner; + r.lock(|inner| { + // Read from the RX FIFO until it is indicating empty. + while !inner.FR.matches_all(FR::RXFE::SET) { + inner.DR.get(); + } + }) + } +} + +impl console::interface::Statistics for PL011Uart { + fn chars_written(&self) -> usize { + let mut r = &self.inner; + r.lock(|inner| inner.chars_written) + } + + fn chars_read(&self) -> usize { + let mut r = &self.inner; + r.lock(|inner| inner.chars_read) + } +} + +impl exception::asynchronous::interface::IRQHandler for PL011Uart { + fn handle(&self) -> Result<(), &'static str> { + let mut r = &self.inner; + r.lock(|inner| { + // Echo any received characters. + loop { + match inner.read_char_converting(false) { + None => break, + Some(c) => inner.write_char(c), + } + } + }); + + Ok(()) + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/common.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/common.rs new file mode 100644 index 00000000..8a83399a --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/common.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Common device driver code. + +use core::{marker::PhantomData, ops}; + +pub struct MMIODerefWrapper { + base_addr: usize, + phantom: PhantomData, +} + +impl MMIODerefWrapper { + /// Create an instance. + pub const unsafe fn new(base_addr: usize) -> Self { + Self { + base_addr, + phantom: PhantomData, + } + } + + /// Return a pointer to the associated MMIO register block. + fn ptr(&self) -> *const T { + self.base_addr as *const _ + } +} + +impl ops::Deref for MMIODerefWrapper { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.ptr() } + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs new file mode 100644 index 00000000..aa395411 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Top-level BSP file for the Raspberry Pi 3 and 4. + +pub mod console; +pub mod cpu; +pub mod driver; +pub mod exception; +pub mod memory; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- +use super::device_driver; + +static GPIO: device_driver::GPIO = + unsafe { device_driver::GPIO::new(memory::map::mmio::GPIO_BASE) }; + +static PL011_UART: device_driver::PL011Uart = unsafe { + device_driver::PL011Uart::new( + memory::map::mmio::PL011_UART_BASE, + exception::asynchronous::irq_map::PL011_UART, + ) +}; + +#[cfg(feature = "bsp_rpi3")] +static INTERRUPT_CONTROLLER: device_driver::InterruptController = unsafe { + device_driver::InterruptController::new( + memory::map::mmio::LOCAL_INTERRUPT_CONTROLLER_BASE, + memory::map::mmio::PERIPHERAL_INTERRUPT_CONTROLLER_BASE, + ) +}; + +#[cfg(feature = "bsp_rpi4")] +static INTERRUPT_CONTROLLER: device_driver::GICv2 = unsafe { + device_driver::GICv2::new(memory::map::mmio::GICD_BASE, memory::map::mmio::GICC_BASE) +}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Board identification. +pub fn board_name() -> &'static str { + #[cfg(feature = "bsp_rpi3")] + { + "Raspberry Pi 3" + } + + #[cfg(feature = "bsp_rpi4")] + { + "Raspberry Pi 4" + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/console.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/console.rs new file mode 100644 index 00000000..90011da6 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/console.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP console facilities. + +use super::memory; +use crate::{bsp::device_driver, console}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// In case of a panic, the panic handler uses this function to take a last shot at printing +/// something before the system is halted. +/// +/// # Safety +/// +/// - Use only for printing during a panic. +pub unsafe fn panic_console_out() -> impl fmt::Write { + let mut uart = device_driver::PanicUart::new(memory::map::mmio::PL011_UART_BASE); + uart.init(); + uart +} + +/// Return a reference to the console. +pub fn console() -> &'static impl console::interface::All { + &super::PL011_UART +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps +/// than on real hardware due to QEMU's abstractions. +/// +/// For the RPi, nothing needs to be done. +pub fn qemu_bring_up_console() {} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs new file mode 100644 index 00000000..c24204f0 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/cpu.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP Processor code. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Used by `arch` code to find the early boot core. +pub const BOOT_CORE_ID: usize = 0; + +/// The early boot core's stack address. +pub const BOOT_CORE_STACK_START: u64 = 0x80_000; + +/// The number of processor cores. +pub const NUM_CORES: usize = 4; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs new file mode 100644 index 00000000..b9e1e171 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP driver support. + +use crate::driver; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Device Driver Manager type. +pub struct BSPDriverManager { + device_drivers: [&'static (dyn DeviceDriver + Sync); 3], +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static BSP_DRIVER_MANAGER: BSPDriverManager = BSPDriverManager { + device_drivers: [ + &super::GPIO, + &super::PL011_UART, + &super::INTERRUPT_CONTROLLER, + ], +}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the driver manager. +pub fn driver_manager() -> &'static impl driver::interface::DriverManager { + &BSP_DRIVER_MANAGER +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use driver::interface::DeviceDriver; + +impl driver::interface::DriverManager for BSPDriverManager { + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[..] + } + + fn post_device_driver_init(&self) { + // Configure PL011Uart's output pins. + super::GPIO.map_pl011_uart(); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception.rs new file mode 100644 index 00000000..55e82119 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! BSP synchronous and asynchronous exception handling. + +pub mod asynchronous; diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception/asynchronous.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception/asynchronous.rs new file mode 100644 index 00000000..787d5926 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/exception/asynchronous.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! BSP asynchronous exception handling. + +use crate::{bsp, exception}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +#[cfg(feature = "bsp_rpi3")] +pub(in crate::bsp) mod irq_map { + use super::bsp::device_driver::{IRQNumber, PeripheralIRQ}; + + pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57)); +} + +#[cfg(feature = "bsp_rpi4")] +pub(in crate::bsp) mod irq_map { + use super::bsp::device_driver::IRQNumber; + + pub const PL011_UART: IRQNumber = IRQNumber::new(153); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the IRQ manager. +pub fn irq_manager() -> &'static impl exception::asynchronous::interface::IRQManager< + IRQNumberType = bsp::device_driver::IRQNumber, +> { + &super::super::INTERRUPT_CONTROLLER +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld new file mode 100644 index 00000000..fac86cda --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT OR Apache-2.0 + * + * Copyright (c) 2018-2020 Andre Richter + */ + +SECTIONS +{ + /* Set current address to the value from which the RPi starts execution */ + . = 0x80000; + + __ro_start = .; + .text : + { + *(.text._start) *(.text*) + } + + .rodata : + { + *(.rodata*) + } + . = ALIGN(65536); /* Fill up to 64 KiB */ + __ro_end = .; + + .data : + { + *(.data*) + } + + /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ + .bss ALIGN(8): + { + __bss_start = .; + *(.bss*); + . = ALIGN(8); + __bss_end = .; + } + + /DISCARD/ : { *(.comment*) } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs new file mode 100644 index 00000000..43c54094 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP Memory Management. + +pub mod mmu; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// The board's memory map. +#[rustfmt::skip] +pub(super) mod map { + pub const END_INCLUSIVE: usize = 0xFFFF_FFFF; + + pub const GPIO_OFFSET: usize = 0x0020_0000; + pub const UART_OFFSET: usize = 0x0020_1000; + + /// Physical devices. + #[cfg(feature = "bsp_rpi3")] + pub mod mmio { + use super::*; + + pub const BASE: usize = 0x3F00_0000; + pub const PERIPHERAL_INTERRUPT_CONTROLLER_BASE: usize = BASE + 0x0000_B200; + pub const GPIO_BASE: usize = BASE + GPIO_OFFSET; + pub const PL011_UART_BASE: usize = BASE + UART_OFFSET; + pub const LOCAL_INTERRUPT_CONTROLLER_BASE: usize = 0x4000_0000; + pub const END_INCLUSIVE: usize = 0x4000_FFFF; + } + + /// Physical devices. + #[cfg(feature = "bsp_rpi4")] + pub mod mmio { + use super::*; + + pub const BASE: usize = 0xFE00_0000; + pub const GPIO_BASE: usize = BASE + GPIO_OFFSET; + pub const PL011_UART_BASE: usize = BASE + UART_OFFSET; + pub const GICD_BASE: usize = 0xFF84_1000; + pub const GICC_BASE: usize = 0xFF84_2000; + pub const END_INCLUSIVE: usize = 0xFF84_FFFF; + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs new file mode 100644 index 00000000..31c6f9a6 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP Memory Management Unit. + +use super::map as memory_map; +use crate::memory::mmu::*; +use core::ops::RangeInclusive; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +const NUM_MEM_RANGES: usize = 2; + +/// The virtual memory layout. +/// +/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. +/// It is agnostic of the paging granularity that the architecture's MMU will use. +pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( + memory_map::END_INCLUSIVE, + [ + RangeDescriptor { + name: "Kernel code and RO data", + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 64 KiB + // aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the first + // byte of the area. + static __ro_start: usize; + + // The exclusive end of the read-only area, aka the address of the first + // byte _after_ the RO area. + static __ro_end: usize; + } + + unsafe { + // Notice the subtraction to turn the exclusive end into an inclusive end. + #[allow(clippy::range_minus_one)] + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + RangeDescriptor { + name: "Device MMIO", + virtual_range: || { + RangeInclusive::new(memory_map::mmio::BASE, memory_map::mmio::END_INCLUSIVE) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + ], +); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return the address space size in bytes. +pub const fn addr_space_size() -> usize { + memory_map::END_INCLUSIVE + 1 +} + +/// Return a reference to the virtual memory layout. +pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { + &LAYOUT +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check 64 KiB alignment of the kernel's virtual memory layout sections. + #[kernel_test] + fn virt_mem_layout_sections_are_64KiB_aligned() { + const SIXTYFOUR_KIB: usize = 65536; + + for i in LAYOUT.inner().iter() { + let start: usize = *(i.virtual_range)().start(); + let end: usize = *(i.virtual_range)().end() + 1; + + assert_eq!(start % SIXTYFOUR_KIB, 0); + assert_eq!(end % SIXTYFOUR_KIB, 0); + assert!(end >= start); + } + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/console.rs b/14_exceptions_part2_peripheral_IRQs/src/console.rs new file mode 100644 index 00000000..e6323a20 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/console.rs @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! System console. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Console interfaces. +pub mod interface { + use core::fmt; + + /// Console write functions. + pub trait Write { + /// Write a single character. + fn write_char(&self, c: char); + + /// Write a Rust format string. + fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; + + /// Block execution until the last character has been physically put on the TX wire + /// (draining TX buffers/FIFOs, if any). + fn flush(&self); + } + + /// Console read functions. + pub trait Read { + /// Read a single character. + fn read_char(&self) -> char { + ' ' + } + + /// Clear RX buffers, if any. + fn clear(&self); + } + + /// Console statistics. + pub trait Statistics { + /// Return the number of characters written. + fn chars_written(&self) -> usize { + 0 + } + + /// Return the number of characters read. + fn chars_read(&self) -> usize { + 0 + } + } + + /// Trait alias for a full-fledged console. + pub trait All = Write + Read + Statistics; +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/cpu.rs b/14_exceptions_part2_peripheral_IRQs/src/cpu.rs new file mode 100644 index 00000000..9c67c0e7 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/cpu.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Processor code. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/cpu.rs"] +mod arch_cpu; +pub use arch_cpu::*; + +pub mod smp; diff --git a/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs b/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs new file mode 100644 index 00000000..b1428884 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/cpu/smp.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Symmetric multiprocessing. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/smp.rs"] +mod arch_cpu_smp; +pub use arch_cpu_smp::*; diff --git a/14_exceptions_part2_peripheral_IRQs/src/driver.rs b/14_exceptions_part2_peripheral_IRQs/src/driver.rs new file mode 100644 index 00000000..0e24b403 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/driver.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Driver support. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Driver interfaces. +pub mod interface { + + /// Device Driver functions. + pub trait DeviceDriver { + /// Return a compatibility string for identifying the driver. + fn compatible(&self) -> &str; + + /// Called by the kernel to bring up the device. + fn init(&self) -> Result<(), ()> { + Ok(()) + } + + /// Called by the kernel to register and enable the device's IRQ handlers, if any. + /// + /// Rust's type system will prevent a call to this function unless the calling instance + /// itself has static lifetime. + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + Ok(()) + } + } + + /// Device driver management functions. + /// + /// The `BSP` is supposed to supply one global instance. + pub trait DriverManager { + /// Return a slice of references to all `BSP`-instantiated drivers. + /// + /// # Safety + /// + /// - The order of devices is the order in which `DeviceDriver::init()` is called. + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Initialization code that runs after driver init. + /// + /// For example, device driver code that depends on other drivers already being online. + fn post_device_driver_init(&self); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/exception.rs b/14_exceptions_part2_peripheral_IRQs/src/exception.rs new file mode 100644 index 00000000..81ea2b26 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/exception.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Synchronous and asynchronous exception handling. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/exception.rs"] +mod arch_exception; +pub use arch_exception::*; + +pub mod asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Kernel privilege levels. +#[allow(missing_docs)] +#[derive(PartialEq)] +pub enum PrivilegeLevel { + User, + Kernel, + Hypervisor, + Unknown, +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Libkernel unit tests must execute in kernel mode. + #[kernel_test] + fn test_runner_executes_in_kernel_mode() { + let (level, _) = current_privilege_level(); + + assert!(level == PrivilegeLevel::Kernel) + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs b/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs new file mode 100644 index 00000000..25a01736 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/exception/asynchronous.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Asynchronous exception handling. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/exception/asynchronous.rs"] +mod arch_exception_async; +pub use arch_exception_async::*; + +use core::{fmt, marker::PhantomData}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Asynchronous exception handling interfaces. +pub mod interface { + + /// Implemented by types that handle IRQs. + pub trait IRQHandler { + /// Called when the corresponding interrupt is asserted. + fn handle(&self) -> Result<(), &'static str>; + } + + /// IRQ management functions. + /// + /// The `BSP` is supposed to supply one global instance. Typically implemented by the + /// platform's interrupt controller. + pub trait IRQManager { + /// The IRQ number type depends on the implementation. + type IRQNumberType; + + /// Register a handler. + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: super::IRQDescriptor, + ) -> Result<(), &'static str>; + + /// Enable an interrupt in the controller. + fn enable(&self, irq_number: Self::IRQNumberType); + + /// Handle pending interrupts. + /// + /// This function is called directly from the CPU's IRQ exception vector. On AArch64, + /// this means that the respective CPU core has disabled exception handling. + /// This function can therefore not be preempted and runs start to finish. + /// + /// Takes an IRQContext token to ensure it can only be called from IRQ context. + #[allow(clippy::trivially_copy_pass_by_ref)] + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &super::IRQContext<'irq_context>, + ); + + /// Print list of registered handlers. + fn print_handler(&self); + } +} + +/// Interrupt descriptor. +#[derive(Copy, Clone)] +pub struct IRQDescriptor { + /// Descriptive name. + pub name: &'static str, + + /// Reference to handler trait object. + pub handler: &'static (dyn interface::IRQHandler + Sync), +} + +/// IRQContext token. +/// +/// An instance of this type indicates that the local core is currently executing in IRQ +/// context, aka executing an interrupt vector or subcalls of it. +/// +/// Concept and implementation derived from the `CriticalSection` introduced in +/// https://github.com/rust-embedded/bare-metal +#[derive(Clone, Copy)] +pub struct IRQContext<'irq_context> { + _0: PhantomData<&'irq_context ()>, +} + +/// A wrapper type for IRQ numbers with integrated range sanity check. +#[derive(Copy, Clone)] +pub struct IRQNumber(usize); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl<'irq_context> IRQContext<'irq_context> { + /// Creates an IRQContext token. + /// + /// # Safety + /// + /// - This must only be called when the current core is in an interrupt context and will not + /// live beyond the end of it. That is, creation is allowed in interrupt vector functions. For + /// example, in the ARMv8-A case, in `extern "C" fn current_elx_irq()`. + /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code + /// must not be able to influence the lifetime picked for this type, since that might cause it + /// to be inferred to `'static`. + #[inline(always)] + pub unsafe fn new() -> Self { + IRQContext { _0: PhantomData } + } +} + +impl IRQNumber<{ MAX_INCLUSIVE }> { + /// Creates a new instance if number <= MAX_INCLUSIVE. + pub const fn new(number: usize) -> Self { + assert!(number <= MAX_INCLUSIVE); + + Self { 0: number } + } + + /// Return the wrapped number. + pub fn get(self) -> usize { + self.0 + } +} + +impl fmt::Display for IRQNumber<{ MAX_INCLUSIVE }> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Executes the provided closure while IRQs are masked on the executing core. +/// +/// While the function temporarily changes the HW state of the executing core, it restores it to the +/// previous state before returning, so this is deemed safe. +#[inline(always)] +pub fn exec_with_irq_masked(f: impl FnOnce() -> T) -> T { + let ret: T; + + unsafe { + let saved = local_irq_mask_save(); + ret = f(); + local_irq_restore(saved); + } + + ret +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/lib.rs b/14_exceptions_part2_peripheral_IRQs/src/lib.rs new file mode 100644 index 00000000..004bb212 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/lib.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc(html_logo_url = "https://git.io/JeGIp")] + +//! The `kernel` library. +//! +//! Used by `main.rs` to compose the final kernel binary. +//! +//! # TL;DR - Overview of important Kernel entities +//! +//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. +//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. +//! - [`bsp::exception::asynchronous::irq_manager()`] - Returns a reference to the kernel's [IRQ +//! Handling interface]. +//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. +//! - [`state::state_manager()`] - Returns a reference to the kernel's [state management] instance. +//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. +//! +//! [console interface]: ../libkernel/console/interface/index.html +//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html +//! [IRQ Handling interface]: ../libkernel/exception/asynchronous/interface/trait.IRQManager.html +//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html +//! [state management]: ../libkernel/state/struct.StateManager.html +//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html +//! +//! # Code organization and architecture +//! +//! The code is divided into different *modules*, each representing a typical **subsystem** of the +//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example, +//! `src/memory.rs` contains code that is concerned with all things memory management. +//! +//! ## Visibility of processor architecture code +//! +//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target +//! processor architecture. For each supported processor architecture, there exists a subfolder in +//! `src/_arch`, for example, `src/_arch/aarch64`. +//! +//! The architecture folders mirror the subsystem modules laid out in `src`. For example, +//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go +//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in +//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's +//! module organization. That means a public function `foo()` defined in +//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! +//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. +//! Rather, it's contents are conditionally pulled into respective files using the `#[path = +//! "_arch/xxx/yyy.rs"]` attribute. +//! +//! ## BSP code +//! +//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains +//! target board specific definitions and functions. These are things such as the board's memory map +//! or instances of drivers for devices that are featured on the respective board. +//! +//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the +//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means +//! whatever is provided must be called starting from the `bsp` namespace, e.g. +//! `bsp::driver::driver_manager()`. +//! +//! ## Kernel interfaces +//! +//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target +//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of +//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel` +//! code to play nicely with any of the two without much hassle. +//! +//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`, +//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined +//! in the respective subsystem module and help to enforce the idiom of *program to an interface, +//! not an implementation*. For example, there will be a common IRQ handling interface which the two +//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the +//! interface to the rest of the `kernel`. +//! +//! ``` +//! +-------------------+ +//! | Interface (Trait) | +//! | | +//! +--+-------------+--+ +//! ^ ^ +//! | | +//! | | +//! +----------+--+ +--+----------+ +//! | kernel code | | bsp code | +//! | | | arch code | +//! +-------------+ +-------------+ +//! ``` +//! +//! # Summary +//! +//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical +//! locations. Here is an example for the **memory** subsystem: +//! +//! - `src/memory.rs` and `src/memory/**/*` +//! - Common code that is agnostic of target processor architecture and `BSP` characteristics. +//! - Example: A function to zero a chunk of memory. +//! - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code. +//! - Example: An `MMU` interface that defines `MMU` function prototypes. +//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*` +//! - `BSP` specific code. +//! - Example: The board's memory map (physical addresses of DRAM and MMIO devices). +//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*` +//! - Processor architecture specific code. +//! - Example: Implementation of the `MMU` interface for the `__arch_name__` processor +//! architecture. +//! +//! From a namespace perspective, **memory** subsystem code lives in: +//! +//! - `crate::memory::*` +//! - `crate::bsp::memory::*` + +#![allow(incomplete_features)] +#![feature(asm)] +#![feature(const_fn)] +#![feature(const_generics)] +#![feature(const_if_match)] +#![feature(const_panic)] +#![feature(core_intrinsics)] +#![feature(custom_inner_attributes)] +#![feature(format_args_nl)] +#![feature(global_asm)] +#![feature(linkage)] +#![feature(naked_functions)] +#![feature(panic_info_message)] +#![feature(slice_ptr_range)] +#![feature(trait_alias)] +#![no_std] +// Testing +#![cfg_attr(test, no_main)] +#![feature(custom_test_frameworks)] +#![reexport_test_harness_main = "test_main"] +#![test_runner(crate::test_runner)] + +// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls +// `runtime_init()`, which jumps to `kernel_init()` (defined in `main.rs`). + +mod panic_wait; +mod runtime_init; +mod synchronization; + +pub mod bsp; +pub mod console; +pub mod cpu; +pub mod driver; +pub mod exception; +pub mod memory; +pub mod print; +pub mod state; +pub mod time; + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// The default runner for unit tests. +pub fn test_runner(tests: &[&test_types::UnitTest]) { + println!("Running {} tests", tests.len()); + println!("-------------------------------------------------------------------\n"); + for (i, test) in tests.iter().enumerate() { + print!("{:>3}. {:.<58}", i + 1, test.name); + + // Run the actual test. + (test.test_func)(); + + // Failed tests call panic!(). Execution reaches here only if the test has passed. + println!("[ok]") + } +} + +/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +#[cfg(test)] +#[no_mangle] +unsafe fn kernel_init() -> ! { + bsp::console::qemu_bring_up_console(); + + test_main(); + + cpu::qemu_exit_success() +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/main.rs b/14_exceptions_part2_peripheral_IRQs/src/main.rs new file mode 100644 index 00000000..df81e718 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/main.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc(html_logo_url = "https://git.io/JeGIp")] + +//! The `kernel` binary. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +use libkernel::{bsp, cpu, driver, exception, info, memory, state, time, warn}; + +/// Early init code. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +/// - The init calls in this function must appear in the correct order: +/// - Virtual memory must be activated before the device drivers. +/// - Without it, any atomic operations, e.g. the yet-to-be-introduced spinlocks in the device +/// drivers (which currently employ IRQSafeNullLocks instead of spinlocks), will fail to +/// work on the RPi SoCs. +#[no_mangle] +unsafe fn kernel_init() -> ! { + use driver::interface::DriverManager; + use memory::mmu::interface::MMU; + + exception::handling_init(); + + if let Err(string) = memory::mmu::mmu().init() { + panic!("MMU: {}", string); + } + + for i in bsp::driver::driver_manager().all_device_drivers().iter() { + if i.init().is_err() { + panic!("Error loading driver: {}", i.compatible()) + } + } + bsp::driver::driver_manager().post_device_driver_init(); + // println! is usable from here on. + + // Let device drivers register and enable their handlers with the interrupt controller. + for i in bsp::driver::driver_manager().all_device_drivers() { + if let Err(msg) = i.register_and_enable_irq_handler() { + warn!("Error registering IRQ handler: {}", msg); + } + } + + // Unmask interrupts on the boot CPU core. + exception::asynchronous::local_irq_unmask(); + + // Announce conclusion of the kernel_init() phase. + state::state_manager().transition_to_single_core_main(); + + // Transition from unsafe to safe. + kernel_main() +} + +/// The main function running after the early init. +fn kernel_main() -> ! { + use driver::interface::DriverManager; + use exception::asynchronous::interface::IRQManager; + + info!("Booting on: {}", bsp::board_name()); + + info!("MMU online. Special regions:"); + bsp::memory::mmu::virt_mem_layout().print_layout(); + + let (_, privilege_level) = exception::current_privilege_level(); + info!("Current privilege level: {}", privilege_level); + + info!("Exception handling state:"); + exception::asynchronous::print_state(); + + info!( + "Architectural timer resolution: {} ns", + time::time_manager().resolution().as_nanos() + ); + + info!("Drivers loaded:"); + for (i, driver) in bsp::driver::driver_manager() + .all_device_drivers() + .iter() + .enumerate() + { + info!(" {}. {}", i + 1, driver.compatible()); + } + + info!("Registered IRQ handlers:"); + bsp::exception::asynchronous::irq_manager().print_handler(); + + info!("Echoing input now"); + cpu::wait_forever(); +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/memory.rs b/14_exceptions_part2_peripheral_IRQs/src/memory.rs new file mode 100644 index 00000000..4a68a6d9 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/memory.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Memory Management. + +pub mod mmu; + +use core::ops::Range; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Zero out a memory region. +/// +/// # Safety +/// +/// - `range.start` and `range.end` must be valid. +/// - `range.start` and `range.end` must be `T` aligned. +pub unsafe fn zero_volatile(range: Range<*mut T>) +where + T: From, +{ + let mut ptr = range.start; + + while ptr < range.end { + core::ptr::write_volatile(ptr, T::from(0)); + ptr = ptr.offset(1); + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check `zero_volatile()`. + #[kernel_test] + fn zero_volatile_works() { + let mut x: [usize; 3] = [10, 11, 12]; + let x_range = x.as_mut_ptr_range(); + + unsafe { zero_volatile(x_range) }; + + assert_eq!(x, [0, 0, 0]); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs b/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs new file mode 100644 index 00000000..8d789704 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Memory Management Unit. +//! +//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file +//! provides types for composing an architecture-agnostic description of the kernel 's virtual +//! memory layout. +//! +//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` +//! function. +//! +//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and +//! install respective page tables. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/memory/mmu.rs"] +mod arch_mmu; +pub use arch_mmu::*; + +use core::{fmt, ops::RangeInclusive}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Memory Management interfaces. +pub mod interface { + + /// MMU functions. + pub trait MMU { + /// Called by the kernel during early init. Supposed to take the page tables from the + /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU. + /// + /// # Safety + /// + /// - Changes the HW's global state. + unsafe fn init(&self) -> Result<(), &'static str>; + } +} + +/// Architecture agnostic translation types. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +pub enum Translation { + Identity, + Offset(usize), +} + +/// Architecture agnostic memory attributes. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +pub enum MemAttributes { + CacheableDRAM, + Device, +} + +/// Architecture agnostic access permissions. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +pub enum AccessPermissions { + ReadOnly, + ReadWrite, +} + +/// Collection of memory attributes. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, +} + +/// Architecture agnostic descriptor for a memory range. +#[allow(missing_docs)] +pub struct RangeDescriptor { + pub name: &'static str, + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, +} + +/// Type for expressing the kernel's virtual memory layout. +pub struct KernelVirtualLayout { + /// The last (inclusive) address of the address space. + max_virt_addr_inclusive: usize, + + /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. + inner: [RangeDescriptor; NUM_SPECIAL_RANGES], +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } +} + +/// Human-readable output of a RangeDescriptor. +impl fmt::Display for RangeDescriptor { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Call the function to which self.range points, and dereference the result, which causes + // Rust to copy the value. + let start = *(self.virtual_range)().start(); + let end = *(self.virtual_range)().end(); + let size = end - start + 1; + + // log2(1024). + const KIB_RSHIFT: u32 = 10; + + // log2(1024 * 1024). + const MIB_RSHIFT: u32 = 20; + + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; + + let attr = match self.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::Device => "Dev", + }; + + let acc_p = match self.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if self.attribute_fields.execute_never { + "PXN" + } else { + "PX" + }; + + write!( + f, + " {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}", + start, end, size, unit, attr, acc_p, xn, self.name + ) + } +} + +impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { + /// Create a new instance. + pub const fn new(max: usize, layout: [RangeDescriptor; NUM_SPECIAL_RANGES]) -> Self { + Self { + max_virt_addr_inclusive: max, + inner: layout, + } + } + + /// For a virtual address, find and return the output address and corresponding attributes. + /// + /// If the address is not found in `inner`, return an identity mapped default with normal + /// cacheable DRAM attributes. + pub fn get_virt_addr_properties( + &self, + virt_addr: usize, + ) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > self.max_virt_addr_inclusive { + return Err("Address out of range"); + } + + for i in self.inner.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } + } + + Ok((virt_addr, AttributeFields::default())) + } + + /// Print the memory layout. + pub fn print_layout(&self) { + use crate::info; + + for i in self.inner.iter() { + info!("{}", i); + } + } + + #[cfg(test)] + pub fn inner(&self) -> &[RangeDescriptor; NUM_SPECIAL_RANGES] { + &self.inner + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/panic_wait.rs b/14_exceptions_part2_peripheral_IRQs/src/panic_wait.rs new file mode 100644 index 00000000..218c0a88 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/panic_wait.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! A panic handler that infinitely waits. + +use crate::{bsp, cpu}; +use core::{fmt, panic::PanicInfo}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +fn _panic_print(args: fmt::Arguments) { + use fmt::Write; + + unsafe { bsp::console::panic_console_out().write_fmt(args).unwrap() }; +} + +/// The point of exit for the "standard" (non-testing) `libkernel`. +/// +/// This code will be used by the release kernel binary and the `integration tests`. It is linked +/// weakly, so that the integration tests can overload it to exit `QEMU` instead of spinning +/// forever. +/// +/// This is one possible approach to solve the problem that `cargo` can not know who the consumer of +/// the library will be: +/// - The release kernel binary that should safely park the paniced core, +/// - or an `integration test` that is executed in QEMU, which should just exit QEMU. +#[cfg(not(test))] +#[linkage = "weak"] +#[no_mangle] +fn _panic_exit() -> ! { + cpu::wait_forever() +} + +/// Prints with a newline - only use from the panic handler. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! panic_println { + ($($arg:tt)*) => ({ + _panic_print(format_args_nl!($($arg)*)); + }) +} + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + if let Some(args) = info.message() { + panic_println!("\nKernel panic: {}", args); + } else { + panic_println!("\nKernel panic!"); + } + + _panic_exit() +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// The point of exit when the library is compiled for testing. +#[cfg(test)] +#[no_mangle] +fn _panic_exit() -> ! { + cpu::qemu_exit_failure() +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/print.rs b/14_exceptions_part2_peripheral_IRQs/src/print.rs new file mode 100644 index 00000000..cc303bfc --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/print.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Printing facilities. + +use crate::{bsp, console}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +#[doc(hidden)] +pub fn _print(args: fmt::Arguments) { + use console::interface::Write; + + bsp::console::console().write_fmt(args).unwrap(); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Prints without a newline. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! print { + ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*))); +} + +/// Prints with a newline. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! println { + () => ($crate::print!("\n")); + ($($arg:tt)*) => ({ + $crate::print::_print(format_args_nl!($($arg)*)); + }) +} + +/// Prints an info, with a newline. +#[macro_export] +macro_rules! info { + ($string:expr) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:03}{:03}] ", $string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000 + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:03}{:03}] ", $format_string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000, + $($arg)* + )); + }) +} + +/// Prints a warning, with a newline. +#[macro_export] +macro_rules! warn { + ($string:expr) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:03}{:03}] ", $string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000 + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:03}{:03}] ", $format_string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000, + $($arg)* + )); + }) +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/runtime_init.rs b/14_exceptions_part2_peripheral_IRQs/src/runtime_init.rs new file mode 100644 index 00000000..04cca287 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/runtime_init.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Rust runtime initialization code. + +use crate::memory; +use core::ops::Range; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Return the range spanning the .bss section. +/// +/// # Safety +/// +/// - The symbol-provided addresses must be valid. +/// - The symbol-provided addresses must be usize aligned. +unsafe fn bss_range() -> Range<*mut usize> { + extern "C" { + // Boundaries of the .bss section, provided by linker script symbols. + static mut __bss_start: usize; + static mut __bss_end: usize; + } + + Range { + start: &mut __bss_start, + end: &mut __bss_end, + } +} + +/// Zero out the .bss section. +/// +/// # Safety +/// +/// - Must only be called pre `kernel_init()`. +#[inline(always)] +unsafe fn zero_bss() { + memory::zero_volatile(bss_range()); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel +/// init code. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +pub unsafe fn runtime_init() -> ! { + extern "Rust" { + fn kernel_init() -> !; + } + + zero_bss(); + kernel_init() +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check `bss` section layout. + #[kernel_test] + fn bss_section_is_sane() { + use core::mem; + + let start = unsafe { bss_range().start } as *const _ as usize; + let end = unsafe { bss_range().end } as *const _ as usize; + + assert_eq!(start % mem::size_of::(), 0); + assert_eq!(end % mem::size_of::(), 0); + assert!(end >= start); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/state.rs b/14_exceptions_part2_peripheral_IRQs/src/state.rs new file mode 100644 index 00000000..5edb9fc5 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/state.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! State information about the kernel itself. + +use core::sync::atomic::{AtomicU8, Ordering}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Different stages in the kernel execution. +#[derive(Copy, Clone, Eq, PartialEq)] +pub enum State { + /// The kernel starts booting in this state. + Init, + + /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of + /// `kernel_init()`, after all init calls are done). + SingleCoreMain, + + /// The kernel transitions to this state when it boots the secondary cores, aka switches + /// exectution mode to symmetric multiprocessing (SMP). + MultiCoreMain, +} + +/// Maintains the kernel state and state transitions. +pub struct StateManager(AtomicU8); + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static STATE_MANAGER: StateManager = StateManager::new(); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the global StateManager. +pub fn state_manager() -> &'static StateManager { + &STATE_MANAGER +} + +impl StateManager { + const INIT: u8 = 0; + const SINGLE_CORE_MAIN: u8 = 1; + const MULTI_CORE_MAIN: u8 = 2; + + /// Create a new instance. + pub const fn new() -> Self { + Self(AtomicU8::new(Self::INIT)) + } + + /// Return the current state. + pub fn state(&self) -> State { + let state = self.0.load(Ordering::Acquire); + + match state { + Self::INIT => State::Init, + Self::SINGLE_CORE_MAIN => State::SingleCoreMain, + Self::MULTI_CORE_MAIN => State::MultiCoreMain, + _ => panic!("Invalid KERNEL_STATE"), + } + } + + /// Transition from Init to SingleCoreMain. + pub fn transition_to_single_core_main(&self) { + if self + .0 + .compare_exchange( + Self::INIT, + Self::SINGLE_CORE_MAIN, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_err() + { + panic!("transition_to_single_core_main() called while state != Init"); + } + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/synchronization.rs b/14_exceptions_part2_peripheral_IRQs/src/synchronization.rs new file mode 100644 index 00000000..08edd68d --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/synchronization.rs @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Synchronization primitives. + +use core::cell::UnsafeCell; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Synchronization interfaces. +pub mod interface { + + /// Any object implementing this trait guarantees exclusive access to the data contained within + /// the Mutex for the duration of the provided closure. + /// + /// The trait follows the [Rust embedded WG's + /// proposal](https://github.com/korken89/wg/blob/master/rfcs/0377-mutex-trait.md) and therefore + /// provides some goodness such as [deadlock + /// prevention](https://github.com/korken89/wg/blob/master/rfcs/0377-mutex-trait.md#design-decisions-and-compatibility). + /// + /// # Example + /// + /// Since the lock function takes an `&mut self` to enable deadlock-prevention, the trait is + /// best implemented **for a reference to a container struct**, and has a usage pattern that + /// might feel strange at first: + /// + /// ``` + /// static MUT: Mutex> = Mutex::new(RefCell::new(0)); + /// + /// fn foo() { + /// let mut r = &MUT; // Note that r is mutable + /// r.lock(|data| *data += 1); + /// } + /// ``` + pub trait Mutex { + /// The type of encapsulated data. + type Data; + + /// Creates a critical section and grants temporary mutable access to the encapsulated data. + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + } + + /// A reader-writer exclusion type. + /// + /// The implementing object allows either a number of readers or at most one writer at any point + /// in time. + pub trait ReadWriteEx { + /// The type of encapsulated data. + type Data; + + /// Grants temporary mutable access to the encapsulated data. + fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + + /// Grants temporary immutable access to the encapsulated data. + fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R; + } +} + +/// A pseudo-lock for teaching purposes. +/// +/// Used to introduce [interior mutability]. +/// +/// In contrast to a real Mutex implementation, does not protect against concurrent access from +/// other cores to the contained data. This part is preserved for later lessons. +/// +/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is +/// executing on a single core. +/// +/// [interior mutability]: https://doc.rust-lang.org/std/cell/index.html +pub struct IRQSafeNullLock { + data: UnsafeCell, +} + +/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards. +/// +/// Intended to encapsulate data that is populated during kernel init when no concurrency exists. +pub struct InitStateLock { + data: UnsafeCell, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +unsafe impl Sync for IRQSafeNullLock {} + +impl IRQSafeNullLock { + /// Wraps `data` into a new `IRQSafeNullLock`. + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), + } + } +} + +unsafe impl Sync for InitStateLock {} + +impl InitStateLock { + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use crate::{exception, state}; + +impl interface::Mutex for &IRQSafeNullLock { + type Data = T; + + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + // In a real lock, there would be code encapsulating this line that ensures that this + // mutable reference will ever only be given out once at a time. + let data = unsafe { &mut *self.data.get() }; + + // Execute the closure while IRQs are masked. + exception::asynchronous::exec_with_irq_masked(|| f(data)) + } +} + +impl interface::ReadWriteEx for &InitStateLock { + type Data = T; + + fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + assert!( + state::state_manager().state() == state::State::Init, + "InitStateLock::write called after kernel init phase" + ); + assert!( + !exception::asynchronous::is_local_irq_masked(), + "InitStateLock::write called with IRQs unmasked" + ); + + let data = unsafe { &mut *self.data.get() }; + + f(data) + } + + fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R { + let data = unsafe { &*self.data.get() }; + + f(data) + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/src/time.rs b/14_exceptions_part2_peripheral_IRQs/src/time.rs new file mode 100644 index 00000000..cd3ceec3 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/src/time.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Timer primitives. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/time.rs"] +mod arch_time; +pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Timekeeping interfaces. +pub mod interface { + use core::time::Duration; + + /// Time management functions. + /// + /// The `BSP` is supposed to supply one global instance. + pub trait TimeManager { + /// The timer's resolution. + fn resolution(&self) -> Duration; + + /// The uptime since power-on of the device. + /// + /// This includes time consumed by firmware and bootloaders. + fn uptime(&self) -> Duration; + + /// Spin for a given duration. + fn spin_for(&self, duration: Duration); + } +} diff --git a/14_exceptions_part2_peripheral_IRQs/test-macros/Cargo.toml b/14_exceptions_part2_peripheral_IRQs/test-macros/Cargo.toml new file mode 100644 index 00000000..a570f72b --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/test-macros/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "test-macros" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.x" +quote = "1.x" +syn = { version = "1.x", features = ["full"] } +test-types = { path = "../test-types" } diff --git a/14_exceptions_part2_peripheral_IRQs/test-macros/src/lib.rs b/14_exceptions_part2_peripheral_IRQs/test-macros/src/lib.rs new file mode 100644 index 00000000..092c4806 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/test-macros/src/lib.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::{parse_macro_input, Ident, ItemFn}; + +#[proc_macro_attribute] +pub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream { + let f = parse_macro_input!(input as ItemFn); + + let test_name = &format!("{}", f.sig.ident.to_string()); + let test_ident = Ident::new( + &format!("{}_TEST_CONTAINER", f.sig.ident.to_string().to_uppercase()), + Span::call_site(), + ); + let test_code_block = f.block; + + quote!( + #[test_case] + const #test_ident: test_types::UnitTest = test_types::UnitTest { + name: #test_name, + test_func: || #test_code_block, + }; + ) + .into() +} diff --git a/14_exceptions_part2_peripheral_IRQs/test-types/Cargo.toml b/14_exceptions_part2_peripheral_IRQs/test-types/Cargo.toml new file mode 100644 index 00000000..a0be2c57 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/test-types/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "test-types" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" diff --git a/14_exceptions_part2_peripheral_IRQs/test-types/src/lib.rs b/14_exceptions_part2_peripheral_IRQs/test-types/src/lib.rs new file mode 100644 index 00000000..371bb557 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/test-types/src/lib.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Types for the `custom_test_frameworks` implementation. + +#![no_std] + +/// Unit test container. +pub struct UnitTest { + /// Name of the test. + pub name: &'static str, + + /// Function pointer to the test. + pub test_func: fn(), +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rb b/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rb new file mode 100644 index 00000000..a6b549b2 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2019-2020 Andre Richter + +require 'expect' + +TIMEOUT_SECS = 3 + +# Verify sending and receiving works as expected. +class TxRxHandshake + def name + 'Transmit and Receive handshake' + end + + def run(qemu_out, qemu_in) + qemu_in.write_nonblock('ABC') + raise('TX/RX test failed') if qemu_out.expect('OK1234', TIMEOUT_SECS).nil? + end +end + +# Check for correct TX statistics implementation. Depends on test 1 being run first. +class TxStatistics + def name + 'Transmit statistics' + end + + def run(qemu_out, _qemu_in) + raise('chars_written reported wrong') if qemu_out.expect('6', TIMEOUT_SECS).nil? + end +end + +# Check for correct RX statistics implementation. Depends on test 1 being run first. +class RxStatistics + def name + 'Receive statistics' + end + + def run(qemu_out, _qemu_in) + raise('chars_read reported wrong') if qemu_out.expect('3', TIMEOUT_SECS).nil? + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [TxRxHandshake.new, TxStatistics.new, RxStatistics.new] +end diff --git a/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs b/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs new file mode 100644 index 00000000..5aa38f09 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/00_console_sanity.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Console sanity tests - RX, TX and statistics. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +mod panic_exit_failure; + +use libkernel::{bsp, console, print}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + use bsp::console::{console, qemu_bring_up_console}; + use console::interface::*; + + qemu_bring_up_console(); + + // Handshake + assert_eq!(console().read_char(), 'A'); + assert_eq!(console().read_char(), 'B'); + assert_eq!(console().read_char(), 'C'); + print!("OK1234"); + + // 6 + print!("{}", console().chars_written()); + + // 3 + print!("{}", console().chars_read()); + + // The QEMU process running this test will be closed by the I/O test harness. + loop {} +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/01_timer_sanity.rs b/14_exceptions_part2_peripheral_IRQs/tests/01_timer_sanity.rs new file mode 100644 index 00000000..e0b3c162 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/01_timer_sanity.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Timer sanity tests. + +#![feature(custom_test_frameworks)] +#![no_main] +#![no_std] +#![reexport_test_harness_main = "test_main"] +#![test_runner(libkernel::test_runner)] + +mod panic_exit_failure; + +use core::time::Duration; +use libkernel::{bsp, cpu, time, time::interface::TimeManager}; +use test_macros::kernel_test; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + bsp::console::qemu_bring_up_console(); + + // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi. + + test_main(); + + cpu::qemu_exit_success() +} + +/// Simple check that the timer is running. +#[kernel_test] +fn timer_is_counting() { + assert!(time::time_manager().uptime().as_nanos() > 0) +} + +/// Timer resolution must be sufficient. +#[kernel_test] +fn timer_resolution_is_sufficient() { + assert!(time::time_manager().resolution().as_nanos() < 100) +} + +/// Sanity check spin_for() implementation. +#[kernel_test] +fn spin_accuracy_check_1_second() { + let t1 = time::time_manager().uptime(); + time::time_manager().spin_for(Duration::from_secs(1)); + let t2 = time::time_manager().uptime(); + + assert_eq!((t2 - t1).as_secs(), 1) +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs b/14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs new file mode 100644 index 00000000..64fc5486 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Page faults must result in synchronous exceptions. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +/// +/// Reaching this code is a success, because it is called from the synchronous exception handler, +/// which is what this test wants to achieve. +/// +/// It also means that this integration test can not use any other code that calls panic!() directly +/// or indirectly. +mod panic_exit_success; + +use libkernel::{bsp, cpu, exception, memory, println}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + use memory::mmu::interface::MMU; + + bsp::console::qemu_bring_up_console(); + + println!("Testing synchronous exception handling by causing a page fault"); + println!("-------------------------------------------------------------------\n"); + + exception::handling_init(); + + if let Err(string) = memory::mmu::mmu().init() { + println!("MMU: {}", string); + cpu::qemu_exit_failure() + } + + println!("Writing beyond mapped area to address 9 GiB..."); + let big_addr: u64 = 9 * 1024 * 1024 * 1024; + core::ptr::read_volatile(big_addr as *mut u64); + + // If execution reaches here, the memory access above did not cause a page fault exception. + cpu::qemu_exit_failure() +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/03_exception_irq_sanity.rs b/14_exceptions_part2_peripheral_IRQs/tests/03_exception_irq_sanity.rs new file mode 100644 index 00000000..6e2ae8f7 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/03_exception_irq_sanity.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! IRQ handling sanity tests. + +#![feature(custom_test_frameworks)] +#![no_main] +#![no_std] +#![reexport_test_harness_main = "test_main"] +#![test_runner(libkernel::test_runner)] + +mod panic_exit_failure; + +use libkernel::{bsp, cpu, exception}; +use test_macros::kernel_test; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + bsp::console::qemu_bring_up_console(); + + exception::handling_init(); + exception::asynchronous::local_irq_unmask(); + + test_main(); + + cpu::qemu_exit_success() +} + +/// Check that IRQ masking works. +#[kernel_test] +fn local_irq_mask_works() { + // Precondition: IRQs are unmasked. + assert!(exception::asynchronous::is_local_irq_masked()); + + unsafe { exception::asynchronous::local_irq_mask() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + // Restore earlier state. + unsafe { exception::asynchronous::local_irq_unmask() }; +} + +/// Check that IRQ unmasking works. +#[kernel_test] +fn local_irq_unmask_works() { + // Precondition: IRQs are masked. + unsafe { exception::asynchronous::local_irq_mask() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + unsafe { exception::asynchronous::local_irq_unmask() }; + assert!(exception::asynchronous::is_local_irq_masked()); +} + +/// Check that IRQ mask save is saving "something". +#[kernel_test] +fn local_irq_mask_save_works() { + // Precondition: IRQs are unmasked. + assert!(exception::asynchronous::is_local_irq_masked()); + + let first = unsafe { exception::asynchronous::local_irq_mask_save() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + let second = unsafe { exception::asynchronous::local_irq_mask_save() }; + assert_ne!(first, second); + + unsafe { exception::asynchronous::local_irq_restore(first) }; + assert!(exception::asynchronous::is_local_irq_masked()); +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/panic_exit_failure/mod.rs b/14_exceptions_part2_peripheral_IRQs/tests/panic_exit_failure/mod.rs new file mode 100644 index 00000000..b4ac73d1 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/panic_exit_failure/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +#[no_mangle] +fn _panic_exit() -> ! { + libkernel::cpu::qemu_exit_failure() +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/panic_exit_success/mod.rs b/14_exceptions_part2_peripheral_IRQs/tests/panic_exit_success/mod.rs new file mode 100644 index 00000000..54bb072d --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/panic_exit_success/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +#[no_mangle] +fn _panic_exit() -> ! { + libkernel::cpu::qemu_exit_success() +} diff --git a/14_exceptions_part2_peripheral_IRQs/tests/runner.rb b/14_exceptions_part2_peripheral_IRQs/tests/runner.rb new file mode 100755 index 00000000..ef11f316 --- /dev/null +++ b/14_exceptions_part2_peripheral_IRQs/tests/runner.rb @@ -0,0 +1,139 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2019-2020 Andre Richter + +require 'English' +require 'pty' + +# Test base class. +class Test + INDENT = ' ' + + def print_border(status) + puts + puts "#{INDENT}-------------------------------------------------------------------" + puts status + puts "#{INDENT}-------------------------------------------------------------------\n\n\n" + end + + def print_error(error) + puts + print_border("#{INDENT}❌ Failure: #{error}: #{@test_name}") + end + + def print_success + print_border("#{INDENT}✅ Success: #{@test_name}") + end + + def print_output + puts "#{INDENT}-------------------------------------------------------------------" + print INDENT + print '🦀 ' + print @output.join('').gsub("\n", "\n" + INDENT) + end + + def finish(error) + print_output + + exit_code = if error + print_error(error) + false + else + print_success + true + end + + exit(exit_code) + end +end + +# Executes tests with console I/O. +class ConsoleTest < Test + def initialize(binary, qemu_cmd, test_name, console_subtests) + @binary = binary + @qemu_cmd = qemu_cmd + @test_name = test_name + @console_subtests = console_subtests + @cur_subtest = 1 + @output = ["Running #{@console_subtests.length} console-based tests\n", + "-------------------------------------------------------------------\n\n"] + end + + def format_test_name(number, name) + formatted_name = number.to_s.rjust(3) + '. ' + name + formatted_name.ljust(63, '.') + end + + def run_subtest(subtest, qemu_out, qemu_in) + @output << format_test_name(@cur_subtest, subtest.name) + + subtest.run(qemu_out, qemu_in) + + @output << "[ok]\n" + @cur_subtest += 1 + end + + def exec + error = false + + PTY.spawn(@qemu_cmd) do |qemu_out, qemu_in| + begin + @console_subtests.each { |t| run_subtest(t, qemu_out, qemu_in) } + rescue StandardError => e + error = e.message + end + + finish(error) + end + end +end + +# A wrapper around the bare QEMU invocation. +class RawTest < Test + MAX_WAIT_SECS = 5 + + def initialize(binary, qemu_cmd, test_name) + @binary = binary + @qemu_cmd = qemu_cmd + @test_name = test_name + @output = [] + end + + def exec + error = 'Timed out waiting for test' + io = IO.popen(@qemu_cmd) + + while IO.select([io], nil, nil, MAX_WAIT_SECS) + begin + @output << io.read_nonblock(1024) + rescue EOFError + io.close + error = $CHILD_STATUS.to_i != 0 + break + end + end + + finish(error) + end +end + +##-------------------------------------------------------------------------------------------------- +## Script entry point +##-------------------------------------------------------------------------------------------------- +binary = ARGV.last +test_name = binary.gsub(%r{.*deps/}, '').split('-')[0] +console_test_file = 'tests/' + test_name + '.rb' +qemu_cmd = ARGV.join(' ') + +test_runner = if File.exist?(console_test_file) + load console_test_file + # subtest_collection is provided by console_test_file + ConsoleTest.new(binary, qemu_cmd, test_name, subtest_collection) + else + RawTest.new(binary, qemu_cmd, test_name) + end + +test_runner.exec diff --git a/doc/14_BCM_driver.drawio b/doc/14_BCM_driver.drawio new file mode 100644 index 00000000..d5b8e127 --- /dev/null +++ b/doc/14_BCM_driver.drawio @@ -0,0 +1 @@ +7Vpbc+I2FP41zLSdCeM78Mgtu+k2TTakm+YpI2xhqxGWI4sA/fU9smXHNxqSjYFON8MQ6+j4WD7fuXyy6Zjj5eYTR1FwyTxMO4bmbTrmpGMYumUYHfnRvG0qsbSBmUp8Tjyl9SKYkb+xEmpKuiIejkuKgjEqSFQWuiwMsStKMsQ5W5fVFoyWrxohH9cEMxfRuvSOeCJIpYOi/DMmfqCu3NfUuufIffQ5W4Xqch3DXCR/6fQSZaaUfhwgj61LIrwR5ywU6gb+mK9CsQLxJQtZx54GQkgXDDvGOXwWUrHrM+ZTjCISd122BLEbg8r5Ai0JlQBkNkaJDbiIOe2YY86YSI+WmzGmEsAMmnQl5ztmcx9wHIp9Trh7+nX2rTd9ZtMvX26uru2/rsz1mbLyjOgKZ75KPCK2GQiJH7E0onXM0TogAs8i5MrZNcQdyAKxpDDS4VC64rx6x6nXzFEsOHvEY0YZh9mQhWBjVL+NbE2YC7wpiNRtfcJsiQXfgoqK6zPTyJBfF8NEyYJCiFi2EiKFrJ9be3EdHCjvNXtyZD/dTyYj3A8sfRyi4SSYfjuz9nAlxFkkD2UiYK7cNVPz0n0xOJaE/i2LpED6W6B5Fvq6o6eCPOYtpXHNYiIIC0FE8UJUICGUZj6HRMC6Z+NeDQ2YGTg9Ezmvgfg9eKlZ01SuyfHTnTp+ht2AXw70x+Nn1PDDGxdHqV+H8EHxNnQDDm5YxTVo4dYrfm+M9iIYSoQo8SVyLjg0CQrpSAJFcKgmlsTz5GUac6+cndV4ah1JS6siadeR1K1+E5JtAWm2nYjvzMOWsehXoDAGDUnVcxqSSu+1hoVVw2IeR2k6/fIjh3bg1pBCjtG1D5lDdkMOOVSiE5Uwc55Wkr+Mloj7JKmTUKGjDXwnztFS+ZlgUTpnFeYk3GcKOTmnwMttwpGv/idXJgUBWkrgaH10yxERuaisXrIGfiHVK8x5TZIJLm6+XqIQCCcvGJhX1UEWVWXvclhaPnKP7XJJfcFNKwi4XELG7jNb+m6z7SxZ6nLskxhQfghQCAnJf+p2uz833Y1UxiGaU1xSOZyDpW66yIcIhx40gwfCn+IjribiJBSNjiuvo1JVq6UwycW8Yqo+xUBrQZM9kCyvWRlMY0Y3XiuDhZJdLqb5zkoOYFtEXHVM0RzTUb5Pq9TztO7nxPcjmIptV5lKw57B0BqqrN5emXVOsMzmRW00vgT1C6nNVxEUVW0MMcAZpfJ0bcLJ89uroUeeG+8sc7hcofFvJQ+MJjZOI9ormysP4f7CbdpcOW4fzxcfxLrtXoUy9Ht2Qyw37X9bC+Xe66wbyuhQPhWC0Zwy91GGA4qDpFhIZ8L8eQJHUiFgVESkwqQLrh0OxtZ0JM/YEPGnPBu4Ujq6z2zB8WRTHGwLbG8n0AAR3xYsyuF9vlgYvNhMRpnRnRjHbMVdvEdFEJDQWOzB0LBXeoRWj5niPiwj/xxTJCB7S2trCgll7ppB5ynEnzPoVqqpYVX3EOmtqjOLj6Uqxiy9xn+re/zUGTVTSZzmd/odods/5Sp8jTmJAswRvRi/tdieBPE7k03k5usMLfDvK0p/k4lfoux3HLZkVyHd3mA/3knWD0K05FpvMPIKqzn49S9CAvtPJHDdU59T7ncrafE+jvof9kZHr/bGQSPPa+iNg9Z64+BNvdEjaMlC7zaQcVRoinqpKRpWvSnubmRZZyz2xbxLNnfGvPnl3a5wVjvNr79v83OO2Px6NfKVbS3e2vps81VTLbe+7EnToVjb60yrFGyv8az3EL5DhOcxuZlR52ZO9aXX3gGqGzVjh2ZndtN7tZNhZ9Cj/7PE7JbBzByIhObJpy6AQwhfq0g+pgl96ZWVYJwgeorkgskXNEIqmdqxuMbg9KiGXX+LeSSqob+HauSD/ahGJQjeX9qzIvODeRysrO/xlvawzKMUe20wjyNE6xGJiK7XiUjPfCcRMbVei0QEhi8/w0rVX35QZ07/AQ== \ No newline at end of file diff --git a/doc/14_BCM_driver.png b/doc/14_BCM_driver.png new file mode 100644 index 00000000..642205b5 Binary files /dev/null and b/doc/14_BCM_driver.png differ diff --git a/doc/14_GICv2_driver.drawio b/doc/14_GICv2_driver.drawio new file mode 100644 index 00000000..3b77ec10 --- /dev/null +++ b/doc/14_GICv2_driver.drawio @@ -0,0 +1 @@ +7VpbU+M2FP41mWk7k4zvIY9xLiy7pUsJ3YUnRrEVW0WxjKyQ0F9fyZYd2xIQsoSGtkwGpCP5SDrfdy5y6Nij5eaUgjQ+JyHEHcsINx173LEs07GsjvgY4WMhcQzHKSQRRaGctRXM0F9QCg0pXaEQZo2JjBDMUNoUBiRJYMAaMkApWTenLQhurpqCCCqCWQCwKv2OQhYX0kFd/gmiKJYrnxhy33MQ3EWUrBK5XMeyF/lPMbwEpSo5P4tBSNYNEdywKUmYPMAf81XCVlx8ThLScScxY8IEw4415Z+FmNiLCIkwBCnKegFZcnGQ8SnTBVgiLAAodfi5Dr6IPenYI0oIK1rLzQhiAWAJTbGT6ROjlQ0oTNguD3y//zz71p88kMmXL5dfL9w/v9rrrtTyAPAKlrbKLcIeSxByO0KhxOjY/jpGDM5SEIjRNecdl8VsiXnP5E1himn7xIXVbD9jlNzBEcGE8tGEJFyHrx6j3BOkDG5qInmsU0iWkNFHPkXyumtbJfLrOk2kLK5RxHGlEEhko0rb1nS8Ia2nt6Tv3t+Mxz48iR1zlIDhOJ586zo7mJLzLBVN4QiQSnPN5LgwX8YNi5LoiqRCIOzNwLykvumZhaDivCNnXJAMMUQSLsJwwVqQIIxLm3NHgGbowr6CBh8ZeH0beC+B+CN4yVHblqap8DM9FT/L1eBXAf32+FkKfnATwLSw65B/QPaYBDHlZlhlCrT86C27a9leB0OKAEaRQC7gBs1JIQyJeBAcyoElCkOxjNb3mt7Z5tPBkXSMNpKuiqTpnOiQPBSQrsYRPSzQSRuYefcrEXv9JaARyjHm7Eo3/HduHKOQd5lwRjHm1MYE3F2JnBiT4FU6eSuSf/OVUU0AlgI4rPauKECsEjWnN7Rxu6D2CnOqSErB2eXv5yDhyZLWFMzb07ksbcv2MlgRgiqLPWUSdcO6HcRUbKGsTEpd5tNqD7NlMZfCCGUc5dsYJNwh6U+9Xu9n3WnEZJiAOYaNKe9nYDG32ORtCpOQZ5RbRO+zf3A3KUUJ0xquuY9WVG2HwtwXq4gpcx3hsxY4r99EeC3DYMEZ03opDNZCdjOYVlWh6PCSDgWyjcEcYr+qMVvxvIj7VdJ+iyjruu0oq6l3LMNVo6z5FmH27Az6VhJ3N5/TeXT5EFy709kuleOPlTt7VjsvZzzF8Bp4nsTipJXwrIGmdOl7mtLF7B8KCrVymWdpUbP88u8pVN4YNk2d4lk9jQsdzIOcI6xTqqrg9Gw0fm21cBS5ussHeMEzAwv42wrjX0lw16yyZjFnTXgJo0xfXO1Wlxxq6z5I7qrdHV2CbN0lFwvoBYESQvhI2B/MDeNtfNcz+23n1aY/TcwdHMp3dXeMFjS88BqKd2C8FyKwJEl4FQtu+HxgmsOQm5T3JBKW84pUBjeIXctgKto3os3DV9Ebb2pD48eyk/CjX1fr8k7tKdHdPpb3yueKo8FQeVmn1DQZWdEAvhTzVKTrl8Uyd1KIAUMPzTV1QEp1F4SXmLWiqd9mzUlZRpVKit3K5+pvzlqqXPdFVYw7OGSKqpxb1Tn3p5t35Kli9GFTxf/xVom3g+MLt/1jCbfmPuG26rx7uPU+Yri13zPc6l8hvo5vcyyKTNsPQRbnNxyzQTujQbvc/1u0qznycDByJn49wVcsu6mTTMu45ylc0XHLwJt6MfAsHTXsK0B4zoyulsrvwkdHJVF/Tz46pnJ5bH8LceD0b6o3/OPK/w/WhywASt1ZCpLG/TFBbMYAg+rt8VPxCvNKvN198uV8Xd+u18r/YKGhCfQnO1Yae3yNy7vbL9sLz9z+24Q9+Rs= \ No newline at end of file diff --git a/doc/14_GICv2_driver.png b/doc/14_GICv2_driver.png new file mode 100644 index 00000000..5fca9042 Binary files /dev/null and b/doc/14_GICv2_driver.png differ diff --git a/doc/14_header.drawio b/doc/14_header.drawio new file mode 100644 index 00000000..b067ec79 --- /dev/null +++ b/doc/14_header.drawio @@ -0,0 +1 @@ +7Vxbb+I4FP41SLsPrYhNAnlsaOlWmkpsO6OdeTSJE7wNcdaYFubXr50LIbEpKU2AtlRViU8SX875zh21A4ez5S1D8fSeejjsgK637MDrDgCGMTDFh6SsUkq/10sJASNe9lBBeCS/cUbsZtQF8fC89CCnNOQkLhNdGkXY5SUaYoy+lB/zaVheNUYBVgiPLgpV6j/E49OUaufbk/S/MAmm2cqDbnZjhvJnM8J8ijz6UiLhJR/RiGc7/DFZRHwhyPc0oh3zZsq5PONVB4zEry8fvAwoDUKMYjK/dOlMkN25eGTkoxkJJYPzOZxkDrEIvOnAIaOUp1ez5RCHUkA579OdjLbcXR+e4YjXeWFJLsZXt/BhzLybMf4ds8XPmwtop9M8o3CRcTVjCV/lbGZ0EXlYztLtQOdlSjh+jJEr774IYAnalM9CMTLEpeTFqHrklG3pzQxGhji/M+eMPuEhDSkTpIhG4o6TbQgzjpdbj2qsGSjQjekMc7YSj7xoYDDdgEDPyogoE2ywfrfgnLjImPcGRhoKH53HcXJhhVweNUZRia3WfwspeWeC3KcgYfGFmzJCoKrLgskfhiWEI/YrNtQ1bGt9DaD5Z/G+uArkJ4sJzBcTJ0jXS28lkH/CimAFg3lZelqB+CQMKyQUkiASQ1cIBAu6I8VFhGZeZTdmxPPkMlq4lAG1AQrQ3YWg94Aje6GXQWBVHm5AxzA10AFtIQfs1kAceVfSXhYC8NB8mnDQUNXvjczEnmJl36JnOl7lNIZDxMlzeXodA7MVxpSIhQtJmf2yqKoymNMFc3H21qbp2zVRZR6OWIC5Mk8izvWp95cwbN42ANMs7EG3sA2GaettQ+9sG+rYhsGJGQdLgc4DmscTzJJdj4n4I7FUlZ4IaGJ5OaPeItwUSoj9DHEuiYJvyShl7U6xcRprXHjbEqkIZKAKBPS1AmlLIn3VXA9B58oRMV8sFECcSgwdJ1e3CdNqtmTbxTzho1RpA8TLVF8rmrue5l2zOMN7cftOCpYtYmkYh+JNRsMQy+1dM2Gj2YaBmBTLV4C1YQ88RuPvieFUlXU3NDYNRwdAD+GB7ypWRtyx3AGe+M2AyTDKaDKABk7G4NJUAWW3hafBbvefa3MBMej8izlf5VnPICes051eI6q5XQG2s1hl6Cv28p0BwgUo+/V+eQLq+3PcikevkTXtyJOEFU9TYp8spbt7W+LUgC4Ao3cJzBL7QNdU3Z1GekZrSVR3N1/LAQJlfEoDGqHwG5X+KeFtogoZu9CC0zLn1WBaUEZE7vR6Z4xRFQReEv5TvihsRjr6lU0jr6+Xm4NVNujow26e29Kt3AFbZCz4kwTCr70L9WB4pwICw1ZA1K8XpDelioaaebeKmSkK/SFhbqgiR0iFrTbQIIe/ysMCEclotR7l5TVr7QNz4/7mqooek8bJYdI+GCSrBqttSNZJ6cOQxPNtWdC+BTU53oidusmPLrhXwqz1o5XAzE9+BD2W3Er4Zzod81rnwhgWsSiaJEcsQNW4l9IlZQf1Umo+/ym8f//Y3r+n8PUWR5gRV64WefIvc6fi4wmzKOlluNRTSxdtJL/rmsV7khxseCbu67TPtvoQWU0lOYpooanLczSyrfru5mSrFjK+TmS3V8Bm1/XAg0O5UVgtm7btRjWllnNk11Bkt38ScURMwnL5BlaLfG0Dskat5hzX7Rt/QOPYcd3nrOrA3pHjOqBm6HcPf9+jCAVJ+fk7Q4QrnD6dBtSbtPQxO4HRhDChmspCnTAP2q5Wk9uzNPeUJtQWXA8qTTWX/WINRtMqBxXH7zACNQ9uqsXYQH9RmeLqQTYXb++Gz3LTJ9BL9H1sudpeote3J92GIhjTLuPmFHqJuXn5OM3EFOtfvJsIahRHTj3wtOzTayeCAxcQPnzRKc+vdyb4oJ3ejWUfvZ0IauT456LTAYtOuXE8JUweuJ8IahQmznWnvd3UsfuJudX9bO7/2P1EqNadzv3EfRKdE+wnQrUKdQ7tXg3tzJpuFG7JyJp3o4fuJ8Ia39s4h3YHDO1qpxstYbJf+Tr4ofuJUFPoO8d1TcUfR+8nwhrVuA8Y1x29nwjVctm5A1VHmH3NV2OP3k+EapHuLM09pdlmP1EMi/9VkHrA4r9KwJv/AQ== \ No newline at end of file diff --git a/doc/14_header.png b/doc/14_header.png new file mode 100644 index 00000000..1aaf24f7 Binary files /dev/null and b/doc/14_header.png differ