From 8ff358a50b7b46217dff5c681371467d1af104f0 Mon Sep 17 00:00:00 2001 From: Andre Richter Date: Mon, 2 May 2022 23:24:29 +0200 Subject: [PATCH] Add tutorial 18 --- 18_backtrace/.cargo/config.toml | 2 + 18_backtrace/.vscode/settings.json | 10 + 18_backtrace/Cargo.lock | 96 ++ 18_backtrace/Cargo.toml | 11 + 18_backtrace/Makefile | 389 ++++++ 18_backtrace/README.md | 1244 +++++++++++++++++ 18_backtrace/kernel/Cargo.toml | 70 + 18_backtrace/kernel/build.rs | 20 + .../kernel/src/_arch/aarch64/backtrace.rs | 136 ++ 18_backtrace/kernel/src/_arch/aarch64/cpu.rs | 49 + .../kernel/src/_arch/aarch64/cpu/boot.rs | 113 ++ .../kernel/src/_arch/aarch64/cpu/boot.s | 100 ++ .../kernel/src/_arch/aarch64/cpu/smp.rs | 30 + .../kernel/src/_arch/aarch64/exception.rs | 323 +++++ .../kernel/src/_arch/aarch64/exception.s | 190 +++ .../_arch/aarch64/exception/asynchronous.rs | 152 ++ .../kernel/src/_arch/aarch64/memory/mmu.rs | 158 +++ .../aarch64/memory/mmu/translation_table.rs | 521 +++++++ 18_backtrace/kernel/src/_arch/aarch64/time.rs | 121 ++ 18_backtrace/kernel/src/backtrace.rs | 112 ++ 18_backtrace/kernel/src/bsp.rs | 13 + 18_backtrace/kernel/src/bsp/device_driver.rs | 16 + .../kernel/src/bsp/device_driver/arm.rs | 9 + .../kernel/src/bsp/device_driver/arm/gicv2.rs | 246 ++++ .../src/bsp/device_driver/arm/gicv2/gicc.rs | 156 +++ .../src/bsp/device_driver/arm/gicv2/gicd.rs | 209 +++ .../kernel/src/bsp/device_driver/bcm.rs | 15 + .../src/bsp/device_driver/bcm/bcm2xxx_gpio.rs | 259 ++++ .../bcm/bcm2xxx_interrupt_controller.rs | 138 ++ .../peripheral_ic.rs | 192 +++ .../device_driver/bcm/bcm2xxx_pl011_uart.rs | 536 +++++++ .../kernel/src/bsp/device_driver/common.rs | 38 + 18_backtrace/kernel/src/bsp/raspberrypi.rs | 62 + .../kernel/src/bsp/raspberrypi/console.rs | 98 ++ .../kernel/src/bsp/raspberrypi/cpu.rs | 14 + .../kernel/src/bsp/raspberrypi/driver.rs | 61 + .../kernel/src/bsp/raspberrypi/exception.rs | 7 + .../bsp/raspberrypi/exception/asynchronous.rs | 36 + .../kernel/src/bsp/raspberrypi/kernel.ld | 114 ++ .../kernel_virt_addr_space_size.ld | 1 + .../kernel/src/bsp/raspberrypi/memory.rs | 227 +++ .../kernel/src/bsp/raspberrypi/memory/mmu.rs | 179 +++ 18_backtrace/kernel/src/common.rs | 29 + 18_backtrace/kernel/src/console.rs | 53 + 18_backtrace/kernel/src/cpu.rs | 21 + 18_backtrace/kernel/src/cpu/boot.rs | 9 + 18_backtrace/kernel/src/cpu/smp.rs | 14 + 18_backtrace/kernel/src/driver.rs | 62 + 18_backtrace/kernel/src/exception.rs | 48 + .../kernel/src/exception/asynchronous.rs | 152 ++ 18_backtrace/kernel/src/lib.rs | 188 +++ 18_backtrace/kernel/src/main.rs | 111 ++ 18_backtrace/kernel/src/memory.rs | 191 +++ 18_backtrace/kernel/src/memory/mmu.rs | 270 ++++ 18_backtrace/kernel/src/memory/mmu/alloc.rs | 70 + .../kernel/src/memory/mmu/mapping_record.rs | 233 +++ .../src/memory/mmu/translation_table.rs | 137 ++ 18_backtrace/kernel/src/memory/mmu/types.rs | 378 +++++ 18_backtrace/kernel/src/panic_wait.rs | 106 ++ 18_backtrace/kernel/src/print.rs | 94 ++ 18_backtrace/kernel/src/state.rs | 92 ++ 18_backtrace/kernel/src/symbols.rs | 87 ++ 18_backtrace/kernel/src/synchronization.rs | 159 +++ 18_backtrace/kernel/src/time.rs | 37 + .../kernel/tests/00_console_sanity.rb | 48 + .../kernel/tests/00_console_sanity.rs | 39 + 18_backtrace/kernel/tests/01_timer_sanity.rs | 50 + .../tests/02_exception_sync_page_fault.rs | 37 + .../tests/03_exception_restore_sanity.rb | 25 + .../tests/03_exception_restore_sanity.rs | 49 + .../kernel/tests/04_exception_irq_sanity.rs | 67 + .../kernel/tests/05_backtrace_sanity.rb | 39 + .../kernel/tests/05_backtrace_sanity.rs | 31 + .../tests/06_backtrace_invalid_frame.rb | 26 + .../tests/06_backtrace_invalid_frame.rs | 33 + .../kernel/tests/07_backtrace_invalid_link.rb | 25 + .../kernel/tests/07_backtrace_invalid_link.rs | 38 + 18_backtrace/kernel/tests/boot_test_string.rb | 3 + .../kernel/tests/panic_exit_success/mod.rs | 9 + .../kernel/tests/panic_wait_forever/mod.rs | 9 + 18_backtrace/kernel_symbols.mk | 103 ++ 18_backtrace/kernel_symbols/Cargo.toml | 15 + 18_backtrace/kernel_symbols/build.rs | 14 + 18_backtrace/kernel_symbols/kernel_symbols.ld | 15 + 18_backtrace/kernel_symbols/src/main.rs | 16 + .../libraries/debug-symbol-types/Cargo.toml | 4 + .../libraries/debug-symbol-types/src/lib.rs | 45 + 18_backtrace/libraries/test-macros/Cargo.toml | 14 + 18_backtrace/libraries/test-macros/src/lib.rs | 29 + 18_backtrace/libraries/test-types/Cargo.toml | 5 + 18_backtrace/libraries/test-types/src/lib.rs | 16 + .../tools/kernel_symbols_tool/cmds.rb | 45 + .../tools/kernel_symbols_tool/kernel_elf.rb | 74 + .../tools/kernel_symbols_tool/main.rb | 47 + .../tools/translation_table_tool/arch.rb | 314 +++++ .../tools/translation_table_tool/bsp.rb | 50 + .../tools/translation_table_tool/generic.rb | 179 +++ .../translation_table_tool/kernel_elf.rb | 96 ++ .../tools/translation_table_tool/main.rb | 46 + doc/18_stack_frames.png | Bin 0 -> 47923 bytes 100 files changed, 10669 insertions(+) create mode 100644 18_backtrace/.cargo/config.toml create mode 100644 18_backtrace/.vscode/settings.json create mode 100644 18_backtrace/Cargo.lock create mode 100644 18_backtrace/Cargo.toml create mode 100644 18_backtrace/Makefile create mode 100644 18_backtrace/README.md create mode 100644 18_backtrace/kernel/Cargo.toml create mode 100644 18_backtrace/kernel/build.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/backtrace.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/cpu.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/cpu/boot.s create mode 100644 18_backtrace/kernel/src/_arch/aarch64/cpu/smp.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/exception.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/exception.s create mode 100644 18_backtrace/kernel/src/_arch/aarch64/exception/asynchronous.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/memory/mmu.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs create mode 100644 18_backtrace/kernel/src/_arch/aarch64/time.rs create mode 100644 18_backtrace/kernel/src/backtrace.rs create mode 100644 18_backtrace/kernel/src/bsp.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/arm.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/bcm.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs create mode 100644 18_backtrace/kernel/src/bsp/device_driver/common.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/console.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/cpu.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/driver.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/exception.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/exception/asynchronous.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/memory.rs create mode 100644 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs create mode 100644 18_backtrace/kernel/src/common.rs create mode 100644 18_backtrace/kernel/src/console.rs create mode 100644 18_backtrace/kernel/src/cpu.rs create mode 100644 18_backtrace/kernel/src/cpu/boot.rs create mode 100644 18_backtrace/kernel/src/cpu/smp.rs create mode 100644 18_backtrace/kernel/src/driver.rs create mode 100644 18_backtrace/kernel/src/exception.rs create mode 100644 18_backtrace/kernel/src/exception/asynchronous.rs create mode 100644 18_backtrace/kernel/src/lib.rs create mode 100644 18_backtrace/kernel/src/main.rs create mode 100644 18_backtrace/kernel/src/memory.rs create mode 100644 18_backtrace/kernel/src/memory/mmu.rs create mode 100644 18_backtrace/kernel/src/memory/mmu/alloc.rs create mode 100644 18_backtrace/kernel/src/memory/mmu/mapping_record.rs create mode 100644 18_backtrace/kernel/src/memory/mmu/translation_table.rs create mode 100644 18_backtrace/kernel/src/memory/mmu/types.rs create mode 100644 18_backtrace/kernel/src/panic_wait.rs create mode 100644 18_backtrace/kernel/src/print.rs create mode 100644 18_backtrace/kernel/src/state.rs create mode 100644 18_backtrace/kernel/src/symbols.rs create mode 100644 18_backtrace/kernel/src/synchronization.rs create mode 100644 18_backtrace/kernel/src/time.rs create mode 100644 18_backtrace/kernel/tests/00_console_sanity.rb create mode 100644 18_backtrace/kernel/tests/00_console_sanity.rs create mode 100644 18_backtrace/kernel/tests/01_timer_sanity.rs create mode 100644 18_backtrace/kernel/tests/02_exception_sync_page_fault.rs create mode 100644 18_backtrace/kernel/tests/03_exception_restore_sanity.rb create mode 100644 18_backtrace/kernel/tests/03_exception_restore_sanity.rs create mode 100644 18_backtrace/kernel/tests/04_exception_irq_sanity.rs create mode 100644 18_backtrace/kernel/tests/05_backtrace_sanity.rb create mode 100644 18_backtrace/kernel/tests/05_backtrace_sanity.rs create mode 100644 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb create mode 100644 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs create mode 100644 18_backtrace/kernel/tests/07_backtrace_invalid_link.rb create mode 100644 18_backtrace/kernel/tests/07_backtrace_invalid_link.rs create mode 100644 18_backtrace/kernel/tests/boot_test_string.rb create mode 100644 18_backtrace/kernel/tests/panic_exit_success/mod.rs create mode 100644 18_backtrace/kernel/tests/panic_wait_forever/mod.rs create mode 100644 18_backtrace/kernel_symbols.mk create mode 100644 18_backtrace/kernel_symbols/Cargo.toml create mode 100644 18_backtrace/kernel_symbols/build.rs create mode 100644 18_backtrace/kernel_symbols/kernel_symbols.ld create mode 100644 18_backtrace/kernel_symbols/src/main.rs create mode 100644 18_backtrace/libraries/debug-symbol-types/Cargo.toml create mode 100644 18_backtrace/libraries/debug-symbol-types/src/lib.rs create mode 100644 18_backtrace/libraries/test-macros/Cargo.toml create mode 100644 18_backtrace/libraries/test-macros/src/lib.rs create mode 100644 18_backtrace/libraries/test-types/Cargo.toml create mode 100644 18_backtrace/libraries/test-types/src/lib.rs create mode 100644 18_backtrace/tools/kernel_symbols_tool/cmds.rb create mode 100644 18_backtrace/tools/kernel_symbols_tool/kernel_elf.rb create mode 100755 18_backtrace/tools/kernel_symbols_tool/main.rb create mode 100644 18_backtrace/tools/translation_table_tool/arch.rb create mode 100644 18_backtrace/tools/translation_table_tool/bsp.rb create mode 100644 18_backtrace/tools/translation_table_tool/generic.rb create mode 100644 18_backtrace/tools/translation_table_tool/kernel_elf.rb create mode 100755 18_backtrace/tools/translation_table_tool/main.rb create mode 100644 doc/18_stack_frames.png diff --git a/18_backtrace/.cargo/config.toml b/18_backtrace/.cargo/config.toml new file mode 100644 index 00000000..e3476485 --- /dev/null +++ b/18_backtrace/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.'cfg(target_os = "none")'] +runner = "target/kernel_test_runner.sh" diff --git a/18_backtrace/.vscode/settings.json b/18_backtrace/.vscode/settings.json new file mode 100644 index 00000000..292bf2a9 --- /dev/null +++ b/18_backtrace/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "editor.formatOnSave": true, + "editor.rulers": [100], + "rust-analyzer.cargo.target": "aarch64-unknown-none-softfloat", + "rust-analyzer.cargo.features": ["bsp_rpi3"], + "rust-analyzer.checkOnSave.allTargets": false, + "rust-analyzer.checkOnSave.extraArgs": ["--lib", "--bins"], + "rust-analyzer.lens.debug": false, + "rust-analyzer.lens.run": false +} diff --git a/18_backtrace/Cargo.lock b/18_backtrace/Cargo.lock new file mode 100644 index 00000000..b851d8e3 --- /dev/null +++ b/18_backtrace/Cargo.lock @@ -0,0 +1,96 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cortex-a" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bd91f65ccd348bb2d043d98c5b34af141ecef7f102147f59bf5898f6e734ad" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "debug-symbol-types" +version = "0.1.0" + +[[package]] +name = "kernel_symbols" +version = "0.1.0" +dependencies = [ + "debug-symbol-types", +] + +[[package]] +name = "mingo" +version = "0.18.0" +dependencies = [ + "cortex-a", + "debug-symbol-types", + "qemu-exit", + "test-macros", + "test-types", + "tock-registers", +] + +[[package]] +name = "proc-macro2" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "qemu-exit" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff023245bfcc73fb890e1f8d5383825b3131cc920020a5c487d6f113dfc428a" + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "test-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "test-types", +] + +[[package]] +name = "test-types" +version = "0.1.0" + +[[package]] +name = "tock-registers" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee8fba06c1f4d0b396ef61a54530bb6b28f0dc61c38bc8bc5a5a48161e6282e" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" diff --git a/18_backtrace/Cargo.toml b/18_backtrace/Cargo.toml new file mode 100644 index 00000000..38eeb116 --- /dev/null +++ b/18_backtrace/Cargo.toml @@ -0,0 +1,11 @@ +[workspace] + +members = [ + "libraries/*", + "kernel", + "kernel_symbols" +] + +[profile.release] +lto = true +debug = true diff --git a/18_backtrace/Makefile b/18_backtrace/Makefile new file mode 100644 index 00000000..85826c12 --- /dev/null +++ b/18_backtrace/Makefile @@ -0,0 +1,389 @@ +## SPDX-License-Identifier: MIT OR Apache-2.0 +## +## Copyright (c) 2018-2022 Andre Richter + +include ../common/format.mk +include ../common/docker.mk + +##-------------------------------------------------------------------------------------------------- +## Optional, user-provided configuration values +##-------------------------------------------------------------------------------------------------- + +# Default to the RPi3. +BSP ?= rpi3 + +# Default to a serial device name that is common in Linux. +DEV_SERIAL ?= /dev/ttyUSB0 + +# Optional integration test name. +ifdef TEST + TEST_ARG = --test $(TEST) +else + TEST_ARG = --test '*' +endif + + + +##-------------------------------------------------------------------------------------------------- +## BSP-specific configuration values +##-------------------------------------------------------------------------------------------------- +QEMU_MISSING_STRING = "This board is not yet supported for QEMU." + +ifeq ($(BSP),rpi3) + TARGET = aarch64-unknown-none-softfloat + KERNEL_BIN = kernel8.img + QEMU_BINARY = qemu-system-aarch64 + QEMU_MACHINE_TYPE = raspi3 + QEMU_RELEASE_ARGS = -serial stdio -display none + QEMU_TEST_ARGS = $(QEMU_RELEASE_ARGS) -semihosting + OBJDUMP_BINARY = aarch64-none-elf-objdump + NM_BINARY = aarch64-none-elf-nm + READELF_BINARY = aarch64-none-elf-readelf + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg + JTAG_BOOT_IMAGE = ../X1_JTAG_boot/jtag_boot_rpi3.img + LD_SCRIPT_PATH = $(shell pwd)/kernel/src/bsp/raspberrypi + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C force-frame-pointers +else ifeq ($(BSP),rpi4) + TARGET = aarch64-unknown-none-softfloat + KERNEL_BIN = kernel8.img + QEMU_BINARY = qemu-system-aarch64 + QEMU_MACHINE_TYPE = + QEMU_RELEASE_ARGS = -serial stdio -display none + QEMU_TEST_ARGS = $(QEMU_RELEASE_ARGS) -semihosting + OBJDUMP_BINARY = aarch64-none-elf-objdump + NM_BINARY = aarch64-none-elf-nm + READELF_BINARY = aarch64-none-elf-readelf + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg + JTAG_BOOT_IMAGE = ../X1_JTAG_boot/jtag_boot_rpi4.img + LD_SCRIPT_PATH = $(shell pwd)/kernel/src/bsp/raspberrypi + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -C force-frame-pointers +endif + +# Export for build.rs. +export LD_SCRIPT_PATH + + + +##-------------------------------------------------------------------------------------------------- +## Targets and Prerequisites +##-------------------------------------------------------------------------------------------------- +KERNEL_MANIFEST = kernel/Cargo.toml +KERNEL_LINKER_SCRIPT = kernel.ld +LAST_BUILD_CONFIG = target/$(BSP).build_config + +KERNEL_ELF_RAW = target/$(TARGET)/release/kernel +# This parses cargo's dep-info file. +# https://doc.rust-lang.org/cargo/guide/build-cache.html#dep-info-files +KERNEL_ELF_RAW_DEPS = $(filter-out %: ,$(file < $(KERNEL_ELF_RAW).d)) $(LAST_BUILD_CONFIG) + +##------------------------------------------------------------------------------ +## Translation tables +##------------------------------------------------------------------------------ +TT_TOOL_PATH = tools/translation_table_tool + +KERNEL_ELF_TTABLES = target/$(TARGET)/release/kernel+ttables +KERNEL_ELF_TTABLES_DEPS = $(KERNEL_ELF_RAW) $(wildcard $(TT_TOOL_PATH)/*) + +##------------------------------------------------------------------------------ +## Kernel symbols +##------------------------------------------------------------------------------ +export KERNEL_SYMBOLS_TOOL_PATH = tools/kernel_symbols_tool + +KERNEL_ELF_TTABLES_SYMS = target/$(TARGET)/release/kernel+ttables+symbols + +# Unlike with KERNEL_ELF_RAW, we are not relying on dep-info here. One of the reasons being that the +# name of the generated symbols file varies between runs, which can cause confusion. +KERNEL_ELF_TTABLES_SYMS_DEPS = $(KERNEL_ELF_TTABLES) \ + $(wildcard kernel_symbols/*) \ + $(wildcard $(KERNEL_SYMBOLS_TOOL_PATH)/*) + +export TARGET +export KERNEL_SYMBOLS_INPUT_ELF = $(KERNEL_ELF_TTABLES) +export KERNEL_SYMBOLS_OUTPUT_ELF = $(KERNEL_ELF_TTABLES_SYMS) + +KERNEL_ELF = $(KERNEL_ELF_TTABLES_SYMS) + + + +##-------------------------------------------------------------------------------------------------- +## Command building blocks +##-------------------------------------------------------------------------------------------------- +RUSTFLAGS = $(RUSTC_MISC_ARGS) \ + -C link-arg=--library-path=$(LD_SCRIPT_PATH) \ + -C link-arg=--script=$(KERNEL_LINKER_SCRIPT) + +RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \ + -D warnings \ + -D missing_docs + +FEATURES = --features bsp_$(BSP) +COMPILER_ARGS = --target=$(TARGET) \ + $(FEATURES) \ + --release + +# build-std can be skipped for helper commands that do not rely on correct stack frames and other +# custom compiler options. This results in a huge speedup. +RUSTC_CMD = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST) +DOC_CMD = cargo doc $(COMPILER_ARGS) +CLIPPY_CMD = cargo clippy $(COMPILER_ARGS) +TEST_CMD = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST) +OBJCOPY_CMD = rust-objcopy \ + --strip-all \ + -O binary + +EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) +EXEC_TT_TOOL = ruby $(TT_TOOL_PATH)/main.rb +EXEC_TEST_DISPATCH = ruby ../common/tests/dispatch.rb +EXEC_MINIPUSH = ruby ../common/serial/minipush.rb + +##------------------------------------------------------------------------------ +## Dockerization +##------------------------------------------------------------------------------ +DOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial +DOCKER_CMD_INTERACT = $(DOCKER_CMD) -i +DOCKER_ARG_DIR_COMMON = -v $(shell pwd)/../common:/work/common +DOCKER_ARG_DIR_JTAG = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot +DOCKER_ARG_DEV = --privileged -v /dev:/dev +DOCKER_ARG_NET = --network host + +# DOCKER_IMAGE defined in include file (see top of this file). +DOCKER_QEMU = $(DOCKER_CMD_INTERACT) $(DOCKER_IMAGE) +DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE) +DOCKER_TEST = $(DOCKER_CMD) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE) +DOCKER_GDB = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) + +# Dockerize commands, which require USB device passthrough, only on Linux. +ifeq ($(shell uname -s),Linux) + DOCKER_CMD_DEV = $(DOCKER_CMD_INTERACT) $(DOCKER_ARG_DEV) + + DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_IMAGE) + DOCKER_JTAGBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_COMMON) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE) + DOCKER_OPENOCD = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) +else + DOCKER_OPENOCD = echo "Not yet supported on non-Linux systems."; \# +endif + + + +##-------------------------------------------------------------------------------------------------- +## Targets +##-------------------------------------------------------------------------------------------------- +.PHONY: all doc qemu chainboot clippy clean readelf objdump nm check + +all: $(KERNEL_BIN) + +##------------------------------------------------------------------------------ +## Save the configuration as a file, so make understands if it changed. +##------------------------------------------------------------------------------ +$(LAST_BUILD_CONFIG): + @rm -f target/*.build_config + @mkdir -p target + @touch $(LAST_BUILD_CONFIG) + +##------------------------------------------------------------------------------ +## Compile the kernel ELF +##------------------------------------------------------------------------------ +$(KERNEL_ELF_RAW): $(KERNEL_ELF_RAW_DEPS) + $(call color_header, "Compiling kernel ELF - $(BSP)") + @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(RUSTC_CMD) + +##------------------------------------------------------------------------------ +## Precompute the kernel translation tables and patch them into the kernel ELF +##------------------------------------------------------------------------------ +$(KERNEL_ELF_TTABLES): $(KERNEL_ELF_TTABLES_DEPS) + $(call color_header, "Precomputing kernel translation tables and patching kernel ELF") + @cp $(KERNEL_ELF_RAW) $(KERNEL_ELF_TTABLES) + @$(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $(KERNEL_ELF_TTABLES) + +##------------------------------------------------------------------------------ +## Generate kernel symbols and patch them into the kernel ELF +##------------------------------------------------------------------------------ +$(KERNEL_ELF_TTABLES_SYMS): $(KERNEL_ELF_TTABLES_SYMS_DEPS) + $(call color_header, "Generating kernel symbols and patching kernel ELF") + @time -f "in %es" \ + $(MAKE) --no-print-directory -f kernel_symbols.mk + +##------------------------------------------------------------------------------ +## Generate the stripped kernel binary +##------------------------------------------------------------------------------ +$(KERNEL_BIN): $(KERNEL_ELF_TTABLES_SYMS) + $(call color_header, "Generating stripped binary") + @$(OBJCOPY_CMD) $(KERNEL_ELF_TTABLES_SYMS) $(KERNEL_BIN) + $(call color_progress_prefix, "Name") + @echo $(KERNEL_BIN) + $(call color_progress_prefix, "Size") + @printf '%s KiB\n' `du -k $(KERNEL_BIN) | cut -f1` + +##------------------------------------------------------------------------------ +## Generate the documentation +##------------------------------------------------------------------------------ +doc: + $(call color_header, "Generating docs") + @$(DOC_CMD) --document-private-items --open + +##------------------------------------------------------------------------------ +## Run the kernel in QEMU +##------------------------------------------------------------------------------ +ifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board. + +qemu: + $(call color_header, "$(QEMU_MISSING_STRING)") + +else # QEMU is supported. + +qemu: $(KERNEL_BIN) + $(call color_header, "Launching QEMU") + @$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) + +endif + +##------------------------------------------------------------------------------ +## Push the kernel to the real HW target +##------------------------------------------------------------------------------ +chainboot: $(KERNEL_BIN) + @$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN) + +##------------------------------------------------------------------------------ +## Run clippy +##------------------------------------------------------------------------------ +clippy: + @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) + @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) --features test_build --tests \ + --manifest-path $(KERNEL_MANIFEST) + +##------------------------------------------------------------------------------ +## Clean +##------------------------------------------------------------------------------ +clean: + rm -rf target $(KERNEL_BIN) + +##------------------------------------------------------------------------------ +## Run readelf +##------------------------------------------------------------------------------ +readelf: $(KERNEL_ELF) + $(call color_header, "Launching readelf") + @$(DOCKER_TOOLS) $(READELF_BINARY) --headers $(KERNEL_ELF) + +##------------------------------------------------------------------------------ +## Run objdump +##------------------------------------------------------------------------------ +objdump: $(KERNEL_ELF) + $(call color_header, "Launching objdump") + @$(DOCKER_TOOLS) $(OBJDUMP_BINARY) --disassemble --demangle \ + --section .text \ + --section .rodata \ + --section .got \ + $(KERNEL_ELF) | rustfilt + +##------------------------------------------------------------------------------ +## Run nm +##------------------------------------------------------------------------------ +nm: $(KERNEL_ELF) + $(call color_header, "Launching nm") + @$(DOCKER_TOOLS) $(NM_BINARY) --demangle --print-size $(KERNEL_ELF) | sort | rustfilt + + + +##-------------------------------------------------------------------------------------------------- +## Debugging targets +##-------------------------------------------------------------------------------------------------- +.PHONY: jtagboot openocd gdb gdb-opt0 + +##------------------------------------------------------------------------------ +## Push the JTAG boot image to the real HW target +##------------------------------------------------------------------------------ +jtagboot: + @$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE) + +##------------------------------------------------------------------------------ +## Start OpenOCD session +##------------------------------------------------------------------------------ +openocd: + $(call color_header, "Launching OpenOCD") + @$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG) + +##------------------------------------------------------------------------------ +## Start GDB session +##------------------------------------------------------------------------------ +gdb-opt0: RUSTC_MISC_ARGS += -C opt-level=0 +gdb gdb-opt0: $(KERNEL_ELF) + $(call color_header, "Launching GDB") + @$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF) + + + +##-------------------------------------------------------------------------------------------------- +## Testing targets +##-------------------------------------------------------------------------------------------------- +.PHONY: test test_boot test_unit test_integration + +test_unit test_integration: FEATURES += --features test_build + +ifeq ($(QEMU_MACHINE_TYPE),) # QEMU is not supported for the board. + +test_boot test_unit test_integration test: + $(call color_header, "$(QEMU_MISSING_STRING)") + +else # QEMU is supported. + +##------------------------------------------------------------------------------ +## Run boot test +##------------------------------------------------------------------------------ +test_boot: $(KERNEL_BIN) + $(call color_header, "Boot test - $(BSP)") + @$(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) + +##------------------------------------------------------------------------------ +## Helpers for unit and integration test targets +##------------------------------------------------------------------------------ +define KERNEL_TEST_RUNNER + #!/usr/bin/env bash + + # The cargo test runner seems to change into the crate under test's directory. Therefore, ensure + # this script executes from the root. + cd $(shell pwd) + + TEST_ELF=$$(echo $$1 | sed -e 's/.*target/target/g') + TEST_ELF_SYMS="$${TEST_ELF}_syms" + TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g') + + $(DOCKER_TOOLS) $(EXEC_TT_TOOL) $(BSP) $$TEST_ELF > /dev/null + + # This overrides the two ENV variables. The other ENV variables that are required as input for + # the .mk file are set already because they are exported by this Makefile and this script is + # started by the same. + KERNEL_SYMBOLS_INPUT_ELF=$$TEST_ELF \ + KERNEL_SYMBOLS_OUTPUT_ELF=$$TEST_ELF_SYMS \ + $(MAKE) --no-print-directory -f kernel_symbols.mk > /dev/null 2>&1 + + $(OBJCOPY_CMD) $$TEST_ELF_SYMS $$TEST_BINARY + $(DOCKER_TEST) $(EXEC_TEST_DISPATCH) $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY +endef + +export KERNEL_TEST_RUNNER + +define test_prepare + @mkdir -p target + @echo "$$KERNEL_TEST_RUNNER" > target/kernel_test_runner.sh + @chmod +x target/kernel_test_runner.sh +endef + +##------------------------------------------------------------------------------ +## Run unit test(s) +##------------------------------------------------------------------------------ +test_unit: + $(call color_header, "Compiling unit test(s) - $(BSP)") + $(call test_prepare) + @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(TEST_CMD) --lib + +##------------------------------------------------------------------------------ +## Run integration test(s) +##------------------------------------------------------------------------------ +test_integration: + $(call color_header, "Compiling integration test(s) - $(BSP)") + $(call test_prepare) + @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(TEST_CMD) $(TEST_ARG) + +test: test_boot test_unit test_integration + +endif diff --git a/18_backtrace/README.md b/18_backtrace/README.md new file mode 100644 index 00000000..0963df1d --- /dev/null +++ b/18_backtrace/README.md @@ -0,0 +1,1244 @@ +# Tutorial 18 - Backtracing + +## tl;dr + +- Support for [`backtracing`] is implemented into the kernel. + +```console +[ 0.002782] Writing to bottom of address space to address 1 GiB... +[ 0.004623] Kernel panic! + +Panic location: + File 'kernel/src/_arch/aarch64/exception.rs', line 59, column 5 + +[...] + +Backtrace: + ---------------------------------------------------------------------------------------------- + Address Function containing address + ---------------------------------------------------------------------------------------------- + 1. ffffffffc0001294 | core::fmt::write + 2. ffffffffc0005560 | libkernel::panic_wait::_panic_print + 3. ffffffffc00054a0 | rust_begin_unwind + 4. ffffffffc0002950 | core::panicking::panic_fmt + 5. ffffffffc0004898 | current_elx_synchronous + 6. ffffffffc0000a74 | __vector_current_elx_synchronous + 7. ffffffffc000111c | kernel_init + ---------------------------------------------------------------------------------------------- +``` + +[`backtracing`]: https://en.wikipedia.org/wiki/Stack_trace + +## Table of Contents + +- [Introduction](#introduction) +- [Implementation](#implementation) + - [Chasing Frames](#chasing-frames) + - [Compiler Changes](#compiler-changes) + - [Supporting Changes](#supporting-changes) +- [Test it](#test-it) +- [Diff to previous](#diff-to-previous) + +## Introduction + +Since the kernel gained support for looking up `symbol names` in the previous tutorial, it is now +possible to implement support for printing meaningful backtraces (also called `stack traces`). The +primary use-case will be printing backtraces during a `panic`, which will ease debugging. This is a +good time to add this feature, since some of the upcoming tutorials will cover complex topics and +code changes, so that this will come in handy during development. + +## Implementation + +Since backtracing is a scheme that is usually defined in the [`calling-convention`], and therefore +tightly coupled to the `processor architecture `, the heart of the backtracing code will live in the +`_arch` folder. What can be shared between different architectures is the formatting and printing +part. Hence, the code will be organized as follows: + +[`calling-convention`]: https://en.wikipedia.org/wiki/Calling_convention + +- `src/backtrace.rs` makes a generic definition of a `BacktraceItem`. It also provides code that + uses an `Iterator` to format and print the backtrace. +- `src/__arch_name__/backtrace.rs` contains the code that generates the actual iterator. + +Here is the definition of `BacktraceItem`: + +```rust +pub enum BacktraceItem { + InvalidFramePointer(Address), + InvalidLink(Address), + Link(Address), +} +``` + +In summary, it has two error cases and one valid case. This will become clearer in a minute when we +look at what a `stack frame` and a `frame pointer` is. + +### Chasing Frames + +For `AArch64`, we need to consult the [Procedure Call Standard for the Arm® 64-bit Architecture] +(`AAPCS64`). It has the following to say: + +> Conforming code shall construct a *linked list* of stack-frames. Each frame shall link to the +> frame of its caller by means of a frame record of two 64-bit values on the stack (independent of +> the data model). The frame record for the innermost frame (belonging to the most recent routine +> invocation) shall be pointed to by the frame pointer register (FP). The lowest addressed +> double-word shall point to the previous frame record and the highest addressed double-word shall +> contain the value passed in LR on entry to the current function [...]. The location of the frame +> record within a stack frame is not specified. + +[Procedure Call Standard for the Arm® 64-bit Architecture]: https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst + +The nature of the `linked list` becomes a bit clearer when we look into the corresponding section in +the [ARM Cortex-A Series Programmer’s Guide for ARMv8-A] as well. Here are text and picture +snippets: + +> An AAPC64 stack frame shown in Figure 9-2. The frame pointer (X29) should point to the previous +> frame pointer saved on stack, with the saved LR (X30) stored after it. The final frame pointer in +> the chain should be set to 0. The Stack Pointer must always be aligned on a 16 byte boundary. + +[ARM Cortex-A Series Programmer’s Guide for ARMv8-A]: https://developer.arm.com/documentation/den0024/latest/ + +

+ +

+ +Hence, we can define the following struct in `src/__arch_name__/backtrace.rs` for the stack frame +record: + +```rust +#[repr(C)] +struct StackFrameRecord<'a> { + previous_record: Option<&'a StackFrameRecord<'a>>, + link: Address, +} +``` + +The interesting part is the `previous_record` member. We learned from the two documents which we +inspected above that the lowest addressed double-word is either: + +- Zero. +- Or pointing to the previous stack frame record. + +Thanks to Rust's null pointer optimization [[1]][[2]], this allows us to conveniently type this as +an `Option<&StackFrameRecord>`. So whenever we inspect `previous_record` and observe it to be +`None`, we know that we've reached the end of the backtrace. + +[1]: https://doc.rust-lang.org/std/option/#representation +[2]: https://stackoverflow.com/a/46557737 + +The start of the backtrace is trivially accessed through `x29` (aka the `Frame Pointer Register`). +This is used to generate a `StackFrameIterator`: + +```rust +struct StackFrameRecordIterator<'a> { + cur: &'a StackFrameRecord<'a>, +} + +/// [...] + +fn stack_frame_record_iterator<'a>() -> Option> { + let fp = Address::::new(FP.get() as usize); + if !fp.is_valid_stack_addr() { + return None; + } + + Some(StackFrameRecordIterator { + cur: unsafe { &*(fp.as_usize() as *const _) }, + }) +} +``` + +Although it should be guaranteed by the compiler (and any hand-written assembly) that `x29` points +to a valid stack address, it makes sense to double-check this before generating a reference. There +is always a chance that corruption happens. The implementation of the iterator itself does this +sanity check as well whenever the iterator is advanced. Additionally, it is also checked whether the +`link` address points to a valid `code` section in the kernel before the address is passed on to +the caller of the iterator: + +```rust +impl<'a> Iterator for StackFrameRecordIterator<'a> { + type Item = BacktraceItem; + + fn next(&mut self) -> Option { + static ABORT_FRAME: StackFrameRecord = StackFrameRecord { + previous_record: None, + link: Address::new(0), + }; + + // If previous is None, this is the root frame, so iteration will stop here. + let previous = self.cur.previous_record?; + + // Need to abort if the pointer to the previous frame record is invalid. + let prev_addr = Address::::new(previous as *const _ as usize); + if !prev_addr.is_valid_stack_addr() { + // This allows to return the error and then stop on the next iteration. + self.cur = &ABORT_FRAME; + return Some(BacktraceItem::InvalidFramePointer(prev_addr)); + } + + let ret = if !self.cur.link.is_valid_code_addr() { + Some(BacktraceItem::InvalidLink(self.cur.link)) + } else { + // The link points to the instruction to be executed _after_ returning from a branch. + // However, we want to show the instruction that caused the branch, so subtract by one + // instruction. + // + // This might be called from panic!, so it must not panic itself on the subtraction. + let link = if self.cur.link >= Address::new(4) { + self.cur.link - 4 + } else { + self.cur.link + }; + + Some(BacktraceItem::Link(link)) + }; + + // Advance the iterator. + self.cur = previous; + + ret + } +} +``` + +This already was the gist of the architectural part of the implementation! In the generic part, +where the backtrace is printed, the address returned in `BacktraceItem::Link` is additionally used +to look up the corresponding `symbol`, so that this is conveniently printed together: + +```rust +match backtrace_res { + + // omitted + + BacktraceItem::Link(addr) => { + fmt_res = writeln!( + f, + " {:>2}. {:016x} | {:<50}", + i + 1, + addr.as_usize(), + match symbols::lookup_symbol(addr) { + Some(sym) => sym.name(), + _ => "Symbol not found", + } + ) + } +}; +``` + +Finally, we add printing of a backtrace to `panic!`: + +``` +panic_println!( + "[ {:>3}.{:06}] Kernel panic!\n\n\ + Panic location:\n File '{}', line {}, column {}\n\n\ + {}\n\n\ + {}", + timestamp.as_secs(), + timestamp.subsec_micros(), + location, + line, + column, + info.message().unwrap_or(&format_args!("")), + backtrace::Backtrace +); +``` + +### Compiler Changes + +By default, the `aarch64-unknown-none*` targets *do not* guarantee that a stack frame record is +generated on each function call. Without, the backtracing code will not work. Fortunately, +generation can be forced by modifying the `rustc codegen options`. We add the following to the +`Makefile`: + +```makefile +ifeq ($(BSP),rpi3) + + # omitted + + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C force-frame-pointers +``` + +But there is more! Until now, when we compiled the kernel, cargo was using a **precompiled** version +of the `Rust core library` that comes with rustup whenever a target is added. This is usually very +beneficial in terms of speeding up compilation. Unfortunately, the precompiled version was not +compiled with `-C force-frame-pointers` either. This can be solved using cargo's [`build-std` +feature]. We set it in the Makefile so that cargo also compiles the core library using our compiler +settings, which means we get the frame records thanks to `-C force-frame-pointers` for any core +library functions as well. + +```Makefile +# build-std can be skipped for helper commands that do not rely on correct stack frames and other +# custom compiler options. This results in a huge speedup. +RUSTC_CMD = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST) +DOC_CMD = cargo doc $(COMPILER_ARGS) +CLIPPY_CMD = cargo clippy $(COMPILER_ARGS) +TEST_CMD = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST) +``` + +[`build-std` feature]: https://doc.rust-lang.org/cargo/reference/unstable.html#build-std + +### Supporting Changes + +There's a couple of changes not covered in this tutorial text, but the reader should ideally skim +through them: + +- [`src/_arch/aarch64/exception.s`](`kernel/src/_arch/aarch64/exception.s`) adds some tricky code to + generate a stack frame record on exception entry. The file includes elaborate comments that can be + inspected. +- [`src/_arch/aarch64/cpu/boot.rs`](`kernel/src/_arch/aarch64/cpu/boot.rs`) adds some code that + ensures that `kernel_init()` becomes the root of the backtrace (meaning its is ensured that + `previous_frame` will be zero for `kernel_init()`'s frame record). +- In `$ROOT/Cargo.toml`, `debug = true` has been set, which ensures that the kernel ELF includes the + maximum amount of debug information. Please note that this does *not* change anything for the + kernel at runtime. However, it will allow to dig even deeper on an address that has been reported + by a kernel backtrace. For example, using the `addr2line` tool. The following two snippets show + what `addr2line` reports when the debug flag is not or is set, respectively. + +```console +$ # debug = false +$ addr2line -p -f -s -i -e target/aarch64-unknown-none-softfloat/release/kernel+ttables+symbols 0xffffffffc0001da8 | rustfilt +kernel::kernel_main at kernel.c562062a-cgu.1:? +``` + +```console +$ # debug = true +$ addr2line -p -f -s -i -e target/aarch64-unknown-none-softfloat/release/kernel+ttables+symbols 0xffffffffc0001da8 | rustfilt +libkernel::memory::mmu::mapping_record::MappingRecord::print at mapping_record.rs:136 + (inlined by) libkernel::memory::mmu::mapping_record::kernel_print::{{closure}} at mapping_record.rs:232 + (inlined by) as libkernel::synchronization::interface::ReadWriteEx>::read at synchronization.rs:139 + (inlined by) libkernel::memory::mmu::mapping_record::kernel_print at mapping_record.rs:232 + (inlined by) libkernel::memory::mmu::kernel_print_mappings at mmu.rs:269 + (inlined by) kernel::kernel_main at main.rs:84 +``` + +## Test it + +Three tests were added that check the sanity of the backtracing code. Also, any previous tests that +print a `panic` will now also include a backtrace. For example, `02_exception_sync_page_fault.rs`: + +```console +$ TEST=02_exception_sync_page_fault make test_integration +[...] + ------------------------------------------------------------------- + 🦀 Testing synchronous exception handling by causing a page fault + ------------------------------------------------------------------- + + [ 0.002782] Writing to bottom of address space to address 1 GiB... + [ 0.004623] Kernel panic! + + Panic location: + File 'kernel/src/_arch/aarch64/exception.rs', line 59, column 5 + + CPU Exception! + + ESR_EL1: 0x96000004 + Exception Class (EC) : 0x25 - Data Abort, current EL + Instr Specific Syndrome (ISS): 0x4 + FAR_EL1: 0x0000000040000000 + + [...] + + Backtrace: + ---------------------------------------------------------------------------------------------- + Address Function containing address + ---------------------------------------------------------------------------------------------- + 1. ffffffffc0001294 | core::fmt::write + 2. ffffffffc0005560 | libkernel::panic_wait::_panic_print + 3. ffffffffc00054a0 | rust_begin_unwind + 4. ffffffffc0002950 | core::panicking::panic_fmt + 5. ffffffffc0004898 | current_elx_synchronous + 6. ffffffffc0000a74 | __vector_current_elx_synchronous + 7. ffffffffc000111c | kernel_init + ---------------------------------------------------------------------------------------------- + + ------------------------------------------------------------------- + ✅ Success: 02_exception_sync_page_fault.rs + ------------------------------------------------------------------- +``` + +## Diff to previous +```diff + +diff -uNr 17_kernel_symbols/Cargo.toml 18_backtrace/Cargo.toml +--- 17_kernel_symbols/Cargo.toml ++++ 18_backtrace/Cargo.toml +@@ -8,3 +8,4 @@ + + [profile.release] + lto = true ++debug = true + +diff -uNr 17_kernel_symbols/kernel/Cargo.toml 18_backtrace/kernel/Cargo.toml +--- 17_kernel_symbols/kernel/Cargo.toml ++++ 18_backtrace/kernel/Cargo.toml +@@ -1,6 +1,6 @@ + [package] + name = "mingo" +-version = "0.17.0" ++version = "0.18.0" + authors = ["Andre Richter "] + edition = "2021" + +@@ -56,3 +56,15 @@ + [[test]] + name = "03_exception_restore_sanity" + harness = false ++ ++[[test]] ++name = "05_backtrace_sanity" ++harness = false ++ ++[[test]] ++name = "06_backtrace_invalid_frame" ++harness = false ++ ++[[test]] ++name = "07_backtrace_invalid_link" ++harness = false + +diff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/backtrace.rs 18_backtrace/kernel/src/_arch/aarch64/backtrace.rs +--- 17_kernel_symbols/kernel/src/_arch/aarch64/backtrace.rs ++++ 18_backtrace/kernel/src/_arch/aarch64/backtrace.rs +@@ -0,0 +1,136 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2022 Andre Richter ++ ++//! Architectural backtracing support. ++//! ++//! # Orientation ++//! ++//! Since arch modules are imported into generic modules using the path attribute, the path of this ++//! file is: ++//! ++//! crate::backtrace::arch_backtrace ++ ++use crate::{ ++ backtrace::BacktraceItem, ++ memory::{Address, Virtual}, ++}; ++use cortex_a::registers::*; ++use tock_registers::interfaces::Readable; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// A Stack frame record. ++/// ++/// # Note ++/// ++/// The convention is that `previous_record` is valid as long as it contains a non-null value. ++/// Therefore, it is possible to type the member as `Option<&StackFrameRecord>` because of Rust's ++/// `null-pointer optimization`. ++#[repr(C)] ++struct StackFrameRecord<'a> { ++ previous_record: Option<&'a StackFrameRecord<'a>>, ++ link: Address, ++} ++ ++struct StackFrameRecordIterator<'a> { ++ cur: &'a StackFrameRecord<'a>, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl<'a> Iterator for StackFrameRecordIterator<'a> { ++ type Item = BacktraceItem; ++ ++ fn next(&mut self) -> Option { ++ static ABORT_FRAME: StackFrameRecord = StackFrameRecord { ++ previous_record: None, ++ link: Address::new(0), ++ }; ++ ++ // If previous is None, this is the root frame, so iteration will stop here. ++ let previous = self.cur.previous_record?; ++ ++ // Need to abort if the pointer to the previous frame record is invalid. ++ let prev_addr = Address::::new(previous as *const _ as usize); ++ if !prev_addr.is_valid_stack_addr() { ++ // This allows to return the error and then stop on the next iteration. ++ self.cur = &ABORT_FRAME; ++ return Some(BacktraceItem::InvalidFramePointer(prev_addr)); ++ } ++ ++ let ret = if !self.cur.link.is_valid_code_addr() { ++ Some(BacktraceItem::InvalidLink(self.cur.link)) ++ } else { ++ // The link points to the instruction to be executed _after_ returning from a branch. ++ // However, we want to show the instruction that caused the branch, so subtract by one ++ // instruction. ++ // ++ // This might be called from panic!, so it must not panic itself on the subtraction. ++ let link = if self.cur.link >= Address::new(4) { ++ self.cur.link - 4 ++ } else { ++ self.cur.link ++ }; ++ ++ Some(BacktraceItem::Link(link)) ++ }; ++ ++ // Advance the iterator. ++ self.cur = previous; ++ ++ ret ++ } ++} ++ ++fn stack_frame_record_iterator<'a>() -> Option> { ++ let fp = Address::::new(FP.get() as usize); ++ if !fp.is_valid_stack_addr() { ++ return None; ++ } ++ ++ Some(StackFrameRecordIterator { ++ cur: unsafe { &*(fp.as_usize() as *const _) }, ++ }) ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++/// Architectural implementation of the backtrace. ++pub fn backtrace(f: impl FnOnce(Option<&mut dyn Iterator>)) { ++ f(stack_frame_record_iterator().as_mut().map(|s| s as _)) ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Testing ++//-------------------------------------------------------------------------------------------------- ++ ++#[cfg(feature = "test_build")] ++#[inline(always)] ++/// Hack for corrupting the previous frame address in the current stack frame. ++/// ++/// # Safety ++/// ++/// - To be used only by testing code. ++pub unsafe fn corrupt_previous_frame_addr() { ++ let sf = FP.get() as *mut usize; ++ *sf = 0x123; ++} ++ ++#[cfg(feature = "test_build")] ++#[inline(always)] ++/// Hack for corrupting the link in the current stack frame. ++/// ++/// # Safety ++/// ++/// - To be used only by testing code. ++pub unsafe fn corrupt_link() { ++ let sf = FP.get() as *mut StackFrameRecord; ++ (*sf).link = Address::new(0x456); ++} + +diff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/cpu/boot.rs 18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs +--- 17_kernel_symbols/kernel/src/_arch/aarch64/cpu/boot.rs ++++ 18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs +@@ -12,7 +12,10 @@ + //! crate::cpu::boot::arch_boot + + use crate::{memory, memory::Address}; +-use core::arch::global_asm; ++use core::{ ++ arch::global_asm, ++ sync::atomic::{compiler_fence, Ordering}, ++}; + use cortex_a::{asm, registers::*}; + use tock_registers::interfaces::Writeable; + +@@ -63,6 +66,18 @@ + SP_EL1.set(virt_boot_core_stack_end_exclusive_addr); + } + ++/// Reset the backtrace by setting link register and frame pointer to zero. ++/// ++/// # Safety ++/// ++/// - This function must only be used immediately before entering EL1. ++#[inline(always)] ++unsafe fn prepare_backtrace_reset() { ++ compiler_fence(Ordering::SeqCst); ++ FP.set(0); ++ LR.set(0); ++} ++ + //-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- +@@ -89,6 +104,9 @@ + let addr = Address::new(phys_kernel_tables_base_addr as usize); + memory::mmu::enable_mmu_and_caching(addr).unwrap(); + ++ // Make the function we return to the root of a backtrace. ++ prepare_backtrace_reset(); ++ + // Use `eret` to "return" to EL1. Since virtual memory will already be enabled, this results in + // execution of kernel_init() in EL1 from its _virtual address_. + asm::eret() + +diff -uNr 17_kernel_symbols/kernel/src/_arch/aarch64/exception.s 18_backtrace/kernel/src/_arch/aarch64/exception.s +--- 17_kernel_symbols/kernel/src/_arch/aarch64/exception.s ++++ 18_backtrace/kernel/src/_arch/aarch64/exception.s +@@ -8,10 +8,10 @@ + + /// Call the function provided by parameter `\handler` after saving the exception context. Provide + /// the context as the first parameter to '\handler'. +-.macro CALL_WITH_CONTEXT handler ++.macro CALL_WITH_CONTEXT handler is_lower_el is_sync + __vector_\handler: + // Make room on the stack for the exception context. +- sub sp, sp, #16 * 17 ++ sub sp, sp, #16 * 18 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] +@@ -39,6 +39,42 @@ + stp lr, x1, [sp, #16 * 15] + stp x2, x3, [sp, #16 * 16] + ++ // Build a stack frame for backtracing. ++.if \is_lower_el == 1 ++ // If we came from a lower EL, make it a root frame (by storing zero) so that the kernel ++ // does not attempt to trace into userspace. ++ stp xzr, xzr, [sp, #16 * 17] ++.else ++ // For normal branches, the link address points to the instruction to be executed _after_ ++ // returning from a branch. In a backtrace, we want to show the instruction that caused the ++ // branch, though. That is why code in backtrace.rs subtracts 4 (length of one instruction) ++ // from the link address. ++ // ++ // Here we have a special case, though, because ELR_EL1 is used instead of LR to build the ++ // stack frame, so that it becomes possible to trace beyond an exception. Hence, it must be ++ // considered that semantics for ELR_EL1 differ from case to case. ++ // ++ // Unless an "exception generating instruction" was executed, ELR_EL1 already points to the ++ // the correct instruction, and hence the subtraction by 4 in backtrace.rs would yield wrong ++ // results. To cover for this, 4 is added to ELR_EL1 below unless the cause of exception was ++ // an SVC instruction. BRK and HLT are "exception generating instructions" as well, but they ++ // are not expected and therefore left out for now. ++ // ++ // For reference: Search for "preferred exception return address" in the Architecture ++ // Reference Manual for ARMv8-A. ++.if \is_sync == 1 ++ lsr w3, w3, #26 // w3 = ESR_EL1.EC ++ cmp w3, #0x15 // w3 == SVC64 ? ++ b.eq 1f ++.endif ++ add x1, x1, #4 ++1: ++ stp x29, x1, [sp, #16 * 17] ++.endif ++ ++ // Set the frame pointer to the stack frame record. ++ add x29, sp, #16 * 17 ++ + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + +@@ -81,43 +117,43 @@ + // + // - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. + .org 0x000 +- CALL_WITH_CONTEXT current_el0_synchronous ++ CALL_WITH_CONTEXT current_el0_synchronous, 0, 1 + .org 0x080 +- CALL_WITH_CONTEXT current_el0_irq ++ CALL_WITH_CONTEXT current_el0_irq, 0, 0 + .org 0x100 + FIQ_SUSPEND + .org 0x180 +- CALL_WITH_CONTEXT current_el0_serror ++ CALL_WITH_CONTEXT current_el0_serror, 0, 0 + + // Current exception level with SP_ELx, x > 0. + .org 0x200 +- CALL_WITH_CONTEXT current_elx_synchronous ++ CALL_WITH_CONTEXT current_elx_synchronous, 0, 1 + .org 0x280 +- CALL_WITH_CONTEXT current_elx_irq ++ CALL_WITH_CONTEXT current_elx_irq, 0, 0 + .org 0x300 + FIQ_SUSPEND + .org 0x380 +- CALL_WITH_CONTEXT current_elx_serror ++ CALL_WITH_CONTEXT current_elx_serror, 0, 0 + + // Lower exception level, AArch64 + .org 0x400 +- CALL_WITH_CONTEXT lower_aarch64_synchronous ++ CALL_WITH_CONTEXT lower_aarch64_synchronous, 1, 1 + .org 0x480 +- CALL_WITH_CONTEXT lower_aarch64_irq ++ CALL_WITH_CONTEXT lower_aarch64_irq, 1, 0 + .org 0x500 + FIQ_SUSPEND + .org 0x580 +- CALL_WITH_CONTEXT lower_aarch64_serror ++ CALL_WITH_CONTEXT lower_aarch64_serror, 1, 0 + + // Lower exception level, AArch32 + .org 0x600 +- CALL_WITH_CONTEXT lower_aarch32_synchronous ++ CALL_WITH_CONTEXT lower_aarch32_synchronous, 1, 0 + .org 0x680 +- CALL_WITH_CONTEXT lower_aarch32_irq ++ CALL_WITH_CONTEXT lower_aarch32_irq, 1, 0 + .org 0x700 + FIQ_SUSPEND + .org 0x780 +- CALL_WITH_CONTEXT lower_aarch32_serror ++ CALL_WITH_CONTEXT lower_aarch32_serror, 1, 0 + .org 0x800 + + //------------------------------------------------------------------------------ +@@ -146,7 +182,7 @@ + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + +- add sp, sp, #16 * 17 ++ add sp, sp, #16 * 18 + + eret + + +diff -uNr 17_kernel_symbols/kernel/src/backtrace.rs 18_backtrace/kernel/src/backtrace.rs +--- 17_kernel_symbols/kernel/src/backtrace.rs ++++ 18_backtrace/kernel/src/backtrace.rs +@@ -0,0 +1,112 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2022 Andre Richter ++ ++//! Backtracing support. ++ ++#[cfg(target_arch = "aarch64")] ++#[path = "_arch/aarch64/backtrace.rs"] ++mod arch_backtrace; ++ ++use crate::{ ++ memory::{Address, Virtual}, ++ symbols, ++}; ++use core::fmt; ++ ++//-------------------------------------------------------------------------------------------------- ++// Architectural Public Reexports ++//-------------------------------------------------------------------------------------------------- ++#[cfg(feature = "test_build")] ++pub use arch_backtrace::{corrupt_link, corrupt_previous_frame_addr}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// A backtrace item. ++#[allow(missing_docs)] ++pub enum BacktraceItem { ++ InvalidFramePointer(Address), ++ InvalidLink(Address), ++ Link(Address), ++} ++ ++/// Pseudo-struct for printing a backtrace using its fmt::Display implementation. ++pub struct Backtrace; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl fmt::Display for Backtrace { ++ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { ++ writeln!(f, "Backtrace:")?; ++ writeln!( ++ f, ++ " ----------------------------------------------------------------------------------------------" ++ )?; ++ writeln!( ++ f, ++ " Address Function containing address" ++ )?; ++ writeln!( ++ f, ++ " ----------------------------------------------------------------------------------------------" ++ )?; ++ ++ let mut fmt_res: fmt::Result = Ok(()); ++ let trace_formatter = ++ |maybe_iter: Option<&mut dyn Iterator>| match maybe_iter { ++ None => fmt_res = writeln!(f, "ERROR! No valid stack frame found"), ++ Some(iter) => { ++ for (i, backtrace_res) in iter.enumerate() { ++ match backtrace_res { ++ BacktraceItem::InvalidFramePointer(addr) => { ++ fmt_res = writeln!( ++ f, ++ " {:>2}. ERROR! \ ++ Encountered invalid frame pointer ({}) during backtrace", ++ i + 1, ++ addr ++ ); ++ } ++ BacktraceItem::InvalidLink(addr) => { ++ fmt_res = writeln!( ++ f, ++ " {:>2}. ERROR! \ ++ Link address ({}) is not contained in kernel .text section", ++ i + 1, ++ addr ++ ); ++ } ++ BacktraceItem::Link(addr) => { ++ fmt_res = writeln!( ++ f, ++ " {:>2}. {:016x} | {:<50}", ++ i + 1, ++ addr.as_usize(), ++ match symbols::lookup_symbol(addr) { ++ Some(sym) => sym.name(), ++ _ => "Symbol not found", ++ } ++ ) ++ } ++ }; ++ ++ if fmt_res.is_err() { ++ break; ++ } ++ } ++ } ++ }; ++ ++ arch_backtrace::backtrace(trace_formatter); ++ fmt_res?; ++ ++ writeln!( ++ f, ++ " ----------------------------------------------------------------------------------------------" ++ ) ++ } ++} + +diff -uNr 17_kernel_symbols/kernel/src/bsp/raspberrypi/memory/mmu.rs 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs +--- 17_kernel_symbols/kernel/src/bsp/raspberrypi/memory/mmu.rs ++++ 18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs +@@ -80,16 +80,6 @@ + size >> KernelGranule::SHIFT + } + +-/// The code pages of the kernel binary. +-fn virt_code_region() -> MemoryRegion { +- let num_pages = size_to_num_pages(super::code_size()); +- +- let start_page_addr = super::virt_code_start(); +- let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); +- +- MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +-} +- + /// The data pages of the kernel binary. + fn virt_data_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::data_size()); +@@ -100,16 +90,6 @@ + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) + } + +-/// The boot core stack pages. +-fn virt_boot_core_stack_region() -> MemoryRegion { +- let num_pages = size_to_num_pages(super::boot_core_stack_size()); +- +- let start_page_addr = super::virt_boot_core_stack_start(); +- let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); +- +- MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +-} +- + // There is no reason to expect the following conversions to fail, since they were generated offline + // by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified. + fn kernel_virt_to_phys_region(virt_region: MemoryRegion) -> MemoryRegion { +@@ -132,6 +112,26 @@ + // Public Code + //-------------------------------------------------------------------------------------------------- + ++/// The code pages of the kernel binary. ++pub fn virt_code_region() -> MemoryRegion { ++ let num_pages = size_to_num_pages(super::code_size()); ++ ++ let start_page_addr = super::virt_code_start(); ++ let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); ++ ++ MemoryRegion::new(start_page_addr, end_exclusive_page_addr) ++} ++ ++/// The boot core stack pages. ++pub fn virt_boot_core_stack_region() -> MemoryRegion { ++ let num_pages = size_to_num_pages(super::boot_core_stack_size()); ++ ++ let start_page_addr = super::virt_boot_core_stack_start(); ++ let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); ++ ++ MemoryRegion::new(start_page_addr, end_exclusive_page_addr) ++} ++ + /// Return a reference to the kernel's translation tables. + pub fn kernel_translation_tables() -> &'static InitStateLock { + &KERNEL_TABLES + +diff -uNr 17_kernel_symbols/kernel/src/lib.rs 18_backtrace/kernel/src/lib.rs +--- 17_kernel_symbols/kernel/src/lib.rs ++++ 18_backtrace/kernel/src/lib.rs +@@ -128,6 +128,7 @@ + mod panic_wait; + mod synchronization; + ++pub mod backtrace; + pub mod bsp; + pub mod common; + pub mod console; + +diff -uNr 17_kernel_symbols/kernel/src/memory.rs 18_backtrace/kernel/src/memory.rs +--- 17_kernel_symbols/kernel/src/memory.rs ++++ 18_backtrace/kernel/src/memory.rs +@@ -95,6 +95,18 @@ + } + } + ++impl Sub for Address { ++ type Output = Self; ++ ++ #[inline(always)] ++ fn sub(self, rhs: usize) -> Self::Output { ++ match self.value.checked_sub(rhs) { ++ None => panic!("Overflow on Address::sub"), ++ Some(x) => Self::new(x), ++ } ++ } ++} ++ + impl Sub> for Address { + type Output = Self; + +@@ -107,6 +119,18 @@ + } + } + ++impl Address { ++ /// Checks if the address is part of the boot core stack region. ++ pub fn is_valid_stack_addr(&self) -> bool { ++ bsp::memory::mmu::virt_boot_core_stack_region().contains(*self) ++ } ++ ++ /// Checks if the address is part of the kernel code region. ++ pub fn is_valid_code_addr(&self) -> bool { ++ bsp::memory::mmu::virt_code_region().contains(*self) ++ } ++} ++ + impl fmt::Display for Address { + // Don't expect to see physical addresses greater than 40 bit. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + +diff -uNr 17_kernel_symbols/kernel/src/panic_wait.rs 18_backtrace/kernel/src/panic_wait.rs +--- 17_kernel_symbols/kernel/src/panic_wait.rs ++++ 18_backtrace/kernel/src/panic_wait.rs +@@ -4,7 +4,7 @@ + + //! A panic handler that infinitely waits. + +-use crate::{bsp, cpu, exception}; ++use crate::{backtrace, bsp, cpu, exception}; + use core::{fmt, panic::PanicInfo}; + + //-------------------------------------------------------------------------------------------------- +@@ -91,6 +91,7 @@ + panic_println!( + "[ {:>3}.{:06}] Kernel panic!\n\n\ + Panic location:\n File '{}', line {}, column {}\n\n\ ++ {}\n\n\ + {}", + timestamp.as_secs(), + timestamp.subsec_micros(), +@@ -98,6 +99,7 @@ + line, + column, + info.message().unwrap_or(&format_args!("")), ++ backtrace::Backtrace + ); + + _panic_exit() + +diff -uNr 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rb 18_backtrace/kernel/tests/05_backtrace_sanity.rb +--- 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rb ++++ 18_backtrace/kernel/tests/05_backtrace_sanity.rb +@@ -0,0 +1,39 @@ ++# frozen_string_literal: true ++ ++# SPDX-License-Identifier: MIT OR Apache-2.0 ++# ++# Copyright (c) 2022 Andre Richter ++ ++require 'console_io_test' ++ ++# Verify that panic produces a backtrace. ++class PanicBacktraceTest < SubtestBase ++ def name ++ 'Panic produces backtrace' ++ end ++ ++ def run(qemu_out, _qemu_in) ++ expect_or_raise(qemu_out, 'Kernel panic!') ++ expect_or_raise(qemu_out, 'Backtrace:') ++ end ++end ++ ++# Verify backtrace correctness. ++class BacktraceCorrectnessTest < SubtestBase ++ def name ++ 'Backtrace is correct' ++ end ++ ++ def run(qemu_out, _qemu_in) ++ expect_or_raise(qemu_out, '| core::panicking::panic') ++ expect_or_raise(qemu_out, '| _05_backtrace_sanity::nested') ++ expect_or_raise(qemu_out, '| kernel_init') ++ end ++end ++ ++##-------------------------------------------------------------------------------------------------- ++## Test registration ++##-------------------------------------------------------------------------------------------------- ++def subtest_collection ++ [PanicBacktraceTest.new, BacktraceCorrectnessTest.new] ++end + +diff -uNr 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rs 18_backtrace/kernel/tests/05_backtrace_sanity.rs +--- 17_kernel_symbols/kernel/tests/05_backtrace_sanity.rs ++++ 18_backtrace/kernel/tests/05_backtrace_sanity.rs +@@ -0,0 +1,31 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2022 Andre Richter ++ ++//! Test if backtracing code detects an invalid frame pointer. ++ ++#![feature(format_args_nl)] ++#![no_main] ++#![no_std] ++ ++/// Console tests should time out on the I/O harness in case of panic. ++mod panic_wait_forever; ++ ++use libkernel::{bsp, cpu, exception, memory}; ++ ++#[inline(never)] ++fn nested() { ++ panic!() ++} ++ ++#[no_mangle] ++unsafe fn kernel_init() -> ! { ++ exception::handling_init(); ++ memory::mmu::post_enable_init(); ++ bsp::console::qemu_bring_up_console(); ++ ++ nested(); ++ ++ // The QEMU process running this test will be closed by the I/O test harness. ++ cpu::wait_forever() ++} + +diff -uNr 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rb 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb +--- 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rb ++++ 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb +@@ -0,0 +1,26 @@ ++# frozen_string_literal: true ++ ++# SPDX-License-Identifier: MIT OR Apache-2.0 ++# ++# Copyright (c) 2022 Andre Richter ++ ++require 'console_io_test' ++ ++# Test detection of invalid frame pointers. ++class InvalidFramePointerTest < SubtestBase ++ def name ++ 'Detect invalid frame pointer' ++ end ++ ++ def run(qemu_out, _qemu_in) ++ expect_or_raise(qemu_out, ++ /Encountered invalid frame pointer \(.*\) during backtrace/) ++ end ++end ++ ++##-------------------------------------------------------------------------------------------------- ++## Test registration ++##-------------------------------------------------------------------------------------------------- ++def subtest_collection ++ [InvalidFramePointerTest.new] ++end + +diff -uNr 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rs 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs +--- 17_kernel_symbols/kernel/tests/06_backtrace_invalid_frame.rs ++++ 18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs +@@ -0,0 +1,33 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2022 Andre Richter ++ ++//! Test if backtracing code detects an invalid frame pointer. ++ ++#![feature(format_args_nl)] ++#![no_main] ++#![no_std] ++ ++/// Console tests should time out on the I/O harness in case of panic. ++mod panic_wait_forever; ++ ++use libkernel::{backtrace, bsp, cpu, exception, memory}; ++ ++#[inline(never)] ++fn nested() { ++ unsafe { backtrace::corrupt_previous_frame_addr() }; ++ ++ panic!() ++} ++ ++#[no_mangle] ++unsafe fn kernel_init() -> ! { ++ exception::handling_init(); ++ memory::mmu::post_enable_init(); ++ bsp::console::qemu_bring_up_console(); ++ ++ nested(); ++ ++ // The QEMU process running this test will be closed by the I/O test harness. ++ cpu::wait_forever() ++} + +diff -uNr 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rb 18_backtrace/kernel/tests/07_backtrace_invalid_link.rb +--- 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rb ++++ 18_backtrace/kernel/tests/07_backtrace_invalid_link.rb +@@ -0,0 +1,25 @@ ++# frozen_string_literal: true ++ ++# SPDX-License-Identifier: MIT OR Apache-2.0 ++# ++# Copyright (c) 2022 Andre Richter ++ ++require 'console_io_test' ++ ++# Test detection of invalid link. ++class InvalidLinkTest < SubtestBase ++ def name ++ 'Detect invalid link' ++ end ++ ++ def run(qemu_out, _qemu_in) ++ expect_or_raise(qemu_out, /Link address \(.*\) is not contained in kernel .text section/) ++ end ++end ++ ++##-------------------------------------------------------------------------------------------------- ++## Test registration ++##-------------------------------------------------------------------------------------------------- ++def subtest_collection ++ [InvalidLinkTest.new] ++end + +diff -uNr 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rs 18_backtrace/kernel/tests/07_backtrace_invalid_link.rs +--- 17_kernel_symbols/kernel/tests/07_backtrace_invalid_link.rs ++++ 18_backtrace/kernel/tests/07_backtrace_invalid_link.rs +@@ -0,0 +1,38 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2022 Andre Richter ++ ++//! Test if backtracing code detects an invalid link. ++ ++#![feature(format_args_nl)] ++#![no_main] ++#![no_std] ++ ++/// Console tests should time out on the I/O harness in case of panic. ++mod panic_wait_forever; ++ ++use libkernel::{backtrace, bsp, cpu, exception, memory}; ++ ++#[inline(never)] ++fn nested_2() -> &'static str { ++ unsafe { backtrace::corrupt_link() }; ++ libkernel::println!("{}", libkernel::backtrace::Backtrace); ++ "foo" ++} ++ ++#[inline(never)] ++fn nested_1() { ++ libkernel::println!("{}", nested_2()) ++} ++ ++#[no_mangle] ++unsafe fn kernel_init() -> ! { ++ exception::handling_init(); ++ memory::mmu::post_enable_init(); ++ bsp::console::qemu_bring_up_console(); ++ ++ nested_1(); ++ ++ // The QEMU process running this test will be closed by the I/O test harness. ++ cpu::wait_forever() ++} + +diff -uNr 17_kernel_symbols/Makefile 18_backtrace/Makefile +--- 17_kernel_symbols/Makefile ++++ 18_backtrace/Makefile +@@ -42,7 +42,7 @@ + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg + JTAG_BOOT_IMAGE = ../X1_JTAG_boot/jtag_boot_rpi3.img + LD_SCRIPT_PATH = $(shell pwd)/kernel/src/bsp/raspberrypi +- RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 ++ RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C force-frame-pointers + else ifeq ($(BSP),rpi4) + TARGET = aarch64-unknown-none-softfloat + KERNEL_BIN = kernel8.img +@@ -56,7 +56,7 @@ + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg + JTAG_BOOT_IMAGE = ../X1_JTAG_boot/jtag_boot_rpi4.img + LD_SCRIPT_PATH = $(shell pwd)/kernel/src/bsp/raspberrypi +- RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 ++ RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 -C force-frame-pointers + endif + + # Export for build.rs. +@@ -121,10 +121,12 @@ + $(FEATURES) \ + --release + +-RUSTC_CMD = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST) ++# build-std can be skipped for helper commands that do not rely on correct stack frames and other ++# custom compiler options. This results in a huge speedup. ++RUSTC_CMD = cargo rustc $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST) + DOC_CMD = cargo doc $(COMPILER_ARGS) + CLIPPY_CMD = cargo clippy $(COMPILER_ARGS) +-TEST_CMD = cargo test $(COMPILER_ARGS) --manifest-path $(KERNEL_MANIFEST) ++TEST_CMD = cargo test $(COMPILER_ARGS) -Z build-std=core --manifest-path $(KERNEL_MANIFEST) + OBJCOPY_CMD = rust-objcopy \ + --strip-all \ + -O binary +@@ -303,8 +305,7 @@ + ##------------------------------------------------------------------------------ + ## Start GDB session + ##------------------------------------------------------------------------------ +-gdb: RUSTC_MISC_ARGS += -C debuginfo=2 +-gdb-opt0: RUSTC_MISC_ARGS += -C debuginfo=2 -C opt-level=0 ++gdb-opt0: RUSTC_MISC_ARGS += -C opt-level=0 + gdb gdb-opt0: $(KERNEL_ELF) + $(call color_header, "Launching GDB") + @$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF) + +``` diff --git a/18_backtrace/kernel/Cargo.toml b/18_backtrace/kernel/Cargo.toml new file mode 100644 index 00000000..9e5f55be --- /dev/null +++ b/18_backtrace/kernel/Cargo.toml @@ -0,0 +1,70 @@ +[package] +name = "mingo" +version = "0.18.0" +authors = ["Andre Richter "] +edition = "2021" + +[features] +default = [] +bsp_rpi3 = ["tock-registers"] +bsp_rpi4 = ["tock-registers"] +test_build = ["qemu-exit"] + +##-------------------------------------------------------------------------------------------------- +## Dependencies +##-------------------------------------------------------------------------------------------------- + +[dependencies] +test-types = { path = "../libraries/test-types" } +debug-symbol-types = { path = "../libraries/debug-symbol-types" } + +# Optional dependencies +tock-registers = { version = "0.7.x", default-features = false, features = ["register_types"], optional = true } +qemu-exit = { version = "3.x.x", optional = true } + +# Platform specific dependencies +[target.'cfg(target_arch = "aarch64")'.dependencies] +cortex-a = { version = "7.x.x" } + +##-------------------------------------------------------------------------------------------------- +## Testing +##-------------------------------------------------------------------------------------------------- + +[dev-dependencies] +test-macros = { path = "../libraries/test-macros" } + +# Unit tests are done in the library part of the kernel. +[lib] +name = "libkernel" +test = true + +# Disable unit tests for the kernel binary. +[[bin]] +name = "kernel" +path = "src/main.rs" +test = false + +# List of tests without harness. +[[test]] +name = "00_console_sanity" +harness = false + +[[test]] +name = "02_exception_sync_page_fault" +harness = false + +[[test]] +name = "03_exception_restore_sanity" +harness = false + +[[test]] +name = "05_backtrace_sanity" +harness = false + +[[test]] +name = "06_backtrace_invalid_frame" +harness = false + +[[test]] +name = "07_backtrace_invalid_link" +harness = false diff --git a/18_backtrace/kernel/build.rs b/18_backtrace/kernel/build.rs new file mode 100644 index 00000000..cab00bb3 --- /dev/null +++ b/18_backtrace/kernel/build.rs @@ -0,0 +1,20 @@ +use std::{env, fs, process}; + +fn main() { + let ld_script_path = match env::var("LD_SCRIPT_PATH") { + Ok(var) => var, + _ => process::exit(0), + }; + + let files = fs::read_dir(ld_script_path).unwrap(); + files + .filter_map(Result::ok) + .filter(|d| { + if let Some(e) = d.path().extension() { + e == "ld" + } else { + false + } + }) + .for_each(|f| println!("cargo:rerun-if-changed={}", f.path().display())); +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/backtrace.rs b/18_backtrace/kernel/src/_arch/aarch64/backtrace.rs new file mode 100644 index 00000000..e8860984 --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/backtrace.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Architectural backtracing support. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::backtrace::arch_backtrace + +use crate::{ + backtrace::BacktraceItem, + memory::{Address, Virtual}, +}; +use cortex_a::registers::*; +use tock_registers::interfaces::Readable; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// A Stack frame record. +/// +/// # Note +/// +/// The convention is that `previous_record` is valid as long as it contains a non-null value. +/// Therefore, it is possible to type the member as `Option<&StackFrameRecord>` because of Rust's +/// `null-pointer optimization`. +#[repr(C)] +struct StackFrameRecord<'a> { + previous_record: Option<&'a StackFrameRecord<'a>>, + link: Address, +} + +struct StackFrameRecordIterator<'a> { + cur: &'a StackFrameRecord<'a>, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl<'a> Iterator for StackFrameRecordIterator<'a> { + type Item = BacktraceItem; + + fn next(&mut self) -> Option { + static ABORT_FRAME: StackFrameRecord = StackFrameRecord { + previous_record: None, + link: Address::new(0), + }; + + // If previous is None, this is the root frame, so iteration will stop here. + let previous = self.cur.previous_record?; + + // Need to abort if the pointer to the previous frame record is invalid. + let prev_addr = Address::::new(previous as *const _ as usize); + if !prev_addr.is_valid_stack_addr() { + // This allows to return the error and then stop on the next iteration. + self.cur = &ABORT_FRAME; + return Some(BacktraceItem::InvalidFramePointer(prev_addr)); + } + + let ret = if !self.cur.link.is_valid_code_addr() { + Some(BacktraceItem::InvalidLink(self.cur.link)) + } else { + // The link points to the instruction to be executed _after_ returning from a branch. + // However, we want to show the instruction that caused the branch, so subtract by one + // instruction. + // + // This might be called from panic!, so it must not panic itself on the subtraction. + let link = if self.cur.link >= Address::new(4) { + self.cur.link - 4 + } else { + self.cur.link + }; + + Some(BacktraceItem::Link(link)) + }; + + // Advance the iterator. + self.cur = previous; + + ret + } +} + +fn stack_frame_record_iterator<'a>() -> Option> { + let fp = Address::::new(FP.get() as usize); + if !fp.is_valid_stack_addr() { + return None; + } + + Some(StackFrameRecordIterator { + cur: unsafe { &*(fp.as_usize() as *const _) }, + }) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Architectural implementation of the backtrace. +pub fn backtrace(f: impl FnOnce(Option<&mut dyn Iterator>)) { + f(stack_frame_record_iterator().as_mut().map(|s| s as _)) +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(feature = "test_build")] +#[inline(always)] +/// Hack for corrupting the previous frame address in the current stack frame. +/// +/// # Safety +/// +/// - To be used only by testing code. +pub unsafe fn corrupt_previous_frame_addr() { + let sf = FP.get() as *mut usize; + *sf = 0x123; +} + +#[cfg(feature = "test_build")] +#[inline(always)] +/// Hack for corrupting the link in the current stack frame. +/// +/// # Safety +/// +/// - To be used only by testing code. +pub unsafe fn corrupt_link() { + let sf = FP.get() as *mut StackFrameRecord; + (*sf).link = Address::new(0x456); +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/cpu.rs b/18_backtrace/kernel/src/_arch/aarch64/cpu.rs new file mode 100644 index 00000000..66da661c --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/cpu.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Architectural processor code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::arch_cpu + +use cortex_a::asm; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +pub use asm::nop; + +/// Pause execution on the core. +#[inline(always)] +pub fn wait_forever() -> ! { + loop { + asm::wfe() + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- +#[cfg(feature = "test_build")] +use qemu_exit::QEMUExit; + +#[cfg(feature = "test_build")] +const QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new(); + +/// Make the host QEMU binary execute `exit(1)`. +#[cfg(feature = "test_build")] +pub fn qemu_exit_failure() -> ! { + QEMU_EXIT_HANDLE.exit_failure() +} + +/// Make the host QEMU binary execute `exit(0)`. +#[cfg(feature = "test_build")] +pub fn qemu_exit_success() -> ! { + QEMU_EXIT_HANDLE.exit_success() +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs b/18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs new file mode 100644 index 00000000..c0bc86be --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/cpu/boot.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//! Architectural boot code. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::boot::arch_boot + +use crate::{memory, memory::Address}; +use core::{ + arch::global_asm, + sync::atomic::{compiler_fence, Ordering}, +}; +use cortex_a::{asm, registers::*}; +use tock_registers::interfaces::Writeable; + +// Assembly counterpart to this file. +global_asm!(include_str!("boot.s")); + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Prepares the transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The `bss` section is not initialized yet. The code must not use or reference it in any way. +/// - The HW state of EL1 must be prepared in a sound way. +#[inline(always)] +unsafe fn prepare_el2_to_el1_transition( + virt_boot_core_stack_end_exclusive_addr: u64, + virt_kernel_init_addr: u64, +) { + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to kernel_init(). + ELR_EL2.set(virt_kernel_init_addr); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. Since there + // are no plans to ever return to EL2, just re-use the same stack. + SP_EL1.set(virt_boot_core_stack_end_exclusive_addr); +} + +/// Reset the backtrace by setting link register and frame pointer to zero. +/// +/// # Safety +/// +/// - This function must only be used immediately before entering EL1. +#[inline(always)] +unsafe fn prepare_backtrace_reset() { + compiler_fence(Ordering::SeqCst); + FP.set(0); + LR.set(0); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The Rust entry of the `kernel` binary. +/// +/// The function is called from the assembly `_start` function. +/// +/// # Safety +/// +/// - Exception return from EL2 must must continue execution in EL1 with `kernel_init()`. +#[no_mangle] +pub unsafe extern "C" fn _start_rust( + phys_kernel_tables_base_addr: u64, + virt_boot_core_stack_end_exclusive_addr: u64, + virt_kernel_init_addr: u64, +) -> ! { + prepare_el2_to_el1_transition( + virt_boot_core_stack_end_exclusive_addr, + virt_kernel_init_addr, + ); + + // Turn on the MMU for EL1. + let addr = Address::new(phys_kernel_tables_base_addr as usize); + memory::mmu::enable_mmu_and_caching(addr).unwrap(); + + // Make the function we return to the root of a backtrace. + prepare_backtrace_reset(); + + // Use `eret` to "return" to EL1. Since virtual memory will already be enabled, this results in + // execution of kernel_init() in EL1 from its _virtual address_. + asm::eret() +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/cpu/boot.s b/18_backtrace/kernel/src/_arch/aarch64/cpu/boot.s new file mode 100644 index 00000000..d2c9270d --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/cpu/boot.s @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +// Load the address of a symbol into a register, PC-relative. +// +// The symbol must lie within +/- 4 GiB of the Program Counter. +// +// # Resources +// +// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html +.macro ADR_REL register, symbol + adrp \register, \symbol + add \register, \register, #:lo12:\symbol +.endm + +// Load the address of a symbol into a register, absolute. +// +// # Resources +// +// - https://sourceware.org/binutils/docs-2.36/as/AArch64_002dRelocations.html +.macro ADR_ABS register, symbol + movz \register, #:abs_g3:\symbol + movk \register, #:abs_g2_nc:\symbol + movk \register, #:abs_g1_nc:\symbol + movk \register, #:abs_g0_nc:\symbol +.endm + +.equ _EL2, 0x8 +.equ _core_id_mask, 0b11 + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +.section .text._start + +//------------------------------------------------------------------------------ +// fn _start() +//------------------------------------------------------------------------------ +_start: + // Only proceed if the core executes in EL2. Park it otherwise. + mrs x0, CurrentEL + cmp x0, _EL2 + b.ne .L_parking_loop + + // Only proceed on the boot core. Park it otherwise. + mrs x1, MPIDR_EL1 + and x1, x1, _core_id_mask + ldr x2, BOOT_CORE_ID // provided by bsp/__board_name__/cpu.rs + cmp x1, x2 + b.ne .L_parking_loop + + // If execution reaches here, it is the boot core. + + // Initialize DRAM. + ADR_REL x0, __bss_start + ADR_REL x1, __bss_end_exclusive + +.L_bss_init_loop: + cmp x0, x1 + b.eq .L_prepare_rust + stp xzr, xzr, [x0], #16 + b .L_bss_init_loop + + // Prepare the jump to Rust code. +.L_prepare_rust: + // Load the base address of the kernel's translation tables. + ldr x0, PHYS_KERNEL_TABLES_BASE_ADDR // provided by bsp/__board_name__/memory/mmu.rs + + // Load the _absolute_ addresses of the following symbols. Since the kernel is linked at + // the top of the 64 bit address space, these are effectively virtual addresses. + ADR_ABS x1, __boot_core_stack_end_exclusive + ADR_ABS x2, kernel_init + + // Load the PC-relative address of the stack and set the stack pointer. + // + // Since _start() is the first function that runs after the firmware has loaded the kernel + // into memory, retrieving this symbol PC-relative returns the "physical" address. + // + // Setting the stack pointer to this value ensures that anything that still runs in EL2, + // until the kernel returns to EL1 with the MMU enabled, works as well. After the return to + // EL1, the virtual address of the stack retrieved above will be used. + ADR_REL x4, __boot_core_stack_end_exclusive + mov sp, x4 + + // Jump to Rust code. x0, x1 and x2 hold the function arguments provided to _start_rust(). + b _start_rust + + // Infinitely wait for events (aka "park the core"). +.L_parking_loop: + wfe + b .L_parking_loop + +.size _start, . - _start +.type _start, function +.global _start diff --git a/18_backtrace/kernel/src/_arch/aarch64/cpu/smp.rs b/18_backtrace/kernel/src/_arch/aarch64/cpu/smp.rs new file mode 100644 index 00000000..351fde62 --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/cpu/smp.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Architectural symmetric multiprocessing. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::cpu::smp::arch_smp + +use cortex_a::registers::*; +use tock_registers::interfaces::Readable; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return the executing core's id. +#[inline(always)] +pub fn core_id() -> T +where + T: From, +{ + const CORE_MASK: u64 = 0b11; + + T::from((MPIDR_EL1.get() & CORE_MASK) as u8) +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/exception.rs b/18_backtrace/kernel/src/_arch/aarch64/exception.rs new file mode 100644 index 00000000..6781758a --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/exception.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Architectural synchronous and asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::arch_exception + +use crate::{bsp, exception, memory, symbols}; +use core::{arch::global_asm, cell::UnsafeCell, fmt}; +use cortex_a::{asm::barrier, registers::*}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + registers::InMemoryRegister, +}; + +// Assembly counterpart to this file. +global_asm!(include_str!("exception.s")); + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Wrapper structs for memory copies of registers. +#[repr(transparent)] +struct SpsrEL1(InMemoryRegister); +struct EsrEL1(InMemoryRegister); + +/// The exception context as it is stored on the stack on exception entry. +#[repr(C)] +struct ExceptionContext { + /// General Purpose Registers. + gpr: [u64; 30], + + /// The link register, aka x30. + lr: u64, + + /// Exception link register. The program counter at the time the exception happened. + elr_el1: u64, + + /// Saved program status. + spsr_el1: SpsrEL1, + + /// Exception syndrome register. + esr_el1: EsrEL1, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Prints verbose information about the exception and then panics. +fn default_exception_handler(exc: &ExceptionContext) { + panic!( + "CPU Exception!\n\n\ + {}", + exc + ); +} + +//------------------------------------------------------------------------------ +// Current, EL0 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn current_el0_synchronous(_e: &mut ExceptionContext) { + panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") +} + +#[no_mangle] +unsafe extern "C" fn current_el0_irq(_e: &mut ExceptionContext) { + panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") +} + +#[no_mangle] +unsafe extern "C" fn current_el0_serror(_e: &mut ExceptionContext) { + panic!("Should not be here. Use of SP_EL0 in EL1 is not supported.") +} + +//------------------------------------------------------------------------------ +// Current, ELx +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { + #[cfg(feature = "test_build")] + { + const TEST_SVC_ID: u64 = 0x1337; + + if let Some(ESR_EL1::EC::Value::SVC64) = e.esr_el1.exception_class() { + if e.esr_el1.iss() == TEST_SVC_ID { + return; + } + } + } + + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_elx_irq(_e: &mut ExceptionContext) { + use exception::asynchronous::interface::IRQManager; + + let token = &exception::asynchronous::IRQContext::new(); + bsp::exception::asynchronous::irq_manager().handle_pending_irqs(token); +} + +#[no_mangle] +unsafe extern "C" fn current_elx_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch64 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch32 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Misc +//------------------------------------------------------------------------------ + +/// Human readable SPSR_EL1. +#[rustfmt::skip] +impl fmt::Display for SpsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Raw value. + writeln!(f, "SPSR_EL1: {:#010x}", self.0.get())?; + + let to_flag_str = |x| -> _ { + if x { "Set" } else { "Not set" } + }; + + writeln!(f, " Flags:")?; + writeln!(f, " Negative (N): {}", to_flag_str(self.0.is_set(SPSR_EL1::N)))?; + writeln!(f, " Zero (Z): {}", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?; + writeln!(f, " Carry (C): {}", to_flag_str(self.0.is_set(SPSR_EL1::C)))?; + writeln!(f, " Overflow (V): {}", to_flag_str(self.0.is_set(SPSR_EL1::V)))?; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + writeln!(f, " Exception handling state:")?; + writeln!(f, " Debug (D): {}", to_mask_str(self.0.is_set(SPSR_EL1::D)))?; + writeln!(f, " SError (A): {}", to_mask_str(self.0.is_set(SPSR_EL1::A)))?; + writeln!(f, " IRQ (I): {}", to_mask_str(self.0.is_set(SPSR_EL1::I)))?; + writeln!(f, " FIQ (F): {}", to_mask_str(self.0.is_set(SPSR_EL1::F)))?; + + write!(f, " Illegal Execution State (IL): {}", + to_flag_str(self.0.is_set(SPSR_EL1::IL)) + ) + } +} + +impl EsrEL1 { + #[inline(always)] + fn exception_class(&self) -> Option { + self.0.read_as_enum(ESR_EL1::EC) + } + + #[cfg(feature = "test_build")] + #[inline(always)] + fn iss(&self) -> u64 { + self.0.read(ESR_EL1::ISS) + } +} + +/// Human readable ESR_EL1. +#[rustfmt::skip] +impl fmt::Display for EsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Raw print of whole register. + writeln!(f, "ESR_EL1: {:#010x}", self.0.get())?; + + // Raw print of exception class. + write!(f, " Exception Class (EC) : {:#x}", self.0.read(ESR_EL1::EC))?; + + // Exception class. + let ec_translation = match self.exception_class() { + Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => "Data Abort, current EL", + _ => "N/A", + }; + writeln!(f, " - {}", ec_translation)?; + + // Raw print of instruction specific syndrome. + write!(f, " Instr Specific Syndrome (ISS): {:#x}", self.0.read(ESR_EL1::ISS)) + } +} + +impl ExceptionContext { + #[inline(always)] + fn exception_class(&self) -> Option { + self.esr_el1.exception_class() + } + + #[inline(always)] + fn fault_address_valid(&self) -> bool { + use ESR_EL1::EC::Value::*; + + match self.exception_class() { + None => false, + Some(ec) => matches!( + ec, + InstrAbortLowerEL + | InstrAbortCurrentEL + | PCAlignmentFault + | DataAbortLowerEL + | DataAbortCurrentEL + | WatchpointLowerEL + | WatchpointCurrentEL + ), + } + } +} + +/// Human readable print of the exception context. +impl fmt::Display for ExceptionContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "{}", self.esr_el1)?; + + if self.fault_address_valid() { + writeln!(f, "FAR_EL1: {:#018x}", FAR_EL1.get() as usize)?; + } + + writeln!(f, "{}", self.spsr_el1)?; + writeln!(f, "ELR_EL1: {:#018x}", self.elr_el1)?; + writeln!( + f, + " Symbol: {}", + match symbols::lookup_symbol(memory::Address::new(self.elr_el1 as usize)) { + Some(sym) => sym.name(), + _ => "Symbol not found", + } + )?; + writeln!(f)?; + writeln!(f, "General purpose register:")?; + + #[rustfmt::skip] + let alternating = |x| -> _ { + if x % 2 == 0 { " " } else { "\n" } + }; + + // Print two registers per line. + for (i, reg) in self.gpr.iter().enumerate() { + write!(f, " x{: <2}: {: >#018x}{}", i, reg, alternating(i))?; + } + write!(f, " lr : {:#018x}", self.lr) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::exception::PrivilegeLevel; + +/// The processing element's current privilege level. +pub fn current_privilege_level() -> (PrivilegeLevel, &'static str) { + let el = CurrentEL.read_as_enum(CurrentEL::EL); + match el { + Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, "EL2"), + Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, "EL1"), + Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, "EL0"), + _ => (PrivilegeLevel::Unknown, "Unknown"), + } +} + +/// Init exception handling by setting the exception vector base address register. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must +/// adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference +/// Manual. +pub unsafe fn handling_init() { + // Provided by exception.S. + extern "Rust" { + static __exception_vector_start: UnsafeCell<()>; + } + + VBAR_EL1.set(__exception_vector_start.get() as u64); + + // Force VBAR update to complete before next instruction. + barrier::isb(barrier::SY); +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/exception.s b/18_backtrace/kernel/src/_arch/aarch64/exception.s new file mode 100644 index 00000000..17acaf59 --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/exception.s @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//-------------------------------------------------------------------------------------------------- +// Definitions +//-------------------------------------------------------------------------------------------------- + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler is_lower_el is_sync +__vector_\handler: + // Make room on the stack for the exception context. + sub sp, sp, #16 * 18 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1), saved program status (SPSR_EL1) and exception + // syndrome register (ESR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + mrs x3, ESR_EL1 + + stp lr, x1, [sp, #16 * 15] + stp x2, x3, [sp, #16 * 16] + + // Build a stack frame for backtracing. +.if \is_lower_el == 1 + // If we came from a lower EL, make it a root frame (by storing zero) so that the kernel + // does not attempt to trace into userspace. + stp xzr, xzr, [sp, #16 * 17] +.else + // For normal branches, the link address points to the instruction to be executed _after_ + // returning from a branch. In a backtrace, we want to show the instruction that caused the + // branch, though. That is why code in backtrace.rs subtracts 4 (length of one instruction) + // from the link address. + // + // Here we have a special case, though, because ELR_EL1 is used instead of LR to build the + // stack frame, so that it becomes possible to trace beyond an exception. Hence, it must be + // considered that semantics for ELR_EL1 differ from case to case. + // + // Unless an "exception generating instruction" was executed, ELR_EL1 already points to the + // the correct instruction, and hence the subtraction by 4 in backtrace.rs would yield wrong + // results. To cover for this, 4 is added to ELR_EL1 below unless the cause of exception was + // an SVC instruction. BRK and HLT are "exception generating instructions" as well, but they + // are not expected and therefore left out for now. + // + // For reference: Search for "preferred exception return address" in the Architecture + // Reference Manual for ARMv8-A. +.if \is_sync == 1 + lsr w3, w3, #26 // w3 = ESR_EL1.EC + cmp w3, #0x15 // w3 == SVC64 ? + b.eq 1f +.endif + add x1, x1, #4 +1: + stp x29, x1, [sp, #16 * 17] +.endif + + // Set the frame pointer to the stack frame record. + add x29, sp, #16 * 17 + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via + // `eret`. + b __exception_restore_context + +.size __vector_\handler, . - __vector_\handler +.type __vector_\handler, function +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +.section .text + +//------------------------------------------------------------------------------ +// The exception vector table. +//------------------------------------------------------------------------------ + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous, 0, 1 +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq, 0, 0 +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror, 0, 0 + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous, 0, 1 +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq, 0, 0 +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror, 0, 0 + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous, 1, 1 +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq, 1, 0 +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror, 1, 0 + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous, 1, 0 +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq, 1, 0 +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror, 1, 0 +.org 0x800 + +//------------------------------------------------------------------------------ +// fn __exception_restore_context() +//------------------------------------------------------------------------------ +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 18 + + eret + +.size __exception_restore_context, . - __exception_restore_context +.type __exception_restore_context, function diff --git a/18_backtrace/kernel/src/_arch/aarch64/exception/asynchronous.rs b/18_backtrace/kernel/src/_arch/aarch64/exception/asynchronous.rs new file mode 100644 index 00000000..73b82e65 --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/exception/asynchronous.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Architectural asynchronous exception handling. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::exception::asynchronous::arch_asynchronous + +use core::arch::asm; +use cortex_a::registers::*; +use tock_registers::interfaces::{Readable, Writeable}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +mod daif_bits { + pub const IRQ: u8 = 0b0010; +} + +trait DaifField { + fn daif_field() -> tock_registers::fields::Field; +} + +struct Debug; +struct SError; +struct IRQ; +struct FIQ; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl DaifField for Debug { + fn daif_field() -> tock_registers::fields::Field { + DAIF::D + } +} + +impl DaifField for SError { + fn daif_field() -> tock_registers::fields::Field { + DAIF::A + } +} + +impl DaifField for IRQ { + fn daif_field() -> tock_registers::fields::Field { + DAIF::I + } +} + +impl DaifField for FIQ { + fn daif_field() -> tock_registers::fields::Field { + DAIF::F + } +} + +fn is_masked() -> bool +where + T: DaifField, +{ + DAIF.is_set(T::daif_field()) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Returns whether IRQs are masked on the executing core. +pub fn is_local_irq_masked() -> bool { + !is_masked::() +} + +/// Unmask IRQs on the executing core. +/// +/// It is not needed to place an explicit instruction synchronization barrier after the `msr`. +/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3: +/// +/// "Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional +/// synchronization." +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_unmask() { + #[rustfmt::skip] + asm!( + "msr DAIFClr, {arg}", + arg = const daif_bits::IRQ, + options(nomem, nostack, preserves_flags) + ); +} + +/// Mask IRQs on the executing core. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_mask() { + #[rustfmt::skip] + asm!( + "msr DAIFSet, {arg}", + arg = const daif_bits::IRQ, + options(nomem, nostack, preserves_flags) + ); +} + +/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF). +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_mask_save() -> u64 { + let saved = DAIF.get(); + local_irq_mask(); + + saved +} + +/// Restore the interrupt mask bits (DAIF) using the callee's argument. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +/// - No sanity checks on the input. +#[inline(always)] +pub unsafe fn local_irq_restore(saved: u64) { + DAIF.set(saved); +} + +/// Print the AArch64 exceptions status. +#[rustfmt::skip] +pub fn print_state() { + use crate::info; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + info!(" Debug: {}", to_mask_str(is_masked::())); + info!(" SError: {}", to_mask_str(is_masked::())); + info!(" IRQ: {}", to_mask_str(is_masked::())); + info!(" FIQ: {}", to_mask_str(is_masked::())); +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/memory/mmu.rs b/18_backtrace/kernel/src/_arch/aarch64/memory/mmu.rs new file mode 100644 index 00000000..3d6c18b7 --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/memory/mmu.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Memory Management Unit Driver. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::arch_mmu + +use crate::{ + bsp, memory, + memory::{mmu::TranslationGranule, Address, Physical}, +}; +use core::intrinsics::unlikely; +use cortex_a::{asm::barrier, registers::*}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Memory Management Unit type. +struct MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; + +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +pub mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static MMU: MemoryManagementUnit = MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl memory::mmu::AddressSpace { + /// Checks for architectural restrictions. + pub const fn arch_address_space_size_sanity_checks() { + // Size must be at least one full 512 MiB table. + assert!((AS_SIZE % Granule512MiB::SIZE) == 0); + + // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8 + // version. + assert!(AS_SIZE <= (1 << 48)); + } +} + +impl MemoryManagementUnit { + /// Setup function for the MAIR_EL1 register. + #[inline(always)] + fn set_up_mair(&self) { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + + // Attribute 0 - Device. + MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, + ); + } + + /// Configure various settings of stage 1 of the EL1 translation regime. + #[inline(always)] + fn configure_translation_control(&self) { + let t1sz = (64 - bsp::memory::mmu::KernelVirtAddrSpace::SIZE_SHIFT) as u64; + + TCR_EL1.write( + TCR_EL1::TBI1::Used + + TCR_EL1::IPS::Bits_40 + + TCR_EL1::TG1::KiB_64 + + TCR_EL1::SH1::Inner + + TCR_EL1::ORGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN1::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD1::EnableTTBR1Walks + + TCR_EL1::A1::TTBR1 + + TCR_EL1::T1SZ.val(t1sz) + + TCR_EL1::EPD0::DisableTTBR0Walks, + ); + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the MMU instance. +pub fn mmu() -> &'static impl memory::mmu::interface::MMU { + &MMU +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use memory::mmu::MMUEnableError; + +impl memory::mmu::interface::MMU for MemoryManagementUnit { + unsafe fn enable_mmu_and_caching( + &self, + phys_tables_base_addr: Address, + ) -> Result<(), MMUEnableError> { + if unlikely(self.is_enabled()) { + return Err(MMUEnableError::AlreadyEnabled); + } + + // Fail early if translation granule is not supported. + if unlikely(!ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported)) { + return Err(MMUEnableError::Other( + "Translation granule not supported in HW", + )); + } + + // Prepare the memory attribute indirection register. + self.set_up_mair(); + + // Set the "Translation Table Base Register". + TTBR1_EL1.set_baddr(phys_tables_base_addr.as_usize() as u64); + + self.configure_translation_control(); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction. + barrier::isb(barrier::SY); + + Ok(()) + } + + #[inline(always)] + fn is_enabled(&self) -> bool { + SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) + } +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs b/18_backtrace/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs new file mode 100644 index 00000000..f0b4ac85 --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/memory/mmu/translation_table.rs @@ -0,0 +1,521 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//! Architectural translation table. +//! +//! Only 64 KiB granule is supported. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::memory::mmu::translation_table::arch_translation_table + +use crate::{ + bsp, + memory::{ + self, + mmu::{ + arch_mmu::{Granule512MiB, Granule64KiB}, + AccessPermissions, AttributeFields, MemAttributes, MemoryRegion, PageAddress, + }, + Address, Physical, Virtual, + }, +}; +use core::convert; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, + registers::InMemoryRegister, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Unprivileged execute-never. + UXN OFFSET(54) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Reserved_Invalid = 0, + Page = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(C)] +struct TableDescriptor { + value: u64, +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(C)] +struct PageDescriptor { + value: u64, +} + +trait StartAddr { + fn virt_start_addr(&self) -> Address; +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, so the lvl3 is put first. +#[repr(C)] +#[repr(align(65536))] +pub struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], + + /// Have the tables been initialized? + initialized: bool, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl StartAddr for [T; N] { + fn virt_start_addr(&self) -> Address { + Address::new(self as *const _ as usize) + } +} + +impl TableDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance pointing to the supplied address. + pub fn from_next_lvl_table_addr(phys_next_lvl_table_addr: Address) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = phys_next_lvl_table_addr.as_usize() >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64) + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::VALID::True, + ); + + TableDescriptor { value: val.get() } + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for tock_registers::fields::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(memory::mmu::arch_mmu::mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // The execute-never attribute is mapped to PXN in AArch64. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + // Always set unprivileged exectue-never as long as userspace is not implemented yet. + desc += STAGE1_PAGE_DESCRIPTOR::UXN::True; + + desc + } +} + +/// Convert the HW-specific attributes of the MMU to kernel's generic memory attributes. +impl convert::TryFrom> for AttributeFields { + type Error = &'static str; + + fn try_from( + desc: InMemoryRegister, + ) -> Result { + let mem_attributes = match desc.read(STAGE1_PAGE_DESCRIPTOR::AttrIndx) { + memory::mmu::arch_mmu::mair::NORMAL => MemAttributes::CacheableDRAM, + memory::mmu::arch_mmu::mair::DEVICE => MemAttributes::Device, + _ => return Err("Unexpected memory attribute"), + }; + + let acc_perms = match desc.read_as_enum(STAGE1_PAGE_DESCRIPTOR::AP) { + Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RO_EL1) => AccessPermissions::ReadOnly, + Some(STAGE1_PAGE_DESCRIPTOR::AP::Value::RW_EL1) => AccessPermissions::ReadWrite, + _ => return Err("Unexpected access permission"), + }; + + let execute_never = desc.read(STAGE1_PAGE_DESCRIPTOR::PXN) > 0; + + Ok(AttributeFields { + mem_attributes, + acc_perms, + execute_never, + }) + } +} + +impl PageDescriptor { + /// Create an instance. + /// + /// Descriptor is invalid by default. + pub const fn new_zeroed() -> Self { + Self { value: 0 } + } + + /// Create an instance. + pub fn from_output_page_addr( + phys_output_page_addr: PageAddress, + attribute_fields: &AttributeFields, + ) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = phys_output_page_addr.into_inner().as_usize() >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64) + + STAGE1_PAGE_DESCRIPTOR::AF::True + + STAGE1_PAGE_DESCRIPTOR::TYPE::Page + + STAGE1_PAGE_DESCRIPTOR::VALID::True + + (*attribute_fields).into(), + ); + + Self { value: val.get() } + } + + /// Returns the valid bit. + fn is_valid(&self) -> bool { + InMemoryRegister::::new(self.value) + .is_set(STAGE1_PAGE_DESCRIPTOR::VALID) + } + + /// Returns the output page. + fn output_page_addr(&self) -> PageAddress { + let shifted = InMemoryRegister::::new(self.value) + .read(STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB) as usize; + + PageAddress::from(shifted << Granule64KiB::SHIFT) + } + + /// Returns the attributes. + fn try_attributes(&self) -> Result { + InMemoryRegister::::new(self.value).try_into() + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl memory::mmu::AssociatedTranslationTable + for memory::mmu::AddressSpace +where + [u8; Self::SIZE >> Granule512MiB::SHIFT]: Sized, +{ + type TableStartFromTop = + FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, true>; + + type TableStartFromBottom = + FixedSizeTranslationTable<{ Self::SIZE >> Granule512MiB::SHIFT }, false>; +} + +impl + FixedSizeTranslationTable +{ + const START_FROM_TOP_OFFSET: Address = + Address::new((usize::MAX - (Granule512MiB::SIZE * NUM_TABLES)) + 1); + + /// Create an instance. + #[allow(clippy::assertions_on_constants)] + const fn _new(for_precompute: bool) -> Self { + assert!(bsp::memory::mmu::KernelGranule::SIZE == Granule64KiB::SIZE); + + // Can't have a zero-sized address space. + assert!(NUM_TABLES > 0); + + Self { + lvl3: [[PageDescriptor::new_zeroed(); 8192]; NUM_TABLES], + lvl2: [TableDescriptor::new_zeroed(); NUM_TABLES], + initialized: for_precompute, + } + } + + pub const fn new_for_precompute() -> Self { + Self::_new(true) + } + + #[cfg(test)] + pub fn new_for_runtime() -> Self { + Self::_new(false) + } + + /// Helper to calculate the lvl2 and lvl3 indices from an address. + #[inline(always)] + fn lvl2_lvl3_index_from_page_addr( + &self, + virt_page_addr: PageAddress, + ) -> Result<(usize, usize), &'static str> { + let mut addr = virt_page_addr.into_inner(); + + if START_FROM_TOP { + addr = addr - Self::START_FROM_TOP_OFFSET; + } + + let lvl2_index = addr.as_usize() >> Granule512MiB::SHIFT; + let lvl3_index = (addr.as_usize() & Granule512MiB::MASK) >> Granule64KiB::SHIFT; + + if lvl2_index > (NUM_TABLES - 1) { + return Err("Virtual page is out of bounds of translation table"); + } + + Ok((lvl2_index, lvl3_index)) + } + + /// Returns the PageDescriptor corresponding to the supplied page address. + #[inline(always)] + fn page_descriptor_from_page_addr( + &self, + virt_page_addr: PageAddress, + ) -> Result<&PageDescriptor, &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?; + let desc = &self.lvl3[lvl2_index][lvl3_index]; + + Ok(desc) + } + + /// Sets the PageDescriptor corresponding to the supplied page address. + /// + /// Doesn't allow overriding an already valid page. + #[inline(always)] + fn set_page_descriptor_from_page_addr( + &mut self, + virt_page_addr: PageAddress, + new_desc: &PageDescriptor, + ) -> Result<(), &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from_page_addr(virt_page_addr)?; + let desc = &mut self.lvl3[lvl2_index][lvl3_index]; + + if desc.is_valid() { + return Err("Virtual page is already mapped"); + } + + *desc = *new_desc; + Ok(()) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl + memory::mmu::translation_table::interface::TranslationTable + for FixedSizeTranslationTable +{ + fn init(&mut self) -> Result<(), &'static str> { + if self.initialized { + return Ok(()); + } + + // Populate the l2 entries. + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { + let virt_table_addr = self.lvl3[lvl2_nr].virt_start_addr(); + let phys_table_addr = memory::mmu::try_kernel_virt_addr_to_phys_addr(virt_table_addr)?; + + let new_desc = TableDescriptor::from_next_lvl_table_addr(phys_table_addr); + *lvl2_entry = new_desc; + } + + self.initialized = true; + + Ok(()) + } + + unsafe fn map_at( + &mut self, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + assert!(self.initialized, "Translation tables not initialized"); + + if virt_region.size() != phys_region.size() { + return Err("Tried to map memory regions with unequal sizes"); + } + + if phys_region.end_exclusive_page_addr() > bsp::memory::phys_addr_space_end_exclusive_addr() + { + return Err("Tried to map outside of physical address space"); + } + + let iter = phys_region.into_iter().zip(virt_region.into_iter()); + for (phys_page_addr, virt_page_addr) in iter { + let new_desc = PageDescriptor::from_output_page_addr(phys_page_addr, attr); + let virt_page = virt_page_addr; + + self.set_page_descriptor_from_page_addr(virt_page, &new_desc)?; + } + + Ok(()) + } + + fn try_virt_page_addr_to_phys_page_addr( + &self, + virt_page_addr: PageAddress, + ) -> Result, &'static str> { + let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?; + + if !page_desc.is_valid() { + return Err("Page marked invalid"); + } + + Ok(page_desc.output_page_addr()) + } + + fn try_page_attributes( + &self, + virt_page_addr: PageAddress, + ) -> Result { + let page_desc = self.page_descriptor_from_page_addr(virt_page_addr)?; + + if !page_desc.is_valid() { + return Err("Page marked invalid"); + } + + page_desc.try_attributes() + } + + /// Try to translate a virtual address to a physical address. + /// + /// Will only succeed if there exists a valid mapping for the input address. + fn try_virt_addr_to_phys_addr( + &self, + virt_addr: Address, + ) -> Result, &'static str> { + let virt_page = PageAddress::from(virt_addr.align_down_page()); + let phys_page = self.try_virt_page_addr_to_phys_page_addr(virt_page)?; + + Ok(phys_page.into_inner() + virt_addr.offset_into_page()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +pub type MinSizeTranslationTable = FixedSizeTranslationTable<1, true>; + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check if the size of `struct TableDescriptor` is as expected. + #[kernel_test] + fn size_of_tabledescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if the size of `struct PageDescriptor` is as expected. + #[kernel_test] + fn size_of_pagedescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } +} diff --git a/18_backtrace/kernel/src/_arch/aarch64/time.rs b/18_backtrace/kernel/src/_arch/aarch64/time.rs new file mode 100644 index 00000000..c814219c --- /dev/null +++ b/18_backtrace/kernel/src/_arch/aarch64/time.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Architectural timer primitives. +//! +//! # Orientation +//! +//! Since arch modules are imported into generic modules using the path attribute, the path of this +//! file is: +//! +//! crate::time::arch_time + +use crate::{time, warn}; +use core::time::Duration; +use cortex_a::{asm::barrier, registers::*}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +const NS_PER_S: u64 = 1_000_000_000; + +/// ARMv8 Generic Timer. +struct GenericTimer; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static TIME_MANAGER: GenericTimer = GenericTimer; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl GenericTimer { + #[inline(always)] + fn read_cntpct(&self) -> u64 { + // Prevent that the counter is read ahead of time due to out-of-order execution. + unsafe { barrier::isb(barrier::SY) }; + CNTPCT_EL0.get() + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the time manager. +pub fn time_manager() -> &'static impl time::interface::TimeManager { + &TIME_MANAGER +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl time::interface::TimeManager for GenericTimer { + fn resolution(&self) -> Duration { + Duration::from_nanos(NS_PER_S / (CNTFRQ_EL0.get() as u64)) + } + + fn uptime(&self) -> Duration { + let current_count: u64 = self.read_cntpct() * NS_PER_S; + let frq: u64 = CNTFRQ_EL0.get() as u64; + + Duration::from_nanos(current_count / frq) + } + + fn spin_for(&self, duration: Duration) { + // Instantly return on zero. + if duration.as_nanos() == 0 { + return; + } + + // Calculate the register compare value. + let frq = CNTFRQ_EL0.get(); + let x = match frq.checked_mul(duration.as_nanos() as u64) { + #[allow(unused_imports)] + None => { + warn!("Spin duration too long, skipping"); + return; + } + Some(val) => val, + }; + let tval = x / NS_PER_S; + + // Check if it is within supported bounds. + let warn: Option<&str> = if tval == 0 { + Some("smaller") + // The upper 32 bits of CNTP_TVAL_EL0 are reserved. + } else if tval > u32::max_value().into() { + Some("bigger") + } else { + None + }; + + #[allow(unused_imports)] + if let Some(w) = warn { + warn!( + "Spin duration {} than architecturally supported, skipping", + w + ); + return; + } + + // Set the compare value register. + CNTP_TVAL_EL0.set(tval); + + // Kick off the counting. // Disable timer interrupt. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); + + // ISTATUS will be '1' when cval ticks have passed. Busy-check it. + while !CNTP_CTL_EL0.matches_all(CNTP_CTL_EL0::ISTATUS::SET) {} + + // Disable counting again. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::CLEAR); + } +} diff --git a/18_backtrace/kernel/src/backtrace.rs b/18_backtrace/kernel/src/backtrace.rs new file mode 100644 index 00000000..7dba2e4a --- /dev/null +++ b/18_backtrace/kernel/src/backtrace.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Backtracing support. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/backtrace.rs"] +mod arch_backtrace; + +use crate::{ + memory::{Address, Virtual}, + symbols, +}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +#[cfg(feature = "test_build")] +pub use arch_backtrace::{corrupt_link, corrupt_previous_frame_addr}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// A backtrace item. +#[allow(missing_docs)] +pub enum BacktraceItem { + InvalidFramePointer(Address), + InvalidLink(Address), + Link(Address), +} + +/// Pseudo-struct for printing a backtrace using its fmt::Display implementation. +pub struct Backtrace; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl fmt::Display for Backtrace { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Backtrace:")?; + writeln!( + f, + " ----------------------------------------------------------------------------------------------" + )?; + writeln!( + f, + " Address Function containing address" + )?; + writeln!( + f, + " ----------------------------------------------------------------------------------------------" + )?; + + let mut fmt_res: fmt::Result = Ok(()); + let trace_formatter = + |maybe_iter: Option<&mut dyn Iterator>| match maybe_iter { + None => fmt_res = writeln!(f, "ERROR! No valid stack frame found"), + Some(iter) => { + for (i, backtrace_res) in iter.enumerate() { + match backtrace_res { + BacktraceItem::InvalidFramePointer(addr) => { + fmt_res = writeln!( + f, + " {:>2}. ERROR! \ + Encountered invalid frame pointer ({}) during backtrace", + i + 1, + addr + ); + } + BacktraceItem::InvalidLink(addr) => { + fmt_res = writeln!( + f, + " {:>2}. ERROR! \ + Link address ({}) is not contained in kernel .text section", + i + 1, + addr + ); + } + BacktraceItem::Link(addr) => { + fmt_res = writeln!( + f, + " {:>2}. {:016x} | {:<50}", + i + 1, + addr.as_usize(), + match symbols::lookup_symbol(addr) { + Some(sym) => sym.name(), + _ => "Symbol not found", + } + ) + } + }; + + if fmt_res.is_err() { + break; + } + } + } + }; + + arch_backtrace::backtrace(trace_formatter); + fmt_res?; + + writeln!( + f, + " ----------------------------------------------------------------------------------------------" + ) + } +} diff --git a/18_backtrace/kernel/src/bsp.rs b/18_backtrace/kernel/src/bsp.rs new file mode 100644 index 00000000..824787f6 --- /dev/null +++ b/18_backtrace/kernel/src/bsp.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Conditional reexporting of Board Support Packages. + +mod device_driver; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod raspberrypi; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use raspberrypi::*; diff --git a/18_backtrace/kernel/src/bsp/device_driver.rs b/18_backtrace/kernel/src/bsp/device_driver.rs new file mode 100644 index 00000000..eafaf775 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Device driver. + +#[cfg(feature = "bsp_rpi4")] +mod arm; +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod bcm; +mod common; + +#[cfg(feature = "bsp_rpi4")] +pub use arm::*; +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use bcm::*; diff --git a/18_backtrace/kernel/src/bsp/device_driver/arm.rs b/18_backtrace/kernel/src/bsp/device_driver/arm.rs new file mode 100644 index 00000000..e83e24c9 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/arm.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! ARM driver top level. + +pub mod gicv2; + +pub use gicv2::*; diff --git a/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs b/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs new file mode 100644 index 00000000..4c68a692 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! GICv2 Driver - ARM Generic Interrupt Controller v2. +//! +//! The following is a collection of excerpts with useful information from +//! - `Programmer's Guide for ARMv8-A` +//! - `ARM Generic Interrupt Controller Architecture Specification` +//! +//! # Programmer's Guide - 10.6.1 Configuration +//! +//! The GIC is accessed as a memory-mapped peripheral. +//! +//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core +//! uses the same address to access its own private CPU interface. +//! +//! It is not possible for a core to access the CPU interface of another core. +//! +//! # Architecture Specification - 10.6.2 Initialization +//! +//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized +//! after reset before it can deliver interrupts to the core. +//! +//! In the Distributor, software must configure the priority, target, security and enable individual +//! interrupts. The Distributor must subsequently be enabled through its control register +//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption +//! settings. +//! +//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This +//! prepares the GIC to deliver interrupts to the core. +//! +//! Before interrupts are expected in the core, software prepares the core to take interrupts by +//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in +//! PSTATE, and setting the routing controls. +//! +//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor. +//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface. +//! Individual interrupts can also be disabled (or enabled) in the distributor. +//! +//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must +//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the +//! core's priority mask. +//! +//! # Architecture Specification - 1.4.2 Interrupt types +//! +//! - Peripheral interrupt +//! - Private Peripheral Interrupt (PPI) +//! - This is a peripheral interrupt that is specific to a single processor. +//! - Shared Peripheral Interrupt (SPI) +//! - This is a peripheral interrupt that the Distributor can route to any of a specified +//! combination of processors. +//! +//! - Software-generated interrupt (SGI) +//! - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The +//! system uses SGIs for interprocessor communication. +//! - An SGI has edge-triggered properties. The software triggering of the interrupt is +//! equivalent to the edge transition of the interrupt request signal. +//! - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt +//! Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR, +//! identifies the processor that requested the interrupt. +//! +//! # Architecture Specification - 2.2.1 Interrupt IDs +//! +//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020 +//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by +//! the Distributor. +//! +//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows: +//! - Interrupt numbers 32..1019 are used for SPIs. +//! - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These +//! interrupts are banked in the Distributor. +//! - A banked interrupt is one where the Distributor can have multiple interrupts with the +//! same ID. A banked interrupt is identified uniquely by its ID number and its associated +//! CPU interface number. Of the banked interrupt IDs: +//! - 00..15 SGIs +//! - 16..31 PPIs + +mod gicc; +mod gicd; + +use crate::{bsp, cpu, driver, exception, memory, synchronization, synchronization::InitStateLock}; +use core::sync::atomic::{AtomicBool, Ordering}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +type HandlerTable = [Option; GICv2::NUM_IRQS]; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +pub type IRQNumber = exception::asynchronous::IRQNumber<{ GICv2::MAX_IRQ_NUMBER }>; + +/// Representation of the GIC. +pub struct GICv2 { + gicd_mmio_descriptor: memory::mmu::MMIODescriptor, + gicc_mmio_descriptor: memory::mmu::MMIODescriptor, + + /// The Distributor. + gicd: gicd::GICD, + + /// The CPU Interface. + gicc: gicc::GICC, + + /// Have the MMIO regions been remapped yet? + is_mmio_remapped: AtomicBool, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl GICv2 { + const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space. + const NUM_IRQS: usize = Self::MAX_IRQ_NUMBER + 1; + + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new( + gicd_mmio_descriptor: memory::mmu::MMIODescriptor, + gicc_mmio_descriptor: memory::mmu::MMIODescriptor, + ) -> Self { + Self { + gicd_mmio_descriptor, + gicc_mmio_descriptor, + gicd: gicd::GICD::new(gicd_mmio_descriptor.start_addr().as_usize()), + gicc: gicc::GICC::new(gicc_mmio_descriptor.start_addr().as_usize()), + is_mmio_remapped: AtomicBool::new(false), + handler_table: InitStateLock::new([None; Self::NUM_IRQS]), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::ReadWriteEx; + +impl driver::interface::DeviceDriver for GICv2 { + fn compatible(&self) -> &'static str { + "GICv2 (ARM Generic Interrupt Controller v2)" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let remapped = self.is_mmio_remapped.load(Ordering::Relaxed); + if !remapped { + // GICD + let mut virt_addr = memory::mmu::kernel_map_mmio("GICD", &self.gicd_mmio_descriptor)?; + self.gicd.set_mmio(virt_addr.as_usize()); + + // GICC + virt_addr = memory::mmu::kernel_map_mmio("GICC", &self.gicc_mmio_descriptor)?; + self.gicc.set_mmio(virt_addr.as_usize()); + + // Conclude remapping. + self.is_mmio_remapped.store(true, Ordering::Relaxed); + } + + if bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id() { + self.gicd.boot_core_init(); + } + + self.gicc.priority_accept_all(); + self.gicc.enable(); + + Ok(()) + } +} + +impl exception::asynchronous::interface::IRQManager for GICv2 { + type IRQNumberType = IRQNumber; + + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + self.handler_table.write(|table| { + let irq_number = irq_number.get(); + + if table[irq_number].is_some() { + return Err("IRQ handler already registered"); + } + + table[irq_number] = Some(descriptor); + + Ok(()) + }) + } + + fn enable(&self, irq_number: Self::IRQNumberType) { + self.gicd.enable(irq_number); + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register + // (IAR). + let irq_number = self.gicc.pending_irq_number(ic); + + // Guard against spurious interrupts. + if irq_number > GICv2::MAX_IRQ_NUMBER { + return; + } + + // Call the IRQ handler. Panic if there is none. + self.handler_table.read(|table| { + match table[irq_number] { + None => panic!("No handler registered for IRQ {}", irq_number), + Some(descriptor) => { + // Call the IRQ handler. Panics on failure. + descriptor.handler.handle().expect("Error handling IRQ"); + } + } + }); + + // Signal completion of handling. + self.gicc.mark_comleted(irq_number as u32, ic); + } + + fn print_handler(&self) { + use crate::info; + + info!(" Peripheral handler:"); + + self.handler_table.read(|table| { + for (i, opt) in table.iter().skip(32).enumerate() { + if let Some(handler) = opt { + info!(" {: >3}. {}", i + 32, handler.name); + } + } + }); + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs b/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs new file mode 100644 index 00000000..1a151d24 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicc.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! GICC Driver - GIC CPU interface. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, exception, synchronization::InitStateLock, +}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::ReadWrite, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_bitfields! { + u32, + + /// CPU Interface Control Register + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + + /// Interrupt Priority Mask Register + PMR [ + Priority OFFSET(0) NUMBITS(8) [] + ], + + /// Interrupt Acknowledge Register + IAR [ + InterruptID OFFSET(0) NUMBITS(10) [] + ], + + /// End of Interrupt Register + EOIR [ + EOIINTID OFFSET(0) NUMBITS(10) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + pub RegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => PMR: ReadWrite), + (0x008 => _reserved1), + (0x00C => IAR: ReadWrite), + (0x010 => EOIR: ReadWrite), + (0x014 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Registers = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the GIC CPU interface. +pub struct GICC { + registers: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::synchronization::interface::ReadWriteEx; + +impl GICC { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + registers: InitStateLock::new(Registers::new(mmio_start_addr)), + } + } + + pub unsafe fn set_mmio(&self, new_mmio_start_addr: usize) { + self.registers + .write(|regs| *regs = Registers::new(new_mmio_start_addr)); + } + + /// Accept interrupts of any priority. + /// + /// Quoting the GICv2 Architecture Specification: + /// + /// "Writing 255 to the GICC_PMR always sets it to the largest supported priority field + /// value." + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn priority_accept_all(&self) { + self.registers.read(|regs| { + regs.PMR.write(PMR::Priority.val(255)); // Comment in arch spec. + }); + } + + /// Enable the interface - start accepting IRQs. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn enable(&self) { + self.registers.read(|regs| { + regs.CTLR.write(CTLR::Enable::SET); + }); + } + + /// Extract the number of the highest-priority pending IRQ. + /// + /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + #[allow(clippy::trivially_copy_pass_by_ref)] + pub fn pending_irq_number<'irq_context>( + &self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) -> usize { + self.registers + .read(|regs| regs.IAR.read(IAR::InterruptID) as usize) + } + + /// Complete handling of the currently active IRQ. + /// + /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. + /// + /// To be called after `pending_irq_number()`. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + #[allow(clippy::trivially_copy_pass_by_ref)] + pub fn mark_comleted<'irq_context>( + &self, + irq_number: u32, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + self.registers.read(|regs| { + regs.EOIR.write(EOIR::EOIINTID.val(irq_number)); + }); + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs b/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs new file mode 100644 index 00000000..60bbc468 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/arm/gicv2/gicd.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! GICD Driver - GIC Distributor. +//! +//! # Glossary +//! - SPI - Shared Peripheral Interrupt. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, + state, synchronization, + synchronization::{IRQSafeNullLock, InitStateLock}, +}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite}, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_bitfields! { + u32, + + /// Distributor Control Register + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + + /// Interrupt Controller Type Register + TYPER [ + ITLinesNumber OFFSET(0) NUMBITS(5) [] + ], + + /// Interrupt Processor Targets Registers + ITARGETSR [ + Offset3 OFFSET(24) NUMBITS(8) [], + Offset2 OFFSET(16) NUMBITS(8) [], + Offset1 OFFSET(8) NUMBITS(8) [], + Offset0 OFFSET(0) NUMBITS(8) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + SharedRegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => TYPER: ReadOnly), + (0x008 => _reserved1), + (0x104 => ISENABLER: [ReadWrite; 31]), + (0x108 => _reserved2), + (0x820 => ITARGETSR: [ReadWrite; 248]), + (0x824 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + BankedRegisterBlock { + (0x000 => _reserved1), + (0x100 => ISENABLER: ReadWrite), + (0x104 => _reserved2), + (0x800 => ITARGETSR: [ReadOnly; 8]), + (0x804 => @END), + } +} + +/// Abstraction for the non-banked parts of the associated MMIO registers. +type SharedRegisters = MMIODerefWrapper; + +/// Abstraction for the banked parts of the associated MMIO registers. +type BankedRegisters = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the GIC Distributor. +pub struct GICD { + /// Access to shared registers is guarded with a lock. + shared_registers: IRQSafeNullLock, + + /// Access to banked registers is unguarded. + banked_registers: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl SharedRegisters { + /// Return the number of IRQs that this HW implements. + #[inline(always)] + fn num_irqs(&mut self) -> usize { + // Query number of implemented IRQs. + // + // Refer to GICv2 Architecture Specification, Section 4.3.2. + ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 + } + + /// Return a slice of the implemented ITARGETSR. + #[inline(always)] + fn implemented_itargets_slice(&mut self) -> &[ReadWrite] { + assert!(self.num_irqs() >= 36); + + // Calculate the max index of the shared ITARGETSR array. + // + // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS + // register has four entries, so shift right by two. Subtract one because we start + // counting at zero. + let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1; + + // Rust automatically inserts slice range sanity check, i.e. max >= min. + &self.ITARGETSR[0..spi_itargetsr_max_index] + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::synchronization::interface::ReadWriteEx; +use synchronization::interface::Mutex; + +impl GICD { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)), + banked_registers: InitStateLock::new(BankedRegisters::new(mmio_start_addr)), + } + } + + pub unsafe fn set_mmio(&self, new_mmio_start_addr: usize) { + self.shared_registers + .lock(|regs| *regs = SharedRegisters::new(new_mmio_start_addr)); + self.banked_registers + .write(|regs| *regs = BankedRegisters::new(new_mmio_start_addr)); + } + + /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask. + /// + /// Quoting the GICv2 Architecture Specification: + /// + /// "GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that + /// corresponds only to the processor reading the register." + fn local_gic_target_mask(&self) -> u32 { + self.banked_registers + .read(|regs| regs.ITARGETSR[0].read(ITARGETSR::Offset0)) + } + + /// Route all SPIs to the boot core and enable the distributor. + pub fn boot_core_init(&self) { + assert!( + state::state_manager().is_init(), + "Only allowed during kernel init phase" + ); + + // Target all SPIs to the boot core only. + let mask = self.local_gic_target_mask(); + + self.shared_registers.lock(|regs| { + for i in regs.implemented_itargets_slice().iter() { + i.write( + ITARGETSR::Offset3.val(mask) + + ITARGETSR::Offset2.val(mask) + + ITARGETSR::Offset1.val(mask) + + ITARGETSR::Offset0.val(mask), + ); + } + + regs.CTLR.write(CTLR::Enable::SET); + }); + } + + /// Enable an interrupt. + pub fn enable(&self, irq_num: super::IRQNumber) { + let irq_num = irq_num.get(); + + // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5 + // (division by 32) and arrive at the index for the respective ISENABLER[i]. + let enable_reg_index = irq_num >> 5; + let enable_bit: u32 = 1u32 << (irq_num % 32); + + // Check if we are handling a private or shared IRQ. + match irq_num { + // Private. + 0..=31 => self.banked_registers.read(|regs| { + let enable_reg = ®s.ISENABLER; + enable_reg.set(enable_reg.get() | enable_bit); + }), + // Shared. + _ => { + let enable_reg_index_shared = enable_reg_index - 1; + + self.shared_registers.lock(|regs| { + let enable_reg = ®s.ISENABLER[enable_reg_index_shared]; + enable_reg.set(enable_reg.get() | enable_bit); + }); + } + } + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/bcm.rs b/18_backtrace/kernel/src/bsp/device_driver/bcm.rs new file mode 100644 index 00000000..5a7cc23b --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/bcm.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! BCM driver top level. + +mod bcm2xxx_gpio; +#[cfg(feature = "bsp_rpi3")] +mod bcm2xxx_interrupt_controller; +mod bcm2xxx_pl011_uart; + +pub use bcm2xxx_gpio::*; +#[cfg(feature = "bsp_rpi3")] +pub use bcm2xxx_interrupt_controller::*; +pub use bcm2xxx_pl011_uart::*; diff --git a/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs new file mode 100644 index 00000000..eea07b75 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! GPIO Driver. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, driver, memory, synchronization, + synchronization::IRQSafeNullLock, +}; +use core::sync::atomic::{AtomicUsize, Ordering}; +use tock_registers::{ + interfaces::{ReadWriteable, Writeable}, + register_bitfields, register_structs, + registers::ReadWrite, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// GPIO registers. +// +// Descriptions taken from +// - https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +// - https://datasheets.raspberrypi.org/bcm2711/bcm2711-peripherals.pdf +register_bitfields! { + u32, + + /// GPIO Function Select 1 + GPFSEL1 [ + /// Pin 15 + FSEL15 OFFSET(15) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART RX + + ], + + /// Pin 14 + FSEL14 OFFSET(12) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART TX + ] + ], + + /// GPIO Pull-up/down Register + /// + /// BCM2837 only. + GPPUD [ + /// Controls the actuation of the internal pull-up/down control line to ALL the GPIO pins. + PUD OFFSET(0) NUMBITS(2) [ + Off = 0b00, + PullDown = 0b01, + PullUp = 0b10 + ] + ], + + /// GPIO Pull-up/down Clock Register 0 + /// + /// BCM2837 only. + GPPUDCLK0 [ + /// Pin 15 + PUDCLK15 OFFSET(15) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ], + + /// Pin 14 + PUDCLK14 OFFSET(14) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ] + ], + + /// GPIO Pull-up / Pull-down Register 0 + /// + /// BCM2711 only. + GPIO_PUP_PDN_CNTRL_REG0 [ + /// Pin 15 + GPIO_PUP_PDN_CNTRL15 OFFSET(30) NUMBITS(2) [ + NoResistor = 0b00, + PullUp = 0b01 + ], + + /// Pin 14 + GPIO_PUP_PDN_CNTRL14 OFFSET(28) NUMBITS(2) [ + NoResistor = 0b00, + PullUp = 0b01 + ] + ] +} + +register_structs! { + #[allow(non_snake_case)] + RegisterBlock { + (0x00 => _reserved1), + (0x04 => GPFSEL1: ReadWrite), + (0x08 => _reserved2), + (0x94 => GPPUD: ReadWrite), + (0x98 => GPPUDCLK0: ReadWrite), + (0x9C => _reserved3), + (0xE4 => GPIO_PUP_PDN_CNTRL_REG0: ReadWrite), + (0xE8 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Registers = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub struct GPIOInner { + registers: Registers, +} + +// Export the inner struct so that BSPs can use it for the panic handler. +pub use GPIOInner as PanicGPIO; + +/// Representation of the GPIO HW. +pub struct GPIO { + mmio_descriptor: memory::mmu::MMIODescriptor, + virt_mmio_start_addr: AtomicUsize, + inner: IRQSafeNullLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl GPIOInner { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + registers: Registers::new(mmio_start_addr), + } + } + + /// Init code. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub unsafe fn init(&mut self, new_mmio_start_addr: Option) -> Result<(), &'static str> { + if let Some(addr) = new_mmio_start_addr { + self.registers = Registers::new(addr); + } + + Ok(()) + } + + /// Disable pull-up/down on pins 14 and 15. + #[cfg(feature = "bsp_rpi3")] + fn disable_pud_14_15_bcm2837(&mut self) { + use crate::{time, time::interface::TimeManager}; + use core::time::Duration; + + // The Linux 2837 GPIO driver waits 1 µs between the steps. + const DELAY: Duration = Duration::from_micros(1); + + self.registers.GPPUD.write(GPPUD::PUD::Off); + time::time_manager().spin_for(DELAY); + + self.registers + .GPPUDCLK0 + .write(GPPUDCLK0::PUDCLK15::AssertClock + GPPUDCLK0::PUDCLK14::AssertClock); + time::time_manager().spin_for(DELAY); + + self.registers.GPPUD.write(GPPUD::PUD::Off); + self.registers.GPPUDCLK0.set(0); + } + + /// Disable pull-up/down on pins 14 and 15. + #[cfg(feature = "bsp_rpi4")] + fn disable_pud_14_15_bcm2711(&mut self) { + self.registers.GPIO_PUP_PDN_CNTRL_REG0.write( + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL15::PullUp + + GPIO_PUP_PDN_CNTRL_REG0::GPIO_PUP_PDN_CNTRL14::PullUp, + ); + } + + /// Map PL011 UART as standard output. + /// + /// TX to pin 14 + /// RX to pin 15 + pub fn map_pl011_uart(&mut self) { + // Select the UART on pins 14 and 15. + self.registers + .GPFSEL1 + .modify(GPFSEL1::FSEL15::AltFunc0 + GPFSEL1::FSEL14::AltFunc0); + + // Disable pull-up/down on pins 14 and 15. + #[cfg(feature = "bsp_rpi3")] + self.disable_pud_14_15_bcm2837(); + + #[cfg(feature = "bsp_rpi4")] + self.disable_pud_14_15_bcm2711(); + } +} + +impl GPIO { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new(mmio_descriptor: memory::mmu::MMIODescriptor) -> Self { + Self { + mmio_descriptor, + virt_mmio_start_addr: AtomicUsize::new(0), + inner: IRQSafeNullLock::new(GPIOInner::new(mmio_descriptor.start_addr().as_usize())), + } + } + + /// Concurrency safe version of `GPIOInner.map_pl011_uart()` + pub fn map_pl011_uart(&self) { + self.inner.lock(|inner| inner.map_pl011_uart()) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::Mutex; + +impl driver::interface::DeviceDriver for GPIO { + fn compatible(&self) -> &'static str { + "BCM GPIO" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = memory::mmu::kernel_map_mmio(self.compatible(), &self.mmio_descriptor)?; + + self.inner + .lock(|inner| inner.init(Some(virt_addr.as_usize())))?; + + self.virt_mmio_start_addr + .store(virt_addr.as_usize(), Ordering::Relaxed); + + Ok(()) + } + + fn virt_mmio_start_addr(&self) -> Option { + let addr = self.virt_mmio_start_addr.load(Ordering::Relaxed); + + if addr == 0 { + return None; + } + + Some(addr) + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs new file mode 100644 index 00000000..99961fac --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Interrupt Controller Driver. + +mod peripheral_ic; + +use crate::{driver, exception, memory}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Wrapper struct for a bitmask indicating pending IRQ numbers. +struct PendingIRQs { + bitmask: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub type LocalIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>; +pub type PeripheralIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>; + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +#[derive(Copy, Clone)] +pub enum IRQNumber { + Local(LocalIRQ), + Peripheral(PeripheralIRQ), +} + +/// Representation of the Interrupt Controller. +pub struct InterruptController { + periph: peripheral_ic::PeripheralIC, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl PendingIRQs { + pub fn new(bitmask: u64) -> Self { + Self { bitmask } + } +} + +impl Iterator for PendingIRQs { + type Item = usize; + + fn next(&mut self) -> Option { + use core::intrinsics::cttz; + + let next = cttz(self.bitmask); + if next == 64 { + return None; + } + + self.bitmask &= !(1 << next); + + Some(next as usize) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl InterruptController { + const MAX_LOCAL_IRQ_NUMBER: usize = 11; + const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63; + const NUM_PERIPHERAL_IRQS: usize = Self::MAX_PERIPHERAL_IRQ_NUMBER + 1; + + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new( + _local_mmio_descriptor: memory::mmu::MMIODescriptor, + periph_mmio_descriptor: memory::mmu::MMIODescriptor, + ) -> Self { + Self { + periph: peripheral_ic::PeripheralIC::new(periph_mmio_descriptor), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl driver::interface::DeviceDriver for InterruptController { + fn compatible(&self) -> &'static str { + "BCM Interrupt Controller" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + self.periph.init() + } +} + +impl exception::asynchronous::interface::IRQManager for InterruptController { + type IRQNumberType = IRQNumber; + + fn register_handler( + &self, + irq: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + match irq { + IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), + IRQNumber::Peripheral(pirq) => self.periph.register_handler(pirq, descriptor), + } + } + + fn enable(&self, irq: Self::IRQNumberType) { + match irq { + IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), + IRQNumber::Peripheral(pirq) => self.periph.enable(pirq), + } + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet. + self.periph.handle_pending_irqs(ic) + } + + fn print_handler(&self) { + self.periph.print_handler(); + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs new file mode 100644 index 00000000..f09da862 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Peripheral Interrupt Controller Driver. + +use super::{InterruptController, PendingIRQs, PeripheralIRQ}; +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, + driver, exception, memory, synchronization, + synchronization::{IRQSafeNullLock, InitStateLock}, +}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_structs, + registers::{ReadOnly, WriteOnly}, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_structs! { + #[allow(non_snake_case)] + WORegisterBlock { + (0x00 => _reserved1), + (0x10 => ENABLE_1: WriteOnly), + (0x14 => ENABLE_2: WriteOnly), + (0x24 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + RORegisterBlock { + (0x00 => _reserved1), + (0x04 => PENDING_1: ReadOnly), + (0x08 => PENDING_2: ReadOnly), + (0x0c => @END), + } +} + +/// Abstraction for the WriteOnly parts of the associated MMIO registers. +type WriteOnlyRegisters = MMIODerefWrapper; + +/// Abstraction for the ReadOnly parts of the associated MMIO registers. +type ReadOnlyRegisters = MMIODerefWrapper; + +type HandlerTable = + [Option; InterruptController::NUM_PERIPHERAL_IRQS]; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the peripheral interrupt controller. +pub struct PeripheralIC { + mmio_descriptor: memory::mmu::MMIODescriptor, + + /// Access to write registers is guarded with a lock. + wo_registers: IRQSafeNullLock, + + /// Register read access is unguarded. + ro_registers: InitStateLock, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl PeripheralIC { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new(mmio_descriptor: memory::mmu::MMIODescriptor) -> Self { + let addr = mmio_descriptor.start_addr().as_usize(); + + Self { + mmio_descriptor, + wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(addr)), + ro_registers: InitStateLock::new(ReadOnlyRegisters::new(addr)), + handler_table: InitStateLock::new([None; InterruptController::NUM_PERIPHERAL_IRQS]), + } + } + + /// Query the list of pending IRQs. + fn pending_irqs(&self) -> PendingIRQs { + self.ro_registers.read(|regs| { + let pending_mask: u64 = + (u64::from(regs.PENDING_2.get()) << 32) | u64::from(regs.PENDING_1.get()); + + PendingIRQs::new(pending_mask) + }) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::{Mutex, ReadWriteEx}; + +impl driver::interface::DeviceDriver for PeripheralIC { + fn compatible(&self) -> &'static str { + "BCM Peripheral Interrupt Controller" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = + memory::mmu::kernel_map_mmio(self.compatible(), &self.mmio_descriptor)?.as_usize(); + + self.wo_registers + .lock(|regs| *regs = WriteOnlyRegisters::new(virt_addr)); + self.ro_registers + .write(|regs| *regs = ReadOnlyRegisters::new(virt_addr)); + + Ok(()) + } +} + +impl exception::asynchronous::interface::IRQManager for PeripheralIC { + type IRQNumberType = PeripheralIRQ; + + fn register_handler( + &self, + irq: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + self.handler_table.write(|table| { + let irq_number = irq.get(); + + if table[irq_number].is_some() { + return Err("IRQ handler already registered"); + } + + table[irq_number] = Some(descriptor); + + Ok(()) + }) + } + + fn enable(&self, irq: Self::IRQNumberType) { + self.wo_registers.lock(|regs| { + let enable_reg = if irq.get() <= 31 { + ®s.ENABLE_1 + } else { + ®s.ENABLE_2 + }; + + let enable_bit: u32 = 1 << (irq.get() % 32); + + // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable + // bits are unaffected. So we don't need read and OR'ing here. + enable_reg.set(enable_bit); + }); + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + self.handler_table.read(|table| { + for irq_number in self.pending_irqs() { + match table[irq_number] { + None => panic!("No handler registered for IRQ {}", irq_number), + Some(descriptor) => { + // Call the IRQ handler. Panics on failure. + descriptor.handler.handle().expect("Error handling IRQ"); + } + } + } + }) + } + + fn print_handler(&self) { + use crate::info; + + info!(" Peripheral handler:"); + + self.handler_table.read(|table| { + for (i, opt) in table.iter().enumerate() { + if let Some(handler) = opt { + info!(" {: >3}. {}", i, handler.name); + } + } + }); + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs new file mode 100644 index 00000000..3133047b --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! PL011 UART driver. +//! +//! # Resources +//! +//! - +//! - + +use crate::{ + bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, memory, + synchronization, synchronization::IRQSafeNullLock, +}; +use core::{ + fmt, + sync::atomic::{AtomicUsize, Ordering}, +}; +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_bitfields, register_structs, + registers::{ReadOnly, ReadWrite, WriteOnly}, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// PL011 UART registers. +// +// Descriptions taken from "PrimeCell UART (PL011) Technical Reference Manual" r1p5. +register_bitfields! { + u32, + + /// Flag Register. + FR [ + /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// Line Control Register, LCR_H. + /// + /// - If the FIFO is disabled, this bit is set when the transmit holding register is empty. + /// - If the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty. + /// - This bit does not indicate if there is data in the transmit shift register. + TXFE OFFSET(7) NUMBITS(1) [], + + /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the + /// LCR_H Register. + /// + /// - If the FIFO is disabled, this bit is set when the transmit holding register is full. + /// - If the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full. + TXFF OFFSET(5) NUMBITS(1) [], + + /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// LCR_H Register. + /// + /// - If the FIFO is disabled, this bit is set when the receive holding register is empty. + /// - If the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty. + RXFE OFFSET(4) NUMBITS(1) [], + + /// UART busy. If this bit is set to 1, the UART is busy transmitting data. This bit remains + /// set until the complete byte, including all the stop bits, has been sent from the shift + /// register. + /// + /// This bit is set as soon as the transmit FIFO becomes non-empty, regardless of whether + /// the UART is enabled or not. + BUSY OFFSET(3) NUMBITS(1) [] + ], + + /// Integer Baud Rate Divisor. + IBRD [ + /// The integer baud rate divisor. + BAUD_DIVINT OFFSET(0) NUMBITS(16) [] + ], + + /// Fractional Baud Rate Divisor. + FBRD [ + /// The fractional baud rate divisor. + BAUD_DIVFRAC OFFSET(0) NUMBITS(6) [] + ], + + /// Line Control Register. + LCR_H [ + /// Word length. These bits indicate the number of data bits transmitted or received in a + /// frame. + #[allow(clippy::enum_variant_names)] + WLEN OFFSET(5) NUMBITS(2) [ + FiveBit = 0b00, + SixBit = 0b01, + SevenBit = 0b10, + EightBit = 0b11 + ], + + /// Enable FIFOs: + /// + /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding + /// registers. + /// + /// 1 = Transmit and receive FIFO buffers are enabled (FIFO mode). + FEN OFFSET(4) NUMBITS(1) [ + FifosDisabled = 0, + FifosEnabled = 1 + ] + ], + + /// Control Register. + CR [ + /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled. + /// Data reception occurs for either UART signals or SIR signals depending on the setting of + /// the SIREN bit. When the UART is disabled in the middle of reception, it completes the + /// current character before stopping. + RXE OFFSET(9) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled. + /// Data transmission occurs for either UART signals, or SIR signals depending on the + /// setting of the SIREN bit. When the UART is disabled in the middle of transmission, it + /// completes the current character before stopping. + TXE OFFSET(8) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// UART enable: + /// + /// 0 = UART is disabled. If the UART is disabled in the middle of transmission or + /// reception, it completes the current character before stopping. + /// + /// 1 = The UART is enabled. Data transmission and reception occurs for either UART signals + /// or SIR signals depending on the setting of the SIREN bit + UARTEN OFFSET(0) NUMBITS(1) [ + /// If the UART is disabled in the middle of transmission or reception, it completes the + /// current character before stopping. + Disabled = 0, + Enabled = 1 + ] + ], + + /// Interrupt FIFO Level Select Register. + IFLS [ + /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as + /// follows. + RXIFLSEL OFFSET(3) NUMBITS(5) [ + OneEigth = 0b000, + OneQuarter = 0b001, + OneHalf = 0b010, + ThreeQuarters = 0b011, + SevenEights = 0b100 + ] + ], + + /// Interrupt Mask Set/Clear Register. + IMSC [ + /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR + /// interrupt. + /// + /// - On a write of 1, the mask of the UARTRTINTR interrupt is set. + /// - A write of 0 clears the mask. + RTIM OFFSET(6) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt. + /// + /// - On a write of 1, the mask of the UARTRXINTR interrupt is set. + /// - A write of 0 clears the mask. + RXIM OFFSET(4) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ] + ], + + /// Masked Interrupt Status Register. + MIS [ + /// Receive timeout masked interrupt status. Returns the masked interrupt state of the + /// UARTRTINTR interrupt. + RTMIS OFFSET(6) NUMBITS(1) [], + + /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR + /// interrupt. + RXMIS OFFSET(4) NUMBITS(1) [] + ], + + /// Interrupt Clear Register. + ICR [ + /// Meta field for all pending interrupts. + ALL OFFSET(0) NUMBITS(11) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + pub RegisterBlock { + (0x00 => DR: ReadWrite), + (0x04 => _reserved1), + (0x18 => FR: ReadOnly), + (0x1c => _reserved2), + (0x24 => IBRD: WriteOnly), + (0x28 => FBRD: WriteOnly), + (0x2c => LCR_H: WriteOnly), + (0x30 => CR: WriteOnly), + (0x34 => IFLS: ReadWrite), + (0x38 => IMSC: ReadWrite), + (0x3C => _reserved3), + (0x40 => MIS: ReadOnly), + (0x44 => ICR: WriteOnly), + (0x48 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Registers = MMIODerefWrapper; + +#[derive(PartialEq)] +enum BlockingMode { + Blocking, + NonBlocking, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub struct PL011UartInner { + registers: Registers, + chars_written: usize, + chars_read: usize, +} + +// Export the inner struct so that BSPs can use it for the panic handler. +pub use PL011UartInner as PanicUart; + +/// Representation of the UART. +pub struct PL011Uart { + mmio_descriptor: memory::mmu::MMIODescriptor, + virt_mmio_start_addr: AtomicUsize, + inner: IRQSafeNullLock, + irq_number: bsp::device_driver::IRQNumber, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl PL011UartInner { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + registers: Registers::new(mmio_start_addr), + chars_written: 0, + chars_read: 0, + } + } + + /// Set up baud rate and characteristics. + /// + /// This results in 8N1 and 921_600 baud. + /// + /// The calculation for the BRD is (we set the clock to 48 MHz in config.txt): + /// `(48_000_000 / 16) / 921_600 = 3.2552083`. + /// + /// This means the integer part is `3` and goes into the `IBRD`. + /// The fractional part is `0.2552083`. + /// + /// `FBRD` calculation according to the PL011 Technical Reference Manual: + /// `INTEGER((0.2552083 * 64) + 0.5) = 16`. + /// + /// Therefore, the generated baud rate divider is: `3 + 16/64 = 3.25`. Which results in a + /// genrated baud rate of `48_000_000 / (16 * 3.25) = 923_077`. + /// + /// Error = `((923_077 - 921_600) / 921_600) * 100 = 0.16%`. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub unsafe fn init(&mut self, new_mmio_start_addr: Option) -> Result<(), &'static str> { + if let Some(addr) = new_mmio_start_addr { + self.registers = Registers::new(addr); + } + + // Execution can arrive here while there are still characters queued in the TX FIFO and + // actively being sent out by the UART hardware. If the UART is turned off in this case, + // those queued characters would be lost. + // + // For example, this can happen during runtime on a call to panic!(), because panic!() + // initializes its own UART instance and calls init(). + // + // Hence, flush first to ensure all pending characters are transmitted. + self.flush(); + + // Turn the UART off temporarily. + self.registers.CR.set(0); + + // Clear all pending interrupts. + self.registers.ICR.write(ICR::ALL::CLEAR); + + // From the PL011 Technical Reference Manual: + // + // The LCR_H, IBRD, and FBRD registers form the single 30-bit wide LCR Register that is + // updated on a single write strobe generated by a LCR_H write. So, to internally update the + // contents of IBRD or FBRD, a LCR_H write must always be performed at the end. + // + // Set the baud rate, 8N1 and FIFO enabled. + self.registers.IBRD.write(IBRD::BAUD_DIVINT.val(3)); + self.registers.FBRD.write(FBRD::BAUD_DIVFRAC.val(16)); + self.registers + .LCR_H + .write(LCR_H::WLEN::EightBit + LCR_H::FEN::FifosEnabled); + + // Set RX FIFO fill level at 1/8. + self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth); + + // Enable RX IRQ + RX timeout IRQ. + self.registers + .IMSC + .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled); + + // Turn the UART on. + self.registers + .CR + .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled); + + Ok(()) + } + + /// Send a character. + fn write_char(&mut self, c: char) { + // Spin while TX FIFO full is set, waiting for an empty slot. + while self.registers.FR.matches_all(FR::TXFF::SET) { + cpu::nop(); + } + + // Write the character to the buffer. + self.registers.DR.set(c as u32); + + self.chars_written += 1; + } + + /// Block execution until the last buffered character has been physically put on the TX wire. + fn flush(&self) { + // Spin until the busy bit is cleared. + while self.registers.FR.matches_all(FR::BUSY::SET) { + cpu::nop(); + } + } + + /// Retrieve a character. + fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option { + // If RX FIFO is empty, + if self.registers.FR.matches_all(FR::RXFE::SET) { + // immediately return in non-blocking mode. + if blocking_mode == BlockingMode::NonBlocking { + return None; + } + + // Otherwise, wait until a char was received. + while self.registers.FR.matches_all(FR::RXFE::SET) { + cpu::nop(); + } + } + + // Read one character. + let mut ret = self.registers.DR.get() as u8 as char; + + // Convert carrige return to newline. + if ret == '\r' { + ret = '\n' + } + + // Update statistics. + self.chars_read += 1; + + Some(ret) + } +} + +/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are +/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`, +/// we get `write_fmt()` automatically. +/// +/// The function takes an `&mut self`, so it must be implemented for the inner struct. +/// +/// See [`src/print.rs`]. +/// +/// [`src/print.rs`]: ../../print/index.html +impl fmt::Write for PL011UartInner { + fn write_str(&mut self, s: &str) -> fmt::Result { + for c in s.chars() { + self.write_char(c); + } + + Ok(()) + } +} + +impl PL011Uart { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + /// - The user must ensure to provide correct IRQ numbers. + pub const unsafe fn new( + mmio_descriptor: memory::mmu::MMIODescriptor, + irq_number: bsp::device_driver::IRQNumber, + ) -> Self { + Self { + mmio_descriptor, + virt_mmio_start_addr: AtomicUsize::new(0), + inner: IRQSafeNullLock::new(PL011UartInner::new( + mmio_descriptor.start_addr().as_usize(), + )), + irq_number, + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::Mutex; + +impl driver::interface::DeviceDriver for PL011Uart { + fn compatible(&self) -> &'static str { + "BCM PL011 UART" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = memory::mmu::kernel_map_mmio(self.compatible(), &self.mmio_descriptor)?; + + self.inner + .lock(|inner| inner.init(Some(virt_addr.as_usize())))?; + + self.virt_mmio_start_addr + .store(virt_addr.as_usize(), Ordering::Relaxed); + + Ok(()) + } + + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + use bsp::exception::asynchronous::irq_manager; + use exception::asynchronous::{interface::IRQManager, IRQDescriptor}; + + let descriptor = IRQDescriptor { + name: "BCM PL011 UART", + handler: self, + }; + + irq_manager().register_handler(self.irq_number, descriptor)?; + irq_manager().enable(self.irq_number); + + Ok(()) + } + + fn virt_mmio_start_addr(&self) -> Option { + let addr = self.virt_mmio_start_addr.load(Ordering::Relaxed); + + if addr == 0 { + return None; + } + + Some(addr) + } +} + +impl console::interface::Write for PL011Uart { + /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to + /// serialize access. + fn write_char(&self, c: char) { + self.inner.lock(|inner| inner.write_char(c)); + } + + fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result { + // Fully qualified syntax for the call to `core::fmt::Write::write_fmt()` to increase + // readability. + self.inner.lock(|inner| fmt::Write::write_fmt(inner, args)) + } + + fn flush(&self) { + // Spin until TX FIFO empty is set. + self.inner.lock(|inner| inner.flush()); + } +} + +impl console::interface::Read for PL011Uart { + fn read_char(&self) -> char { + self.inner + .lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap()) + } + + fn clear_rx(&self) { + // Read from the RX FIFO until it is indicating empty. + while self + .inner + .lock(|inner| inner.read_char_converting(BlockingMode::NonBlocking)) + .is_some() + {} + } +} + +impl console::interface::Statistics for PL011Uart { + fn chars_written(&self) -> usize { + self.inner.lock(|inner| inner.chars_written) + } + + fn chars_read(&self) -> usize { + self.inner.lock(|inner| inner.chars_read) + } +} + +impl exception::asynchronous::interface::IRQHandler for PL011Uart { + fn handle(&self) -> Result<(), &'static str> { + self.inner.lock(|inner| { + let pending = inner.registers.MIS.extract(); + + // Clear all pending IRQs. + inner.registers.ICR.write(ICR::ALL::CLEAR); + + // Check for any kind of RX interrupt. + if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) { + // Echo any received characters. + while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) { + inner.write_char(c) + } + } + }); + + Ok(()) + } +} diff --git a/18_backtrace/kernel/src/bsp/device_driver/common.rs b/18_backtrace/kernel/src/bsp/device_driver/common.rs new file mode 100644 index 00000000..fd9e988e --- /dev/null +++ b/18_backtrace/kernel/src/bsp/device_driver/common.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Common device driver code. + +use core::{marker::PhantomData, ops}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub struct MMIODerefWrapper { + start_addr: usize, + phantom: PhantomData T>, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl MMIODerefWrapper { + /// Create an instance. + pub const unsafe fn new(start_addr: usize) -> Self { + Self { + start_addr, + phantom: PhantomData, + } + } +} + +impl ops::Deref for MMIODerefWrapper { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*(self.start_addr as *const _) } + } +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi.rs b/18_backtrace/kernel/src/bsp/raspberrypi.rs new file mode 100644 index 00000000..fb9edf88 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Top-level BSP file for the Raspberry Pi 3 and 4. + +pub mod console; +pub mod cpu; +pub mod driver; +pub mod exception; +pub mod memory; + +use super::device_driver; +use crate::memory::mmu::MMIODescriptor; +use memory::map::mmio; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static GPIO: device_driver::GPIO = + unsafe { device_driver::GPIO::new(MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE)) }; + +static PL011_UART: device_driver::PL011Uart = unsafe { + device_driver::PL011Uart::new( + MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE), + exception::asynchronous::irq_map::PL011_UART, + ) +}; + +#[cfg(feature = "bsp_rpi3")] +static INTERRUPT_CONTROLLER: device_driver::InterruptController = unsafe { + device_driver::InterruptController::new( + MMIODescriptor::new(mmio::LOCAL_IC_START, mmio::LOCAL_IC_SIZE), + MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE), + ) +}; + +#[cfg(feature = "bsp_rpi4")] +static INTERRUPT_CONTROLLER: device_driver::GICv2 = unsafe { + device_driver::GICv2::new( + MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE), + MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE), + ) +}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Board identification. +pub fn board_name() -> &'static str { + #[cfg(feature = "bsp_rpi3")] + { + "Raspberry Pi 3" + } + + #[cfg(feature = "bsp_rpi4")] + { + "Raspberry Pi 4" + } +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/console.rs b/18_backtrace/kernel/src/bsp/raspberrypi/console.rs new file mode 100644 index 00000000..a0d2e687 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/console.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! BSP console facilities. + +use crate::{bsp::device_driver, console, cpu, driver}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// In case of a panic, the panic handler uses this function to take a last shot at printing +/// something before the system is halted. +/// +/// We try to init panic-versions of the GPIO and the UART. The panic versions are not protected +/// with synchronization primitives, which increases chances that we get to print something, even +/// when the kernel's default GPIO or UART instances happen to be locked at the time of the panic. +/// +/// # Safety +/// +/// - Use only for printing during a panic. +#[cfg(not(feature = "test_build"))] +pub unsafe fn panic_console_out() -> impl fmt::Write { + use driver::interface::DeviceDriver; + + // If remapping of the driver's MMIO hasn't already happened, we won't be able to print. Just + // park the CPU core in this case. + let gpio_mmio_start_addr = match super::GPIO.virt_mmio_start_addr() { + None => cpu::wait_forever(), + Some(x) => x, + }; + + let uart_mmio_start_addr = match super::PL011_UART.virt_mmio_start_addr() { + None => cpu::wait_forever(), + Some(x) => x, + }; + + let mut panic_gpio = device_driver::PanicGPIO::new(gpio_mmio_start_addr); + let mut panic_uart = device_driver::PanicUart::new(uart_mmio_start_addr); + + panic_gpio + .init(None) + .unwrap_or_else(|_| cpu::wait_forever()); + panic_gpio.map_pl011_uart(); + panic_uart + .init(None) + .unwrap_or_else(|_| cpu::wait_forever()); + + panic_uart +} + +/// Reduced version for test builds. +/// +/// # Safety +/// +/// - Use only for printing during a panic. +#[cfg(feature = "test_build")] +pub unsafe fn panic_console_out() -> impl fmt::Write { + use driver::interface::DeviceDriver; + + let uart_mmio_start_addr = match super::PL011_UART.virt_mmio_start_addr() { + None => cpu::wait_forever(), + Some(x) => x, + }; + let mut panic_uart = device_driver::PanicUart::new(uart_mmio_start_addr); + + panic_uart + .init(None) + .unwrap_or_else(|_| cpu::qemu_exit_failure()); + + panic_uart +} + +/// Return a reference to the console. +pub fn console() -> &'static impl console::interface::All { + &super::PL011_UART +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps +/// than on real hardware due to QEMU's abstractions. +#[cfg(feature = "test_build")] +pub fn qemu_bring_up_console() { + use driver::interface::DeviceDriver; + + // Calling the UART's init ensures that the BSP's instance of the UART does remap the MMIO + // addresses. + unsafe { + super::PL011_UART + .init() + .unwrap_or_else(|_| cpu::qemu_exit_failure()); + } +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/cpu.rs b/18_backtrace/kernel/src/bsp/raspberrypi/cpu.rs new file mode 100644 index 00000000..85fb89e4 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/cpu.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! BSP Processor code. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Used by `arch` code to find the early boot core. +#[no_mangle] +#[link_section = ".text._start_arguments"] +pub static BOOT_CORE_ID: u64 = 0; diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/driver.rs b/18_backtrace/kernel/src/bsp/raspberrypi/driver.rs new file mode 100644 index 00000000..53168752 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/driver.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! BSP driver support. + +use crate::driver; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Device Driver Manager type. +struct BSPDriverManager { + device_drivers: [&'static (dyn DeviceDriver + Sync); 3], +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static BSP_DRIVER_MANAGER: BSPDriverManager = BSPDriverManager { + device_drivers: [ + &super::GPIO, + &super::PL011_UART, + &super::INTERRUPT_CONTROLLER, + ], +}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the driver manager. +pub fn driver_manager() -> &'static impl driver::interface::DriverManager { + &BSP_DRIVER_MANAGER +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use driver::interface::DeviceDriver; + +impl driver::interface::DriverManager for BSPDriverManager { + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[..] + } + + fn early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[0..=1] + } + + fn non_early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[2..] + } + + fn post_early_print_device_driver_init(&self) { + // Configure PL011Uart's output pins. + super::GPIO.map_pl011_uart(); + } +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/exception.rs b/18_backtrace/kernel/src/bsp/raspberrypi/exception.rs new file mode 100644 index 00000000..aa6c5a63 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/exception.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! BSP synchronous and asynchronous exception handling. + +pub mod asynchronous; diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/exception/asynchronous.rs b/18_backtrace/kernel/src/bsp/raspberrypi/exception/asynchronous.rs new file mode 100644 index 00000000..dc5ab421 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/exception/asynchronous.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! BSP asynchronous exception handling. + +use crate::{bsp, exception}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +#[cfg(feature = "bsp_rpi3")] +pub(in crate::bsp) mod irq_map { + use super::bsp::device_driver::{IRQNumber, PeripheralIRQ}; + + pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57)); +} + +#[cfg(feature = "bsp_rpi4")] +pub(in crate::bsp) mod irq_map { + use super::bsp::device_driver::IRQNumber; + + pub const PL011_UART: IRQNumber = IRQNumber::new(153); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the IRQ manager. +pub fn irq_manager() -> &'static impl exception::asynchronous::interface::IRQManager< + IRQNumberType = bsp::device_driver::IRQNumber, +> { + &super::super::INTERRUPT_CONTROLLER +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld b/18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld new file mode 100644 index 00000000..6fcbf31c --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/kernel.ld @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: MIT OR Apache-2.0 + * + * Copyright (c) 2018-2022 Andre Richter + */ + +INCLUDE kernel_virt_addr_space_size.ld; + +PAGE_SIZE = 64K; +PAGE_MASK = PAGE_SIZE - 1; + +/* The kernel's virtual address range will be: + * + * [END_ADDRESS_INCLUSIVE, START_ADDRESS] + * [u64::MAX , (u64::MAX - __kernel_virt_addr_space_size) + 1] + */ +__kernel_virt_start_addr = ((0xffffffffffffffff - __kernel_virt_addr_space_size) + 1); + +__rpi_phys_dram_start_addr = 0; + +/* The physical address at which the the kernel binary will be loaded by the Raspberry's firmware */ +__rpi_phys_binary_load_addr = 0x80000; + + +ENTRY(__rpi_phys_binary_load_addr) + +/* Flags: + * 4 == R + * 5 == RX + * 6 == RW + * + * Segments are marked PT_LOAD below so that the ELF file provides virtual and physical addresses. + * It doesn't mean all of them need actually be loaded. + */ +PHDRS +{ + segment_code PT_LOAD FLAGS(5); + segment_data PT_LOAD FLAGS(6); + segment_boot_core_stack PT_LOAD FLAGS(6); +} + +SECTIONS +{ + . = __kernel_virt_start_addr; + + ASSERT((. & PAGE_MASK) == 0, "Start of address space is not page aligned") + + /*********************************************************************************************** + * Code + RO Data + Global Offset Table + ***********************************************************************************************/ + __code_start = .; + .text : AT(__rpi_phys_binary_load_addr) + { + KEEP(*(.text._start)) + *(.text._start_arguments) /* Constants (or statics in Rust speak) read by _start(). */ + *(.text._start_rust) /* The Rust entry point */ + *(.text*) /* Everything else */ + } :segment_code + + .rodata : ALIGN(8) { *(.rodata*) } :segment_code + .got : ALIGN(8) { *(.got) } :segment_code + .kernel_symbols : ALIGN(8) { + __kernel_symbols_start = .; + . += 32 * 1024; + } :segment_code + + . = ALIGN(PAGE_SIZE); + __code_end_exclusive = .; + + /*********************************************************************************************** + * Data + BSS + ***********************************************************************************************/ + __data_start = .; + .data : { *(.data*) } :segment_data + + /* Section is zeroed in pairs of u64. Align start and end to 16 bytes */ + .bss (NOLOAD) : ALIGN(16) + { + __bss_start = .; + *(.bss*); + . = ALIGN(16); + __bss_end_exclusive = .; + } :segment_data + + . = ALIGN(PAGE_SIZE); + __data_end_exclusive = .; + + /*********************************************************************************************** + * MMIO Remap Reserved + ***********************************************************************************************/ + __mmio_remap_start = .; + . += 8 * 1024 * 1024; + __mmio_remap_end_exclusive = .; + + ASSERT((. & PAGE_MASK) == 0, "MMIO remap reservation is not page aligned") + + /*********************************************************************************************** + * Guard Page + ***********************************************************************************************/ + . += PAGE_SIZE; + + /*********************************************************************************************** + * Boot Core Stack + ***********************************************************************************************/ + .boot_core_stack (NOLOAD) : AT(__rpi_phys_dram_start_addr) + { + __boot_core_stack_start = .; /* ^ */ + /* | stack */ + . += __rpi_phys_binary_load_addr; /* | growth */ + /* | direction */ + __boot_core_stack_end_exclusive = .; /* | */ + } :segment_boot_core_stack + + ASSERT((. & PAGE_MASK) == 0, "End of boot core stack is not page aligned") +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld b/18_backtrace/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld new file mode 100644 index 00000000..c5d58c30 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/kernel_virt_addr_space_size.ld @@ -0,0 +1 @@ +__kernel_virt_addr_space_size = 1024 * 1024 * 1024 diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/memory.rs b/18_backtrace/kernel/src/bsp/raspberrypi/memory.rs new file mode 100644 index 00000000..01aa9441 --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/memory.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! BSP Memory Management. +//! +//! The physical memory layout. +//! +//! The Raspberry's firmware copies the kernel binary to 0x8_0000. The preceding region will be used +//! as the boot core's stack. +//! +//! +---------------------------------------+ +//! | | boot_core_stack_start @ 0x0 +//! | | ^ +//! | Boot-core Stack | | stack +//! | | | growth +//! | | | direction +//! +---------------------------------------+ +//! | | code_start @ 0x8_0000 == boot_core_stack_end_exclusive +//! | .text | +//! | .rodata | +//! | .got | +//! | .kernel_symbols | +//! | | +//! +---------------------------------------+ +//! | | data_start == code_end_exclusive +//! | .data | +//! | .bss | +//! | | +//! +---------------------------------------+ +//! | | data_end_exclusive +//! | | +//! +//! +//! +//! +//! +//! The virtual memory layout is as follows: +//! +//! +---------------------------------------+ +//! | | code_start @ __kernel_virt_start_addr +//! | .text | +//! | .rodata | +//! | .got | +//! | .kernel_symbols | +//! | | +//! +---------------------------------------+ +//! | | data_start == code_end_exclusive +//! | .data | +//! | .bss | +//! | | +//! +---------------------------------------+ +//! | | mmio_remap_start == data_end_exclusive +//! | VA region for MMIO remapping | +//! | | +//! +---------------------------------------+ +//! | | mmio_remap_end_exclusive +//! | Unmapped guard page | +//! | | +//! +---------------------------------------+ +//! | | boot_core_stack_start +//! | | ^ +//! | Boot-core Stack | | stack +//! | | | growth +//! | | | direction +//! +---------------------------------------+ +//! | | boot_core_stack_end_exclusive +//! | | +pub mod mmu; + +use crate::memory::{mmu::PageAddress, Address, Physical, Virtual}; +use core::cell::UnsafeCell; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// Symbols from the linker script. +extern "Rust" { + static __code_start: UnsafeCell<()>; + static __code_end_exclusive: UnsafeCell<()>; + + static __data_start: UnsafeCell<()>; + static __data_end_exclusive: UnsafeCell<()>; + + static __mmio_remap_start: UnsafeCell<()>; + static __mmio_remap_end_exclusive: UnsafeCell<()>; + + static __boot_core_stack_start: UnsafeCell<()>; + static __boot_core_stack_end_exclusive: UnsafeCell<()>; +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// The board's physical memory map. +#[rustfmt::skip] +pub(super) mod map { + use super::*; + + /// Physical devices. + #[cfg(feature = "bsp_rpi3")] + pub mod mmio { + use super::*; + + pub const PERIPHERAL_IC_START: Address = Address::new(0x3F00_B200); + pub const PERIPHERAL_IC_SIZE: usize = 0x24; + + pub const GPIO_START: Address = Address::new(0x3F20_0000); + pub const GPIO_SIZE: usize = 0xA0; + + pub const PL011_UART_START: Address = Address::new(0x3F20_1000); + pub const PL011_UART_SIZE: usize = 0x48; + + pub const LOCAL_IC_START: Address = Address::new(0x4000_0000); + pub const LOCAL_IC_SIZE: usize = 0x100; + + pub const END: Address = Address::new(0x4001_0000); + } + + /// Physical devices. + #[cfg(feature = "bsp_rpi4")] + pub mod mmio { + use super::*; + + pub const GPIO_START: Address = Address::new(0xFE20_0000); + pub const GPIO_SIZE: usize = 0xA0; + + pub const PL011_UART_START: Address = Address::new(0xFE20_1000); + pub const PL011_UART_SIZE: usize = 0x48; + + pub const GICD_START: Address = Address::new(0xFF84_1000); + pub const GICD_SIZE: usize = 0x824; + + pub const GICC_START: Address = Address::new(0xFF84_2000); + pub const GICC_SIZE: usize = 0x14; + + pub const END: Address = Address::new(0xFF85_0000); + } + + pub const END: Address = mmio::END; +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Start page address of the code segment. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn virt_code_start() -> PageAddress { + PageAddress::from(unsafe { __code_start.get() as usize }) +} + +/// Size of the code segment. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn code_size() -> usize { + unsafe { (__code_end_exclusive.get() as usize) - (__code_start.get() as usize) } +} + +/// Start page address of the data segment. +#[inline(always)] +fn virt_data_start() -> PageAddress { + PageAddress::from(unsafe { __data_start.get() as usize }) +} + +/// Size of the data segment. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn data_size() -> usize { + unsafe { (__data_end_exclusive.get() as usize) - (__data_start.get() as usize) } +} + +/// Start page address of the MMIO remap reservation. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn virt_mmio_remap_start() -> PageAddress { + PageAddress::from(unsafe { __mmio_remap_start.get() as usize }) +} + +/// Size of the MMIO remap reservation. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn mmio_remap_size() -> usize { + unsafe { (__mmio_remap_end_exclusive.get() as usize) - (__mmio_remap_start.get() as usize) } +} + +/// Start page address of the boot core's stack. +#[inline(always)] +fn virt_boot_core_stack_start() -> PageAddress { + PageAddress::from(unsafe { __boot_core_stack_start.get() as usize }) +} + +/// Size of the boot core's stack. +#[inline(always)] +fn boot_core_stack_size() -> usize { + unsafe { + (__boot_core_stack_end_exclusive.get() as usize) - (__boot_core_stack_start.get() as usize) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Exclusive end address of the physical address space. +#[inline(always)] +pub fn phys_addr_space_end_exclusive_addr() -> PageAddress { + PageAddress::from(map::END) +} diff --git a/18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs b/18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs new file mode 100644 index 00000000..160c188f --- /dev/null +++ b/18_backtrace/kernel/src/bsp/raspberrypi/memory/mmu.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! BSP Memory Management Unit. + +use crate::{ + memory::{ + mmu::{ + self as generic_mmu, AddressSpace, AssociatedTranslationTable, AttributeFields, + MemoryRegion, PageAddress, TranslationGranule, + }, + Physical, Virtual, + }, + synchronization::InitStateLock, +}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +type KernelTranslationTable = + ::TableStartFromTop; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to +/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. +pub type KernelGranule = TranslationGranule<{ 64 * 1024 }>; + +/// The kernel's virtual address space defined by this BSP. +pub type KernelVirtAddrSpace = AddressSpace<{ kernel_virt_addr_space_size() }>; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +/// The kernel translation tables. +/// +/// It is mandatory that InitStateLock is transparent. +/// +/// That is, `size_of(InitStateLock) == size_of(KernelTranslationTable)`. +/// There is a unit tests that checks this porperty. +#[link_section = ".data"] +#[no_mangle] +static KERNEL_TABLES: InitStateLock = + InitStateLock::new(KernelTranslationTable::new_for_precompute()); + +/// This value is needed during early boot for MMU setup. +/// +/// This will be patched to the correct value by the "translation table tool" after linking. This +/// given value here is just a dummy. +#[link_section = ".text._start_arguments"] +#[no_mangle] +static PHYS_KERNEL_TABLES_BASE_ADDR: u64 = 0xCCCCAAAAFFFFEEEE; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// This is a hack for retrieving the value for the kernel's virtual address space size as a +/// constant from a common place, since it is needed as a compile-time/link-time constant in both, +/// the linker script and the Rust sources. +#[allow(clippy::needless_late_init)] +const fn kernel_virt_addr_space_size() -> usize { + let __kernel_virt_addr_space_size; + + include!("../kernel_virt_addr_space_size.ld"); + + __kernel_virt_addr_space_size +} + +/// Helper function for calculating the number of pages the given parameter spans. +const fn size_to_num_pages(size: usize) -> usize { + assert!(size > 0); + assert!(size % KernelGranule::SIZE == 0); + + size >> KernelGranule::SHIFT +} + +/// The data pages of the kernel binary. +fn virt_data_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::data_size()); + + let start_page_addr = super::virt_data_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +} + +// There is no reason to expect the following conversions to fail, since they were generated offline +// by the `translation table tool`. If it doesn't work, a panic due to the unwraps is justified. +fn kernel_virt_to_phys_region(virt_region: MemoryRegion) -> MemoryRegion { + let phys_start_page_addr = + generic_mmu::try_kernel_virt_page_addr_to_phys_page_addr(virt_region.start_page_addr()) + .unwrap(); + + let phys_end_exclusive_page_addr = phys_start_page_addr + .checked_offset(virt_region.num_pages() as isize) + .unwrap(); + + MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr) +} + +fn kernel_page_attributes(virt_page_addr: PageAddress) -> AttributeFields { + generic_mmu::try_kernel_page_attributes(virt_page_addr).unwrap() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// The code pages of the kernel binary. +pub fn virt_code_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::code_size()); + + let start_page_addr = super::virt_code_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +} + +/// The boot core stack pages. +pub fn virt_boot_core_stack_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::boot_core_stack_size()); + + let start_page_addr = super::virt_boot_core_stack_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +} + +/// Return a reference to the kernel's translation tables. +pub fn kernel_translation_tables() -> &'static InitStateLock { + &KERNEL_TABLES +} + +/// The MMIO remap pages. +pub fn virt_mmio_remap_region() -> MemoryRegion { + let num_pages = size_to_num_pages(super::mmio_remap_size()); + + let start_page_addr = super::virt_mmio_remap_start(); + let end_exclusive_page_addr = start_page_addr.checked_offset(num_pages as isize).unwrap(); + + MemoryRegion::new(start_page_addr, end_exclusive_page_addr) +} + +/// Add mapping records for the kernel binary. +/// +/// The actual translation table entries for the kernel binary are generated using the offline +/// `translation table tool` and patched into the kernel binary. This function just adds the mapping +/// record entries. +pub fn kernel_add_mapping_records_for_precomputed() { + let virt_code_region = virt_code_region(); + generic_mmu::kernel_add_mapping_record( + "Kernel code and RO data", + &virt_code_region, + &kernel_virt_to_phys_region(virt_code_region), + &kernel_page_attributes(virt_code_region.start_page_addr()), + ); + + let virt_data_region = virt_data_region(); + generic_mmu::kernel_add_mapping_record( + "Kernel data and bss", + &virt_data_region, + &kernel_virt_to_phys_region(virt_data_region), + &kernel_page_attributes(virt_data_region.start_page_addr()), + ); + + let virt_boot_core_stack_region = virt_boot_core_stack_region(); + generic_mmu::kernel_add_mapping_record( + "Kernel boot-core stack", + &virt_boot_core_stack_region, + &kernel_virt_to_phys_region(virt_boot_core_stack_region), + &kernel_page_attributes(virt_boot_core_stack_region.start_page_addr()), + ); +} diff --git a/18_backtrace/kernel/src/common.rs b/18_backtrace/kernel/src/common.rs new file mode 100644 index 00000000..678f4a6c --- /dev/null +++ b/18_backtrace/kernel/src/common.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! General purpose code. + +/// Check if a value is aligned to a given size. +#[inline(always)] +pub const fn is_aligned(value: usize, alignment: usize) -> bool { + assert!(alignment.is_power_of_two()); + + (value & (alignment - 1)) == 0 +} + +/// Align down. +#[inline(always)] +pub const fn align_down(value: usize, alignment: usize) -> usize { + assert!(alignment.is_power_of_two()); + + value & !(alignment - 1) +} + +/// Align up. +#[inline(always)] +pub const fn align_up(value: usize, alignment: usize) -> usize { + assert!(alignment.is_power_of_two()); + + (value + alignment - 1) & !(alignment - 1) +} diff --git a/18_backtrace/kernel/src/console.rs b/18_backtrace/kernel/src/console.rs new file mode 100644 index 00000000..e49e241f --- /dev/null +++ b/18_backtrace/kernel/src/console.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! System console. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Console interfaces. +pub mod interface { + use core::fmt; + + /// Console write functions. + pub trait Write { + /// Write a single character. + fn write_char(&self, c: char); + + /// Write a Rust format string. + fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; + + /// Block until the last buffered character has been physically put on the TX wire. + fn flush(&self); + } + + /// Console read functions. + pub trait Read { + /// Read a single character. + fn read_char(&self) -> char { + ' ' + } + + /// Clear RX buffers, if any. + fn clear_rx(&self); + } + + /// Console statistics. + pub trait Statistics { + /// Return the number of characters written. + fn chars_written(&self) -> usize { + 0 + } + + /// Return the number of characters read. + fn chars_read(&self) -> usize { + 0 + } + } + + /// Trait alias for a full-fledged console. + pub trait All = Write + Read + Statistics; +} diff --git a/18_backtrace/kernel/src/cpu.rs b/18_backtrace/kernel/src/cpu.rs new file mode 100644 index 00000000..e1493d1d --- /dev/null +++ b/18_backtrace/kernel/src/cpu.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Processor code. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/cpu.rs"] +mod arch_cpu; + +mod boot; + +pub mod smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_cpu::{nop, wait_forever}; + +#[cfg(feature = "test_build")] +pub use arch_cpu::{qemu_exit_failure, qemu_exit_success}; diff --git a/18_backtrace/kernel/src/cpu/boot.rs b/18_backtrace/kernel/src/cpu/boot.rs new file mode 100644 index 00000000..8091dac3 --- /dev/null +++ b/18_backtrace/kernel/src/cpu/boot.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//! Boot code. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/boot.rs"] +mod arch_boot; diff --git a/18_backtrace/kernel/src/cpu/smp.rs b/18_backtrace/kernel/src/cpu/smp.rs new file mode 100644 index 00000000..57386f79 --- /dev/null +++ b/18_backtrace/kernel/src/cpu/smp.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Symmetric multiprocessing. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/smp.rs"] +mod arch_smp; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_smp::core_id; diff --git a/18_backtrace/kernel/src/driver.rs b/18_backtrace/kernel/src/driver.rs new file mode 100644 index 00000000..7b800dbc --- /dev/null +++ b/18_backtrace/kernel/src/driver.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Driver support. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Driver interfaces. +pub mod interface { + /// Device Driver functions. + pub trait DeviceDriver { + /// Return a compatibility string for identifying the driver. + fn compatible(&self) -> &'static str; + + /// Called by the kernel to bring up the device. + /// + /// # Safety + /// + /// - During init, drivers might do stuff with system-wide impact. + unsafe fn init(&self) -> Result<(), &'static str> { + Ok(()) + } + + /// Called by the kernel to register and enable the device's IRQ handlers, if any. + /// + /// Rust's type system will prevent a call to this function unless the calling instance + /// itself has static lifetime. + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + Ok(()) + } + + /// After MMIO remapping, returns the new virtual start address. + /// + /// This API assumes a driver has only a single, contiguous MMIO aperture, which will not be + /// the case for more complex devices. This API will likely change in future tutorials. + fn virt_mmio_start_addr(&self) -> Option { + None + } + } + + /// Device driver management functions. + /// + /// The `BSP` is supposed to supply one global instance. + pub trait DriverManager { + /// Return a slice of references to all `BSP`-instantiated drivers. + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Return only those drivers needed for the BSP's early printing functionality. + /// + /// For example, the default UART. + fn early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Return all drivers minus early-print drivers. + fn non_early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Initialization code that runs after the early print driver init. + fn post_early_print_device_driver_init(&self); + } +} diff --git a/18_backtrace/kernel/src/exception.rs b/18_backtrace/kernel/src/exception.rs new file mode 100644 index 00000000..f4af8144 --- /dev/null +++ b/18_backtrace/kernel/src/exception.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Synchronous and asynchronous exception handling. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/exception.rs"] +mod arch_exception; + +pub mod asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_exception::{current_privilege_level, handling_init}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Kernel privilege levels. +#[allow(missing_docs)] +#[derive(PartialEq)] +pub enum PrivilegeLevel { + User, + Kernel, + Hypervisor, + Unknown, +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Libkernel unit tests must execute in kernel mode. + #[kernel_test] + fn test_runner_executes_in_kernel_mode() { + let (level, _) = current_privilege_level(); + + assert!(level == PrivilegeLevel::Kernel) + } +} diff --git a/18_backtrace/kernel/src/exception/asynchronous.rs b/18_backtrace/kernel/src/exception/asynchronous.rs new file mode 100644 index 00000000..fb1785c2 --- /dev/null +++ b/18_backtrace/kernel/src/exception/asynchronous.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Asynchronous exception handling. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/exception/asynchronous.rs"] +mod arch_asynchronous; + +use core::{fmt, marker::PhantomData}; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_asynchronous::{ + is_local_irq_masked, local_irq_mask, local_irq_mask_save, local_irq_restore, local_irq_unmask, + print_state, +}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Interrupt descriptor. +#[derive(Copy, Clone)] +pub struct IRQDescriptor { + /// Descriptive name. + pub name: &'static str, + + /// Reference to handler trait object. + pub handler: &'static (dyn interface::IRQHandler + Sync), +} + +/// IRQContext token. +/// +/// An instance of this type indicates that the local core is currently executing in IRQ +/// context, aka executing an interrupt vector or subcalls of it. +/// +/// Concept and implementation derived from the `CriticalSection` introduced in +/// +#[derive(Clone, Copy)] +pub struct IRQContext<'irq_context> { + _0: PhantomData<&'irq_context ()>, +} + +/// Asynchronous exception handling interfaces. +pub mod interface { + + /// Implemented by types that handle IRQs. + pub trait IRQHandler { + /// Called when the corresponding interrupt is asserted. + fn handle(&self) -> Result<(), &'static str>; + } + + /// IRQ management functions. + /// + /// The `BSP` is supposed to supply one global instance. Typically implemented by the + /// platform's interrupt controller. + pub trait IRQManager { + /// The IRQ number type depends on the implementation. + type IRQNumberType; + + /// Register a handler. + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: super::IRQDescriptor, + ) -> Result<(), &'static str>; + + /// Enable an interrupt in the controller. + fn enable(&self, irq_number: Self::IRQNumberType); + + /// Handle pending interrupts. + /// + /// This function is called directly from the CPU's IRQ exception vector. On AArch64, + /// this means that the respective CPU core has disabled exception handling. + /// This function can therefore not be preempted and runs start to finish. + /// + /// Takes an IRQContext token to ensure it can only be called from IRQ context. + #[allow(clippy::trivially_copy_pass_by_ref)] + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &super::IRQContext<'irq_context>, + ); + + /// Print list of registered handlers. + fn print_handler(&self); + } +} + +/// A wrapper type for IRQ numbers with integrated range sanity check. +#[derive(Copy, Clone)] +pub struct IRQNumber(usize); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl<'irq_context> IRQContext<'irq_context> { + /// Creates an IRQContext token. + /// + /// # Safety + /// + /// - This must only be called when the current core is in an interrupt context and will not + /// live beyond the end of it. That is, creation is allowed in interrupt vector functions. For + /// example, in the ARMv8-A case, in `extern "C" fn current_elx_irq()`. + /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code + /// must not be able to influence the lifetime picked for this type, since that might cause it + /// to be inferred to `'static`. + #[inline(always)] + pub unsafe fn new() -> Self { + IRQContext { _0: PhantomData } + } +} + +impl IRQNumber<{ MAX_INCLUSIVE }> { + /// Creates a new instance if number <= MAX_INCLUSIVE. + pub const fn new(number: usize) -> Self { + assert!(number <= MAX_INCLUSIVE); + + Self(number) + } + + /// Return the wrapped number. + pub const fn get(self) -> usize { + self.0 + } +} + +impl fmt::Display for IRQNumber<{ MAX_INCLUSIVE }> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Executes the provided closure while IRQs are masked on the executing core. +/// +/// While the function temporarily changes the HW state of the executing core, it restores it to the +/// previous state before returning, so this is deemed safe. +#[inline(always)] +pub fn exec_with_irq_masked(f: impl FnOnce() -> T) -> T { + let ret: T; + + unsafe { + let saved = local_irq_mask_save(); + ret = f(); + local_irq_restore(saved); + } + + ret +} diff --git a/18_backtrace/kernel/src/lib.rs b/18_backtrace/kernel/src/lib.rs new file mode 100644 index 00000000..4d7a5f5d --- /dev/null +++ b/18_backtrace/kernel/src/lib.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png" +)] + +//! The `kernel` library. +//! +//! Used to compose the final kernel binary. +//! +//! # Code organization and architecture +//! +//! The code is divided into different *modules*, each representing a typical **subsystem** of the +//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example, +//! `src/memory.rs` contains code that is concerned with all things memory management. +//! +//! ## Visibility of processor architecture code +//! +//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target +//! processor architecture. For each supported processor architecture, there exists a subfolder in +//! `src/_arch`, for example, `src/_arch/aarch64`. +//! +//! The architecture folders mirror the subsystem modules laid out in `src`. For example, +//! architectural code that belongs to the `kernel`'s MMU subsystem (`src/memory/mmu.rs`) would go +//! into `src/_arch/aarch64/memory/mmu.rs`. The latter file is loaded as a module in +//! `src/memory/mmu.rs` using the `path attribute`. Usually, the chosen module name is the generic +//! module's name prefixed with `arch_`. +//! +//! For example, this is the top of `src/memory/mmu.rs`: +//! +//! ``` +//! #[cfg(target_arch = "aarch64")] +//! #[path = "../_arch/aarch64/memory/mmu.rs"] +//! mod arch_mmu; +//! ``` +//! +//! Often times, items from the `arch_ module` will be publicly reexported by the parent module. +//! This way, each architecture specific module can provide its implementation of an item, while the +//! caller must not be concerned which architecture has been conditionally compiled. +//! +//! ## BSP code +//! +//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains +//! target board specific definitions and functions. These are things such as the board's memory map +//! or instances of drivers for devices that are featured on the respective board. +//! +//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the +//! `kernel`'s subsystem modules, but there is no reexporting this time. That means whatever is +//! provided must be called starting from the `bsp` namespace, e.g. `bsp::driver::driver_manager()`. +//! +//! ## Kernel interfaces +//! +//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target +//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of +//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel` +//! code to play nicely with any of the two without much hassle. +//! +//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`, +//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined +//! in the respective subsystem module and help to enforce the idiom of *program to an interface, +//! not an implementation*. For example, there will be a common IRQ handling interface which the two +//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the +//! interface to the rest of the `kernel`. +//! +//! ``` +//! +-------------------+ +//! | Interface (Trait) | +//! | | +//! +--+-------------+--+ +//! ^ ^ +//! | | +//! | | +//! +----------+--+ +--+----------+ +//! | kernel code | | bsp code | +//! | | | arch code | +//! +-------------+ +-------------+ +//! ``` +//! +//! # Summary +//! +//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical +//! locations. Here is an example for the **memory** subsystem: +//! +//! - `src/memory.rs` and `src/memory/**/*` +//! - Common code that is agnostic of target processor architecture and `BSP` characteristics. +//! - Example: A function to zero a chunk of memory. +//! - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code. +//! - Example: An `MMU` interface that defines `MMU` function prototypes. +//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*` +//! - `BSP` specific code. +//! - Example: The board's memory map (physical addresses of DRAM and MMIO devices). +//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*` +//! - Processor architecture specific code. +//! - Example: Implementation of the `MMU` interface for the `__arch_name__` processor +//! architecture. +//! +//! From a namespace perspective, **memory** subsystem code lives in: +//! +//! - `crate::memory::*` +//! - `crate::bsp::memory::*` +//! +//! # Boot flow +//! +//! 1. The kernel's entry point is the function `cpu::boot::arch_boot::_start()`. +//! - It is implemented in `src/_arch/__arch_name__/cpu/boot.s`. +//! 2. Once finished with architectural setup, the arch code calls `kernel_init()`. + +#![allow(clippy::upper_case_acronyms)] +#![allow(incomplete_features)] +#![feature(asm_const)] +#![feature(core_intrinsics)] +#![feature(format_args_nl)] +#![feature(generic_const_exprs)] +#![feature(linkage)] +#![feature(panic_info_message)] +#![feature(step_trait)] +#![feature(trait_alias)] +#![no_std] +// Testing +#![cfg_attr(test, no_main)] +#![feature(custom_test_frameworks)] +#![reexport_test_harness_main = "test_main"] +#![test_runner(crate::test_runner)] + +mod panic_wait; +mod synchronization; + +pub mod backtrace; +pub mod bsp; +pub mod common; +pub mod console; +pub mod cpu; +pub mod driver; +pub mod exception; +pub mod memory; +pub mod print; +pub mod state; +pub mod symbols; +pub mod time; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Version string. +pub fn version() -> &'static str { + concat!( + env!("CARGO_PKG_NAME"), + " version ", + env!("CARGO_PKG_VERSION") + ) +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// The default runner for unit tests. +pub fn test_runner(tests: &[&test_types::UnitTest]) { + // This line will be printed as the test header. + println!("Running {} tests", tests.len()); + + for (i, test) in tests.iter().enumerate() { + print!("{:>3}. {:.<58}", i + 1, test.name); + + // Run the actual test. + (test.test_func)(); + + // Failed tests call panic!(). Execution reaches here only if the test has passed. + println!("[ok]") + } +} + +/// The `kernel_init()` for unit tests. +#[cfg(test)] +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + test_main(); + + cpu::qemu_exit_success() +} diff --git a/18_backtrace/kernel/src/main.rs b/18_backtrace/kernel/src/main.rs new file mode 100644 index 00000000..5150f3af --- /dev/null +++ b/18_backtrace/kernel/src/main.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/rust-embedded/wg/master/assets/logo/ewg-logo-blue-white-on-transparent.png" +)] + +//! The `kernel` binary. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +use libkernel::{bsp, cpu, driver, exception, info, memory, state, time, warn}; + +/// Early init code. +/// +/// When this code runs, virtual memory is already enabled. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +/// - Printing will not work until the respective driver's MMIO is remapped. +#[no_mangle] +unsafe fn kernel_init() -> ! { + use driver::interface::DriverManager; + + exception::handling_init(); + memory::mmu::post_enable_init(); + + // Add the mapping records for the precomputed entries first, so that they appear on the top of + // the list. + bsp::memory::mmu::kernel_add_mapping_records_for_precomputed(); + + // Bring up the drivers needed for printing first. + for i in bsp::driver::driver_manager() + .early_print_device_drivers() + .iter() + { + // Any encountered errors cannot be printed yet, obviously, so just safely park the CPU. + i.init().unwrap_or_else(|_| cpu::wait_forever()); + } + bsp::driver::driver_manager().post_early_print_device_driver_init(); + // Printing available from here on. + + // Now bring up the remaining drivers. + for i in bsp::driver::driver_manager() + .non_early_print_device_drivers() + .iter() + { + if let Err(x) = i.init() { + panic!("Error loading driver: {}: {}", i.compatible(), x); + } + } + + // Let device drivers register and enable their handlers with the interrupt controller. + for i in bsp::driver::driver_manager().all_device_drivers() { + if let Err(msg) = i.register_and_enable_irq_handler() { + warn!("Error registering IRQ handler: {}", msg); + } + } + + // Unmask interrupts on the boot CPU core. + exception::asynchronous::local_irq_unmask(); + + // Announce conclusion of the kernel_init() phase. + state::state_manager().transition_to_single_core_main(); + + // Transition from unsafe to safe. + kernel_main() +} + +/// The main function running after the early init. +fn kernel_main() -> ! { + use driver::interface::DriverManager; + use exception::asynchronous::interface::IRQManager; + + info!("{}", libkernel::version()); + info!("Booting on: {}", bsp::board_name()); + + info!("MMU online:"); + memory::mmu::kernel_print_mappings(); + + let (_, privilege_level) = exception::current_privilege_level(); + info!("Current privilege level: {}", privilege_level); + + info!("Exception handling state:"); + exception::asynchronous::print_state(); + + info!( + "Architectural timer resolution: {} ns", + time::time_manager().resolution().as_nanos() + ); + + info!("Drivers loaded:"); + for (i, driver) in bsp::driver::driver_manager() + .all_device_drivers() + .iter() + .enumerate() + { + info!(" {}. {}", i + 1, driver.compatible()); + } + + info!("Registered IRQ handlers:"); + bsp::exception::asynchronous::irq_manager().print_handler(); + + info!("Echoing input now"); + cpu::wait_forever(); +} diff --git a/18_backtrace/kernel/src/memory.rs b/18_backtrace/kernel/src/memory.rs new file mode 100644 index 00000000..5e8cdbce --- /dev/null +++ b/18_backtrace/kernel/src/memory.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Memory Management. + +pub mod mmu; + +use crate::{bsp, common}; +use core::{ + fmt, + marker::PhantomData, + ops::{Add, Sub}, +}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Metadata trait for marking the type of an address. +pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} + +/// Zero-sized type to mark a physical address. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub enum Physical {} + +/// Zero-sized type to mark a virtual address. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub enum Virtual {} + +/// Generic address type. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub struct Address { + value: usize, + _address_type: PhantomData ATYPE>, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl AddressType for Physical {} +impl AddressType for Virtual {} + +impl Address { + /// Create an instance. + pub const fn new(value: usize) -> Self { + Self { + value, + _address_type: PhantomData, + } + } + + /// Convert to usize. + pub const fn as_usize(self) -> usize { + self.value + } + + /// Align down to page size. + #[must_use] + pub const fn align_down_page(self) -> Self { + let aligned = common::align_down(self.value, bsp::memory::mmu::KernelGranule::SIZE); + + Self::new(aligned) + } + + /// Align up to page size. + #[must_use] + pub const fn align_up_page(self) -> Self { + let aligned = common::align_up(self.value, bsp::memory::mmu::KernelGranule::SIZE); + + Self::new(aligned) + } + + /// Checks if the address is page aligned. + pub const fn is_page_aligned(&self) -> bool { + common::is_aligned(self.value, bsp::memory::mmu::KernelGranule::SIZE) + } + + /// Return the address' offset into the corresponding page. + pub const fn offset_into_page(&self) -> usize { + self.value & bsp::memory::mmu::KernelGranule::MASK + } +} + +impl Add for Address { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: usize) -> Self::Output { + match self.value.checked_add(rhs) { + None => panic!("Overflow on Address::add"), + Some(x) => Self::new(x), + } + } +} + +impl Sub for Address { + type Output = Self; + + #[inline(always)] + fn sub(self, rhs: usize) -> Self::Output { + match self.value.checked_sub(rhs) { + None => panic!("Overflow on Address::sub"), + Some(x) => Self::new(x), + } + } +} + +impl Sub> for Address { + type Output = Self; + + #[inline(always)] + fn sub(self, rhs: Address) -> Self::Output { + match self.value.checked_sub(rhs.value) { + None => panic!("Overflow on Address::sub"), + Some(x) => Self::new(x), + } + } +} + +impl Address { + /// Checks if the address is part of the boot core stack region. + pub fn is_valid_stack_addr(&self) -> bool { + bsp::memory::mmu::virt_boot_core_stack_region().contains(*self) + } + + /// Checks if the address is part of the kernel code region. + pub fn is_valid_code_addr(&self) -> bool { + bsp::memory::mmu::virt_code_region().contains(*self) + } +} + +impl fmt::Display for Address { + // Don't expect to see physical addresses greater than 40 bit. + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let q3: u8 = ((self.value >> 32) & 0xff) as u8; + let q2: u16 = ((self.value >> 16) & 0xffff) as u16; + let q1: u16 = (self.value & 0xffff) as u16; + + write!(f, "0x")?; + write!(f, "{:02x}_", q3)?; + write!(f, "{:04x}_", q2)?; + write!(f, "{:04x}", q1) + } +} + +impl fmt::Display for Address { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let q4: u16 = ((self.value >> 48) & 0xffff) as u16; + let q3: u16 = ((self.value >> 32) & 0xffff) as u16; + let q2: u16 = ((self.value >> 16) & 0xffff) as u16; + let q1: u16 = (self.value & 0xffff) as u16; + + write!(f, "0x")?; + write!(f, "{:04x}_", q4)?; + write!(f, "{:04x}_", q3)?; + write!(f, "{:04x}_", q2)?; + write!(f, "{:04x}", q1) + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Sanity of [Address] methods. + #[kernel_test] + fn address_type_method_sanity() { + let addr = Address::::new(bsp::memory::mmu::KernelGranule::SIZE + 100); + + assert_eq!( + addr.align_down_page().as_usize(), + bsp::memory::mmu::KernelGranule::SIZE + ); + + assert_eq!( + addr.align_up_page().as_usize(), + bsp::memory::mmu::KernelGranule::SIZE * 2 + ); + + assert!(!addr.is_page_aligned()); + + assert_eq!(addr.offset_into_page(), 100); + } +} diff --git a/18_backtrace/kernel/src/memory/mmu.rs b/18_backtrace/kernel/src/memory/mmu.rs new file mode 100644 index 00000000..dfc29993 --- /dev/null +++ b/18_backtrace/kernel/src/memory/mmu.rs @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Memory Management Unit. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/memory/mmu.rs"] +mod arch_mmu; + +mod alloc; +mod mapping_record; +mod translation_table; +mod types; + +use crate::{ + bsp, + memory::{Address, Physical, Virtual}, + synchronization::{self, interface::Mutex}, + warn, +}; +use core::{fmt, num::NonZeroUsize}; + +pub use types::*; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// MMU enable errors variants. +#[allow(missing_docs)] +#[derive(Debug)] +pub enum MMUEnableError { + AlreadyEnabled, + Other(&'static str), +} + +/// Memory Management interfaces. +pub mod interface { + use super::*; + + /// MMU functions. + pub trait MMU { + /// Turns on the MMU for the first time and enables data and instruction caching. + /// + /// # Safety + /// + /// - Changes the HW's global state. + unsafe fn enable_mmu_and_caching( + &self, + phys_tables_base_addr: Address, + ) -> Result<(), MMUEnableError>; + + /// Returns true if the MMU is enabled, false otherwise. + fn is_enabled(&self) -> bool; + } +} + +/// Describes the characteristics of a translation granule. +pub struct TranslationGranule; + +/// Describes properties of an address space. +pub struct AddressSpace; + +/// Intended to be implemented for [`AddressSpace`]. +pub trait AssociatedTranslationTable { + /// A translation table whose address range is: + /// + /// [u64::MAX, (u64::MAX - AS_SIZE) + 1] + type TableStartFromTop; + + /// A translation table whose address range is: + /// + /// [AS_SIZE - 1, 0] + type TableStartFromBottom; +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +use interface::MMU; +use synchronization::interface::ReadWriteEx; +use translation_table::interface::TranslationTable; + +/// Query the BSP for the reserved virtual addresses for MMIO remapping and initialize the kernel's +/// MMIO VA allocator with it. +fn kernel_init_mmio_va_allocator() { + let region = bsp::memory::mmu::virt_mmio_remap_region(); + + alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.initialize(region)); +} + +/// Map a region in the kernel's translation tables. +/// +/// No input checks done, input is passed through to the architectural implementation. +/// +/// # Safety +/// +/// - See `map_at()`. +/// - Does not prevent aliasing. +unsafe fn kernel_map_at_unchecked( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, +) -> Result<(), &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .write(|tables| tables.map_at(virt_region, phys_region, attr))?; + + kernel_add_mapping_record(name, virt_region, phys_region, attr); + + Ok(()) +} + +/// Try to translate a kernel virtual address to a physical address. +/// +/// Will only succeed if there exists a valid mapping for the input address. +fn try_kernel_virt_addr_to_phys_addr( + virt_addr: Address, +) -> Result, &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_virt_addr_to_phys_addr(virt_addr)) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl fmt::Display for MMUEnableError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MMUEnableError::AlreadyEnabled => write!(f, "MMU is already enabled"), + MMUEnableError::Other(x) => write!(f, "{}", x), + } + } +} + +impl TranslationGranule { + /// The granule's size. + pub const SIZE: usize = Self::size_checked(); + + /// The granule's mask. + pub const MASK: usize = Self::SIZE - 1; + + /// The granule's shift, aka log2(size). + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(GRANULE_SIZE.is_power_of_two()); + + GRANULE_SIZE + } +} + +impl AddressSpace { + /// The address space size. + pub const SIZE: usize = Self::size_checked(); + + /// The address space shift, aka log2(size). + pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize; + + const fn size_checked() -> usize { + assert!(AS_SIZE.is_power_of_two()); + + // Check for architectural restrictions as well. + Self::arch_address_space_size_sanity_checks(); + + AS_SIZE + } +} + +/// Add an entry to the mapping info record. +pub fn kernel_add_mapping_record( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, +) { + if let Err(x) = mapping_record::kernel_add(name, virt_region, phys_region, attr) { + warn!("{}", x); + } +} + +/// MMIO remapping in the kernel translation tables. +/// +/// Typically used by device drivers. +/// +/// # Safety +/// +/// - Same as `kernel_map_at_unchecked()`, minus the aliasing part. +pub unsafe fn kernel_map_mmio( + name: &'static str, + mmio_descriptor: &MMIODescriptor, +) -> Result, &'static str> { + let phys_region = MemoryRegion::from(*mmio_descriptor); + let offset_into_start_page = mmio_descriptor.start_addr().offset_into_page(); + + // Check if an identical region has been mapped for another driver. If so, reuse it. + let virt_addr = if let Some(addr) = + mapping_record::kernel_find_and_insert_mmio_duplicate(mmio_descriptor, name) + { + addr + // Otherwise, allocate a new region and map it. + } else { + let num_pages = match NonZeroUsize::new(phys_region.num_pages()) { + None => return Err("Requested 0 pages"), + Some(x) => x, + }; + + let virt_region = + alloc::kernel_mmio_va_allocator().lock(|allocator| allocator.alloc(num_pages))?; + + kernel_map_at_unchecked( + name, + &virt_region, + &phys_region, + &AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + virt_region.start_addr() + }; + + Ok(virt_addr + offset_into_start_page) +} + +/// Try to translate a kernel virtual page address to a physical page address. +/// +/// Will only succeed if there exists a valid mapping for the input page. +pub fn try_kernel_virt_page_addr_to_phys_page_addr( + virt_page_addr: PageAddress, +) -> Result, &'static str> { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_virt_page_addr_to_phys_page_addr(virt_page_addr)) +} + +/// Try to get the attributes of a kernel page. +/// +/// Will only succeed if there exists a valid mapping for the input page. +pub fn try_kernel_page_attributes( + virt_page_addr: PageAddress, +) -> Result { + bsp::memory::mmu::kernel_translation_tables() + .read(|tables| tables.try_page_attributes(virt_page_addr)) +} + +/// Enable the MMU and data + instruction caching. +/// +/// # Safety +/// +/// - Crucial function during kernel init. Changes the the complete memory view of the processor. +#[inline(always)] +pub unsafe fn enable_mmu_and_caching( + phys_tables_base_addr: Address, +) -> Result<(), MMUEnableError> { + arch_mmu::mmu().enable_mmu_and_caching(phys_tables_base_addr) +} + +/// Finish initialization of the MMU subsystem. +pub fn post_enable_init() { + kernel_init_mmio_va_allocator(); +} + +/// Human-readable print of all recorded kernel mappings. +pub fn kernel_print_mappings() { + mapping_record::kernel_print() +} diff --git a/18_backtrace/kernel/src/memory/mmu/alloc.rs b/18_backtrace/kernel/src/memory/mmu/alloc.rs new file mode 100644 index 00000000..aadb72ef --- /dev/null +++ b/18_backtrace/kernel/src/memory/mmu/alloc.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//! Allocation. + +use super::MemoryRegion; +use crate::{ + memory::{AddressType, Virtual}, + synchronization::IRQSafeNullLock, + warn, +}; +use core::num::NonZeroUsize; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// A page allocator that can be lazyily initialized. +pub struct PageAllocator { + pool: Option>, +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static KERNEL_MMIO_VA_ALLOCATOR: IRQSafeNullLock> = + IRQSafeNullLock::new(PageAllocator::new()); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the kernel's MMIO virtual address allocator. +pub fn kernel_mmio_va_allocator() -> &'static IRQSafeNullLock> { + &KERNEL_MMIO_VA_ALLOCATOR +} + +impl PageAllocator { + /// Create an instance. + pub const fn new() -> Self { + Self { pool: None } + } + + /// Initialize the allocator. + pub fn initialize(&mut self, pool: MemoryRegion) { + if self.pool.is_some() { + warn!("Already initialized"); + return; + } + + self.pool = Some(pool); + } + + /// Allocate a number of pages. + pub fn alloc( + &mut self, + num_requested_pages: NonZeroUsize, + ) -> Result, &'static str> { + if self.pool.is_none() { + return Err("Allocator not initialized"); + } + + self.pool + .as_mut() + .unwrap() + .take_first_n_pages(num_requested_pages) + } +} diff --git a/18_backtrace/kernel/src/memory/mmu/mapping_record.rs b/18_backtrace/kernel/src/memory/mmu/mapping_record.rs new file mode 100644 index 00000000..d171c6e6 --- /dev/null +++ b/18_backtrace/kernel/src/memory/mmu/mapping_record.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! A record of mapped pages. + +use super::{ + AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, MemoryRegion, + Physical, Virtual, +}; +use crate::{bsp, info, synchronization, synchronization::InitStateLock, warn}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Type describing a virtual memory mapping. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +struct MappingRecordEntry { + pub users: [Option<&'static str>; 5], + pub phys_start_addr: Address, + pub virt_start_addr: Address, + pub num_pages: usize, + pub attribute_fields: AttributeFields, +} + +struct MappingRecord { + inner: [Option; 12], +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static KERNEL_MAPPING_RECORD: InitStateLock = + InitStateLock::new(MappingRecord::new()); + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl MappingRecordEntry { + pub fn new( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Self { + Self { + users: [Some(name), None, None, None, None], + phys_start_addr: phys_region.start_addr(), + virt_start_addr: virt_region.start_addr(), + num_pages: phys_region.num_pages(), + attribute_fields: *attr, + } + } + + fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> { + if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) { + return Ok(x); + }; + + Err("Storage for user info exhausted") + } + + pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> { + let x = self.find_next_free_user()?; + *x = Some(user); + Ok(()) + } +} + +impl MappingRecord { + pub const fn new() -> Self { + Self { inner: [None; 12] } + } + + fn find_next_free(&mut self) -> Result<&mut Option, &'static str> { + if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) { + return Ok(x); + } + + Err("Storage for mapping info exhausted") + } + + fn find_duplicate( + &mut self, + phys_region: &MemoryRegion, + ) -> Option<&mut MappingRecordEntry> { + self.inner + .iter_mut() + .filter(|x| x.is_some()) + .map(|x| x.as_mut().unwrap()) + .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device) + .find(|x| { + if x.phys_start_addr != phys_region.start_addr() { + return false; + } + + if x.num_pages != phys_region.num_pages() { + return false; + } + + true + }) + } + + pub fn add( + &mut self, + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + let x = self.find_next_free()?; + + *x = Some(MappingRecordEntry::new( + name, + virt_region, + phys_region, + attr, + )); + Ok(()) + } + + pub fn print(&self) { + const KIB_RSHIFT: u32 = 10; // log2(1024). + const MIB_RSHIFT: u32 = 20; // log2(1024 * 1024). + + info!(" -------------------------------------------------------------------------------------------------------------------------------------------"); + info!( + " {:^44} {:^30} {:^7} {:^9} {:^35}", + "Virtual", "Physical", "Size", "Attr", "Entity" + ); + info!(" -------------------------------------------------------------------------------------------------------------------------------------------"); + + for i in self.inner.iter().flatten() { + let size = i.num_pages * bsp::memory::mmu::KernelGranule::SIZE; + let virt_start = i.virt_start_addr; + let virt_end_inclusive = virt_start + (size - 1); + let phys_start = i.phys_start_addr; + let phys_end_inclusive = phys_start + (size - 1); + + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; + + let attr = match i.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::Device => "Dev", + }; + + let acc_p = match i.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if i.attribute_fields.execute_never { + "XN" + } else { + "X" + }; + + info!( + " {}..{} --> {}..{} | \ + {: >3} {} | {: <3} {} {: <2} | {}", + virt_start, + virt_end_inclusive, + phys_start, + phys_end_inclusive, + size, + unit, + attr, + acc_p, + xn, + i.users[0].unwrap() + ); + + for k in i.users[1..].iter() { + if let Some(additional_user) = *k { + info!( + " | {}", + additional_user + ); + } + } + } + + info!(" -------------------------------------------------------------------------------------------------------------------------------------------"); + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use synchronization::interface::ReadWriteEx; + +/// Add an entry to the mapping info record. +pub fn kernel_add( + name: &'static str, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, +) -> Result<(), &'static str> { + KERNEL_MAPPING_RECORD.write(|mr| mr.add(name, virt_region, phys_region, attr)) +} + +pub fn kernel_find_and_insert_mmio_duplicate( + mmio_descriptor: &MMIODescriptor, + new_user: &'static str, +) -> Option> { + let phys_region: MemoryRegion = (*mmio_descriptor).into(); + + KERNEL_MAPPING_RECORD.write(|mr| { + let dup = mr.find_duplicate(&phys_region)?; + + if let Err(x) = dup.add_user(new_user) { + warn!("{}", x); + } + + Some(dup.virt_start_addr) + }) +} + +/// Human-readable print of all recorded kernel mappings. +pub fn kernel_print() { + KERNEL_MAPPING_RECORD.read(|mr| mr.print()); +} diff --git a/18_backtrace/kernel/src/memory/mmu/translation_table.rs b/18_backtrace/kernel/src/memory/mmu/translation_table.rs new file mode 100644 index 00000000..9d627f97 --- /dev/null +++ b/18_backtrace/kernel/src/memory/mmu/translation_table.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2021-2022 Andre Richter + +//! Translation table. + +#[cfg(target_arch = "aarch64")] +#[path = "../../_arch/aarch64/memory/mmu/translation_table.rs"] +mod arch_translation_table; + +use super::{AttributeFields, MemoryRegion}; +use crate::memory::{Address, Physical, Virtual}; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +#[cfg(target_arch = "aarch64")] +pub use arch_translation_table::FixedSizeTranslationTable; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Translation table interfaces. +pub mod interface { + use crate::memory::mmu::PageAddress; + + use super::*; + + /// Translation table operations. + pub trait TranslationTable { + /// Anything that needs to run before any of the other provided functions can be used. + /// + /// # Safety + /// + /// - Implementor must ensure that this function can run only once or is harmless if invoked + /// multiple times. + fn init(&mut self) -> Result<(), &'static str>; + + /// Map the given virtual memory region to the given physical memory region. + /// + /// # Safety + /// + /// - Using wrong attributes can cause multiple issues of different nature in the system. + /// - It is not required that the architectural implementation prevents aliasing. That is, + /// mapping to the same physical memory using multiple virtual addresses, which would + /// break Rust's ownership assumptions. This should be protected against in the kernel's + /// generic MMU code. + unsafe fn map_at( + &mut self, + virt_region: &MemoryRegion, + phys_region: &MemoryRegion, + attr: &AttributeFields, + ) -> Result<(), &'static str>; + + /// Try to translate a virtual page address to a physical page address. + /// + /// Will only succeed if there exists a valid mapping for the input page. + fn try_virt_page_addr_to_phys_page_addr( + &self, + virt_page_addr: PageAddress, + ) -> Result, &'static str>; + + /// Try to get the attributes of a page. + /// + /// Will only succeed if there exists a valid mapping for the input page. + fn try_page_attributes( + &self, + virt_page_addr: PageAddress, + ) -> Result; + + /// Try to translate a virtual address to a physical address. + /// + /// Will only succeed if there exists a valid mapping for the input address. + fn try_virt_addr_to_phys_addr( + &self, + virt_addr: Address, + ) -> Result, &'static str>; + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::mmu::{AccessPermissions, MemAttributes, PageAddress}; + use arch_translation_table::MinSizeTranslationTable; + use interface::TranslationTable; + use test_macros::kernel_test; + + /// Sanity checks for the TranslationTable implementation. + #[kernel_test] + fn translationtable_implementation_sanity() { + // This will occupy a lot of space on the stack. + let mut tables = MinSizeTranslationTable::new_for_runtime(); + + assert!(tables.init().is_ok()); + + let virt_end_exclusive_page_addr: PageAddress = PageAddress::MAX; + let virt_start_page_addr: PageAddress = + virt_end_exclusive_page_addr.checked_offset(-5).unwrap(); + + let phys_start_page_addr: PageAddress = PageAddress::from(0); + let phys_end_exclusive_page_addr: PageAddress = + phys_start_page_addr.checked_offset(5).unwrap(); + + let virt_region = MemoryRegion::new(virt_start_page_addr, virt_end_exclusive_page_addr); + let phys_region = MemoryRegion::new(phys_start_page_addr, phys_end_exclusive_page_addr); + + let attr = AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }; + + unsafe { assert_eq!(tables.map_at(&virt_region, &phys_region, &attr), Ok(())) }; + + assert_eq!( + tables.try_virt_page_addr_to_phys_page_addr(virt_start_page_addr), + Ok(phys_start_page_addr) + ); + + assert_eq!( + tables.try_page_attributes(virt_start_page_addr.checked_offset(-1).unwrap()), + Err("Page marked invalid") + ); + + assert_eq!(tables.try_page_attributes(virt_start_page_addr), Ok(attr)); + + let virt_addr = virt_start_page_addr.into_inner() + 0x100; + let phys_addr = phys_start_page_addr.into_inner() + 0x100; + assert_eq!(tables.try_virt_addr_to_phys_addr(virt_addr), Ok(phys_addr)); + } +} diff --git a/18_backtrace/kernel/src/memory/mmu/types.rs b/18_backtrace/kernel/src/memory/mmu/types.rs new file mode 100644 index 00000000..85c852b3 --- /dev/null +++ b/18_backtrace/kernel/src/memory/mmu/types.rs @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Memory Management Unit types. + +use crate::{ + bsp, common, + memory::{Address, AddressType, Physical}, +}; +use core::{convert::From, iter::Step, num::NonZeroUsize, ops::Range}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// A wrapper type around [Address] that ensures page alignment. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub struct PageAddress { + inner: Address, +} + +/// A type that describes a region of memory in quantities of pages. +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub struct MemoryRegion { + start: PageAddress, + end_exclusive: PageAddress, +} + +/// Architecture agnostic memory attributes. +#[allow(missing_docs)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub enum MemAttributes { + CacheableDRAM, + Device, +} + +/// Architecture agnostic access permissions. +#[allow(missing_docs)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub enum AccessPermissions { + ReadOnly, + ReadWrite, +} + +/// Collection of memory attributes. +#[allow(missing_docs)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq)] +pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, +} + +/// An MMIO descriptor for use in device drivers. +#[derive(Copy, Clone)] +pub struct MMIODescriptor { + start_addr: Address, + end_addr_exclusive: Address, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +//------------------------------------------------------------------------------ +// PageAddress +//------------------------------------------------------------------------------ +impl PageAddress { + /// The largest value that can be represented by this type. + pub const MAX: Self = PageAddress { + inner: Address::new(usize::MAX).align_down_page(), + }; + + /// Unwraps the value. + pub fn into_inner(self) -> Address { + self.inner + } + + /// Calculates the offset from the page address. + /// + /// `count` is in units of [PageAddress]. For example, a count of 2 means `result = self + 2 * + /// page_size`. + pub fn checked_offset(self, count: isize) -> Option { + if count == 0 { + return Some(self); + } + + let delta = count + .unsigned_abs() + .checked_mul(bsp::memory::mmu::KernelGranule::SIZE)?; + let result = if count.is_positive() { + self.inner.as_usize().checked_add(delta)? + } else { + self.inner.as_usize().checked_sub(delta)? + }; + + Some(Self { + inner: Address::new(result), + }) + } +} + +impl From for PageAddress { + fn from(addr: usize) -> Self { + assert!( + common::is_aligned(addr, bsp::memory::mmu::KernelGranule::SIZE), + "Input usize not page aligned" + ); + + Self { + inner: Address::new(addr), + } + } +} + +impl From> for PageAddress { + fn from(addr: Address) -> Self { + assert!(addr.is_page_aligned(), "Input Address not page aligned"); + + Self { inner: addr } + } +} + +impl Step for PageAddress { + fn steps_between(start: &Self, end: &Self) -> Option { + if start > end { + return None; + } + + // Since start <= end, do unchecked arithmetic. + Some( + (end.inner.as_usize() - start.inner.as_usize()) + >> bsp::memory::mmu::KernelGranule::SHIFT, + ) + } + + fn forward_checked(start: Self, count: usize) -> Option { + start.checked_offset(count as isize) + } + + fn backward_checked(start: Self, count: usize) -> Option { + start.checked_offset(-(count as isize)) + } +} + +//------------------------------------------------------------------------------ +// MemoryRegion +//------------------------------------------------------------------------------ +impl MemoryRegion { + /// Create an instance. + pub fn new(start: PageAddress, end_exclusive: PageAddress) -> Self { + assert!(start <= end_exclusive); + + Self { + start, + end_exclusive, + } + } + + fn as_range(&self) -> Range> { + self.into_iter() + } + + /// Returns the start page address. + pub fn start_page_addr(&self) -> PageAddress { + self.start + } + + /// Returns the start address. + pub fn start_addr(&self) -> Address { + self.start.into_inner() + } + + /// Returns the exclusive end page address. + pub fn end_exclusive_page_addr(&self) -> PageAddress { + self.end_exclusive + } + + /// Returns the exclusive end page address. + pub fn end_inclusive_page_addr(&self) -> PageAddress { + self.end_exclusive.checked_offset(-1).unwrap() + } + + /// Checks if self contains an address. + pub fn contains(&self, addr: Address) -> bool { + let page_addr = PageAddress::from(addr.align_down_page()); + self.as_range().contains(&page_addr) + } + + /// Checks if there is an overlap with another memory region. + pub fn overlaps(&self, other_region: &Self) -> bool { + let self_range = self.as_range(); + + self_range.contains(&other_region.start_page_addr()) + || self_range.contains(&other_region.end_inclusive_page_addr()) + } + + /// Returns the number of pages contained in this region. + pub fn num_pages(&self) -> usize { + PageAddress::steps_between(&self.start, &self.end_exclusive).unwrap() + } + + /// Returns the size in bytes of this region. + pub fn size(&self) -> usize { + // Invariant: start <= end_exclusive, so do unchecked arithmetic. + let end_exclusive = self.end_exclusive.into_inner().as_usize(); + let start = self.start.into_inner().as_usize(); + + end_exclusive - start + } + + /// Splits the MemoryRegion like: + /// + /// -------------------------------------------------------------------------------- + /// | | | | | | | | | | | | | | | | | | | + /// -------------------------------------------------------------------------------- + /// ^ ^ ^ + /// | | | + /// left_start left_end_exclusive | + /// | + /// ^ | + /// | | + /// right_start right_end_exclusive + /// + /// Left region is returned to the caller. Right region is the new region for this struct. + pub fn take_first_n_pages(&mut self, num_pages: NonZeroUsize) -> Result { + let count: usize = num_pages.into(); + + let left_end_exclusive = self.start.checked_offset(count as isize); + let left_end_exclusive = match left_end_exclusive { + None => return Err("Overflow while calculating left_end_exclusive"), + Some(x) => x, + }; + + if left_end_exclusive > self.end_exclusive { + return Err("Not enough free pages"); + } + + let allocation = Self { + start: self.start, + end_exclusive: left_end_exclusive, + }; + self.start = left_end_exclusive; + + Ok(allocation) + } +} + +impl IntoIterator for MemoryRegion { + type Item = PageAddress; + type IntoIter = Range; + + fn into_iter(self) -> Self::IntoIter { + Range { + start: self.start, + end: self.end_exclusive, + } + } +} + +impl From for MemoryRegion { + fn from(desc: MMIODescriptor) -> Self { + let start = PageAddress::from(desc.start_addr.align_down_page()); + let end_exclusive = PageAddress::from(desc.end_addr_exclusive().align_up_page()); + + Self { + start, + end_exclusive, + } + } +} + +//------------------------------------------------------------------------------ +// MMIODescriptor +//------------------------------------------------------------------------------ + +impl MMIODescriptor { + /// Create an instance. + pub const fn new(start_addr: Address, size: usize) -> Self { + assert!(size > 0); + let end_addr_exclusive = Address::new(start_addr.as_usize() + size); + + Self { + start_addr, + end_addr_exclusive, + } + } + + /// Return the start address. + pub const fn start_addr(&self) -> Address { + self.start_addr + } + + /// Return the exclusive end address. + pub fn end_addr_exclusive(&self) -> Address { + self.end_addr_exclusive + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::Virtual; + use test_macros::kernel_test; + + /// Sanity of [PageAddress] methods. + #[kernel_test] + fn pageaddress_type_method_sanity() { + let page_addr: PageAddress = + PageAddress::from(bsp::memory::mmu::KernelGranule::SIZE * 2); + + assert_eq!( + page_addr.checked_offset(-2), + Some(PageAddress::::from(0)) + ); + + assert_eq!( + page_addr.checked_offset(2), + Some(PageAddress::::from( + bsp::memory::mmu::KernelGranule::SIZE * 4 + )) + ); + + assert_eq!( + PageAddress::::from(0).checked_offset(0), + Some(PageAddress::::from(0)) + ); + assert_eq!(PageAddress::::from(0).checked_offset(-1), None); + + let max_page_addr = Address::::new(usize::MAX).align_down_page(); + assert_eq!( + PageAddress::::from(max_page_addr).checked_offset(1), + None + ); + + let zero = PageAddress::::from(0); + let three = PageAddress::::from(bsp::memory::mmu::KernelGranule::SIZE * 3); + assert_eq!(PageAddress::steps_between(&zero, &three), Some(3)); + } + + /// Sanity of [MemoryRegion] methods. + #[kernel_test] + fn memoryregion_type_method_sanity() { + let zero = PageAddress::::from(0); + let zero_region = MemoryRegion::new(zero, zero); + assert_eq!(zero_region.num_pages(), 0); + assert_eq!(zero_region.size(), 0); + + let one = PageAddress::::from(bsp::memory::mmu::KernelGranule::SIZE); + let one_region = MemoryRegion::new(zero, one); + assert_eq!(one_region.num_pages(), 1); + assert_eq!(one_region.size(), bsp::memory::mmu::KernelGranule::SIZE); + + let three = PageAddress::::from(bsp::memory::mmu::KernelGranule::SIZE * 3); + let mut three_region = MemoryRegion::new(zero, three); + assert!(three_region.contains(zero.into_inner())); + assert!(!three_region.contains(three.into_inner())); + assert!(three_region.overlaps(&one_region)); + + let allocation = three_region + .take_first_n_pages(NonZeroUsize::new(2).unwrap()) + .unwrap(); + assert_eq!(allocation.num_pages(), 2); + assert_eq!(three_region.num_pages(), 1); + + for (i, alloc) in allocation.into_iter().enumerate() { + assert_eq!( + alloc.into_inner().as_usize(), + i * bsp::memory::mmu::KernelGranule::SIZE + ); + } + } +} diff --git a/18_backtrace/kernel/src/panic_wait.rs b/18_backtrace/kernel/src/panic_wait.rs new file mode 100644 index 00000000..1b67c533 --- /dev/null +++ b/18_backtrace/kernel/src/panic_wait.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! A panic handler that infinitely waits. + +use crate::{backtrace, bsp, cpu, exception}; +use core::{fmt, panic::PanicInfo}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +fn _panic_print(args: fmt::Arguments) { + use fmt::Write; + + unsafe { bsp::console::panic_console_out().write_fmt(args).unwrap() }; +} + +/// The point of exit for `libkernel`. +/// +/// It is linked weakly, so that the integration tests can overload its standard behavior. +#[linkage = "weak"] +#[no_mangle] +fn _panic_exit() -> ! { + #[cfg(not(feature = "test_build"))] + { + cpu::wait_forever() + } + + #[cfg(feature = "test_build")] + { + cpu::qemu_exit_failure() + } +} + +/// Prints with a newline - only use from the panic handler. +/// +/// Carbon copy from +#[macro_export] +macro_rules! panic_println { + ($($arg:tt)*) => ({ + _panic_print(format_args_nl!($($arg)*)); + }) +} + +/// Stop immediately if called a second time. +/// +/// # Note +/// +/// Using atomics here relieves us from needing to use `unsafe` for the static variable. +/// +/// On `AArch64`, which is the only implemented architecture at the time of writing this, +/// [`AtomicBool::load`] and [`AtomicBool::store`] are lowered to ordinary load and store +/// instructions. They are therefore safe to use even with MMU + caching deactivated. +/// +/// [`AtomicBool::load`]: core::sync::atomic::AtomicBool::load +/// [`AtomicBool::store`]: core::sync::atomic::AtomicBool::store +fn panic_prevent_reenter() { + use core::sync::atomic::{AtomicBool, Ordering}; + + #[cfg(not(target_arch = "aarch64"))] + compile_error!("Add the target_arch to above's check if the following code is safe to use"); + + static PANIC_IN_PROGRESS: AtomicBool = AtomicBool::new(false); + + if !PANIC_IN_PROGRESS.load(Ordering::Relaxed) { + PANIC_IN_PROGRESS.store(true, Ordering::Relaxed); + + return; + } + + _panic_exit() +} + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + use crate::time::interface::TimeManager; + + unsafe { exception::asynchronous::local_irq_mask() }; + + // Protect against panic infinite loops if any of the following code panics itself. + panic_prevent_reenter(); + + let timestamp = crate::time::time_manager().uptime(); + let (location, line, column) = match info.location() { + Some(loc) => (loc.file(), loc.line(), loc.column()), + _ => ("???", 0, 0), + }; + + panic_println!( + "[ {:>3}.{:06}] Kernel panic!\n\n\ + Panic location:\n File '{}', line {}, column {}\n\n\ + {}\n\n\ + {}", + timestamp.as_secs(), + timestamp.subsec_micros(), + location, + line, + column, + info.message().unwrap_or(&format_args!("")), + backtrace::Backtrace + ); + + _panic_exit() +} diff --git a/18_backtrace/kernel/src/print.rs b/18_backtrace/kernel/src/print.rs new file mode 100644 index 00000000..9ec13a28 --- /dev/null +++ b/18_backtrace/kernel/src/print.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2022 Andre Richter + +//! Printing. + +use crate::{bsp, console}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +#[doc(hidden)] +pub fn _print(args: fmt::Arguments) { + use console::interface::Write; + + bsp::console::console().write_fmt(args).unwrap(); +} + +/// Prints without a newline. +/// +/// Carbon copy from +#[macro_export] +macro_rules! print { + ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*))); +} + +/// Prints with a newline. +/// +/// Carbon copy from +#[macro_export] +macro_rules! println { + () => ($crate::print!("\n")); + ($($arg:tt)*) => ({ + $crate::print::_print(format_args_nl!($($arg)*)); + }) +} + +/// Prints an info, with a newline. +#[macro_export] +macro_rules! info { + ($string:expr) => ({ + use $crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:06}] ", $string), + timestamp.as_secs(), + timestamp.subsec_micros(), + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + use $crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:06}] ", $format_string), + timestamp.as_secs(), + timestamp.subsec_micros(), + $($arg)* + )); + }) +} + +/// Prints a warning, with a newline. +#[macro_export] +macro_rules! warn { + ($string:expr) => ({ + use $crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:06}] ", $string), + timestamp.as_secs(), + timestamp.subsec_micros(), + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + use $crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:06}] ", $format_string), + timestamp.as_secs(), + timestamp.subsec_micros(), + $($arg)* + )); + }) +} diff --git a/18_backtrace/kernel/src/state.rs b/18_backtrace/kernel/src/state.rs new file mode 100644 index 00000000..0af3688c --- /dev/null +++ b/18_backtrace/kernel/src/state.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! State information about the kernel itself. + +use core::sync::atomic::{AtomicU8, Ordering}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Different stages in the kernel execution. +#[derive(Copy, Clone, Eq, PartialEq)] +enum State { + /// The kernel starts booting in this state. + Init, + + /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of + /// `kernel_init()`, after all init calls are done). + SingleCoreMain, + + /// The kernel transitions to this state when it boots the secondary cores, aka switches + /// exectution mode to symmetric multiprocessing (SMP). + MultiCoreMain, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Maintains the kernel state and state transitions. +pub struct StateManager(AtomicU8); + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static STATE_MANAGER: StateManager = StateManager::new(); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the global StateManager. +pub fn state_manager() -> &'static StateManager { + &STATE_MANAGER +} + +impl StateManager { + const INIT: u8 = 0; + const SINGLE_CORE_MAIN: u8 = 1; + const MULTI_CORE_MAIN: u8 = 2; + + /// Create a new instance. + pub const fn new() -> Self { + Self(AtomicU8::new(Self::INIT)) + } + + /// Return the current state. + fn state(&self) -> State { + let state = self.0.load(Ordering::Acquire); + + match state { + Self::INIT => State::Init, + Self::SINGLE_CORE_MAIN => State::SingleCoreMain, + Self::MULTI_CORE_MAIN => State::MultiCoreMain, + _ => panic!("Invalid KERNEL_STATE"), + } + } + + /// Return if the kernel is init state. + pub fn is_init(&self) -> bool { + self.state() == State::Init + } + + /// Transition from Init to SingleCoreMain. + pub fn transition_to_single_core_main(&self) { + if self + .0 + .compare_exchange( + Self::INIT, + Self::SINGLE_CORE_MAIN, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_err() + { + panic!("transition_to_single_core_main() called while state != Init"); + } + } +} diff --git a/18_backtrace/kernel/src/symbols.rs b/18_backtrace/kernel/src/symbols.rs new file mode 100644 index 00000000..22001389 --- /dev/null +++ b/18_backtrace/kernel/src/symbols.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Debug symbol support. + +use crate::memory::{Address, Virtual}; +use core::{cell::UnsafeCell, slice}; +use debug_symbol_types::Symbol; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// Symbol from the linker script. +extern "Rust" { + static __kernel_symbols_start: UnsafeCell<()>; +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +/// This will be patched to the correct value by the "kernel symbols tool" after linking. This given +/// value here is just a (safe) dummy. +#[no_mangle] +static NUM_KERNEL_SYMBOLS: u64 = 0; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +fn kernel_symbol_section_virt_start_addr() -> Address { + Address::new(unsafe { __kernel_symbols_start.get() as usize }) +} + +fn kernel_symbols_slice() -> &'static [Symbol] { + let ptr = kernel_symbol_section_virt_start_addr().as_usize() as *const Symbol; + + unsafe { + let num = core::ptr::read_volatile(&NUM_KERNEL_SYMBOLS as *const u64) as usize; + slice::from_raw_parts(ptr, num) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Retrieve the symbol corresponding to a virtual address, if any. +pub fn lookup_symbol(addr: Address) -> Option<&'static Symbol> { + for i in kernel_symbols_slice() { + if i.contains(addr.as_usize()) { + return Some(i); + } + } + + None +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Sanity of symbols module. + #[kernel_test] + fn symbols_sanity() { + let first_sym = lookup_symbol(Address::new( + crate::common::is_aligned as *const usize as usize, + )) + .unwrap() + .name(); + + assert_eq!(first_sym, "libkernel::common::is_aligned"); + + let second_sym = lookup_symbol(Address::new(crate::version as *const usize as usize)) + .unwrap() + .name(); + + assert_eq!(second_sym, "libkernel::version"); + } +} diff --git a/18_backtrace/kernel/src/synchronization.rs b/18_backtrace/kernel/src/synchronization.rs new file mode 100644 index 00000000..4b4c4c3f --- /dev/null +++ b/18_backtrace/kernel/src/synchronization.rs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Synchronization primitives. +//! +//! # Resources +//! +//! - +//! - +//! - + +use core::cell::UnsafeCell; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Synchronization interfaces. +pub mod interface { + + /// Any object implementing this trait guarantees exclusive access to the data wrapped within + /// the Mutex for the duration of the provided closure. + pub trait Mutex { + /// The type of the data that is wrapped by this mutex. + type Data; + + /// Locks the mutex and grants the closure temporary mutable access to the wrapped data. + fn lock(&self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + } + + /// A reader-writer exclusion type. + /// + /// The implementing object allows either a number of readers or at most one writer at any point + /// in time. + pub trait ReadWriteEx { + /// The type of encapsulated data. + type Data; + + /// Grants temporary mutable access to the encapsulated data. + fn write(&self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + + /// Grants temporary immutable access to the encapsulated data. + fn read(&self, f: impl FnOnce(&Self::Data) -> R) -> R; + } +} + +/// A pseudo-lock for teaching purposes. +/// +/// In contrast to a real Mutex implementation, does not protect against concurrent access from +/// other cores to the contained data. This part is preserved for later lessons. +/// +/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is +/// executing on a single core. +pub struct IRQSafeNullLock +where + T: ?Sized, +{ + data: UnsafeCell, +} + +/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards. +/// +/// Intended to encapsulate data that is populated during kernel init when no concurrency exists. +pub struct InitStateLock +where + T: ?Sized, +{ + data: UnsafeCell, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +unsafe impl Send for IRQSafeNullLock where T: ?Sized + Send {} +unsafe impl Sync for IRQSafeNullLock where T: ?Sized + Send {} + +impl IRQSafeNullLock { + /// Create an instance. + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), + } + } +} + +unsafe impl Send for InitStateLock where T: ?Sized + Send {} +unsafe impl Sync for InitStateLock where T: ?Sized + Send {} + +impl InitStateLock { + /// Create an instance. + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use crate::{exception, state}; + +impl interface::Mutex for IRQSafeNullLock { + type Data = T; + + fn lock(&self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + // In a real lock, there would be code encapsulating this line that ensures that this + // mutable reference will ever only be given out once at a time. + let data = unsafe { &mut *self.data.get() }; + + // Execute the closure while IRQs are masked. + exception::asynchronous::exec_with_irq_masked(|| f(data)) + } +} + +impl interface::ReadWriteEx for InitStateLock { + type Data = T; + + fn write(&self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + assert!( + state::state_manager().is_init(), + "InitStateLock::write called after kernel init phase" + ); + assert!( + !exception::asynchronous::is_local_irq_masked(), + "InitStateLock::write called with IRQs unmasked" + ); + + let data = unsafe { &mut *self.data.get() }; + + f(data) + } + + fn read(&self, f: impl FnOnce(&Self::Data) -> R) -> R { + let data = unsafe { &*self.data.get() }; + + f(data) + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// InitStateLock must be transparent. + #[kernel_test] + fn init_state_lock_is_transparent() { + use core::mem::size_of; + + assert_eq!(size_of::>(), size_of::()); + } +} diff --git a/18_backtrace/kernel/src/time.rs b/18_backtrace/kernel/src/time.rs new file mode 100644 index 00000000..6d92b196 --- /dev/null +++ b/18_backtrace/kernel/src/time.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! Timer primitives. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/time.rs"] +mod arch_time; + +//-------------------------------------------------------------------------------------------------- +// Architectural Public Reexports +//-------------------------------------------------------------------------------------------------- +pub use arch_time::time_manager; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Timekeeping interfaces. +pub mod interface { + use core::time::Duration; + + /// Time management functions. + pub trait TimeManager { + /// The timer's resolution. + fn resolution(&self) -> Duration; + + /// The uptime since power-on of the device. + /// + /// This includes time consumed by firmware and bootloaders. + fn uptime(&self) -> Duration; + + /// Spin for a given duration. + fn spin_for(&self, duration: Duration); + } +} diff --git a/18_backtrace/kernel/tests/00_console_sanity.rb b/18_backtrace/kernel/tests/00_console_sanity.rb new file mode 100644 index 00000000..4dde5576 --- /dev/null +++ b/18_backtrace/kernel/tests/00_console_sanity.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2019-2022 Andre Richter + +require 'console_io_test' + +# Verify sending and receiving works as expected. +class TxRxHandshakeTest < SubtestBase + def name + 'Transmit and Receive handshake' + end + + def run(qemu_out, qemu_in) + qemu_in.write_nonblock('ABC') + expect_or_raise(qemu_out, 'OK1234') + end +end + +# Check for correct TX statistics implementation. Depends on test 1 being run first. +class TxStatisticsTest < SubtestBase + def name + 'Transmit statistics' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, '6') + end +end + +# Check for correct RX statistics implementation. Depends on test 1 being run first. +class RxStatisticsTest < SubtestBase + def name + 'Receive statistics' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, '3') + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [TxRxHandshakeTest.new, TxStatisticsTest.new, RxStatisticsTest.new] +end diff --git a/18_backtrace/kernel/tests/00_console_sanity.rs b/18_backtrace/kernel/tests/00_console_sanity.rs new file mode 100644 index 00000000..6595aac1 --- /dev/null +++ b/18_backtrace/kernel/tests/00_console_sanity.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2022 Andre Richter + +//! Console sanity tests - RX, TX and statistics. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Console tests should time out on the I/O harness in case of panic. +mod panic_wait_forever; + +use libkernel::{bsp, console, cpu, exception, memory, print}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + use bsp::console::console; + use console::interface::*; + + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + // Handshake + assert_eq!(console().read_char(), 'A'); + assert_eq!(console().read_char(), 'B'); + assert_eq!(console().read_char(), 'C'); + print!("OK1234"); + + // 6 + print!("{}", console().chars_written()); + + // 3 + print!("{}", console().chars_read()); + + // The QEMU process running this test will be closed by the I/O test harness. + cpu::wait_forever(); +} diff --git a/18_backtrace/kernel/tests/01_timer_sanity.rs b/18_backtrace/kernel/tests/01_timer_sanity.rs new file mode 100644 index 00000000..9b2b228d --- /dev/null +++ b/18_backtrace/kernel/tests/01_timer_sanity.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2022 Andre Richter + +//! Timer sanity tests. + +#![feature(custom_test_frameworks)] +#![no_main] +#![no_std] +#![reexport_test_harness_main = "test_main"] +#![test_runner(libkernel::test_runner)] + +use core::time::Duration; +use libkernel::{bsp, cpu, exception, memory, time, time::interface::TimeManager}; +use test_macros::kernel_test; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi. + + test_main(); + + cpu::qemu_exit_success() +} + +/// Simple check that the timer is running. +#[kernel_test] +fn timer_is_counting() { + assert!(time::time_manager().uptime().as_nanos() > 0) +} + +/// Timer resolution must be sufficient. +#[kernel_test] +fn timer_resolution_is_sufficient() { + assert!(time::time_manager().resolution().as_nanos() < 100) +} + +/// Sanity check spin_for() implementation. +#[kernel_test] +fn spin_accuracy_check_1_second() { + let t1 = time::time_manager().uptime(); + time::time_manager().spin_for(Duration::from_secs(1)); + let t2 = time::time_manager().uptime(); + + assert_eq!((t2 - t1).as_secs(), 1) +} diff --git a/18_backtrace/kernel/tests/02_exception_sync_page_fault.rs b/18_backtrace/kernel/tests/02_exception_sync_page_fault.rs new file mode 100644 index 00000000..0d2a1e63 --- /dev/null +++ b/18_backtrace/kernel/tests/02_exception_sync_page_fault.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2022 Andre Richter + +//! Page faults must result in synchronous exceptions. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Overwrites libkernel's `panic_wait::_panic_exit()` so that it returns a "success" code. +/// +/// In this test, reaching the panic is a success, because it is called from the synchronous +/// exception handler, which is what this test wants to achieve. +/// +/// It also means that this integration test can not use any other code that calls panic!() directly +/// or indirectly. +mod panic_exit_success; + +use libkernel::{bsp, cpu, exception, info, memory, println}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + // This line will be printed as the test header. + println!("Testing synchronous exception handling by causing a page fault"); + + info!("Writing to bottom of address space to address 1 GiB..."); + let big_addr: u64 = 1024 * 1024 * 1024; + core::ptr::read_volatile(big_addr as *mut u64); + + // If execution reaches here, the memory access above did not cause a page fault exception. + cpu::qemu_exit_failure() +} diff --git a/18_backtrace/kernel/tests/03_exception_restore_sanity.rb b/18_backtrace/kernel/tests/03_exception_restore_sanity.rb new file mode 100644 index 00000000..5f52e0c7 --- /dev/null +++ b/18_backtrace/kernel/tests/03_exception_restore_sanity.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2022 Andre Richter + +require 'console_io_test' + +# Verify that exception restore works. +class ExceptionRestoreTest < SubtestBase + def name + 'Exception restore' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, 'Back from system call!') + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [ExceptionRestoreTest.new] +end diff --git a/18_backtrace/kernel/tests/03_exception_restore_sanity.rs b/18_backtrace/kernel/tests/03_exception_restore_sanity.rs new file mode 100644 index 00000000..983d488f --- /dev/null +++ b/18_backtrace/kernel/tests/03_exception_restore_sanity.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! A simple sanity test to see if exception restore code works. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Console tests should time out on the I/O harness in case of panic. +mod panic_wait_forever; + +use core::arch::asm; +use libkernel::{bsp, cpu, exception, info, memory, println}; + +#[inline(never)] +fn nested_system_call() { + #[cfg(target_arch = "aarch64")] + unsafe { + asm!("svc #0x1337", options(nomem, nostack, preserves_flags)); + } + + #[cfg(not(target_arch = "aarch64"))] + { + info!("Not supported yet"); + cpu::wait_forever(); + } +} + +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + // This line will be printed as the test header. + println!("Testing exception restore"); + + info!("Making a dummy system call"); + + // Calling this inside a function indirectly tests if the link register is restored properly. + nested_system_call(); + + info!("Back from system call!"); + + // The QEMU process running this test will be closed by the I/O test harness. + cpu::wait_forever(); +} diff --git a/18_backtrace/kernel/tests/04_exception_irq_sanity.rs b/18_backtrace/kernel/tests/04_exception_irq_sanity.rs new file mode 100644 index 00000000..9030424d --- /dev/null +++ b/18_backtrace/kernel/tests/04_exception_irq_sanity.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020-2022 Andre Richter + +//! IRQ handling sanity tests. + +#![feature(custom_test_frameworks)] +#![no_main] +#![no_std] +#![reexport_test_harness_main = "test_main"] +#![test_runner(libkernel::test_runner)] + +use libkernel::{bsp, cpu, exception, memory}; +use test_macros::kernel_test; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + exception::handling_init(); + exception::asynchronous::local_irq_unmask(); + + test_main(); + + cpu::qemu_exit_success() +} + +/// Check that IRQ masking works. +#[kernel_test] +fn local_irq_mask_works() { + // Precondition: IRQs are unmasked. + assert!(exception::asynchronous::is_local_irq_masked()); + + unsafe { exception::asynchronous::local_irq_mask() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + // Restore earlier state. + unsafe { exception::asynchronous::local_irq_unmask() }; +} + +/// Check that IRQ unmasking works. +#[kernel_test] +fn local_irq_unmask_works() { + // Precondition: IRQs are masked. + unsafe { exception::asynchronous::local_irq_mask() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + unsafe { exception::asynchronous::local_irq_unmask() }; + assert!(exception::asynchronous::is_local_irq_masked()); +} + +/// Check that IRQ mask save is saving "something". +#[kernel_test] +fn local_irq_mask_save_works() { + // Precondition: IRQs are unmasked. + assert!(exception::asynchronous::is_local_irq_masked()); + + let first = unsafe { exception::asynchronous::local_irq_mask_save() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + let second = unsafe { exception::asynchronous::local_irq_mask_save() }; + assert_ne!(first, second); + + unsafe { exception::asynchronous::local_irq_restore(first) }; + assert!(exception::asynchronous::is_local_irq_masked()); +} diff --git a/18_backtrace/kernel/tests/05_backtrace_sanity.rb b/18_backtrace/kernel/tests/05_backtrace_sanity.rb new file mode 100644 index 00000000..5650f97c --- /dev/null +++ b/18_backtrace/kernel/tests/05_backtrace_sanity.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2022 Andre Richter + +require 'console_io_test' + +# Verify that panic produces a backtrace. +class PanicBacktraceTest < SubtestBase + def name + 'Panic produces backtrace' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, 'Kernel panic!') + expect_or_raise(qemu_out, 'Backtrace:') + end +end + +# Verify backtrace correctness. +class BacktraceCorrectnessTest < SubtestBase + def name + 'Backtrace is correct' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, '| core::panicking::panic') + expect_or_raise(qemu_out, '| _05_backtrace_sanity::nested') + expect_or_raise(qemu_out, '| kernel_init') + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [PanicBacktraceTest.new, BacktraceCorrectnessTest.new] +end diff --git a/18_backtrace/kernel/tests/05_backtrace_sanity.rs b/18_backtrace/kernel/tests/05_backtrace_sanity.rs new file mode 100644 index 00000000..24229f95 --- /dev/null +++ b/18_backtrace/kernel/tests/05_backtrace_sanity.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Test if backtracing code detects an invalid frame pointer. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Console tests should time out on the I/O harness in case of panic. +mod panic_wait_forever; + +use libkernel::{bsp, cpu, exception, memory}; + +#[inline(never)] +fn nested() { + panic!() +} + +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + nested(); + + // The QEMU process running this test will be closed by the I/O test harness. + cpu::wait_forever() +} diff --git a/18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb b/18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb new file mode 100644 index 00000000..7601cf97 --- /dev/null +++ b/18_backtrace/kernel/tests/06_backtrace_invalid_frame.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2022 Andre Richter + +require 'console_io_test' + +# Test detection of invalid frame pointers. +class InvalidFramePointerTest < SubtestBase + def name + 'Detect invalid frame pointer' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, + /Encountered invalid frame pointer \(.*\) during backtrace/) + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [InvalidFramePointerTest.new] +end diff --git a/18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs b/18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs new file mode 100644 index 00000000..a1874c4e --- /dev/null +++ b/18_backtrace/kernel/tests/06_backtrace_invalid_frame.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Test if backtracing code detects an invalid frame pointer. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Console tests should time out on the I/O harness in case of panic. +mod panic_wait_forever; + +use libkernel::{backtrace, bsp, cpu, exception, memory}; + +#[inline(never)] +fn nested() { + unsafe { backtrace::corrupt_previous_frame_addr() }; + + panic!() +} + +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + nested(); + + // The QEMU process running this test will be closed by the I/O test harness. + cpu::wait_forever() +} diff --git a/18_backtrace/kernel/tests/07_backtrace_invalid_link.rb b/18_backtrace/kernel/tests/07_backtrace_invalid_link.rb new file mode 100644 index 00000000..0fabcf4c --- /dev/null +++ b/18_backtrace/kernel/tests/07_backtrace_invalid_link.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2022 Andre Richter + +require 'console_io_test' + +# Test detection of invalid link. +class InvalidLinkTest < SubtestBase + def name + 'Detect invalid link' + end + + def run(qemu_out, _qemu_in) + expect_or_raise(qemu_out, /Link address \(.*\) is not contained in kernel .text section/) + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [InvalidLinkTest.new] +end diff --git a/18_backtrace/kernel/tests/07_backtrace_invalid_link.rs b/18_backtrace/kernel/tests/07_backtrace_invalid_link.rs new file mode 100644 index 00000000..a0731091 --- /dev/null +++ b/18_backtrace/kernel/tests/07_backtrace_invalid_link.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Test if backtracing code detects an invalid link. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Console tests should time out on the I/O harness in case of panic. +mod panic_wait_forever; + +use libkernel::{backtrace, bsp, cpu, exception, memory}; + +#[inline(never)] +fn nested_2() -> &'static str { + unsafe { backtrace::corrupt_link() }; + libkernel::println!("{}", libkernel::backtrace::Backtrace); + "foo" +} + +#[inline(never)] +fn nested_1() { + libkernel::println!("{}", nested_2()) +} + +#[no_mangle] +unsafe fn kernel_init() -> ! { + exception::handling_init(); + memory::mmu::post_enable_init(); + bsp::console::qemu_bring_up_console(); + + nested_1(); + + // The QEMU process running this test will be closed by the I/O test harness. + cpu::wait_forever() +} diff --git a/18_backtrace/kernel/tests/boot_test_string.rb b/18_backtrace/kernel/tests/boot_test_string.rb new file mode 100644 index 00000000..f778b3d8 --- /dev/null +++ b/18_backtrace/kernel/tests/boot_test_string.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +EXPECTED_PRINT = 'Echoing input now' diff --git a/18_backtrace/kernel/tests/panic_exit_success/mod.rs b/18_backtrace/kernel/tests/panic_exit_success/mod.rs new file mode 100644 index 00000000..908fac51 --- /dev/null +++ b/18_backtrace/kernel/tests/panic_exit_success/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2022 Andre Richter + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +#[no_mangle] +fn _panic_exit() -> ! { + libkernel::cpu::qemu_exit_success() +} diff --git a/18_backtrace/kernel/tests/panic_wait_forever/mod.rs b/18_backtrace/kernel/tests/panic_wait_forever/mod.rs new file mode 100644 index 00000000..7a4effa5 --- /dev/null +++ b/18_backtrace/kernel/tests/panic_wait_forever/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with wait_forever. +#[no_mangle] +fn _panic_exit() -> ! { + libkernel::cpu::wait_forever() +} diff --git a/18_backtrace/kernel_symbols.mk b/18_backtrace/kernel_symbols.mk new file mode 100644 index 00000000..5b51ccfe --- /dev/null +++ b/18_backtrace/kernel_symbols.mk @@ -0,0 +1,103 @@ +## SPDX-License-Identifier: MIT OR Apache-2.0 +## +## Copyright (c) 2018-2022 Andre Richter + +include ../common/format.mk +include ../common/docker.mk + +##-------------------------------------------------------------------------------------------------- +## Check for input variables that need be exported by the calling Makefile +##-------------------------------------------------------------------------------------------------- +ifndef KERNEL_SYMBOLS_TOOL_PATH +$(error KERNEL_SYMBOLS_TOOL_PATH is not set) +endif + +ifndef TARGET +$(error TARGET is not set) +endif + +ifndef KERNEL_SYMBOLS_INPUT_ELF +$(error KERNEL_SYMBOLS_INPUT_ELF is not set) +endif + +ifndef KERNEL_SYMBOLS_OUTPUT_ELF +$(error KERNEL_SYMBOLS_OUTPUT_ELF is not set) +endif + + + +##-------------------------------------------------------------------------------------------------- +## Targets and Prerequisites +##-------------------------------------------------------------------------------------------------- +KERNEL_SYMBOLS_MANIFEST = kernel_symbols/Cargo.toml +KERNEL_SYMBOLS_LINKER_SCRIPT = kernel_symbols/kernel_symbols.ld + +KERNEL_SYMBOLS_RS = $(KERNEL_SYMBOLS_INPUT_ELF)_symbols.rs +KERNEL_SYMBOLS_DEMANGLED_RS = $(shell pwd)/$(KERNEL_SYMBOLS_INPUT_ELF)_symbols_demangled.rs + +KERNEL_SYMBOLS_ELF = target/$(TARGET)/release/kernel_symbols +KERNEL_SYMBOLS_STRIPPED = target/$(TARGET)/release/kernel_symbols_stripped + +# Export for build.rs of kernel_symbols crate. +export KERNEL_SYMBOLS_DEMANGLED_RS + + + +##-------------------------------------------------------------------------------------------------- +## Command building blocks +##-------------------------------------------------------------------------------------------------- +GET_SYMBOLS_SECTION_VIRT_ADDR = $(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) \ + --get_symbols_section_virt_addr $(KERNEL_SYMBOLS_OUTPUT_ELF) + +RUSTFLAGS = -C link-arg=--script=$(KERNEL_SYMBOLS_LINKER_SCRIPT) \ + -C link-arg=--section-start=.rodata=$$($(GET_SYMBOLS_SECTION_VIRT_ADDR)) + +RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) \ + -D warnings \ + -D missing_docs + +COMPILER_ARGS = --target=$(TARGET) \ + --release + +RUSTC_CMD = cargo rustc $(COMPILER_ARGS) --manifest-path $(KERNEL_SYMBOLS_MANIFEST) +OBJCOPY_CMD = rust-objcopy \ + --strip-all \ + -O binary + +EXEC_SYMBOLS_TOOL = ruby $(KERNEL_SYMBOLS_TOOL_PATH)/main.rb + +##------------------------------------------------------------------------------ +## Dockerization +##------------------------------------------------------------------------------ +DOCKER_CMD = docker run -t --rm -v $(shell pwd):/work/tutorial -w /work/tutorial + +# DOCKER_IMAGE defined in include file (see top of this file). +DOCKER_TOOLS = $(DOCKER_CMD) $(DOCKER_IMAGE) + + + +##-------------------------------------------------------------------------------------------------- +## Targets +##-------------------------------------------------------------------------------------------------- +.PHONY: all + +all: + @cp $(KERNEL_SYMBOLS_INPUT_ELF) $(KERNEL_SYMBOLS_OUTPUT_ELF) + + @$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --gen_symbols $(KERNEL_SYMBOLS_OUTPUT_ELF) \ + $(KERNEL_SYMBOLS_RS) + + $(call color_progress_prefix, "Demangling") + @echo Symbol names + @cat $(KERNEL_SYMBOLS_RS) | rustfilt > $(KERNEL_SYMBOLS_DEMANGLED_RS) + + @RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(RUSTC_CMD) + + $(call color_progress_prefix, "Stripping") + @echo Symbols ELF file + @$(OBJCOPY_CMD) $(KERNEL_SYMBOLS_ELF) $(KERNEL_SYMBOLS_STRIPPED) + + @$(DOCKER_TOOLS) $(EXEC_SYMBOLS_TOOL) --patch_data $(KERNEL_SYMBOLS_OUTPUT_ELF) \ + $(KERNEL_SYMBOLS_STRIPPED) + + $(call color_progress_prefix, "Finished") diff --git a/18_backtrace/kernel_symbols/Cargo.toml b/18_backtrace/kernel_symbols/Cargo.toml new file mode 100644 index 00000000..3407aa7e --- /dev/null +++ b/18_backtrace/kernel_symbols/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "kernel_symbols" +version = "0.1.0" +edition = "2021" + +[features] +default = [] +generated_symbols_available = [] + +##-------------------------------------------------------------------------------------------------- +## Dependencies +##-------------------------------------------------------------------------------------------------- + +[dependencies] +debug-symbol-types = { path = "../libraries/debug-symbol-types" } diff --git a/18_backtrace/kernel_symbols/build.rs b/18_backtrace/kernel_symbols/build.rs new file mode 100644 index 00000000..5062df44 --- /dev/null +++ b/18_backtrace/kernel_symbols/build.rs @@ -0,0 +1,14 @@ +use std::{env, path::Path}; + +fn main() { + if let Ok(path) = env::var("KERNEL_SYMBOLS_DEMANGLED_RS") { + if Path::new(&path).exists() { + println!("cargo:rustc-cfg=feature=\"generated_symbols_available\"") + } + } + + println!( + "cargo:rerun-if-changed={}", + Path::new("kernel_symbols.ld").display() + ); +} diff --git a/18_backtrace/kernel_symbols/kernel_symbols.ld b/18_backtrace/kernel_symbols/kernel_symbols.ld new file mode 100644 index 00000000..0625f008 --- /dev/null +++ b/18_backtrace/kernel_symbols/kernel_symbols.ld @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT OR Apache-2.0 + * + * Copyright (c) 2022 Andre Richter + */ + +SECTIONS +{ + .rodata : { + ASSERT(. > 0xffffffff00000000, "Expected higher half address") + + KEEP(*(.rodata.symbol_desc*)) + . = ALIGN(8); + *(.rodata*) + } +} diff --git a/18_backtrace/kernel_symbols/src/main.rs b/18_backtrace/kernel_symbols/src/main.rs new file mode 100644 index 00000000..bd90b535 --- /dev/null +++ b/18_backtrace/kernel_symbols/src/main.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Generation of kernel symbols. + +#![no_std] +#![no_main] + +#[cfg(feature = "generated_symbols_available")] +include!(env!("KERNEL_SYMBOLS_DEMANGLED_RS")); + +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + unimplemented!() +} diff --git a/18_backtrace/libraries/debug-symbol-types/Cargo.toml b/18_backtrace/libraries/debug-symbol-types/Cargo.toml new file mode 100644 index 00000000..e5b1fd1f --- /dev/null +++ b/18_backtrace/libraries/debug-symbol-types/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "debug-symbol-types" +version = "0.1.0" +edition = "2021" diff --git a/18_backtrace/libraries/debug-symbol-types/src/lib.rs b/18_backtrace/libraries/debug-symbol-types/src/lib.rs new file mode 100644 index 00000000..b6dff082 --- /dev/null +++ b/18_backtrace/libraries/debug-symbol-types/src/lib.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2022 Andre Richter + +//! Types for implementing debug symbol support. + +#![no_std] + +use core::ops::Range; + +/// A symbol containing a size. +#[repr(C)] +#[derive(Clone)] +pub struct Symbol { + addr_range: Range, + name: &'static str, +} + +impl Symbol { + /// Create an instance. + pub const fn new(start: usize, size: usize, name: &'static str) -> Symbol { + Symbol { + addr_range: Range { + start, + end: start + size, + }, + name, + } + } + + /// Returns true if addr is contained in the range. + pub fn contains(&self, addr: usize) -> bool { + self.addr_range.contains(&addr) + } + + /// Returns the symbol's name. + pub fn name(&self) -> &'static str { + self.name + } + + /// Returns the symbol's size. + pub fn size(&self) -> usize { + self.addr_range.end - self.addr_range.start + } +} diff --git a/18_backtrace/libraries/test-macros/Cargo.toml b/18_backtrace/libraries/test-macros/Cargo.toml new file mode 100644 index 00000000..fff98a1f --- /dev/null +++ b/18_backtrace/libraries/test-macros/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "test-macros" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.x" +quote = "1.x" +syn = { version = "1.x", features = ["full"] } +test-types = { path = "../test-types" } diff --git a/18_backtrace/libraries/test-macros/src/lib.rs b/18_backtrace/libraries/test-macros/src/lib.rs new file mode 100644 index 00000000..9879677c --- /dev/null +++ b/18_backtrace/libraries/test-macros/src/lib.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2022 Andre Richter + +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::{parse_macro_input, Ident, ItemFn}; + +#[proc_macro_attribute] +pub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream { + let f = parse_macro_input!(input as ItemFn); + + let test_name = &format!("{}", f.sig.ident); + let test_ident = Ident::new( + &format!("{}_TEST_CONTAINER", f.sig.ident.to_string().to_uppercase()), + Span::call_site(), + ); + let test_code_block = f.block; + + quote!( + #[test_case] + const #test_ident: test_types::UnitTest = test_types::UnitTest { + name: #test_name, + test_func: || #test_code_block, + }; + ) + .into() +} diff --git a/18_backtrace/libraries/test-types/Cargo.toml b/18_backtrace/libraries/test-types/Cargo.toml new file mode 100644 index 00000000..2f20f060 --- /dev/null +++ b/18_backtrace/libraries/test-types/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "test-types" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2021" diff --git a/18_backtrace/libraries/test-types/src/lib.rs b/18_backtrace/libraries/test-types/src/lib.rs new file mode 100644 index 00000000..922c2a1c --- /dev/null +++ b/18_backtrace/libraries/test-types/src/lib.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2022 Andre Richter + +//! Types for the `custom_test_frameworks` implementation. + +#![no_std] + +/// Unit test container. +pub struct UnitTest { + /// Name of the test. + pub name: &'static str, + + /// Function pointer to the test. + pub test_func: fn(), +} diff --git a/18_backtrace/tools/kernel_symbols_tool/cmds.rb b/18_backtrace/tools/kernel_symbols_tool/cmds.rb new file mode 100644 index 00000000..fe66ea71 --- /dev/null +++ b/18_backtrace/tools/kernel_symbols_tool/cmds.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2022 Andre Richter + +def generate_symbols(kernel_elf, output_file) + File.open(output_file, 'w') do |file| + header = <<~HEREDOC + use debug_symbol_types::Symbol; + + # [no_mangle] + # [link_section = ".rodata.symbol_desc"] + static KERNEL_SYMBOLS: [Symbol; #{kernel_elf.num_symbols}] = [ + HEREDOC + + file.write(header) + kernel_elf.symbols.each do |sym| + value = sym.header.st_value + size = sym.header.st_size + name = sym.name + + file.write(" Symbol::new(#{value}, #{size}, \"#{name}\"),\n") + end + file.write("];\n") + end +end + +def get_symbols_section_virt_addr(kernel_elf) + kernel_elf.kernel_symbols_section_virt_addr +end + +def patch_symbol_data(kernel_elf, symbols_blob_path) + symbols_blob = File.binread(symbols_blob_path) + + raise if symbols_blob.size > kernel_elf.kernel_symbols_section_size + + File.binwrite(kernel_elf.path, File.binread(symbols_blob_path), + kernel_elf.kernel_symbols_section_offset_in_file) +end + +def patch_num_symbols(kernel_elf) + num_packed = [kernel_elf.num_symbols].pack('Q<*') # "Q" == uint64_t, "<" == little endian + File.binwrite(kernel_elf.path, num_packed, kernel_elf.num_kernel_symbols_offset_in_file) +end diff --git a/18_backtrace/tools/kernel_symbols_tool/kernel_elf.rb b/18_backtrace/tools/kernel_symbols_tool/kernel_elf.rb new file mode 100644 index 00000000..b1649767 --- /dev/null +++ b/18_backtrace/tools/kernel_symbols_tool/kernel_elf.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2021-2022 Andre Richter + +# KernelELF +class KernelELF + attr_reader :path + + def initialize(kernel_elf_path, kernel_symbols_section, num_kernel_symbols) + @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path)) + @symtab_section = @elf.section_by_name('.symtab') + + @path = kernel_elf_path + fetch_values(kernel_symbols_section, num_kernel_symbols) + end + + private + + def fetch_values(kernel_symbols_section, num_kernel_symbols) + sym = @symtab_section.symbol_by_name(num_kernel_symbols) + raise "Symbol \"#{num_kernel_symbols}\" not found" if sym.nil? + + @num_kernel_symbols = sym + + section = @elf.section_by_name(kernel_symbols_section) + raise "Section \"#{kernel_symbols_section}\" not found" if section.nil? + + @kernel_symbols_section = section + end + + def num_kernel_symbols_virt_addr + @num_kernel_symbols.header.st_value + end + + def segment_containing_virt_addr(virt_addr) + @elf.each_segments do |segment| + return segment if segment.vma_in?(virt_addr) + end + end + + def virt_addr_to_file_offset(virt_addr) + segment = segment_containing_virt_addr(virt_addr) + segment.vma_to_offset(virt_addr) + end + + public + + def symbols + non_zero_symbols = @symtab_section.symbols.reject { |sym| sym.header.st_size.zero? } + non_zero_symbols.sort_by { |sym| sym.header.st_value } + end + + def num_symbols + symbols.size + end + + def kernel_symbols_section_virt_addr + @kernel_symbols_section.header.sh_addr.to_i + end + + def kernel_symbols_section_size + @kernel_symbols_section.header.sh_size.to_i + end + + def kernel_symbols_section_offset_in_file + virt_addr_to_file_offset(kernel_symbols_section_virt_addr) + end + + def num_kernel_symbols_offset_in_file + virt_addr_to_file_offset(num_kernel_symbols_virt_addr) + end +end diff --git a/18_backtrace/tools/kernel_symbols_tool/main.rb b/18_backtrace/tools/kernel_symbols_tool/main.rb new file mode 100755 index 00000000..30a8be6f --- /dev/null +++ b/18_backtrace/tools/kernel_symbols_tool/main.rb @@ -0,0 +1,47 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2022 Andre Richter + +require 'rubygems' +require 'bundler/setup' +require 'colorize' +require 'elftools' + +require_relative 'kernel_elf' +require_relative 'cmds' + +KERNEL_SYMBOLS_SECTION = '.kernel_symbols' +NUM_KERNEL_SYMBOLS = 'NUM_KERNEL_SYMBOLS' + +cmd = ARGV[0] + +kernel_elf_path = ARGV[1] +kernel_elf = KernelELF.new(kernel_elf_path, KERNEL_SYMBOLS_SECTION, NUM_KERNEL_SYMBOLS) + +case cmd +when '--gen_symbols' + output_file = ARGV[2] + + print 'Generating'.rjust(12).green.bold + puts ' Symbols source file' + + generate_symbols(kernel_elf, output_file) +when '--get_symbols_section_virt_addr' + addr = get_symbols_section_virt_addr(kernel_elf) + + puts "0x#{addr.to_s(16)}" +when '--patch_data' + symbols_blob_path = ARGV[2] + num_symbols = kernel_elf.num_symbols + + print 'Patching'.rjust(12).green.bold + puts " Symbols blob and number of symbols (#{num_symbols}) into ELF" + + patch_symbol_data(kernel_elf, symbols_blob_path) + patch_num_symbols(kernel_elf) +else + raise +end diff --git a/18_backtrace/tools/translation_table_tool/arch.rb b/18_backtrace/tools/translation_table_tool/arch.rb new file mode 100644 index 00000000..deceb6d0 --- /dev/null +++ b/18_backtrace/tools/translation_table_tool/arch.rb @@ -0,0 +1,314 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2021-2022 Andre Richter + +# Bitfield manipulation. +class BitField + def initialize + @value = 0 + end + + def self.attr_bitfield(name, offset, num_bits) + define_method("#{name}=") do |bits| + mask = (2**num_bits) - 1 + + raise "Input out of range: #{name} = 0x#{bits.to_s(16)}" if (bits & ~mask).positive? + + # Clear bitfield + @value &= ~(mask << offset) + + # Set it + @value |= (bits << offset) + end + end + + def to_i + @value + end + + def size_in_byte + 8 + end +end + +# An array class that knows its memory location. +class CArray < Array + attr_reader :phys_start_addr + + def initialize(phys_start_addr, size, &block) + @phys_start_addr = phys_start_addr + + super(size, &block) + end + + def size_in_byte + inject(0) { |sum, n| sum + n.size_in_byte } + end +end + +#--------------------------------------------------------------------------------------------------- +# Arch:: +#--------------------------------------------------------------------------------------------------- +module Arch +#--------------------------------------------------------------------------------------------------- +# Arch::ARMv8 +#--------------------------------------------------------------------------------------------------- +module ARMv8 +# ARMv8 Table Descriptor. +class Stage1TableDescriptor < BitField + module NextLevelTableAddr + OFFSET = 16 + NUMBITS = 32 + end + + module Type + OFFSET = 1 + NUMBITS = 1 + + BLOCK = 0 + TABLE = 1 + end + + module Valid + OFFSET = 0 + NUMBITS = 1 + + FALSE = 0 + TRUE = 1 + end + + attr_bitfield(:__next_level_table_addr, NextLevelTableAddr::OFFSET, NextLevelTableAddr::NUMBITS) + attr_bitfield(:type, Type::OFFSET, Type::NUMBITS) + attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS) + + def next_level_table_addr=(addr) + addr = addr >> Granule64KiB::SHIFT + + self.__next_level_table_addr = addr + end + + private :__next_level_table_addr= +end + +# ARMv8 level 3 page descriptor. +class Stage1PageDescriptor < BitField + module UXN + OFFSET = 54 + NUMBITS = 1 + + FALSE = 0 + TRUE = 1 + end + + module PXN + OFFSET = 53 + NUMBITS = 1 + + FALSE = 0 + TRUE = 1 + end + + module OutputAddr + OFFSET = 16 + NUMBITS = 32 + end + + module AF + OFFSET = 10 + NUMBITS = 1 + + FALSE = 0 + TRUE = 1 + end + + module SH + OFFSET = 8 + NUMBITS = 2 + + INNER_SHAREABLE = 0b11 + end + + module AP + OFFSET = 6 + NUMBITS = 2 + + RW_EL1 = 0b00 + RO_EL1 = 0b10 + end + + module AttrIndx + OFFSET = 2 + NUMBITS = 3 + end + + module Type + OFFSET = 1 + NUMBITS = 1 + + RESERVED_INVALID = 0 + PAGE = 1 + end + + module Valid + OFFSET = 0 + NUMBITS = 1 + + FALSE = 0 + TRUE = 1 + end + + attr_bitfield(:uxn, UXN::OFFSET, UXN::NUMBITS) + attr_bitfield(:pxn, PXN::OFFSET, PXN::NUMBITS) + attr_bitfield(:__output_addr, OutputAddr::OFFSET, OutputAddr::NUMBITS) + attr_bitfield(:af, AF::OFFSET, AF::NUMBITS) + attr_bitfield(:sh, SH::OFFSET, SH::NUMBITS) + attr_bitfield(:ap, AP::OFFSET, AP::NUMBITS) + attr_bitfield(:attr_indx, AttrIndx::OFFSET, AttrIndx::NUMBITS) + attr_bitfield(:type, Type::OFFSET, Type::NUMBITS) + attr_bitfield(:valid, Valid::OFFSET, Valid::NUMBITS) + + def output_addr=(addr) + addr = addr >> Granule64KiB::SHIFT + + self.__output_addr = addr + end + + private :__output_addr= +end + +# Translation table representing the structure defined in translation_table.rs. +class TranslationTable + module MAIR + NORMAL = 1 + end + + def initialize + do_sanity_checks + + num_lvl2_tables = BSP.kernel_virt_addr_space_size >> Granule512MiB::SHIFT + + @lvl3 = new_lvl3(num_lvl2_tables, BSP.phys_addr_of_kernel_tables) + + @lvl2_phys_start_addr = @lvl3.phys_start_addr + @lvl3.size_in_byte + @lvl2 = new_lvl2(num_lvl2_tables, @lvl2_phys_start_addr) + + populate_lvl2_entries + end + + def map_at(virt_region, phys_region, attributes) + return if virt_region.empty? + + raise if virt_region.size != phys_region.size + raise if phys_region.last > BSP.phys_addr_space_end_page + + virt_region.zip(phys_region).each do |virt_page, phys_page| + desc = page_descriptor_from(virt_page) + set_lvl3_entry(desc, phys_page, attributes) + end + end + + def to_binary + data = @lvl3.flatten.map(&:to_i) + @lvl2.map(&:to_i) + data.pack('Q<*') # "Q" == uint64_t, "<" == little endian + end + + def phys_tables_base_addr_binary + [@lvl2_phys_start_addr].pack('Q<*') # "Q" == uint64_t, "<" == little endian + end + + def phys_tables_base_addr + @lvl2_phys_start_addr + end + + private + + def do_sanity_checks + raise unless BSP.kernel_granule::SIZE == Granule64KiB::SIZE + raise unless (BSP.kernel_virt_addr_space_size % Granule512MiB::SIZE).zero? + end + + def new_lvl3(num_lvl2_tables, start_addr) + CArray.new(start_addr, num_lvl2_tables) do + temp = CArray.new(start_addr, 8192) do + Stage1PageDescriptor.new + end + start_addr += temp.size_in_byte + + temp + end + end + + def new_lvl2(num_lvl2_tables, start_addr) + CArray.new(start_addr, num_lvl2_tables) do + Stage1TableDescriptor.new + end + end + + def populate_lvl2_entries + @lvl2.each_with_index do |descriptor, i| + descriptor.next_level_table_addr = @lvl3[i].phys_start_addr + descriptor.type = Stage1TableDescriptor::Type::TABLE + descriptor.valid = Stage1TableDescriptor::Valid::TRUE + end + end + + def lvl2_lvl3_index_from(addr) + addr -= BSP.kernel_virt_start_addr + + lvl2_index = addr >> Granule512MiB::SHIFT + lvl3_index = (addr & Granule512MiB::MASK) >> Granule64KiB::SHIFT + + raise unless lvl2_index < @lvl2.size + + [lvl2_index, lvl3_index] + end + + def page_descriptor_from(virt_addr) + lvl2_index, lvl3_index = lvl2_lvl3_index_from(virt_addr) + + @lvl3[lvl2_index][lvl3_index] + end + + # rubocop:disable Metrics/MethodLength + def set_attributes(desc, attributes) + case attributes.mem_attributes + when :CacheableDRAM + desc.sh = Stage1PageDescriptor::SH::INNER_SHAREABLE + desc.attr_indx = MAIR::NORMAL + else + raise 'Invalid input' + end + + desc.ap = case attributes.acc_perms + when :ReadOnly + Stage1PageDescriptor::AP::RO_EL1 + when :ReadWrite + Stage1PageDescriptor::AP::RW_EL1 + else + raise 'Invalid input' + + end + + desc.pxn = if attributes.execute_never + Stage1PageDescriptor::PXN::TRUE + else + Stage1PageDescriptor::PXN::FALSE + end + + desc.uxn = Stage1PageDescriptor::UXN::TRUE + end + # rubocop:enable Metrics/MethodLength + + def set_lvl3_entry(desc, output_addr, attributes) + desc.output_addr = output_addr + desc.af = Stage1PageDescriptor::AF::TRUE + desc.type = Stage1PageDescriptor::Type::PAGE + desc.valid = Stage1PageDescriptor::Valid::TRUE + + set_attributes(desc, attributes) + end +end +end +end diff --git a/18_backtrace/tools/translation_table_tool/bsp.rb b/18_backtrace/tools/translation_table_tool/bsp.rb new file mode 100644 index 00000000..536a2f21 --- /dev/null +++ b/18_backtrace/tools/translation_table_tool/bsp.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2021-2022 Andre Richter + +# Raspberry Pi 3 + 4 +class RaspberryPi + attr_reader :kernel_granule, :kernel_virt_addr_space_size, :kernel_virt_start_addr + + MEMORY_SRC = File.read('kernel/src/bsp/raspberrypi/memory.rs').split("\n") + + def initialize + @kernel_granule = Granule64KiB + + @kernel_virt_addr_space_size = KERNEL_ELF.symbol_value('__kernel_virt_addr_space_size') + @kernel_virt_start_addr = KERNEL_ELF.symbol_value('__kernel_virt_start_addr') + + @virt_addr_of_kernel_tables = KERNEL_ELF.symbol_value('KERNEL_TABLES') + @virt_addr_of_phys_kernel_tables_base_addr = KERNEL_ELF.symbol_value( + 'PHYS_KERNEL_TABLES_BASE_ADDR' + ) + end + + def phys_addr_of_kernel_tables + KERNEL_ELF.virt_to_phys(@virt_addr_of_kernel_tables) + end + + def kernel_tables_offset_in_file + KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_kernel_tables) + end + + def phys_kernel_tables_base_addr_offset_in_file + KERNEL_ELF.virt_addr_to_file_offset(@virt_addr_of_phys_kernel_tables_base_addr) + end + + def phys_addr_space_end_page + x = MEMORY_SRC.grep(/pub const END/) + x = case BSP_TYPE + when :rpi3 + x[0] + when :rpi4 + x[1] + else + raise + end + + x.scan(/\d+/).join.to_i(16) + end +end diff --git a/18_backtrace/tools/translation_table_tool/generic.rb b/18_backtrace/tools/translation_table_tool/generic.rb new file mode 100644 index 00000000..13df0658 --- /dev/null +++ b/18_backtrace/tools/translation_table_tool/generic.rb @@ -0,0 +1,179 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2021-2022 Andre Richter + +module Granule64KiB + SIZE = 64 * 1024 + SHIFT = Math.log2(SIZE).to_i +end + +module Granule512MiB + SIZE = 512 * 1024 * 1024 + SHIFT = Math.log2(SIZE).to_i + MASK = SIZE - 1 +end + +# Monkey-patch Integer with some helper functions. +class Integer + def power_of_two? + self[0].zero? + end + + def aligned?(alignment) + raise unless alignment.power_of_two? + + (self & (alignment - 1)).zero? + end + + def align_up(alignment) + raise unless alignment.power_of_two? + + (self + alignment - 1) & ~(alignment - 1) + end + + def to_hex_underscore(with_leading_zeros: false) + fmt = with_leading_zeros ? '%016x' : '%x' + value = format(fmt, self).to_s.reverse.scan(/.{4}|.+/).join('_').reverse + + format('0x%s', value) + end +end + +# An array where each value is the start address of a Page. +class MemoryRegion < Array + def initialize(start_addr, size, granule_size) + raise unless start_addr.aligned?(granule_size) + raise unless size.positive? + raise unless (size % granule_size).zero? + + num_pages = size / granule_size + super(num_pages) do |i| + (i * granule_size) + start_addr + end + end +end + +# Collection of memory attributes. +class AttributeFields + attr_reader :mem_attributes, :acc_perms, :execute_never + + def initialize(mem_attributes, acc_perms, execute_never) + @mem_attributes = mem_attributes + @acc_perms = acc_perms + @execute_never = execute_never + end + + def to_s + x = case @mem_attributes + when :CacheableDRAM + 'C' + else + '?' + end + + y = case @acc_perms + when :ReadWrite + 'RW' + when :ReadOnly + 'RO' + else + '??' + end + + z = @execute_never ? 'XN' : 'X ' + + "#{x} #{y} #{z}" + end +end + +# A container that describes a virt-to-phys region mapping. +class MappingDescriptor + @max_section_name_length = 'Sections'.length + + class << self + attr_accessor :max_section_name_length + + def update_max_section_name_length(length) + @max_section_name_length = [@max_section_name_length, length].max + end + end + + attr_reader :name, :virt_region, :phys_region, :attributes + + def initialize(name, virt_region, phys_region, attributes) + @name = name + @virt_region = virt_region + @phys_region = phys_region + @attributes = attributes + end + + def to_s + name = @name.ljust(self.class.max_section_name_length) + virt_start = @virt_region.first.to_hex_underscore(with_leading_zeros: true) + phys_start = @phys_region.first.to_hex_underscore(with_leading_zeros: true) + size = ((@virt_region.size * 65_536) / 1024).to_s.rjust(3) + + "#{name} | #{virt_start} | #{phys_start} | #{size} KiB | #{@attributes}" + end + + def self.print_divider + print ' ' + print '-' * max_section_name_length + puts '--------------------------------------------------------------------' + end + + def self.print_header + print_divider + print ' ' + print 'Sections'.center(max_section_name_length) + print ' ' + print 'Virt Start Addr'.center(21) + print ' ' + print 'Phys Start Addr'.center(21) + print ' ' + print 'Size'.center(7) + print ' ' + print 'Attr'.center(7) + puts + print_divider + end +end + +def kernel_map_binary + mapping_descriptors = KERNEL_ELF.generate_mapping_descriptors + + # Generate_mapping_descriptors updates the header being printed with this call. So it must come + # afterwards. + MappingDescriptor.print_header + + mapping_descriptors.each do |i| + print 'Generating'.rjust(12).green.bold + print ' ' + puts i.to_s + + TRANSLATION_TABLES.map_at(i.virt_region, i.phys_region, i.attributes) + end + + MappingDescriptor.print_divider +end + +def kernel_patch_tables(kernel_elf_path) + print 'Patching'.rjust(12).green.bold + print ' Kernel table struct at ELF file offset ' + puts BSP.kernel_tables_offset_in_file.to_hex_underscore + + File.binwrite(kernel_elf_path, TRANSLATION_TABLES.to_binary, BSP.kernel_tables_offset_in_file) +end + +def kernel_patch_base_addr(kernel_elf_path) + print 'Patching'.rjust(12).green.bold + print ' Kernel tables physical base address start argument to value ' + print TRANSLATION_TABLES.phys_tables_base_addr.to_hex_underscore + print ' at ELF file offset ' + puts BSP.phys_kernel_tables_base_addr_offset_in_file.to_hex_underscore + + File.binwrite(kernel_elf_path, TRANSLATION_TABLES.phys_tables_base_addr_binary, + BSP.phys_kernel_tables_base_addr_offset_in_file) +end diff --git a/18_backtrace/tools/translation_table_tool/kernel_elf.rb b/18_backtrace/tools/translation_table_tool/kernel_elf.rb new file mode 100644 index 00000000..f2d5b0b7 --- /dev/null +++ b/18_backtrace/tools/translation_table_tool/kernel_elf.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2021-2022 Andre Richter + +# KernelELF +class KernelELF + SECTION_FLAG_ALLOC = 2 + + def initialize(kernel_elf_path) + @elf = ELFTools::ELFFile.new(File.open(kernel_elf_path)) + @symtab_section = @elf.section_by_name('.symtab') + end + + def machine + @elf.machine.to_sym + end + + def symbol_value(symbol_name) + @symtab_section.symbol_by_name(symbol_name).header.st_value + end + + def segment_containing_virt_addr(virt_addr) + @elf.each_segments do |segment| + return segment if segment.vma_in?(virt_addr) + end + end + + def virt_to_phys(virt_addr) + segment = segment_containing_virt_addr(virt_addr) + translation_offset = segment.header.p_vaddr - segment.header.p_paddr + + virt_addr - translation_offset + end + + def virt_addr_to_file_offset(virt_addr) + segment = segment_containing_virt_addr(virt_addr) + segment.vma_to_offset(virt_addr) + end + + def sections_in_segment(segment) + head = segment.mem_head + tail = segment.mem_tail + + sections = @elf.each_sections.select do |section| + file_offset = section.header.sh_addr + flags = section.header.sh_flags + + file_offset >= head && file_offset < tail && (flags & SECTION_FLAG_ALLOC != 0) + end + + sections.map(&:name).join(' ') + end + + def select_load_segments + @elf.each_segments.select do |segment| + segment.instance_of?(ELFTools::Segments::LoadSegment) + end + end + + def segment_get_acc_perms(segment) + if segment.readable? && segment.writable? + :ReadWrite + elsif segment.readable? + :ReadOnly + else + :Invalid + end + end + + def update_max_section_name_length(descriptors) + MappingDescriptor.update_max_section_name_length(descriptors.map { |i| i.name.size }.max) + end + + def generate_mapping_descriptors + descriptors = select_load_segments.map do |segment| + # Assume each segment is page aligned. + size = segment.mem_size.align_up(BSP.kernel_granule::SIZE) + virt_start_addr = segment.header.p_vaddr + phys_start_addr = segment.header.p_paddr + acc_perms = segment_get_acc_perms(segment) + execute_never = !segment.executable? + section_names = sections_in_segment(segment) + + virt_region = MemoryRegion.new(virt_start_addr, size, BSP.kernel_granule::SIZE) + phys_region = MemoryRegion.new(phys_start_addr, size, BSP.kernel_granule::SIZE) + attributes = AttributeFields.new(:CacheableDRAM, acc_perms, execute_never) + + MappingDescriptor.new(section_names, virt_region, phys_region, attributes) + end + + update_max_section_name_length(descriptors) + descriptors + end +end diff --git a/18_backtrace/tools/translation_table_tool/main.rb b/18_backtrace/tools/translation_table_tool/main.rb new file mode 100755 index 00000000..6419e364 --- /dev/null +++ b/18_backtrace/tools/translation_table_tool/main.rb @@ -0,0 +1,46 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2021-2022 Andre Richter + +require 'rubygems' +require 'bundler/setup' +require 'colorize' +require 'elftools' + +require_relative 'generic' +require_relative 'kernel_elf' +require_relative 'bsp' +require_relative 'arch' + +BSP_TYPE = ARGV[0].to_sym +kernel_elf_path = ARGV[1] + +start = Time.now + +KERNEL_ELF = KernelELF.new(kernel_elf_path) + +BSP = case BSP_TYPE + when :rpi3, :rpi4 + RaspberryPi.new + else + raise + end + +TRANSLATION_TABLES = case KERNEL_ELF.machine + when :AArch64 + Arch::ARMv8::TranslationTable.new + else + raise + end + +kernel_map_binary +kernel_patch_tables(kernel_elf_path) +kernel_patch_base_addr(kernel_elf_path) + +elapsed = Time.now - start + +print 'Finished'.rjust(12).green.bold +puts " in #{elapsed.round(2)}s" diff --git a/doc/18_stack_frames.png b/doc/18_stack_frames.png new file mode 100644 index 0000000000000000000000000000000000000000..781732f6c155981444b6afac4ab0cb067694cf70 GIT binary patch literal 47923 zcmdqJby$?`w>CT`7VxNqk}9Qyh?KO7fI~M(N=r8bLmG(Ef`GI%44p%VNJvTyNDLs& z49w6y@Lhw?^V{#<-`?-L_xIoTavUDW+;iVotvJ_O=ej>CE6R{wqPYZtKuDp_pQ%D1 zXY3%5v!oZ!gHKqZZ_;60-aY*6cW z-AO8fe`fONMXTc|aHml2}y;h@TmToAP%2UKoe*C?R z+5h1o{Y-~m_kZ{9m&S0naj`btI3saHzcW04&gJj3ghVp^yHRe@x~z<9sAb9*=idLb znI*4Vj6c;`OJ4i4ng7#SC}%0spF4kklsF5#^y!m1*ff3k-^S+}kl>f6k|3NkW8SuY4wD$9_U%y`8*eJ~{h;K12 zepD$zoCk!&`1^mRX#YIzvA~T-4qxjD+qyNFKgV#3_MN}~()wKIze$Nlb@iF{7t_P99oo*r>VddnM?~WH7 z?n{$N^vRFr(zWW2#bXlEqW3hT?L)aX3E~tHMMv}+&CP6 zaAng{M1TMzBV%_gugqL$3=G2`%X+b~D`{n;OjsP%ghE3gHTk-065c#kh}@b_X@8AR zsl(n^WXQz;bD>#)qk>eb|0p?y@HDg~4z=ty-F{IEcDv08!JP2^C((u+W@#52cO*+R z=)2x@e5>}?DEQ3pbN%l0GJ_hWkfv*m2!A1d6jq#(#BNTWCUhqXlVG9C}>l+ z&I&#{n?l^;A{=37ayWuTZW+-z&D>s6_EfhsnoA$e1+F!!lhF&OPqze(X~bL=G`)Co zgM;`R+GXjJ)s~Pnk-AwA7to(vXg_~e@Yl+Cm2*V;+DzN^lY!apRNn3kq_{VYxeuNP z+2Q;5cC_S4z1QCC%#4Ya{`QIB0d3zJHXPFj!x6v#U)kM}mz#@yI@K;Y`mQe$h*knp z(o~>d^I5{z=Wl@s&ty6X!{}`CLse7moYN4f%u`r^JYpbfaO215`#PCOLpozRd(jF_ zpCcbPBn+edMMult-q{04!(4Z=jK{`;#-~(&jT$2^eya)P?lJzz`1p8KwWJ|!W;Yi3 zJVMyTwy?K8_SJw8B=p1!IHZ1m7G(Dw?V#;RB{x476!H<3vGH5jXYpMHpk7EtR6_Ro`MeM1!Geb8ogs8#E5xnBIT}>eQD~>+!O1Zh{3#8&?oi*E@sgGX>|yL> z%OTIrrBHaKGc8n>pWR}Gjwm}onr3fF1yc5{a*OkY&e*qD69SJjbbV0V*38hYxWVp0 zAxMq~9Ny8~++1HxhPw<#kuUgtW@~PK-n@3l4Wq3cH|JwmqDtI_kbM*?ea>4T`Am3) zef?f5t~&T+fCnDybDwJ3d?n6Yzjm#kQ_vUl6qRSFKj5?bw5rx=#EZ;t=EPEmH^0OE z%6hkZ{7t9D>b7NceD$s~<=xK`-Gqs}#4}UwL<8wZ$IQc{68YXcdb$Qto}2!n;^ptt zOBE)><(MFtA&|#pj%(vATUhbO*a$A^^tq>E~aH`7xbg>g|Q$Z#~a8-aI)?p=TpgCDjd5Cgqr4Pl13ryTTx* zzyuNRb6XIx`uOo97;Ydb-?PkU%dpkJR>NAW@7;)rMlpPpH_K_KOgRLm0c^z2}ohJce^m zPqRRgb`$+I+;#Rlo{kW`fGJ>I9LPDeEB!LLV(-{9+C!mg_ei$AWR7tcTpfEMrsq$l z$DoxWTXa5*-jr^#+C$De8Ll>5XaF-R{QLSXlEuCSOMaiC6i2W1c4!E_=!C!6X{V%f zm#G$`v<&`sq_;>MoSg?M3)|WS!@}0kZwl*C+YPYlBCW?g({pnyN=gSq0n8_o@mI!a zIjgA;f*~eBEvZd)lf(J(NWEVikcwNx%cf~xu*I?aYyzx9DZ}_ZI62Bjx%rWJE&fA2 zjbL-iFl5U81d@e^#A&>W%2_vi+i8Vv(jcp;uc@gKPZCh2rx7gx2N{vhE$y2Vko+ol zJ?age8$BdPAznb?_U+qN6SW?hM7s)sgg2`rR8;CFF=lj=wT^0U4R5jswM>6AIJHwR zakML$lCo{v@`MPZ+Fu*^HiQ-PAen8^MG8UYKKN>~Q$>L}5kNB(H(%|L!OJ{J0?M^2 zyl&DfJ^C`|=H$6BG)my0|H%XydHugPe*Xe0CK^zpKDVFeHSba`(5qTT{DE+OHp*Wg zp8~Ex;~Me$fAGkDHV^D4Lz_I*=|&G#T~SGQQlQ`s(VR0Ti!x6%`YU60|pf_6IQfWWPCgS*K3AdgYe}(X2lXg&iGwP_nDv zMQ@n2g`&pn8*J(D=o+nZD^$toeE4liKC8jIGrO}tA}?LKRHCy`K5Q^RZRll0XISMV zB^5|ftdk`Z#tT^dupaJs3_w*_qGO_ON z?xxypXtAw$bFeR(GUNRS6Y@WzT~k?A^`o=1({j$9*EvzI%E_!cR80F1h>#CG(Y*Af z_M^0FDsP1d_3Y8c=|2B)|4UgCy*pP;lf_=yskE>Pj=LS<^O_ZM|U;yeeS5uD9iZgdy%lhKZw9=Gd3} zHrII7%5DD_*Q=HzEwx3z9_UC%dp#G`aP-N1M57_^6En(MRZwjKqDu^pjf{*6aPOB+ zB0ZrFGv!;gEUUpkYWJW`kLH8s}x_8O8m(*OL4Yh`H$n0<|cK2LN3uaG{5d2l*tnqea_C# z4x!AVt)GO?vjdoN;?RSBRNN`Lz5O`|rtbm=DYqj3<-rre!46Ehu*-IJstGfcubYWI zIlB^-9H$KRTqA`5R~`Pgow!r^BIT-e2iVGL+IAyRQBgjRbyHHh zN6Ob(<%>ib!K|`9Hs%!wZ^$c(GTku4LG3x0Ij^05!&L_JE#7(}yu2>UPD%cHZWdOS|_5keOTSvbLwz*#@>bEYG1JOq+;r_qf5#MVEqxYalkcH*dR>9UYJ z)Oqei$Hc_s)|kToWPci2IzHb^E3Dt|K%px+9{inPXd?njzv~`z+x=_r-jdlP;b?@S zCAFW>)p5VPqVKZRzcB{mW=?W;;srPm)HLe*Pn|dQpNliCng$p2$1k1i=);e&|6byu zr$U3}x{_S<4K~=hk!H2^3r?wXB|dw{JgbYBJLh&knvs*?~vcohmXuK1vI$hXdo7?(pOsNsOyw7D2R$081ntJvo9=9tjQ0dc9$?T!n+jXLf+8p+m*f=0Xx*RdS;g_WczTP%zZ&4jnBw90(ABPTj*f!N(6x&fW5vmTVYZXg@}5`D^nPtI z_N+m6>Q_^wcpro=BImGtyLA^aM}tiJP8~8G@xy|h1Ym;wOibk*MBG;^g<&50DFu=Spr@VUA%xgH+gQ@P= zfRW#~$Oy3xX3yu{3ZoI)m(#EQOmXFW9j~It`Yr~gWfOn;`~r=HPwo3FH}62L#wzj| zJZ7&_?=0eW))tY2*c1PNCHjYwjDf*yqkQ#AT~4Z&HnzL>aqoK)8?q@m1Q$zt7dRe1 z)QKoF!!m^2;MYi#u8mEd2;z28A?)*S{@9HLX$bqw10pmyc*I``Qh8fmztI#aD>Hre zONmHBS!5^@X1C=@cu{WUksK%f3!Obyczi*u|L957ef@8-`qJlH!x#&nygd(^L%MTzi<+L|2q`as4^Y;oB=NE})w=-wt5RPf1m_v73e(R&)s}tghqFQe!mlBOT zW_guhgVFh(#Mvk=alaH;TU%S-bQuvgfIw~}2)C_laL;4I%HxA)f}iR-X&}yGvsl!E zXF$ZQaO9QLd~jjx>L|{ZAE7nq`PPsp)Wq1hd~;YrHR@n5hCd>mb!WJ{qW)CDdY=sA z(uhSe>JyWU9-0!khlR2!kUHb@ndY3SbedbI+0>k$Q1kq1=}lazqbN9(z`$xNHMAUX z<*QSUwv$&8OfrfzX%`Bd=48vZ@e&9zPjbvR|5Z*TY;D?%KnvIA;VC9t)Nj3DDsXWUH9%}2}2Ep`t8P35WNn|4T+&+``mBe zdTzEclC4+|-=TUt0av+tRR!ct3*yENl^7TpmZJ3!7R(jm>z`wfk^J2b>Qzo2Vxu`x zKsjIR9UPcdbH`HBdn-hWC#JPX_mozA&pIDm{|zt zo;ayB=wlLV@9 zD$c`}-2@S23{ZMYEs=aitJEQK{$d?g;r+O3ag-s4rd^Sa9WY6w4aeE_aoZqcmsoH> zGhC6YRIzKa!Ya142{CycB?5BK(lcyc`8RnRhHHgp28$(w_&O8Cy$z1vv-;Ot4;v4r zsM(t^a^15u-555rMgi?mylD5XB4KWQ?m1H#RQ2`gJ=KKc-v_-K*~&guRaIuNVTjr- z;`X;#HrY0}+O{sdC$$o`eX;iL-Pv;Q$`}FL2}>`nTyryiYfVFzKx#D`y@XIkiF}|{ zH_}M73Z7x-CM3&YJB5mSr}nWW<(Tm~S9CQcH*|e-Q@N(1TfN53Y|q`(8e37PgFIrC zdiLxdI|jkvn_5`^;=vOoH%p6hGJ*4qx-FYlmPdkG7vNh6A?(9A0fGoq8CZ$sM1muHeknAkL%)V9?Fl}3 zKx?2rirF70l4sUH$RP_$7wb1G1kM0cwJ^!^RJR;K$!CvygeuTrk*We_6_tUd9W8Q< zzjp1~Aa-s?9jT}h#cjxnu4vFNlaJV3m(xcrpl}^Af?*Xl=rpj^vrDl@6sn$ojXEV^ zhfPqvjdy2$q0#!}<aRCZ!fd+?`SW zjltR1S)0Al=xX>@wRFgyY^ga2k$lDk6!=#AY}1e!V>#C>#B9Qk~w0sFhe<3XOUy4I#mh`>I#5U5(s7UTDOZ!SrJfDA`hYK zGbAtfFo-Byjn9bVGjc%e1V$WKVOCvwg-)k*^Y2Aq0ea#HDf8+k>Q1SH}waOOSJpQoZlkXnH=@q+e3GY{9d zw)&>=;+^V=*H{8?Cfh4nxO`TKhh=L2YKrcwC5h)($Bi2hU6kEKZ3eiIV;i4o8BKce zqWVZl5ACVjWoS>${E071!cU%LYz;)*(>(4!<>z|#5yUS|h^3N}JIN`Bu-d7W;CM6s zU4UWV>$h!X&I{tUht{5fQEVXilq@Va>-4=Wa`r66z{f}nm+9HqE7&M5+L!VwV->dc z)lK=*TY+Rt7CNoLH`8uQzJDGuinK3ek&C_yvs>W2`*!d=nfnv{q*vC?9x_T2ZqCkX zQ5{Y2-+MinEo17r-sItw6*i>(q`26~3V&8P`6JcXhf57Gn;?-h;~*8l0tKPe;BHE? zaB2`_>Ke$eMWnB;=HZpDj6Jgbp0Av8*sA`0Gx|HMY(?bh)|frEY-jSVu-9&SX!4`{ z#YO88CtGNBxJ_?vsRcK};B6Z(`h9sgb|D>hj@z(4??)t?ob3K73Dfq!p6Uv4P;1MQ z(hj<~*VvPKsWdHdy3ed6>h5hUL?s*EMr9lHVJP%v0z*PHUcGu%p?A3*kt#*cyY3~@ zh@J@I1EA~T#fxN?v=F_I$1!G%S^4ZY)eFih459mu71qN?oaL??^IW;{1UqzvmgGcy zHoxzoZF%CZM+X=Hs_%Wiudf60piZuWkoe_^y0P0UWmYIz5L>Au z?a(gM-!GQq24#1UHp}4^ry42UY)8_PsSZ=U_DUAGeSUwR+Fot$F7uXAbr>sg4NFZc z9N{OhgSgYILo8dBg@CH{U=rJ2l)NXytRpw19C|8qMr z-?$}Er^YS+>Vyu`ExR1X5AOTXYH%u7T^Zs#IKIXVN z^f8rc38dUwNcLq1^(qw=6{~^FkfXf?MlIN72<{TttN!`!_;UD`)l*tXV+vR^_}T0W zD5PyW*rxBJR--aUga9<~b0^4GFm$Zk`*9RUj9@4XFZ9x@K>gnH5L#3;e{r!R(fm8o zG2(y4++%#DGr8-9)vS#^tbEuk3^ps*q3`_qhJE(=kfb8(JvW)l7cbtfJ@uINuV27G z&6iEFLdB9j^+++p4?;<5`K2Qz;w{wB`ZYV)J?csR&^}rr{eQRs&I{Z9?N{$j20@P$ zV{1HHkq?ar)%6<~tm#DHzEzEl5mtS*pBxW|#!S@eJYiWpCRz6c1QcVqWiXRJn-N;b zYRLmr?r*dZ)8NMQkoNnEq#(4li(79ASYLyVeis5+Gy+MmYD@tEpxq=s>@#U;>z^Vs zgGG+iOr}IfIn>pMP^+;DEGf6v5D>>t`VL|c5!czehbooKGk0&@S_ z0~>+qYdcu%0yULrf^o+iiYFCqyosu{YztP%X5AkVEFr5E9vkMwlh6w#&c{wsH(|B{ z?wV_f3Veb2xkM=?&ANi!ZjOlBH~Jw<)C_iwHyWApkNc%ZNJ1^~4J58BIvtPso!ot0 zeG!s~3K7x%r+CuD^ba5I0vs+jw8vjJ(vUS;W`~$Q@~Nh~nv*E9W7*n;523t&=5*i^ zzycBJLDcjO4SJ;{G>e2C1JvqOHD;*w@pzsUKV#qJUYKTnTomPB%JRg(?&yu7s{B}< zMrlc_w`lus!VkpQ?|cwv3p>vTk5<^`0%glqNt3>*b>7eON>C^S0HdWQg$RH?wT^X> z%1WYp)u5Va&Tm*Ialhy{1TML6Mrl*S*f_O7KdA4+ zC$$h1XQVR>HarFM#74!``BwqRG_yY_O79?vQhEHCC(90zC_w%r&8xMg-*I=<$YhpFDM?hP-m8ki zed-j+HvWF3JG4a4(Ulu_-_DxIWVyn0LJ5xD zgHeLqEw4onNWQ^B*pj_e$wgQF_P1B!m#Rk|YCPluA!)X1#w{@M*&+b~fiR25XC2^k zXNm;qZ!0k{iv7xaom=z!FQm+bPT8xJfh?t5$pEtb1AK!9NGu0)5Nqp2auo_u-|J@F z+&8wjHCTHszIrwnQ)Z`a><(miDd)ib_6&?N zuER?ccwTvlE`mzLrVHMVM%eK~kmp&*Jswc}@}rxT9w#MT-V057wf{N#xabc0Co7aK zEUAAuxksgwePwgx!#xnZYK|#_+{-u1NrXqnQ1vyRmOgzLF~~-Ls7ax96r^BGp2FZD zkJ*WZ>TpSBX69^rc9de0dYvOZ!tAtqs=8y#WZ>{pqy>fBVcE3t3^UZR*fEZ8 znO92s8Rjh%pE0uCO)I$dY`|``Mr?3t=M+TfDiBdWI^&~3|9ek$(Fa!8tsO(PQUL|E zsF0wLfkW_7$^0u76xqe!Mp^2Or4WKDDN@=$+|_TON;^EajtDorJ;(iJ61Qz^V(fZ}(NhK_hQO%1h&wHTzr1asfx#d8 zN7O8$*a`~+DL7JV_cqLP#B?k7Z%XTb9Lm^;q44D}@Hvv2UBFxHl(V!=lH&}BwSOnS z*#Y|N*RQ(*N=JwP*->>QsQtrd-qu&Uav1mrQmAiLqk^5~sJR9{^BCUaxjnb}gTXdg z@LPXCa(&@JoS-ZUH|c9PBsf5|lzeQfvFpF8zVMbKMhmN$zvZ7``KYYTW$WI%AmbH}6XDVov>6~Fyr;cg@pSH`!WUivIa}dr zpc%rMrMfImHs{n!5K9$% z`zY+a9{r&D%zE^`HKBu4v(UrzScN!8P~pfej8W7=AM`WM>4KWmzANL`#r8$@1&}as z0f!{rAf{N#+KcKktXD;iYUCM+aE%>}rX+>crhl>KiAV=L6sz3@#$WWKbbE9?iQ@K$ zo|Qga(%f##`GARYV*$jKN7@ydXt*o%sxT*2kTf|>ZfvA^mdA=@N&x?6xsgu&E^O=I zYoTLE>XFm{r-akg4R69yf(w3bYC=-Dawpu}S3oP7dtaT4!RIHY&f@!(-qphL_`$*h zh?HL7^NrCgrG&oVi-y*HU3h*++@o^L&pWy}k{t$OT|nqDD50RL)j0BZEu)y1dMAgm z#Rp01cqBY$j;VWYWyV&k=E?2XaKNgmcHM~f(LJnL^9HgMtWrPC5=1-6#>~7}72s*v zmYLWJ)3DOeY$n0vYJW5aH7m7WYXih09Z(m}dzP4yaIN=NEjU9!s)Y%5A1Rr(i%`_QVPA({Ris#LxX@;N_W$+ zr|X;}l1$=C-pH1#O6(nLT-8!>R$bR|59rH*(h>8>k*fDoa0Wk7##Q8ptap7|2;oHY_E&6&SG|P zA$+`Fs*@S2R5z$0X0TEqy0`vnXGkotii(N6_gGq6uy8WsP8qRe%-TaAGc8sIoX@6F z!_|Epwnf-^-KKgtxQZ&x5wE}(%GS+TC{qJ_u3>kKJA)ZLyr~~jjQ}i`I3A;DX+`>I zyP6?jDU-xpHP3j~o4l(Cx1!{8?5R{wtNINxc3B`}S8YVh*lqU6;d68n5jm%GdGRDW z=TBG{0T=FG|GLLv&XH|gMKRgd`rT?}-Ia#0{YcDhG7_OunFKHMcrwt|E6+36ceOsY zzI}ImV~fG^=s{G(#1C45J#Vl}wq85KjPW3g4nj1O-n-k;8EfIZ&q*`-vQR=MOWkk*$ zptV6&%)k5fwJrc--CxF-m98Vo3@k3u1R|T(WxlMb{k?;wG5!r=of0sf0ENa3nvi7m+pW53lyzGtdIJ?z{FL8I&3#z&D`I|hBcIWt^E3;9hLi6HhVcc zhf=*L9FtL?+37jIE`Rl#-SKlp+zz7_$X}AKSi7zTw34$bocbK`bOM*}nP(sY*jih! z4r&Xe^`n^it%e>u`zB`q;n2>HxoROAAQH0gREgS<*=9u5>mP*7JV`sRooKZ)U*w~i zH8Am^#>h*Y+D(yX*kpj~VD-A~*6%F5XZoW+l&)@GTwOb!IW^B~0N?YUWNi0PBl%cagM?r>yYk$T@Q8sKUVaxC$B| z8P;xGoX+0h*~uU_uq2#!!v<7-Z5@E5Aj)iy)lJ1Oua&@7;^2nlKQ?BN~epQMr~|a+TW7WZdjIo=j%+VFJw``l&w;IaKVO{_rf%F6<+$C2c=l9P3 z7$-jXOtt^OIDYszO~G0a>zjKlTby3c%yn$!pL!>CbE|Nmr0lScU+Y`p;^J$QOXzYX zT|qmg;_8XSl=ao4wAKaH*DQb$(R5`DDZuVD_Q_jj(*>IZu*<`G&;2MnvWvDA9bL|q zY8e*8}9 z;k}vGLT`AJf8)50Lx%QAC&-2_GxZm$WP4lBJO(5S)4ypV<)lmA@LCr(c;M)dtO@%gB+d1V>M#xzV{8Y&VVx1 z_fL_RM0|u%^@c;tzB1hsU%x~}#BS^)$E$e@=1pEe57&ND6A8D3Y6;tV29e6vGC1fWAqYk@%%(|mq_0F0q`#}AS-}|FDI)a45j}X8?SnY?Nng_chQ0254*=O-ht`GgQmLI0G>knDb$M-#P@Bolf3Fn;T66=F319a zpi+3McdV9VJaO9woePX$*J@o=Nt4;!@81O`Bw<+|N3tc1^%LUT3X^4-FXA(a-U$sz zdVmozC_|93{FYEmW<9CW`&+R>-mLE0RZFiP!=Du@MR#u6d|6q*XH8Zd=jAXe4GRuX zXN(SW-WGonKSW4Vz#jFjo=QW_pt82M7f-FPl^;(g<|Y}f?+h$c(#KYlTA=?HDV*a% zuino%!(sNrr-cF9d z+!Y8YcM^0 zum#4QaDp6oBz#8kUS;^=)cjfBFtk6>7)2MZf4CRVLmRkX z>dWlK0o(F!Lzzwoax{HDAE7}kjow2RwEm0{_L=PhS zoyxD_jTCKwLY=rIz2eHaz2$gP>?- z8W{-^&p|*;-St`yw@niu*=Fb}l0VGfe0^dP0fgAz+t*KvyRoliAYO+r+>%D)GuoylCMo*>W*M zM<~5$0YG@rokOBC%tfrI-?Jy}juRcW(!H(0k*&W_(dxL~Xw0D{(R^M3OPdUs<5=JY z&9&KpidgOOD<>M0qH_AKj~2nx9D7hZZ)^S_yIXtg;W0e{xfTZo0!rMXZmX58q_HpQ zB(`0$)W=S_yJvbG+!0{iI^XqU^0E`GcbBbwfhjBVSNq;PeHx8;y*T{Q3&@xKH9nRJ zo`+w?*Cg4>9J_tfL~I$Y6O9<{Jb@p-RG>HUh+)0sC&kz~)Awn~Z^fP>G?kmpNrPv0 z_PxH~GDmweeM)sSRRjIy9*Wp%IT3g{0RwJ_-`jHor(hHzK zd9~9-7qparB{KG2eNY`kBtj4Wq%IhX+qXdOyJ3^(OO^nKaI=;7OmL1BKkcA0z)<~l zM~Ssgo|Z)E48zDRG;e)MRAWtCqg&>2`S#Mp>?eJ^8>UZMX>qll>0U>IgtviGIb741 zvsC=s7q5*C@0xAv*~YK|(mq;EdwCw?JwU6C#ns$QGHOUEyy?quKcOx=h7lWyJAHp6 zlk?|t@BGa+AlUa>TvT|+$-BZ@Kt8Sg6i5Z|5wddN>rGi9mX0b~sr>Mt+94jE34jWg zeJw*a8-(is5vP@cjHgRmJp~gRATR7M$A7f6`@TFspE=br=jwpJ4TUoOMYMl_khrzp zNR2{BUM}}a_it^GfPT8F0+KrLy|v*@mX%&N zPgHX!jFX$2MZtt_wU2gxwhmMTCu8yk0n9?Bj4>e$kce?KnzGtC8Cc2Q8@D-TwVtar z#3w9SXxFH4iA~~P5Kj!+-#dAT2&gF=kfi6O76qZyvP!udCF$NQeaMXmI;#COI{CRs zttCK&E7tj-NGFt&MiP2+z#u4wy}z=zhMv-3=fau0_q>vh3~Hcg zxL8ifoLuMFu*`x|^6)+-ZQ1g{oOE^#Fn=azcDo)l^V)#S>3;nVLY$UIO11yU*Valj zl`R>c2T*9P_G2D@`pW}hmwoz%NRJh)^Hg(7h_wCcgV%c}$+p|;lqlXBP&=ke zOkC|6sooIr*!*07BdS^f%C1*dl<6ZVq(NIj^C-1p=&)K7r0OC*mvxvDShrMzlT5$A z049jLb}hllW3K-Th|&ghn?sW|L6|Ik6W}>gkdOx`IAkHt%WBe=QJyAsfUW1%ioLIF z1erg>(5j$JhUZEgsnud9qc}2m(%j>{JsCDjuS%4Et-05SedS zxZ2*O^l_11&Yd&PfAV0@K`YVLk31fo!v=d)hJ^(XD~ob3pjdpb1|j)JuqqT_CY;NT zIbR!YwJ}XR9Up9cO%klk+9B#*m*G&7l?m5#$UZtvPio^Hb{wwOxM)Qf$oLy~rLo+4ka^ z6^uCae*LfvF?|4n8X^Tbqcf^sjoTHqzS6J=NXkR`rk-)nVLp2nA9c#?3jn9NpHmG)U#&fK`IE5w zQOUle$KA1WJ}(XIsD{Y>6G3}d>W2>(ym!zr`1+47d(n#!A%&IEa!@U}Ps$+joVx+V zF>`685fBTBylQt`YJpzpAc&|t;<>{Dsdm&}I(!8=s4kcIVxD4QS3Y@ghD6Z{JVznN zEAMZ3VfZ6c7KNLrJZJ;TarbLL4Ch8${f_J+<_9NMJYDyU-#H!3YWjCHUAl7LI`6Ti z0N9sM0b=536@`pQU)bONS@Z+1%f;w(TMx+KKx|PdG31u;l{n}BxfGNzDW8kt@<_edz@`0EB~O2@m?L$RuB_Q>RV|JGWL)>f9FFn~j7| zy(Sq1oCEn`{)OWvKpU`kqc<05au3d7hn$$u0jfSk(VAweB~BurLQ@wrT> zL=3M`6aATFn2v8_ zYq|4y`(Y=cQSYQkASB;ajwR+j($l^7+zbxetlWtTRdHya>!?@i6XMc&+JWp3S==ZP zn470Q{>&`4w-UW__C;DiT>mrZZAgnvM>KbuaENt=E`y)*2^rRj*c9oyG&1ri31qPA zjt1jTay`Hy0DJ9vceAKIaV>Fl*n3dBX97@NOe}&s1`l)~bZ<|Hq(VPw88=a(-6G|- zGj5;~$#vjU744mT)bTEEsYE(sqDnmEG3ibQUczOg<_J1%b*6)yicj%oQ2~h&Nztu=lv)Dmy}qDf!}}9oW8JxA%nn>?mw>KF^Ygc zEZTAm2j6=bV5sz1liA=Cr0U%LbgNrD;&JXlbhAtr4Vq}vF`i? z00q>`5n)yVgX+WgS6nQP8LWO4xg{Nl@~?B{>o#aaxZuT**Z?6$DvUPRjl1I2vZu)+ zA|en*VYPYj)#F*oJ_gYiuOia{4RkRrr*tse(y&hKi1!6&v}mUe=L=Wrc|$L+H+Ps! z%6yNFxc0mg4Scq|`7M3o8R!J0FJ6RhKe7eSr2rtAWxJER;Vu(Xb6@vewTBNMy7YSY zT5rQH=Rf_DoQ$-4g;s}c<5R}=-P9skx76!v1B%6jgrtt?cRm{2*eXDef_En5;1@Vm z!*m1gv9O@*+Q6nf{@tf>twq3V*Fhmk^aXf3K}>FLu6b_~7d17tj9{+(uFXCj*S>8? z@kEY^b=}(j0A`>@F0|ywk9%=PdwO+(dva1zEwT~fN2V2{6_y1gAfTKors9rEArq|V zSgQhDH8}jes;a81z;tai*W_2FOz*79N_S}gkgX@5&o>u^8^>=@K4D_Q+Q51IJYekl zoMn-5Cy^*wfdOoGcPpTgtV^JsD3ZAwbWVZtum&7n{Z`k^XKCr?{#3Jl@RptanDq(< z?*uq8#5GIT?Pd=Z?ioa}YdjE7ez4|jW^PX9F3QcVw6_nR+g(^#$j6`v6xUQG=|#$h zDv!l;x2zigk3XvzKNxn*EfezttfBA;Y$uu-+(5ZB4mL?y^sM0L_IDSrgH{*cBQLb? zG2tEZG~EReL?l};4VF>S0~B%QE{zvLBjNk6I;^!YLJsP>6-TxM73HrcluZl?VZ}+G zKj%p`yCiQZ2hr$%t%^f zcNqDXA317Sn}cdfIiR~@e|IJEMIY%l~IA=LJ41;Y&QSK@+r{j1qYlW z)Ym6x0C%(T_Kk$I0=!1w&O-ZZ@b$TX@+KEC-xbTGrR8Hj!f&ak)YH?`dSJ;N4A%;x z@-bL^tv}&4W9ccN)Ys=It61-(rjsFSmb)U+o6Kr&uOuWSG*V9QcT=Ct0lb5u4=_wN zzh75sZ*ESTqCW+>dpoJJDU((H%hzI)HrWpNdaECc(7ChfC_fQT8f*i%ej!=n`&UOw za`~qt(`?WcZB}(gXRtpiy}b~BjXI!bF4951mR%ix>5Q-42)#Ptv%kdlP4~Wd+&TiG zwZl7v#Q)0j#Cd`$BfX2vG23R-XTV>?PJ+I3mqJ}*fqG&vgdcaB8sp zuVsIYzodSLJUmB2V%1V&k19&p8pumeHzlfbYpv)DdSTK}G4{G!n2qd-<5UlMPh7ncCY}ykh;x&IruSQhXqP6;-(H>udo8IHSdmF>K2?V z!}2ggV?RZ(EJT(>1crtVRB!2)nXj1VYgefRkTdqGWVUh9m%H~}@o=xy^{zKrkgyu_ zdHE@9X?8Z9H3YlVCxaVWZ9CsNP^wxymYi(T#=Y2$%pA$EYa9ZWC-~Tf{Uc~80-b}h z`1*r_o@5DbC2l2+l7Yz^kk9YazcaT;wdDYEp6v7I4ld3e!GMCA4c>1+BjjZIX=kpO zM79M8`y3bxkUgyjG8Nn;4CLkIoBZXwhKJ=~TpTM@ zr_MZ5Ja#+LfQDCevRXO}F2992zfj%X+^nCJ6g3OX5LGwr{EhudhSapQEi`O$gz(6Z z%LPAyX5G^BIlZu)uw~`pr3c7A)3&ODkVn_>o6(^5DBGP5ja09LyV5^$XvDQy4Q78r zhpEKT2;}8kGB%{hFjnb$+cTy(FQhU<)g)pwRJg?>@N^DmaLYOI!f0h31+MVI!a@eG z)h9B7XYNwyqQmukrUgKIWugd7QGs?yrEYY4#RG_-%k-wu$|!B-`rKt7+cOnU93*q# zzx}D{&}RdLz4sHS2o=%`Ju*wcg%VAM)M$fiE-Ty@oblrC7-pzNuQX8F!G5!$L;FzFrlIzoi`Mgd{?)Lw30!O9 zUwylOTN8Vks({mpJ~lR16I(^PMc&Bs=FJ;ESByHb$*@;!hYoNlUh;}MO6CZAtfvEk zu-}AA35V@OiEhN^(dStB?C+Yn75!bi^Kw3h7Mw903y5l0 zCr0Iv}Fqr({zvGr9n!T`-Z zxz44_2b@f8Ho?=Et?X1rID37E+m|lpsATVtj{j7T-h3c}q^T4aC2S6a+JUz9hzoW_ zXP)Y=JrWYi0dC!@KRqcep4iSFnUc$F6L|7UmS5o2A_9^pZ$(*tJ9R$(*2ybawj~`T zKmK_I%fX$>lxG)+_|1=SbzIrzodEdnSCisqF-lT-Ic<>E~-oXEfEVcyki*VSE=1dB;lNywIJ>@vzS-!3e8$$BAMN%nYx`HHiU|GC1^hyIhjX&?3}D0 zwzCbuf)|V#m<`G44Z52&X1c3m*)7Plz6 z6|sQ8Rw)7kO7AEN2uSaQfYN&pNNAR=s0c`xj?y6nsR2R=7C=BiS^@+JN>6}D2_>P- zFYNET_kJ^HX3jl#W}Q2Cjcd7NAte9&%3Gf2ecsn_=Z`iSmO7SwNvMHc&=?*D{*D2& zin9FYW=_5YdEp1l+q#D>tuWA9CN4I``K2c6o|bNZ{nzvITC7D&g?^|OO#Vx(kKyt; z_N2s&_u3pHUpuSfh0RM0-q)tSRPLs(1X*6mDN!07yGLv-nv*j{8rYcbG)|j!hoc*p zkG^~rQ=hZ;`uG;YRH)Zmao6ts4^7QlSf@RQj1u9@?C}1%*~|7)8-!^q z#_WkqWnRykE?h}$L!d|5gWFyS5vJK{gXg$IZbc<~e)krUnVU*oZJh;?BI~b>!5l@e z-Z#6-Q!{2_Bt-4viNSImyzA`vp;s@Xdn&MjJ~<6=w=xFHS|xGd_tM>=l2re*TJfW! z9rKjf*0aK)dRum=+QkuZE%F%lDNj8$=AxqqJUlyWwMxcgSX7I1uYXI+#rpizEvZC!X~h4Bq(@~nb$mDdK)1j05}jkpyMeVgAvR;Wag-wy=}E~?D2jcogj*zuZp8@mhwsmePj)jT9TJWnu}=GgSWvd z{P6c2X_6(`Kg*WO{l)IQM)?V+yKJwz_2$_$O-JSGcD5qy`%<*^dpHBbC-KVU@%iVW z8(R0Y@lQwuE<&|#r-pnjab>$(y=Cre1=1isBctRTDWiltpBiCP@Jxr`KKk|)d87iK1rL*&JkqrTSI;7zjLh!)@yj?3 zeLsJkn~|r0_O@II%w5=TwUM5tyceX)pbOfA$~E1xG{Ba!T=MhtGn)HoKZ2#^7#Z~^ z1`5Fv3cAd;TYghU*y5}yP+FT=blAo*N2_b`d}u$ETYqH*=U_R1PX`A_;O1XuI~ArN z$9(nr)!;F2y+Ne@xQVo?cW%tLZ}*f}M) zY<1aIcL-O*W+oSl(^)SFEDX1Tb>O;U#j?qmZ{{vH)fLLonKgbFcVR9n3mouo!ks8VS`|TAHp_y=JYre$imNYpHstb+h)}-a6#w4t=a~!Kg+Z!W`NfmA z*K7A095PRx2M0vT>)Y?N#oim5pl-#O)^4s!PT_aw*V2s5#%Du!&A{z>6{58=P;Tyg zrEaI*5q{L< z;i~AA+fWh@H;uj;YXMJ*#=OZPjAx=eF}L}({39$;A9oNIhZ&z<$nln;$Fp7ks%^L= z4W8-6Mg1UqyGdGyVzw)x>1hYJf14+9rnw;Qd@Ld)Z`zN;t^`T!PPYxtCT9?Se(y~{ zgsw4HUqo5?HfgW=Ubzy(AZ{eu{YWmeJx==80@2oCOV;7A$tv@deLXdjhZNMPQVG6& z>#?P5RqEL&l~j3E&HD1$-GaJfN8|ZyslsWr<#c&9;xxS#w^CQp&Pn5)e!5Hsy#+VRwg+Z17ihb_*x2$)|862*uoy|hu1u3IxuT`5 zt=B8-A5kUDj<`Ddd-|nY2kM{8XqDIs-qlwLMOW++YL(T!WKrDvzJvWpY{qxh2b$3| z!RKT7D?CfV$wB@i&TwPI#4q2);fSGk3YWH6RjBy-=)n zEhuTSX|3&3z?o-rnHBlfR|IioW@gf+J|9{g+x>+_b5Wv_y}!Z@5*zvW3thBo1+&DJ z!pp(+U%5v8`O||&w5fHfI(@qQxKey_V7rCNxeK{di33U$@^F5phO)egaT_kjm0j~( zLs#88i1Hzjp)b=Ymw1z|9%xudfUr+?7K}JwbA@u}mQLFGtKjTL7AwZ0@R871aTn%B3 z&jtruA#erxDdr+!DJ&AHp}f6uD+M;dR{XA&P3;Wg*IQulKxl?Ma%7`MxN7fyT}<|? zH;IGl;y6RrRinPuQd^8uT&$%DzpMS?MDsyPkgu;9%}u5CYs$OGm~#qsJM3q@u9?$l zeX0J23jSgx%{wIw#{3r?X>9jt+v|z56V$>O&vdTY^Wd#`9@0e*Z!(H+r#q3q7Zi`6 zz0#7_r$#luLd;?F9VWl%#wUq}5Srszl>_c^p?AwZ|0rap z&zUb@bliwAb0r74J3r%55)dlYnHv1)+1??dPo$6cRW!HfE&7JrWYuQg{)YEF{!4t6 z3c{oEQ=?*E1kW_=z8yN&>gN!U!NVz!a$!rPc|+sJIhC!iXy{8l$xM_^Y6d>q^&^!s z_46kZ-^EFu>Gm9`a#x%TyK9YO<_FCJmAKEwUFALrZaS9KTv&3vU%sN%Iv4C@5c+hj z7^QvCnY4{8!>sVmUB3DuEh95Cy3l$Z&qzLzMxR#7hiMbu{Yp@{7_Lp*4f~XomGzDT z6ka=)FA1b(D))%vf;gf+*?6vrO!18k3$E&EbuqA&rH*@^=ga}I^9TK5h+FrXZ<@4s z1+YtEZH+z`E~Jj^1{Kxraf3SMx23?k&Vc}Vc*^^LBRiA38Tw1JHl0zLEjA^nKKZsJf^#MLB`+;&-t9pkP|MG zwaOz6?zj_?*%BDNoX8(*IoQDgRllQGQqQ@7u`cjqH@&pJ>B zYUy1r^07-28s@7$hYze7I(CjWI&?C+dE&=X!K}t;E)@99Du(5{;I*?MO;hXT;aD$; zC&U#vbZss7>D1JegLM3ZJsX(c82Va9HkN2H8WTmZ634+G5>jd8a%_0JP;KRL*H z{CIJ(D>HJe$cl?Iobc5lleKEhy@ycX=Q&ry;Pn-zS^h&w&sM8~xCp!Tw^I-2VI19M zx%9W0*T_|nb@KTPTlDnW+VQgC54p`@c6O8S&YU>1{N@F=kQql$vYkx>fW76-T)I@@&0%oN?mfpgViPtb#}A!m}vE4)#JTdr1Ksr3j=R zT1KA{A2sUj1$$-`-~RyK&wYJ6-yD`ut)xu^V%EWELBm3TN5*zpe(WM(UMcJ-?wNaa zeA-IV;1Ud%fMP0@^D)=KmV~oa{Z=z9>k};-eK>* z77bZ^fz*21}n^F)0R##?I`!$Mgk~pns{XhxyPCc5>Kiw3~_Qoo+onRz*Lx)w3w6 z@dGO*Mp{(tZla+kfeY%2n6^C=al;TmZp@GN?odc6@M$Zw;`dZw;j@O40`osBv zJ9*)@v=md&u2R;b=LD~vsrll>O}2T`?bUN-$$VYVIwNS%A^{eLDb~0z9bAW*4>fr$ z<4Gz_qSSJ}Ljj`(e6nR6F~*%)H3%6RfZA&m0pvk+Gb|JI!cASJ&qR{^L@5B}WKfsN z;$ly9S2aaav9S3I7t(HoJm#Ap7+$3#yq0Le=EcYjn+U7H(k;$xQ`#y%J(s6nXR<)3 z-_8)AEl%B*c>q(G9wLErdXf|pxzsf2G#0)W6<;|{h?3*=G%duAj+9|lHbYTHRmz|Q zW=ZDnE42CT*T=NgRf$X`f15q%dgLy8A=bK6dFSh&%05BacG+$0C)wVVmPR>!v-uMg zUPT?%N?Yw*QR*a|I5ALtG}olqZtEhE87%^<==wPsdB7!xN3~4I_FGpK3Y>VC6|O5$ zV}Xfb`-qRsH1Pg}EUp4(TnJLR=q1PD`i2IO;RU8Ge-uG@eNjH#ts}j?yGwT>Thw@u z-RbU@?@acG4~7NN`pQXqeQctfGehHL_U>V-VHr0@>4Sjr260447h{E6dF$6r)74e8 z#R+8GZy?_{LlWaD;k9NJYQbZn#$5JT%UlHw#TDbT-91^JYbYbjssd~=5fLE4Qr(UU zo@}7Ib+{uo$B8i^O^u+PIgG`W1T&0A!>Abh}d{7`jebOYB5)KnsR+8}9Mt`xN z;sS1aAAOiu874%2@WcrdgPaJI>_Yv^m(hfF$=Lvf%SO}IFw;Q-RG0IwDMgPEOQZcy z`x?M!dw+48dW@@YqLlyDwFPtloeAafEy?+fktm`tTo;7S$hq#*`=p!74w>k}N1 z8<3kq%Ruq!=T})m5AJO1?CehQ*mjg%n->6;sUy4!qe`x>y@0cE@bmqUzB>B(ca%RbM z%*SAWp%SlRusJVQZ@dDi8GuR;AEuXT+&2w2E18>{IRIg03elUT=hlYridr?@!j%o> ze4VSa?%aGBQWCUrqJfHna!gRoJ46$0BB;m8#_EjY*M2FZ-M<_PI_s(kvDRm%X{l2~fnE9~hV=o(b?`N#fK?yyYDw7VmsW>R#X0cN}J4xQh($rhWO+A|kW&9^k!6kr?RXY0t26 zr`?zbip(YK_>dX$8QCBc^Ke2>UQwQ-J`12arU0R?aT~u`mL~J2%6~W0@Bf1G5l%{9 z2J8l)1)c$vUbJoE8u01N8iMC+^)MS#VfnnKPZl`WlyIwRD-fcZf~;b&7E$9~q7!Kb zj+o5~%Ba4%L7TpV1P}p;^Z0~Hp^F6GXj6H@6K}4<&>Mt@bRs6?h_WAvTfRf?=ZJRg zR#*6XSI@`?M>L=7wupX_2F(o-(s8eB%RdqzQvj?U*lJBi_zm40EN6G9DKG0GdPF?^ zurDQb9O@fPf9(hu1~R?^UO}G!UXZF#?P_-aUlatWWvN9Sjx19(e ziN40eBDAF;!NK~lPV3HLGQ$-x8{==+^85O$3LXsA!DAbO8ZMC(*c0MwCN^k*Xkwsa zH1!Z)RYN!bH-$G$iI0A6cANESoLC?>mL-Z8(wMb+LV3Kx9>BK z6Kg<{JX&oCtFKS0B34L9O47c$^SXJ@J(#Rwfj@=cj(Wbbx@_d;mezgej%)uq_SMTy z#KAwZ-xM7+c6J?qz3eC4n$Y~jW2g}-9=$j4`+0!Nk_S-FIPqotK^uM}yk}8>ZDQUw zfbUDsEb2@D7!(BYDZ{9^{CfB{Nx1YozQi)D1|00BannSQq&(Hsb6IT71i-wBpLR*> z>-2(br>4@h1A|C%#q`F1%fvit`d20f5tB3V!ri@xw6LuXs1c$g2(B80pdV_dgVUG2 z|H$9(F@NDK2ejVoz>b_y9zGzhg9pA81n#Qn z?J_ZLtKn*80EN)-49RG&pMNt6zXPJKv&0(EIr~ZPNRR=b%{cddpg~&4r}?V_9`-CQ zF{5OjwCjP4jo9*~)$`9vD+jFQfz~;OM_;xB+V1;?s%XDRkBWn7E1ske0Ga5C_1WYx z?YsB7H?VCa^Bu<;@4%S4+W4@Ds!oDe9N!om8pO5~6eXOv!Nk@9`zw6zoHHf^J-o|e z*l?cNKSe@VNM2p5y|}f({L*-s@Jrhiq+~Mn!gommmxfk}!8j~8yTO);M`U}BVO0s0 z{LA~O|JSsGAFKS-!xVf2Vz}&tC`Z4}CfJ%)X~6j$VQBA}U8fB?>ydc7d^{ZDgRaWKko+>g> z9W7mBAKeJ)A}%DLf~0=cE^Q?T!QqCJ^h>SumMAS5Pkkf5SIG_`pH@FjZgljtiS(?1 zvuO6w8E^#JcN--jFUlw)NWkE?t^it=zc6pcver2zkdroLpv%4Yq(Ju~ptpHQu~AV~ zrq?ZMo&2+CqA)D8;Kc~fh&K%@w^r1I5&Jf~n&_h+l+eTfc+mZbyram^9NQB(Pg=5A zXTpIXF>ScZaKLHl_S{9ghA^@CMt%Nb16cD&wF{H2)W#&Dbmr#)4QT=R8njN>Gmg_| z?q3ru%pN%Zh~ z#gf2$k-IyUv9MM58E8t~#)J}5ux0Xdvd^UwL z<-+uwFTeNurtGADqvG7_>92Eg%2IyTag-NPI6tfh%+|0`8d0b5I?H7gsbl&1vp9eg z&Tcp9fdk++hJ5F>G^(MDsg)d8?2Hw~Wrw2!%>c6;U)gZShI=#_*|<_&Miu1-jxcT6C5CDW(@d@fQOi^yi9!Cd!9}u4vS~O}n@hp1)Fy|#3)cyh{ zB58qZxf%kO)oQOGm{pRzYa1g&nuv?=ai=Y=JM+xM&*1skK+`j@ehGX#?13gPwc3G) zpBp2CKnmV=D0hlFn=2#AH9wO+GW9l+()?o*v69~oxV;K`V@Woe{A|DkJ0fk03w)CX z$`U8i2BoK?>H{&Z0sa`jWBqmEFId(V&88FpvQ8{(dJmqsKjJytz>6a+F0)yOM(R9c zYnMjK0f+MXmTDf7>N@@@i3u~xt|&fCWh^623}K4nIi?XFNqnNvDy=KLT-Su$ z13H=XXr)Hex5Y6fO&cg?*Xh{X(_K8Xo(2DP3kWe~(*E6-wJc?6>9thj9@OG1>z)Gc z5Ds>(^0z||pYbRZyyED99!CEYo#)bElI*fSdB*zkJ+ojNm8!lUhBgm#^>ie?o(45A zY>hzdMivnRR^9s(0t7J@}-rMeCx`f|1U|-^>4(H4AIRYU_8jyG*d+MjqQJ;8jm?ssk8qLPJb!4GrDZbG~WotP#}*e&}H2ztAT4N8Zda2{M5Ylw?-_2K#8XyFVEXtFe}7-=C!|{@WeEa);@-Q zPXYTzpNm!GU-0ma@P7*rN7Gq50dv#_1WVwy0`SaeqQ7t9P|9y+8rSJBoC+V%AcPjM%@%I>BRl8AXTfTpd4zppC!p_Ny^cxX zD-H7Zn0|hi4diCKY4WoX@p+)C1Kz`4UK&9aa|6h;X;>o@*~697bug9i3-12_qd|z0 z3l8h^HR`*qsLBR(_rD8{PBqkb4O86-3Br$m{~YgdJkbx@qi2>2ooPKqz#qq1LbK2p zx{rX#e1{+toV_4e4cq!yirC8RK??Y*0Dd*{v;K5_HU^WfMPq^Oi|lf zU!5!9y%Iqb2(ai2SFD_77pC$6oP9fpiw*gR`|N+t;ErOg;go0fGiU6twFYW4-Cx*(U&vsp}| z2hU*A6m}28-3XUSK@UnEn)NZ!p=>P z2%dTh0PZ2{&-YwIRXIcw30<-^H}HV%OR)bleLHczM!M4*Z#L_hy!zN)&hqWRftD!> zrS$FF^E*Ht!zq#XaO^x}0@2q1cV8UGLIE-H9weorN%jIDpt5#&y)1s^%54yV2=QIL z33oeu*6q#HQKpfa0*C?ksbW7As)S4_6JS#VX*_BobMdM?zbVsn3dq>FriclBC+SNc zDrxT_lv>QXft>9M`2MN5UM5JAH)XPg2q`fGu@};&<}Up0GNT<;U0!C1dRf*DOl%b> zhmU65FF1JU5FCt2F_NFDx&QUeCGtp3DHxDq>GPwMSxEB8-6Hs85U9ht%6GN7fCkw2 z4iGSirsL7BSbo#-&oECbDGAT8XTDn_XPx<4N-5Wg8woI1TWFOvJ2Vq**?lBG62+WT z_V1m3`=5n0{r?c*|M|+J%+L1cf3s+-`7Qs8rQbR{;0kj?&wNapaB6>Z?KiZ0b`;+} z^lgy5UwpLRDD}A=7}AjhTFi2f9S1)>(J*1<6Zew|0H3Ly-;Esst)% zF+QHPu5eBJBN)3hfQyL8FDzuX5+{$=SLLSv27ceV--Bl~hDW*B+iB;$xAP9DyYHU0 zY6UIoH^xFRSs_ku)zsXE8jc?3q&Hs`wWu?SU+d6`rQp0{uJr-Bvm;3aKARaNEH7^f zW(-+@Cj;l6d+ZwIDm&0r@@XkTuw;HvPWev#>lu|3Nc_`>%zLg7vfTCi)+Fo zEpyYP-4z-v!D=pf;VW6o`3R;W&;8J}tzh+}Vu5?cNnP#< zQ&JkGEl8d$c8=8}_k|TzP^)@k3Eb!)a|45$T4}#=fl;%M!hkSBzTqf<3-a#tAGpU@ zQ!KNoYPS%-TJ1>4(^3}QoR+j#1{III`suLvnda7Haa+B8EyKN>q5U^~dJ=a7&n1g0 zy^$NWo@E8ZwF6a}g7!O|06cki?A$SI1o*_CK7FbIdN0tAzwYPHi;CqIK&_&7JJtm# zS27jPIX2K=*=?;%Gj|Xuy#|07QH`mJp19evBwW86>j~sN$ckN(tVYLh{hYkdx4E`6 z?Ebf44b_IT=_4I2E&QYvcf1$=wfL;-4lmJ<4PY|_ri_qW@5~dA!`a3WjZK!9kAf~| z*FYz`%8O5{{`za+dk%&%H+|y@&~Pojp}(>pu5@gmpfE@^uG_oJmc=o~3lr-*8G9Ex zmzKO5sv=_T2Ky=aZZh2cBfDH%o8Vj)Ba2Wu&9x$WjuUa`-8;>-me;)14|`L{0{|Z! zF+E>O@aPNMz_}c+^sBLN)1y2UvSrH3q(kryG-{AeG2Z;*zL2Cj^cDB0%y#{lMN>tX z4iLu?^KR9*fBkBZO?>X&KLcxZF0+y`LwkXFV%RjmOP=8g*X~UA{thIaAa%W|AR#wRol!()Qct zt#}|NylvI!P(~knKom3<)XV7oplKtcW>!@|lVrbiRNq8MR{wKVvnaKbn8f2J8Rm!Wnm;_$VN4c=N^q z2};E(ET8&GPId;ju2l6Xy>$m#Z0#iBxrE~KQoYPSC#x26ZALmU)Gh-L5ByCqD$lgq zZWt}E2K2uShHF>Ja06IeElYyur!V3QHnxW9>guj@-`@e)GzomZ@Xa4b_Rrk?%Pp*4 zK=x);#@IBe(6aYS0D@l@Hno8gM|)G0wPEZ@oFHjsx}(0Ni%mD0*7^9WnV8MJfd%SW zPKxn(g!+L4rmNSeWb*rWYBB(+qjK3G{Z7XW=;bzbnJOy0vfYmNp$(u|@@cu!t zhTzhytglNoT$+&%#VRiVk!XYW3G^X4}-*7haP z-5?%xH*h$z60Ur*XG~Jje@4-s*pn$an+LLZDP&a1^WVtH(2@C=1Im@6X*+}%EE#WK zCuV)Oa+v4z>7v$Xa*6MxkF@JWluTKAk=Ga}V!ghY|LkaG6Nyl%ep^||&*dGX$hc7B zCCynJ47J09KJ`-fn<43f-hTnM29mSJAzw$M5@5!=l29Y3O{;phTp(D6&4rtx=rAx2 zy+2iQwtA-=yMi?76{u_Z@};<^>?HIybZdnt?Z!?p8_$4Fk_cmZ$XV^v;JbwN@HSD( z@idU?Bi77QXC7*FB_<9S z&G(EWXykm}h;2jA%^LqJkZiJ>P{`c8580QG$R@PswU?_ndQyq!-ko&r5;x(EYJ$G= zJ|~KpeBsK+RsbQve&>B}+i&8W1YZbfNYCT!XG%!6D%*%fIi^-PC-Hdyh&KG@N z#lX`_wxTwhOHr4K-E({#Gq$x6uKer!Lw90V*Pzpbc_gvi$FBo7DrD>|A|y0CQoF(F zFYWeM5k+(nxmeMo-?P3nR#Tw6cL#K30Mt7n3mZ9D;y8JlQl7WAFl?-oe68ZBx1fI3 zwdK|Q+w9~cnR+jc_M;RGj!EH5?Q&9PjGD03uey2;H;A;^|D0r(UV0`wSiAFwd$UC9 zJ-SCX1npuzVZ@rlAsil>|8V1`6o}LygyCfk@MzS0a zRV&oXZ1*LB^{hHtwZTU5UbTU0t(hC4!dCDV_L@Kfjx^~8^;?icNA;|XsaIyATwnl3 zPe90C8@NFAC}L2QIT_O;GFWE_lXfwHzI*pEHOhfk?)_NE3Gz>M;&oa!A!|@Mh?Hf& zhbG}o*=@r$eSMDtt!IP%_XN(Jh2Lr9VZ&RGcV9(E5n{L*Yx5c2WmVQJN6gJk1}wGu zhO5?X?NR=PG6>h7lL*g_Yg&CH@btl=c~8Wz$G6<@)lE;Hicr4B-3^nbYNkI(?@GfH z(!F1cEbnR?#qubxfqg8A*xOYITzJcr`jWyK(eQEbaUPvESpc(5WJv=?T=*Ws#5?`@ z@9*sKisUJMR46;9gK?o0ZG2Zk1S3ztbSHdmqgSLb=~fRxT&c0=uk#LHSME65_Zk4fCt zrwjsz`0KQcCN<#j`&3p@+lpJf-D!m!A`v}s4NX;5?=e{o^OBvUJRW7l*ZzKEU*GkJ_W)gp_Ly!^ zrHy>svm$mjh3Yh^WT`JvO#)s&SCOb_W+p z>WWez{fMnA1IOM*Tj(=07F05uYB+EvX6#9N;ugw^q>?#^)_3OT=VuN&j1$b|xDaeL zKR*v>jsl0gX20|z%+ecNA6Gv-bQh>k8@dJG01$ALmQk987; zGHcDSs{Xa3)RRQou&nXuHRufAw$RdQA8GJJgQ%!*U_ffH=+t{F>DwF^t?La4Zq!|{ z5g^5K!sul=#pQqB$p>nc_0T7J8A@fi4IX@Bi^u@dkQ*!E#HZg2d0Of2(^;Z+_~c-z zxiPRo)$X{*k9S#*uMu>)t%hnyprP2u+b{+64UY(5tv!lNDjKa})_yxGpEKV5^(D{a zN^aniJhIR>6q=3Ot87#fNe)Pbuoi#WPX)C}rS_U7{)|QfRu_|jl61DOS&Chx|0LQb zZc>fz2jr2t8j8x$RS9Q19&1XI&f`|P8 z|MO>^gv2*|U>WBn`6bz%X+a{OL zXkvU?Nmyiz+v+;D(n6w;ezF_}OK6QTS^Q8EBg}F0(tf8euRmdmwMMh+UH;ok77o=m z8T7alywL+6UraVc3^?dMbgJ{B2Kx({AIo;W&c&F_eslyPmn$o)cYX`%x%1qRxJ#cciS84G1T z?snNCfcX)=WDl6Op6`8A$jZJ{;_B1qqvNJPxQVX4lG{ywx+ms$U;V%Q-#vi@n+vNs zK#q(gp53d~!>p^zeGrD!?^l#38k!#5p>4i+jIV96>x~#sEd_6I0?B$)Z-3oRBxb~0 z6{!jpy9O%nWgbElJ$R_=&X{|7HLT*-IZW1}o68k3;3Gad#CCo;^xb;-S!+vDPo@`o z6kIRPzb<7r)7@a3xA!cB=@BT@ zaIvVCpUou@2PWDJd89Gk-_vj%%?GL|?&>7(%{0G)IJ6jm63jDt3m_lwa%=992fgzCNF@INjvHSx8d-{(G&5!@6Gn?4| zw2=Eg%%cRla>?yq@Bi#y};Q(|+FcXf0Uz+l`{`9OrCPzS0H8*cAqyx{}zkz!qZ z<=^Fn@`$w}zWG8^E?R#{^a&IT-mq$7+ld^`ClS zT6(7;H{*cJ!qhvW`X=bp7bBMcjoGu**#l{=MxlU`IU5b z2~;coPq)0Y3v{G4-eh^f(CgGXrqUg_qxDnbnASjVRBel#CxqCDa& zc222HQ}|Kw-WKvm?e9BOSzX=Pvh>+Q#+eAl+p>-ZS&caIYbj*=+|ty)<2sp)lcrQr z2TI))qa)nvIzs9=FdC5c&x)Wv05l&@6+J7u2T#a)_(K=-mk#(oJM#!0;iP^4gKciC z_Y z3d`_C0xn2U`uCIl=26L|YQeE;gyT7Le_v^n@iASnE;@NsZH|svVo=h}2x*ip6d1LE zTzk@o=jd{rZT_jCaVJTKN`3S?tMd7x^0T5TDbCMM7hky9eV$oK4H-OIT5yEgG&Z6K zS|dE~QmkA=??yyu#is%3v|G&q<_UsK*oI-?A6$7APdWDQ{_h-*lk3gY{k!;xubt3l z>}5Hg85M=Qzq<1;o+o&ch3R>igQm2M)PAbx7Zg|$IYfr^Zc6;a?WEo-=o4eRy1M_- z-XzMWJD3r~S{E%}9@T0}I#8>A`-s_Ze~M%=A5iYQv+Z79 zi)y*ZuU+)$fOhP-YBNfDK{fQAq^eC0yP!T44X2K3+)CP6K=44+_n+4%2e>YUbLVU) zcDM*VO*w&HGv925d8yjNwa{xuefbV88H(kSH+&Vc?<*dCSsq)cp^7kn+;b>rYTwj& z8Kk4BLTfiJnI4n^4brEm6xczI16PHa_6Ar{t^w$nLXFt3T@{<*>>Q4ib_XL1mk)Fr z8Hk7#iorf^pFFthD3i_&Q%l`|N2U&4a`?#1W+sqF2qy=nq?q%A zZA$heN96qZ{{4H)#v!DmSXY^uP9N)PaI07=7hojphQ6X6}fB0sXy4B8SGnggS-E;8!pTEc zI}Et-!-p0X6ku>}E`zw>>0K=?(Lmk{7cR(nM&(cJ*bIvc2M*v6q(R*+{A6bJ8vMU{DGKM&g*7*QPfSbL*H(t+?wc#y|JIN-r<~!Yq zncdpa*HJYDBaWH({~9P=Gtky9d9R;6_$moU**V!{6dL-T&$SZ|OOt@t7U*4o&82}g zGYI`nw)v5;*)?&HhJfjii;Jn=IA-n}hM57)no<-<65lw-$HS<{bRYgOe17l!y_c_F zo7}zoA|vB`&OwOJwOyUx{?ydvjCB9`R~vFsCndfnV7UAQ^7pG@Rn9$TVBG22bc(H0 zNaPvNF$u(xZ%P@@X=qofs?V2ONsySTAgT=;uZp#9x(t8w0m#MY6D)t4gW!ARR~>zP zbhxVG1d_$)8nXdj9Js!I99&%GQ&VT$iJhhKhe{8;ehdMVf`5C$^~f4z)}001HYqK2 zk41j>`Nw;^Wtw%YTw|Rr8=$Y6(68Oa5l34SMZt32)clU5u&Vv7xOcCw*x30@u!Pl(8@@86BP?|H z87Moe^+|(d=ELxV=?saNm>|a54KUTZ|74VNv33#!GjLxfJt;BZqgdDh{xZ4GJpQac z=5(DpA48BcjFjGkT!qiFOYA|tz*V=_8V1Fp$6dbppLF{=F@6{rDOHAX_0Q|6_rSo{ zTAtARZ!}b-?qp?gEaEp}xD^LVba^qfBZ+xpu!b|3Smc4FUq3#_=ZT-)eGmQ`Ei@fy z?zOXHRZ;uB!3oieR=2STS~Zx;8HtCS8E6b%tq$Lz+tCb*jrf(9zp!(v zhBq0Tr|DC-Y0r{V(^wo3{VuLGay){iK2QQ8Np~XP?#!X3n>K{Ro7KH^o>^ z@*tNd+d9FY-h5DU4WhyO5HBxOWgc&J2o7}6x)vAyxbnSYXL5`fa3fmr3CuVR{F#dW ze>cp`;D$kbV3PHBS9}6E2)Ym9)fH$+XvmeP#qHS z*V#~+y=-G3zyQ2T0w0Ky<`cwIuKoa=ky4<3Oc*n{`r2z; zv-hhazUesRY69~bB%UIP?F9CuTU2Y*ULDaiL#)sEGpcp6c`HR=3n0mL63tE zR&3*G~3VdzgIUcG$Qygd&uwE-L%+!9WiBJ?h` z?5Hr+mjeGTYp4ey9qKj`k`2wSR==oR+)8sN)-*cxl0J;Yf(74kSm=;Gz$eB{y6Q4EZdyQ}NhU$I; z-1pZWB0$qxFp=v`Jk~f=Y(&%n$Rj(UY)7|nu$jia&Ky?KMK{|JW+ge1R7yF3XM%&x zOm-R`;HhRPhOs}+QlM%$aC55YsCRoyeFh+=CpSvqRrF!{Sc5;VX)hSNI}&!&cb2^> zjQ1=jvYR`NvUEWC*xIBlfVq-Uo3Ku^9n{yNYqtj5wLtnVwONbe94Su+rAw>q57=~%CY^>v2gl(CVKbRj-|;h+QaG|3K4_AEh>C(zcyZYtY#w86ydxg2|SB(kN@lx6&p!g(2<0L zMnv%@g2=QiUTNglcU@TJzJQ~mRP>TfMNl3QX zz;=s>ztTqBWTDi~Q*j#Qc^RiXGMjcWT*XFyX|G$**8Sr~sUJ4^MXn%+?^<5d)`J?G z?9DnauYRc>_nz2OOF!`33@P7{Hjq1BYV+63mM;t@m3@z&#<{N4MGh8+ts2$yj6U4I zz6U^BP9A;t48KzWi(hL?hH)-`J2R%57Yo|U2%rxhKhUY=GWaKL9&eK(?(EX_Tok_u zQ>cE^hD%5@S>LGel1=YIkXuqmu&SRL%Dl=9YFti>Mc8@+;~_*|llL5{h59b1>T{kJ z7jPwGvMK5M1z#N`97`X7S4!KYx8V)QOY)dyqSNmUd1d5I+aMo0SK`yAbvnN+PCLzl=4M@GX7jAc?{3+2v5bXv+h-gx~V# zc|-GoF|4ZH3BwLny4JawmY3Ma^)eS6++|lHi2)bx^dV+k+xCstb#ccR#iYreNXd>( zi5KIid{-5$UW&yzfXE7@e7*^)-`~rZm%1f&1e1`~=YR0*HU$@EWv{&J>z5?#ikNq6 zaa+k6J}e{q`TO@aJ=LrUBOKtjx!c#0=MWvTcw0B~S5ZTdn{f#^m`t_&BZ;5&ZA*Am z*r`{_*?(${M9jSvxVLk6$qVH+SlwhLQ%$U>V%vfpmdTl?^lT9=|)QBE?DPwD!E zEG>taNYo}8^~Y|aNlp}l!E_aZsJ?L?W~EwnjI|l^%9Z4AaUvLKMkj%R>f5uGO z_V#BP>O0_d@UMqgs~xsj)U%_zXc^#sNrx;2;nx$-@fcE3!{xKT;_fO-1g41z@H0Dd zt)4CWzFA1^sUbDW(Rs2|xJtDoXDn_*0W_SZwDiM6-+!`O&}}=y$+a!(HXv!zB>Yk2 z!Qwd456M%siPmN$sG3VW1rqGFAJr8l;awO{05y)(O?zWE=GBycfdeh^>u7Y^ujD4H z>QSe!xyqq?s|d=h;*o%!<;Y^bqz#|`A;BfnyGamAt*GLLBawF>ulCfBp`L)b@4d6Q+~!B~Ngy}d};R*n$A-$O`|?lswtF7I{x_NLGR!n4%NUHAIrr1=ob&%VpYz}OWB&2s@yz#mzW4pT z@9VnmtIu(&(bf`EdKYVZ#wO_az{DYFG`?B>v7s-jn!>NwAs8NUG0Baw17?J(Qtr&$ zuE%*yf!k0li#YMMgL7~8EauqUt(dp194$)rl@*!cz%@DciRC#m7PL9Dq*vkhSLliZc2)3=JwE$(&eGg^~)3hT6uAevo4 zynqlCH*me1yh0plo_O1=r^KMK3rid-Os#R?se76W)TKl2z$@z9v*pFnl@)&{)~6$W zDS{hcJdlNzm$I*IdFzZqiH7T5Ja6)}Bu<|`^O)M04Z4Pu#>p($q(mw8GwNhkbjshI z(2_SFFjQAKM^5*S9`w;ksdwt?l()pBhKC!&Y?6GgH>=mePP|r!Ww4)sejiCHY8!+X z!v-dokzM$XZqdOL@+(|_T17~?r-AZATth#JQd{u0vs~Ot@b82_J~f5cVbyBGk)(j2 z#+iveQCWVbfjf_Z0OkR6dv}-$7_wZd8d+6 zUff+6f^NLBOBZ{(TXlr5nWT(`G(HiqBP{wNJ$&v=o}me{JxirEtS5Zd6@BCcm0|Dn zq#X>)abOR;7o(7O(4SsU-83mxN4C^ZB~lD?1>felqCI$X3?dQg@NmEL{mc>1p!GQF z1988UlbEQQFzBkwC@GQp<-aL3z!Em~G+cLz@4~5QSXXj%vV(HJ#$4h1-?RJGTD=VA z0a{s%yR@0L+eJe{X<%^aqlUv|=VEuNzsr z-+h;VwlwCb7{$%4Hbgy5VN8jE^#L@V!0}ickvatG$_-~&Z8>-RN`K8v7pkg?LF=(S{ z57d{c@K@48Qbr6#)&4?9Cp#?I++IxZS{;jLBcJ0C;YkNV8F7i0_<$@*gUz5?Zf8xH z2d!YZ7NX}6#@RESyeswf!u+}zUcu(Gzsw-N8L{!_ML81+Y(|ffVXb19YCw196)xureI1MgvP6`j5qeTUlW-uUjGs& zn6&2m5KtX`L(Ow#<{`|3ICN1>D%M2A%iOE*ZT^4W?b))|UNrW=RPr;m)U*iQCjZ$F zl|r~cUnbpuYD|qyPfWcM)7H=+yLvT`l;^z>LGHvu_$1c|&&|u+Z0|@sUJPrB`;5=I z>H8YMsTpRp$i(V+H;GM0^IPJX+$b_($h}k6PP|)0D5T>DSwt@;*k&ONz)1UeCo0oV z;^VP8PaE-BGimeia1GnEyX2IX5RA`RGY#dyG;widtq`Va8m45nGfs@V7Y;lm?#uoA z9fFE(6ZtB^S0t{A*%e(qe_mx?yrRN}x7E7VUWvs%6QBWoeF5Hr8m;5XqFR9+(9aa$^~V<#~q6 z`N`)b8Q-aM;Ue}YnU+eFTsw&vNWDC5E}a@Va9`nj+0hv`r>6b$W!%K|4|jyz^_FVU zm&RBOd!E&EEnvlvrvy8x!jsJ^Ik<&Q2!43u-;ne-i& z-P}0u(CpzkPJ2;j59e6dnbV3tDe?_;atHM>V?ztnwR0DzsC>fDEq zRGBpfSOe$0<~fk&1BuA)N-OA#!oIj(wHMvxL&|UL3tyA zZ0^BPKrM1Tcovi;z!NO9%1>rv_-g~JeSPIgcx+A*XPwGgjC$Y%z2c@4-V{Wk#DyP+ zp_GJZ^|9p4>C>(??j{9sNy#ldZgQVPT=ifc*RIl=8PR^SmTzd3CA9Qg4xB)DNciUx ztJkRd{m<=Hd7mLH%RGa*f;I%&owtCnw%V~;!=2~1t+B`}SnbO-*@;|@jDisJSVvo> z+^}rR#$3I#Um9qdx;l3c2B9q#ca{blfX26fq?8tLb#Ub(PE7sEP?-9=F~jQjr0eT* zGa=`0M*8mdbv*4=b4+83Vf&dqj>k3AZ`2D#r>Yas7;antY!@-8WD=Dhq;pKNMbyl! ztjtP7s-PjI;C0cB9MXlh8LE2EJs&SqC}r}PE11~@2Ayfp6px?#Kr@Gv}M#*Zf zy_&aopFjIp3xgGOv&c&8=C4OYq91bZm~dWxy)+zOF8?O^#Z3wn5wH4O+la^}5{vi; z3w?`RiXeej6XPl+zjbGGKuL4bxm9s26i*BXHi5~}4$WATFpei3HVR#t&gjaJCGVwTy=Cr3eMZ%7u# z$IGkdrtGj z?gLEv$H6Rp3}9X-tLMUT*%tLb-{6glEb?i!ak&viHo>#!Re0P?Rj6Pc_IiA~tveW& z;XB(=NjUR(c32`E9*RSea>}a}{=hidkP16%s?RLC%`Gn#&GWjz3Y^!^Kjgo#q@ipm zr&j82GuC`(qOz*BKS0%c=?~C0hJ9i-l3hA%_ zzyDJvoO^BW7%`X8r!(8#8eJO)Bpph7nDp^;GFr&DoA2Ha?^20Dpi$pQqw0$$qG2e~ zb1Akx8&tVyrrUW|59`tr6;zRF=otE&SsXn0wv>o2D5r`3ynQa&E)e%{>y|$&eAaM< zRu6i(%g}2|Aal%%SF0uwqzgk+O@=JexBX;Oa=K2f+LHvuM_LNM!pW4q_$~curT8Bb zzsjeKTN}|qi&G2!oUYJSKehx#Vgl6N$dT;li@_1EZTKHqb1-uk*cnZ@)VpGqmX8zr zTua)%^j3M4K@Tn;6(Ui6dg_n{W@Q6lra*w`Nm|iUa^!cq2ElrXdQja&?+=`)U=P!d z+HZ>Ja>_}zlr5z)x5b!R^%ats`tZzekL#I%FP;qi+qHum1S77 zb{@VufVQ1T!(a&BE^|nRna_mPxBF=*C_nu*f5g`h+nsAC-kFqelD?F3wgu7D^BCQE z@1-UF$aBXezd?MlHqTb**KcCCmH5tm_;!auYhxz54F71NIDIdGE+>9AxqmuQz>OA{ zICU+S7nH|}1I`=4Lf{zH7+F=Csu|#u(XZ(KeEjpMJWL4SMy0F<&)!e;sod5Axpa?C zbfKmP94)f<7flIQq{dXBFP&Om%3t+vWB5;}449T;v!RxM4%_?D_ z3s`G|TGe$=6D^lE;*&vAxisOpI3=5DQi{D(!YbGzw1Xm)IosvPGHBg*V_Urp=VyAr zU8;ygRR%=+<^~m-<6(eD7w5gubVhX6H>xx(;c%p!5)l#?rQ`n8mypR`k{yT-R8^T zN1(8}f0~{L#D3Gh5gkAmRG#LPErdV&M>W$p%WmTPk31AcLMhww$iNrrP&icUa(9aL ztSk#bv%m--SpZrs9Mcrvz?v*6BfXv>L&i~Ea1hJ?U^tja<3sZfrz(ZbB7cRTf?o?% zH?4NQmQMX$ZcFB1{lqI1rrLcI_?=kqm5+}$@JxwU>{qycB6m7p$o}K0;kV+%mUy#9 zIS7tO3_xq62cK(K84-dLZmk8Q8bt)T4yXE&u0USN>=Bx5`WFx+0|;9papVW66O zv(Xj0=7c1KBpB?O>-^~|Ze+e)z?3YJRN;4rL1LBf1HUdBXL@^C@G@+l$wky+OD~kx$2zyoN-ynnuMC*463v zo(~GDs*hi3w$fT<$Qk$8%1SEf7gV!t8Ml>9b4^OFw~J}b{9@y1?H`92yp$rw?#XCn z*YR1olAug~wQ0EjQu?9uo<2T3<-NYVYEI$Q37Z@Qe2)%TDc>x_uGxp;{^Aq7I*&#( z3Wm5xioM7;k@uMI7J(I%NK$#p$t_N;cAUK~ScCtB>17@R-${`XNh97~abI=r@J!Y` zkT@PJ^->q;aEzI-u(;-nM^K)Z5uTAuCeZJYXt?9Yz$GU{3V^n=^uUOAlQ*r zXtz!8!Dl;=oTKE?aQ%0Gk8b*-^*u>#v8^)IrH77r^&Hx$LC!tfOlt=!>3vyYvWkl5 z$b5eP2eY%ds!A=IHxBnAF)?@j`aof9CSDciO5pP8lGAhZIgIfZ(oEYHRwcKYmLyn^r^XwTerR!yW2hq z=G#}>yjIZOActa?ePAh;+3x%ak3I2c z{K+gv@0_u!|KeK^oE+5!V!A(Wp8HHqqQJ+|K<-6~^waGO0Ne_D;($wU zHE&*n`ak3rbhjWhgT8;Kx4R1soWbTP zOyjJeqeXp5td**2jY%3UnHdep@}oy}w|Th3 z*ucO{aB8@k``p^)<=g+JG0f6X|1`rem5z){&h2UI*6l=%UaK14bk3YI76mW@Fq-k8 z3E8Mh0W&j%Hxl2~HA!O-umsV*>Y6%e&Vc8d^hK2bWeReoHdkdAxROPc?}tr0MBg4W zelG(kQ-}RJRP*nT4vgN|L8r#SO|f~!9d_rL*nWYpQD zY0gFR=0El$HzNZSn1gvhDA+MQGSttvUvYP)-psDCE*_LpP-%`MRJ!($;jaI4pltW( z9~ZWwWKO`hM-BT00M%!Wz;qknOv6rz4Y{Ex zZffA+q{G?y&SJxj+rOPdX*mJuZPV|fqIssWQ^(Z@mz*fD-m5F|Y*EhVMC9LB@hoy2 zS(}x{2`Z}hr_IADh`xnB#5t6>uwATE2h<3X{x+lj7awduy;20H-R&73nHBvX+`iY?=U!O~4ZP>t5S|DrB13tGW*gkq zPfRTKv3&wV@Je{4u0!v{tHs-&f2vmE)VgP1*ccs1{@~31dVy+Mdh4BVCHdT`XDrtX^|cp9jx9#F<$^vLLRaFI_xX+85H{ zBfD25J}osa_%JXlfcpVNw6tJ1jpe$3MF!saTU|r8|J{g2z|x(%x-e#^$hA&7B>lYc zf*tpAa&mw>JF@oHJ^tfsjRy=fn8}r?C*yTBE{X3Rn1;G=;lf^F;dh`Xb{?w0;jJr6 zGWCuJP5sdn6ZEH@uA1-9Y|SJM!_>AFEKKC4FuSrMyB5YT_y4GbC*s`WkVe$-Zq>$H z%PKYogZInC%ylC^U(>nsT-C+*{1HBN7ty(^ahA7UGF>lT-l8_R+IkfvE%Wp9z&EP0 zUfqB?vSsH^3b8|{l&VVg2LXqm4UFy{>z%nK30D7#!^}DAA?z%rJ8x5 zYSwMGviR40ue-8Ko%Ml1<-5xkXK{Co=tP@cUh>uvA-$u83mZZ|*thHhVz9yt41 zgLz;Vmlb_N zqE#xkr)fn=U{VAIer3CRK};>FOA%~D0jYZ@s=-sT%Fo~5qJDtS-6qED&MZFvymAzA1)RAJf@cfuGxV%ED+qnj??dpuO^K4JsT zjIq(t=rO|YAKTXZtuCst%W=1(BHU>cqn|S0D9pLJ1VBn)mK;PqsM^&F85KrORyy~6 ze1|Qt(f5^o0<5207>!O@LOf)KRyEj5tz=lgm*c0lZWTeYZmxEt8gBLe)ITxe;B^>Nk-NI$>ukQ*e65QTirGM7jKj_tq1aw#XQdyl-mW~OidYG)euc1yVTTtkCRn7 z5XVgk7Q$d4i)2hN+@#p5Ey3{p`hJ~q&&fshTEaFo+N;|hbygjS0haF7rocoyoVN~j zMa*#U+F{>$>^U2o*kW%|yy&UGG{ClTl>pCBzG29`MCF_;s$d~$ ziOH870QBm|`1{1Woj|+V;)+6rfidyEh|Gw(NH=&}=|h#JyZ7u7-n~0xvL!y(xd$)l zZ{X0=F^X9-10?AVeF-2 VzF-T;amdegFBx3Szi{Kxe*h(}+*$wt literal 0 HcmV?d00001