diff --git a/11_virtual_memory/.vscode/settings.json b/11_virtual_memory/.vscode/settings.json new file mode 100644 index 00000000..f2fa6961 --- /dev/null +++ b/11_virtual_memory/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "editor.formatOnSave": true, + "rust.features": [ + "bsp_rpi3" + ], + "rust.all_targets": false, + "editor.rulers": [ + 100 + ], +} \ No newline at end of file diff --git a/11_virtual_memory/Cargo.lock b/11_virtual_memory/Cargo.lock new file mode 100644 index 00000000..9a17339b --- /dev/null +++ b/11_virtual_memory/Cargo.lock @@ -0,0 +1,42 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "cortex-a" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "register 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel" +version = "0.1.0" +dependencies = [ + "cortex-a 2.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "r0 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "register 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "r0" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "register" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tock-registers 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tock-registers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum cortex-a 2.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cbb16c411ab74044f174746a6cbae67bcdebea126e376b5441e5986e6a6aa950" +"checksum r0 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2a38df5b15c8d5c7e8654189744d8e396bddc18ad48041a500ce52d6948941f" +"checksum register 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "469bb5ddde81d67fb8bba4e14d77689b8166cfd077abe7530591cefe29d05823" +"checksum tock-registers 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c758f5195a2e0df9d9fecf6f506506b2766ff74cf64db1e995c87e2761a5c3e2" diff --git a/11_virtual_memory/Cargo.toml b/11_virtual_memory/Cargo.toml new file mode 100644 index 00000000..cf0f0636 --- /dev/null +++ b/11_virtual_memory/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "kernel" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" + +[package.metadata.cargo-xbuild] +sysroot_path = "../xbuild_sysroot" + +# The features section is used to select the target board. +[features] +default = [] +bsp_rpi3 = ["cortex-a", "register"] +bsp_rpi4 = ["cortex-a", "register"] + +[dependencies] +r0 = "0.2.*" + +# Optional dependencies +cortex-a = { version = "2.*", optional = true } +register = { version = "0.3.*", optional = true } diff --git a/11_virtual_memory/Makefile b/11_virtual_memory/Makefile new file mode 100644 index 00000000..4cf5e508 --- /dev/null +++ b/11_virtual_memory/Makefile @@ -0,0 +1,127 @@ +## SPDX-License-Identifier: MIT +## +## Copyright (c) 2018-2019 Andre Richter + +# Default to the RPi3 +ifndef BSP + BSP = rpi3 +endif + +# BSP-specific arguments +ifeq ($(BSP),rpi3) + TARGET = aarch64-unknown-none-softfloat + OUTPUT = kernel8.img + QEMU_BINARY = qemu-system-aarch64 + QEMU_MACHINE_TYPE = raspi3 + QEMU_MISC_ARGS = -serial stdio + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg + JTAG_BOOT_IMAGE = jtag_boot_rpi3.img + LINKER_FILE = src/bsp/rpi/link.ld + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 +else ifeq ($(BSP),rpi4) + TARGET = aarch64-unknown-none-softfloat + OUTPUT = kernel8.img +# QEMU_BINARY = qemu-system-aarch64 +# QEMU_MACHINE_TYPE = +# QEMU_MISC_ARGS = -serial stdio + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg + JTAG_BOOT_IMAGE = jtag_boot_rpi4.img + LINKER_FILE = src/bsp/rpi/link.ld + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 +endif + +SOURCES = $(wildcard **/*.rs) $(wildcard **/*.S) $(wildcard **/*.ld) + +XRUSTC_CMD = cargo xrustc \ + --target=$(TARGET) \ + --features bsp_$(BSP) \ + --release \ + -- \ + -C link-arg=-T$(LINKER_FILE) \ + $(RUSTC_MISC_ARGS) + +CARGO_OUTPUT = target/$(TARGET)/release/kernel + +OBJCOPY_CMD = cargo objcopy \ + -- \ + --strip-all \ + -O binary + +CONTAINER_UTILS = rustembedded/osdev-utils + +DOCKER_CMD = docker run -it --rm +DOCKER_ARG_CURDIR = -v $(shell pwd):/work -w /work +DOCKER_ARG_TTY = --privileged -v /dev:/dev +DOCKER_ARG_JTAG = -v $(shell pwd)/../X1_JTAG_boot:/jtag +DOCKER_ARG_NET = --network host + +DOCKER_EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) -kernel $(OUTPUT) +DOCKER_EXEC_RASPBOOT = raspbootcom +DOCKER_EXEC_RASPBOOT_DEV = /dev/ttyUSB0 +# DOCKER_EXEC_RASPBOOT_DEV = /dev/ttyACM0 + +.PHONY: all doc qemu chainboot clippy clean readelf objdump nm + +all: clean $(OUTPUT) + +$(CARGO_OUTPUT): $(SOURCES) + RUSTFLAGS="-D warnings -D missing_docs" $(XRUSTC_CMD) + +$(OUTPUT): $(CARGO_OUTPUT) + cp $< . + $(OBJCOPY_CMD) $< $(OUTPUT) + +doc: + cargo xdoc --target=$(TARGET) --features bsp_$(BSP) --document-private-items + xdg-open target/$(TARGET)/doc/kernel/index.html + +ifeq ($(QEMU_MACHINE_TYPE),) +qemu: + @echo "This board is not yet supported for QEMU." +else +qemu: all + $(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(CONTAINER_UTILS) \ + $(DOCKER_EXEC_QEMU) $(QEMU_MISC_ARGS) +endif + +chainboot: all + $(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(DOCKER_ARG_TTY) \ + $(CONTAINER_UTILS) $(DOCKER_EXEC_RASPBOOT) $(DOCKER_EXEC_RASPBOOT_DEV) \ + $(OUTPUT) + +jtagboot: + $(DOCKER_CMD) $(DOCKER_ARG_TTY) $(DOCKER_ARG_JTAG) $(CONTAINER_UTILS) \ + $(DOCKER_EXEC_RASPBOOT) $(DOCKER_EXEC_RASPBOOT_DEV) \ + /jtag/$(JTAG_BOOT_IMAGE) + +openocd: + $(DOCKER_CMD) $(DOCKER_ARG_TTY) $(DOCKER_ARG_NET) $(CONTAINER_UTILS) \ + openocd $(OPENOCD_ARG) + +define gen_gdb + RUSTFLAGS="-D warnings -D missing_docs" $(XRUSTC_CMD) $1 + cp $(CARGO_OUTPUT) kernel_for_jtag + $(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(DOCKER_ARG_NET) $(CONTAINER_UTILS) \ + gdb-multiarch -q kernel_for_jtag +endef + +gdb: clean $(SOURCES) + $(call gen_gdb,-C debuginfo=2) + +gdb-opt0: clean $(SOURCES) + $(call gen_gdb,-C debuginfo=2 -C opt-level=0) + +clippy: + cargo xclippy --target=$(TARGET) --features bsp_$(BSP) + +clean: + cargo clean + +readelf: + readelf -a kernel + +objdump: + cargo objdump --target $(TARGET) -- -disassemble -print-imm-hex kernel + +nm: + cargo nm --target $(TARGET) -- kernel | sort diff --git a/11_virtual_memory/README.md b/11_virtual_memory/README.md new file mode 100644 index 00000000..47d93244 --- /dev/null +++ b/11_virtual_memory/README.md @@ -0,0 +1,967 @@ +# Tutorial 11 - Virtual Memory + +## tl;dr + +The `MMU` is turned on; A simple scheme is used: static `64 KiB` page tables; For educational +purposes, we write to a remapped `UART`. + +## Introduction + +Virtual memory is an immensely complex, but important and powerful topic. In this tutorial, we start +slow and easy by switching on the `MMU`, using static page tables and mapping everything at once. + +## MMU and paging theory + +At this point, we will not re-invent the wheel and go into detailed descriptions of how paging in +modern application-grade processors works. The internet is full of great resources regarding this +topic, and we encourage you to read some of it to get a high-level understanding of the topic. + +To follow the rest of this `AArch64` specific tutorial, I strongly recommend that you stop right +here and first read `Chapter 12` of the [ARM Cortex-A Series Programmer's Guide for ARMv8-A] before +you continue. This will set you up with all the `AArch64`-specific knowledge needed to follow along. + +Back from reading `Chapter 12` already? Good job :+1:! + +[ARM Cortex-A Series Programmer's Guide for ARMv8-A]: http://infocenter.arm.com/help/topic/com.arm.doc.den0024a/DEN0024A_v8_architecture_PG.pdf + +## Approach + +- Everything is mapped using a `64 KiB` granule. +- Attributes of different regions (e.g. R/W, no-execute, cached/uncached, etc...) are set in a + high-level data structure in `BSP` code. +- `Arch` code picks up this high-level description and maps it using its specific MMU HW. + +### BSP: `bsp/rpi/virt_mem_layout.rs` + +This file is used to describe our kernel's memory layout in a high-level abstraction using our own +descriptor format. We can define ranges of arbitrary length and set respective attributes, for +example if the bits and bytes in this range should be executable or not. + +The descriptors we use here are agnostic of the hardware `MMU`'s actual descriptors, and we are also +agnostic of the paging granule the `MMU` will use. Having this distinction is less of a technical +need and more a convenience feature for us in order to easily describe the kernel's memory layout, +and hopefully it makes the whole concept a bit more graspable for the reader. + +The file contains an instance of `memory::KernelVirtualLayout`,which stores these descriptors. The +policy is to only store regions that are **not** ordinary, normal chacheable DRAM. However, nothing +prevents you from defining those too if you wish to. Here is an example for the device MMIO region: + +```rust +// Device MMIO. +RangeDescriptor { + name: "Device MMIO", + virtual_range: || { + RangeInclusive::new(memory_map::mmio::BASE, memory_map::mmio::END_INCLUSIVE) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, +}, +``` + +`KernelVirtualLayout` also provides the following method: + +```rust +pub fn get_virt_addr_properties( + &self, + virt_addr: usize, + ) -> Result<(usize, AttributeFields), &'static str> +``` + +It will be used by the `arch`'s MMU code to request attributes for a virtual address and the +translation of the address. The function scans for a descriptor that contains the queried address, +and returns the respective findings for the first entry that is a hit. If no entry is found, it +returns default attributes for normal chacheable DRAM and the input address, hence telling the `MMU` +code that the requested address should be `identity mapped`. + +Due to this default return, it is technicall not needed to define normal cacheable DRAM regions. + +### Arch: `arch/aarch64/mmu.rs` + +This file contains the `AArch64` specific code. It is a driver, if you like, and the paging granule +is hardcoded here (`64 KiB` page descriptors). + +The actual page tables are stored in a global instance of the `PageTables` struct: + +```rust +// Two newtypes for added type safety, so that you cannot accidentally place a TableDescriptor into +// a PageDescriptor slot in `struct PageTables`, and vice versa. +#[derive(Copy, Clone)] +#[repr(transparent)] +struct RawTableDescriptor(u64); + +#[derive(Copy, Clone)] +#[repr(transparent)] +struct RawPageDescriptor(u64); + +/// Big monolithic struct for storing the page tables. Individual levels must be 64 KiB aligned, +/// hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +struct PageTables { + // Page descriptors, covering 64 KiB windows per entry. + lvl3: [[RawPageDescriptor; 8192]; N], + // Table descriptors, covering 512 MiB windows. + lvl2: [RawTableDescriptor; N], +} + +/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. +const ENTRIES_512_MIB: usize = bsp::addr_space_size() >> FIVETWELVE_MIB_SHIFT; + +/// The page tables. +/// +/// Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: PageTables<{ ENTRIES_512_MIB }> = PageTables { + lvl3: [[RawPageDescriptor(0); 8192]; ENTRIES_512_MIB], + lvl2: [RawTableDescriptor(0); ENTRIES_512_MIB], +}; +``` + +They are populated using `get_virt_addr_properties()` and a bunch of utility functions that convert +our own descriptors to the actual `64 bit` integer entries needed by the MMU hardware for the page +table arrays. + +Each page table has an entry (`AttrIndex`) that indexes into the [MAIR_EL1] register, which holds +information about the cacheability of the respective page. We currently define normal cacheable +memory and device memory (which is not cached). + +[MAIR_EL1]: http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500d/CIHDHJBB.html + +```rust +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc + + // Attribute 0 - Device + + MAIR_EL1::Attr0_HIGH::Device + + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, + ); +} +``` + +Afterwards, the [Translation Table Base Register 0 - EL1] is set up with the base address of the +`lvl2` and the [Translation Control Register - EL1] is configured. + +Finally, the MMU is turned on through the [System Control Register - EL1]. The last step also +enables caching for data and instructions. + +[Translation Table Base Register 0 - EL1]: https://docs.rs/crate/cortex-a/2.4.0/source/src/regs/ttbr0_el1.rs +[Translation Control Register - EL1]: https://docs.rs/crate/cortex-a/2.4.0/source/src/regs/tcr_el1.rs +[System Control Register - EL1]: https://docs.rs/crate/cortex-a/2.4.0/source/src/regs/sctlr_el1.rs + +### `link.ld` + +We need to align the `ro` section to `64 KiB` so that it doesn't overlap with the next section that +needs read/write attributes. This blows up the binary in size, but is a small price to pay +considering that it reduces the amount of static paging entries significantly, when compared to the +classical `4 KiB` granule. + +## Address translation examples + +For educational purposes, a layout is defined which allows to access the `UART` via two different +virtual addresses: +- Since we identity map the whole `Device MMIO` region, it is accessible by asserting its physical + base address (`0x3F20_1000` or `0xFA20_1000` depending on which RPi you use) after the `MMU` is + turned on. +- Additionally, it is also mapped into the last `64 KiB` entry of the `lvl3` table, making it + accessible through base address `0x1FFF_1000`. + +The following block diagram visualizes the underlying translation for the second mapping. + +### Address translation using a 64 KiB page descriptor + +![4 KiB translation block diagram](../doc/page_tables_64KiB.png) + +## Zero-cost abstraction + +The MMU init code is again a good example to see the great potential of Rust's zero-cost +abstractions[[1]][[2]] for embedded programming. + +Take this piece of code for setting up the `MAIR_EL1` register using the [cortex-a] crate: + +[1]: https://blog.rust-lang.org/2015/05/11/traits.html +[2]: https://ruudvanasseldonk.com/2016/11/30/zero-cost-abstractions +[cortex-a]: https://crates.io/crates/cortex-a + +```rust +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc + + // Attribute 0 - Device + + MAIR_EL1::Attr0_HIGH::Device + + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, + ); +} +``` + +This piece of code is super expressive, and it makes use of `traits`, different `types` and +`constants` to provide type-safe register manipulation. + +In the end, this code sets the first four bytes of the register to certain values according to the +data sheet. Looking at the generated code, we can see that despite all the type-safety and +abstractions, it boils down to two assembly instructions: + +```text +00000000000818d8 kernel::arch::aarch64::mmu::init::h97006e19222f36e2: + ... + 8191c: 88 e0 9f 52 mov w8, #0xff04 + ... + 81924: 08 a2 18 d5 msr MAIR_EL1, x8 +``` + +## Test it + +```console +make chainbot +[...] +### Listening on /dev/ttyUSB0 + __ __ _ _ _ _ +| \/ (_)_ _ (_) | ___ __ _ __| | +| |\/| | | ' \| | |__/ _ \/ _` / _` | +|_| |_|_|_||_|_|____\___/\__,_\__,_| + + Raspberry Pi 3 + +[ML] Requesting binary +### sending kernel kernel8.img [65560 byte] +### finished sending +[ML] Loaded! Executing the payload now + +[ 5.732854] Booting on: Raspberry Pi 3 +[ 5.792441] MMU online +[ 5.793306] Special memory regions: +[ 5.796778] 0x00080000 - 0x0008FFFF | 64 KiB | C RO PX | Kernel code and RO data +[ 5.805026] 0x1FFF0000 - 0x1FFFFFFF | 64 KiB | Dev RW PXN | Remapped Device MMIO +[ 5.813014] 0x3F000000 - 0x3FFFFFFF | 16 MiB | Dev RW PXN | Device MMIO +[ 5.820220] Current privilege level: EL1 +[ 5.824127] Exception handling state: +[ 5.827774] Debug: Masked +[ 5.830986] SError: Masked +[ 5.834199] IRQ: Masked +[ 5.837412] FIQ: Masked +[ 5.840624] Architectural timer resolution: 52 ns +[ 5.845312] Drivers loaded: +[ 5.848091] 1. GPIO +[ 5.850695] 2. PL011Uart +[ 5.853734] Timer test, spinning for 1 second +[ !!! ] Writing through the remapped UART at 0x1FFF_1000 +[ 6.862236] Echoing input now +``` + +## Diff to previous +```diff + +diff -uNr 10_privilege_level/src/arch/aarch64/mmu.rs 11_virtual_memory/src/arch/aarch64/mmu.rs +--- 10_privilege_level/src/arch/aarch64/mmu.rs ++++ 11_virtual_memory/src/arch/aarch64/mmu.rs +@@ -0,0 +1,292 @@ ++// SPDX-License-Identifier: MIT ++// ++// Copyright (c) 2018-2019 Andre Richter ++ ++//! Memory Management Unit. ++//! ++//! Static page tables, compiled on boot; Everything 64 KiB granule. ++ ++use crate::{ ++ bsp, ++ memory::{AccessPermissions, AttributeFields, MemAttributes}, ++}; ++use core::convert; ++use cortex_a::{barrier, regs::*}; ++use register::register_bitfields; ++ ++// A table descriptor, as per AArch64 Reference Manual Figure D4-15. ++register_bitfields! {u64, ++ STAGE1_TABLE_DESCRIPTOR [ ++ /// Physical address of the next page table. ++ NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] ++ ++ TYPE OFFSET(1) NUMBITS(1) [ ++ Block = 0, ++ Table = 1 ++ ], ++ ++ VALID OFFSET(0) NUMBITS(1) [ ++ False = 0, ++ True = 1 ++ ] ++ ] ++} ++ ++// A level 3 page descriptor, as per AArch64 Reference Manual Figure D4-17. ++register_bitfields! {u64, ++ STAGE1_PAGE_DESCRIPTOR [ ++ /// Privileged execute-never ++ PXN OFFSET(53) NUMBITS(1) [ ++ False = 0, ++ True = 1 ++ ], ++ ++ /// Physical address of the next page table (lvl2) or the page descriptor (lvl3). ++ OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] ++ ++ /// Access flag ++ AF OFFSET(10) NUMBITS(1) [ ++ False = 0, ++ True = 1 ++ ], ++ ++ /// Shareability field ++ SH OFFSET(8) NUMBITS(2) [ ++ OuterShareable = 0b10, ++ InnerShareable = 0b11 ++ ], ++ ++ /// Access Permissions ++ AP OFFSET(6) NUMBITS(2) [ ++ RW_EL1 = 0b00, ++ RW_EL1_EL0 = 0b01, ++ RO_EL1 = 0b10, ++ RO_EL1_EL0 = 0b11 ++ ], ++ ++ /// Memory attributes index into the MAIR_EL1 register ++ AttrIndx OFFSET(2) NUMBITS(3) [], ++ ++ TYPE OFFSET(1) NUMBITS(1) [ ++ Block = 0, ++ Table = 1 ++ ], ++ ++ VALID OFFSET(0) NUMBITS(1) [ ++ False = 0, ++ True = 1 ++ ] ++ ] ++} ++ ++// Two newtypes for added type safety, so that you cannot accidentally place a TableDescriptor into ++// a PageDescriptor slot in `struct PageTables`, and vice versa. ++#[derive(Copy, Clone)] ++#[repr(transparent)] ++struct RawTableDescriptor(u64); ++ ++#[derive(Copy, Clone)] ++#[repr(transparent)] ++struct RawPageDescriptor(u64); ++ ++const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) ++const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) ++ ++/// Big monolithic struct for storing the page tables. Individual levels must be 64 KiB aligned, ++/// hence the "reverse" order of appearance. ++#[repr(C)] ++#[repr(align(65536))] ++struct PageTables { ++ // Page descriptors, covering 64 KiB windows per entry. ++ lvl3: [[RawPageDescriptor; 8192]; N], ++ // Table descriptors, covering 512 MiB windows. ++ lvl2: [RawTableDescriptor; N], ++} ++ ++/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. ++const ENTRIES_512_MIB: usize = bsp::addr_space_size() >> FIVETWELVE_MIB_SHIFT; ++ ++/// The page tables. ++/// ++/// Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. ++static mut TABLES: PageTables<{ ENTRIES_512_MIB }> = PageTables { ++ lvl3: [[RawPageDescriptor(0); 8192]; ENTRIES_512_MIB], ++ lvl2: [RawTableDescriptor(0); ENTRIES_512_MIB], ++}; ++ ++trait BaseAddr { ++ fn base_addr_u64(&self) -> u64; ++ fn base_addr_usize(&self) -> usize; ++} ++ ++impl BaseAddr for [T; N] { ++ fn base_addr_u64(&self) -> u64 { ++ self as *const T as u64 ++ } ++ ++ fn base_addr_usize(&self) -> usize { ++ self as *const T as usize ++ } ++} ++ ++/// A descriptor pointing to the next page table. ++struct TableDescriptor(register::FieldValue); ++ ++impl TableDescriptor { ++ fn new(next_lvl_table_addr: usize) -> TableDescriptor { ++ let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; ++ ++ TableDescriptor( ++ STAGE1_TABLE_DESCRIPTOR::VALID::True ++ + STAGE1_TABLE_DESCRIPTOR::TYPE::Table ++ + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), ++ ) ++ } ++} ++ ++impl convert::From for RawTableDescriptor { ++ fn from(desc: TableDescriptor) -> Self { ++ RawTableDescriptor(desc.0.value) ++ } ++} ++ ++/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. ++impl convert::From ++ for register::FieldValue ++{ ++ fn from(attribute_fields: AttributeFields) -> Self { ++ // Memory attributes ++ let mut desc = match attribute_fields.mem_attributes { ++ MemAttributes::CacheableDRAM => { ++ STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable ++ + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) ++ } ++ MemAttributes::Device => { ++ STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable ++ + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) ++ } ++ }; ++ ++ // Access Permissions ++ desc += match attribute_fields.acc_perms { ++ AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, ++ AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, ++ }; ++ ++ // Execute Never ++ desc += if attribute_fields.execute_never { ++ STAGE1_PAGE_DESCRIPTOR::PXN::True ++ } else { ++ STAGE1_PAGE_DESCRIPTOR::PXN::False ++ }; ++ ++ desc ++ } ++} ++ ++/// A page descriptor with 64 KiB aperture. ++/// ++/// The output points to physical memory. ++struct PageDescriptor(register::FieldValue); ++ ++impl PageDescriptor { ++ fn new(output_addr: usize, attribute_fields: AttributeFields) -> PageDescriptor { ++ let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; ++ ++ PageDescriptor( ++ STAGE1_PAGE_DESCRIPTOR::VALID::True ++ + STAGE1_PAGE_DESCRIPTOR::AF::True ++ + attribute_fields.into() ++ + STAGE1_PAGE_DESCRIPTOR::TYPE::Table ++ + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64), ++ ) ++ } ++} ++ ++impl convert::From for RawPageDescriptor { ++ fn from(desc: PageDescriptor) -> Self { ++ RawPageDescriptor(desc.0.value) ++ } ++} ++ ++/// Constants for indexing the MAIR_EL1. ++#[allow(dead_code)] ++mod mair { ++ pub const DEVICE: u64 = 0; ++ pub const NORMAL: u64 = 1; ++} ++ ++/// Setup function for the MAIR_EL1 register. ++fn set_up_mair() { ++ // Define the memory types being mapped. ++ MAIR_EL1.write( ++ // Attribute 1 - Cacheable normal DRAM ++ MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc ++ + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc ++ ++ // Attribute 0 - Device ++ + MAIR_EL1::Attr0_HIGH::Device ++ + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, ++ ); ++} ++ ++/// Compile the page tables from the `BSP`-supplied `virt_mem_layout()`. ++/// ++/// # Safety ++/// ++/// - User must ensure that the hardware supports the paremeters being set here. ++pub unsafe fn init() -> Result<(), &'static str> { ++ // Fail early if translation granule is not supported. Both RPis support it, though. ++ if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { ++ return Err("MMU does not support 64 KiB translation granule"); ++ } ++ ++ // Prepare the memory attribute indirection register. ++ set_up_mair(); ++ ++ // Iterate over all page table entries and fill them at once. ++ for (l2_nr, l2_entry) in TABLES.lvl2.iter_mut().enumerate() { ++ *l2_entry = TableDescriptor::new(TABLES.lvl3[l2_nr].base_addr_usize()).into(); ++ ++ for (l3_nr, l3_entry) in TABLES.lvl3[l2_nr].iter_mut().enumerate() { ++ let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); ++ ++ let (output_addr, attribute_fields) = ++ match bsp::virt_mem_layout().get_virt_addr_properties(virt_addr) { ++ Err(string) => return Err(string), ++ Ok((a, b)) => (a, b), ++ }; ++ ++ *l3_entry = PageDescriptor::new(output_addr, attribute_fields).into(); ++ } ++ } ++ ++ // Set the "Translation Table Base Register". ++ TTBR0_EL1.set_baddr(TABLES.lvl2.base_addr_u64()); ++ ++ // Configure various settings of stage 1 of the EL1 translation regime. ++ let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); ++ TCR_EL1.write( ++ TCR_EL1::TBI0::Ignored ++ + TCR_EL1::IPS.val(ips) ++ + TCR_EL1::TG0::KiB_64 ++ + TCR_EL1::SH0::Inner ++ + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable ++ + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable ++ + TCR_EL1::EPD0::EnableTTBR0Walks ++ + TCR_EL1::T0SZ.val(32), // TTBR0 spans 4 GiB total. ++ ); ++ ++ // Switch the MMU on. ++ // ++ // First, force all previous changes to be seen before the MMU is enabled. ++ barrier::isb(barrier::SY); ++ ++ // Enable the MMU and turn on data and instruction caching. ++ SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); ++ ++ // Force MMU init to complete before next instruction ++ barrier::isb(barrier::SY); ++ ++ Ok(()) ++} + +diff -uNr 10_privilege_level/src/arch/aarch64.rs 11_virtual_memory/src/arch/aarch64.rs +--- 10_privilege_level/src/arch/aarch64.rs ++++ 11_virtual_memory/src/arch/aarch64.rs +@@ -5,6 +5,7 @@ + //! AArch64. + + mod exception; ++pub mod mmu; + pub mod sync; + mod time; + + +diff -uNr 10_privilege_level/src/bsp/rpi/link.ld 11_virtual_memory/src/bsp/rpi/link.ld +--- 10_privilege_level/src/bsp/rpi/link.ld ++++ 11_virtual_memory/src/bsp/rpi/link.ld +@@ -8,6 +8,7 @@ + /* Set current address to the value from which the RPi starts execution */ + . = 0x80000; + ++ __ro_start = .; + .text : + { + *(.text._start) *(.text*) +@@ -17,6 +18,8 @@ + { + *(.rodata*) + } ++ . = ALIGN(65536); /* Fill up to 64 KiB */ ++ __ro_end = .; + + .data : + { + +diff -uNr 10_privilege_level/src/bsp/rpi/memory_map.rs 11_virtual_memory/src/bsp/rpi/memory_map.rs +--- 10_privilege_level/src/bsp/rpi/memory_map.rs ++++ 11_virtual_memory/src/bsp/rpi/memory_map.rs +@@ -4,6 +4,14 @@ + + //! The board's memory map. + ++#[cfg(feature = "bsp_rpi3")] ++#[rustfmt::skip] ++pub const END_INCLUSIVE: usize = 0x3FFF_FFFF; ++ ++#[cfg(feature = "bsp_rpi4")] ++#[rustfmt::skip] ++pub const END_INCLUSIVE: usize = 0xFFFF_FFFF; ++ + /// Physical devices. + #[rustfmt::skip] + pub mod mmio { +@@ -15,4 +23,5 @@ + + pub const GPIO_BASE: usize = BASE + 0x0020_0000; + pub const PL011_UART_BASE: usize = BASE + 0x0020_1000; ++ pub const END_INCLUSIVE: usize = super::END_INCLUSIVE; + } + +diff -uNr 10_privilege_level/src/bsp/rpi/virt_mem_layout.rs 11_virtual_memory/src/bsp/rpi/virt_mem_layout.rs +--- 10_privilege_level/src/bsp/rpi/virt_mem_layout.rs ++++ 11_virtual_memory/src/bsp/rpi/virt_mem_layout.rs +@@ -0,0 +1,78 @@ ++// SPDX-License-Identifier: MIT ++// ++// Copyright (c) 2018-2019 Andre Richter ++ ++//! The virtual memory layout. ++//! ++//! The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. ++//! It is agnostic of the paging granularity that the architecture's MMU will use. ++ ++use super::memory_map; ++use crate::memory::*; ++use core::ops::RangeInclusive; ++ ++pub const NUM_MEM_RANGES: usize = 3; ++ ++pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( ++ memory_map::END_INCLUSIVE, ++ [ ++ RangeDescriptor { ++ name: "Kernel code and RO data", ++ virtual_range: || { ++ // Using the linker script, we ensure that the RO area is consecutive and 4 KiB ++ // aligned, and we export the boundaries via symbols: ++ // ++ // [__ro_start, __ro_end) ++ extern "C" { ++ // The inclusive start of the read-only area, aka the address of the first ++ // byte of the area. ++ static __ro_start: u64; ++ ++ // The exclusive end of the read-only area, aka the address of the first ++ // byte _after_ the RO area. ++ static __ro_end: u64; ++ } ++ ++ unsafe { ++ // Notice the subtraction to turn the exclusive end into an inclusive end. ++ #[allow(clippy::range_minus_one)] ++ RangeInclusive::new( ++ &__ro_start as *const _ as usize, ++ &__ro_end as *const _ as usize - 1, ++ ) ++ } ++ }, ++ translation: Translation::Identity, ++ attribute_fields: AttributeFields { ++ mem_attributes: MemAttributes::CacheableDRAM, ++ acc_perms: AccessPermissions::ReadOnly, ++ execute_never: false, ++ }, ++ }, ++ RangeDescriptor { ++ name: "Remapped Device MMIO", ++ virtual_range: || { ++ // The last 64 KiB slot in the first 512 MiB ++ RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF) ++ }, ++ translation: Translation::Offset(memory_map::mmio::BASE + 0x20_0000), ++ attribute_fields: AttributeFields { ++ mem_attributes: MemAttributes::Device, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ }, ++ }, ++ RangeDescriptor { ++ name: "Device MMIO", ++ virtual_range: || { ++ RangeInclusive::new(memory_map::mmio::BASE, memory_map::mmio::END_INCLUSIVE) ++ }, ++ translation: Translation::Identity, ++ attribute_fields: AttributeFields { ++ mem_attributes: MemAttributes::Device, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ }, ++ }, ++ ], ++); + +diff -uNr 10_privilege_level/src/bsp/rpi.rs 11_virtual_memory/src/bsp/rpi.rs +--- 10_privilege_level/src/bsp/rpi.rs ++++ 11_virtual_memory/src/bsp/rpi.rs +@@ -5,9 +5,10 @@ + //! Board Support Package for the Raspberry Pi. + + mod memory_map; ++mod virt_mem_layout; + + use super::driver; +-use crate::interface; ++use crate::{interface, memory::KernelVirtualLayout}; + + pub const BOOT_CORE_ID: u64 = 0; + pub const BOOT_CORE_STACK_START: u64 = 0x80_000; +@@ -56,3 +57,13 @@ + // Configure PL011Uart's output pins. + GPIO.map_pl011_uart(); + } ++ ++/// Return the address space size in bytes. ++pub const fn addr_space_size() -> usize { ++ memory_map::END_INCLUSIVE + 1 ++} ++ ++/// Return a reference to the virtual memory layout. ++pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ virt_mem_layout::NUM_MEM_RANGES }> { ++ &virt_mem_layout::LAYOUT ++} + +diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_memory/src/bsp.rs +--- 10_privilege_level/src/bsp.rs ++++ 11_virtual_memory/src/bsp.rs +@@ -4,7 +4,7 @@ + + //! Conditional exporting of Board Support Packages. + +-mod driver; ++pub mod driver; + + #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] + mod rpi; + +diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs +--- 10_privilege_level/src/main.rs ++++ 11_virtual_memory/src/main.rs +@@ -19,6 +19,8 @@ + //! [Architecture-specific code]: arch/index.html + //! [`kernel::interface`]: interface/index.html + ++#![allow(incomplete_features)] ++#![feature(const_generics)] + #![feature(format_args_nl)] + #![feature(panic_info_message)] + #![feature(trait_alias)] +@@ -36,6 +38,7 @@ + mod bsp; + + mod interface; ++mod memory; + mod panic_wait; + mod print; + +@@ -48,6 +51,7 @@ + /// - Only a single core must be active and running this function. + /// - The init calls in this function must appear in the correct order. + unsafe fn kernel_init() -> ! { ++ // Bring up device drivers first, so that eventual MMU errors can be printed. + for i in bsp::device_drivers().iter() { + if let Err(()) = i.init() { + // This message will only be readable if, at the time of failure, the return value of +@@ -58,6 +62,13 @@ + + bsp::post_driver_init(); + ++ println!("Booting on: {}", bsp::board_name()); ++ ++ if let Err(string) = arch::mmu::init() { ++ panic!("MMU: {}", string); ++ } ++ println!("MMU online"); ++ + // Transition from unsafe to safe. + kernel_main() + } +@@ -67,7 +78,7 @@ + use core::time::Duration; + use interface::{console::All, time::Timer}; + +- println!("Booting on: {}", bsp::board_name()); ++ bsp::virt_mem_layout().print_layout(); + + println!( + "Current privilege level: {}", +@@ -89,6 +100,13 @@ + println!("Timer test, spinning for 1 second"); + arch::timer().spin_for(Duration::from_secs(1)); + ++ let remapped_uart = unsafe { bsp::driver::PL011Uart::new(0x1FFF_1000) }; ++ writeln!( ++ remapped_uart, ++ "[ !!! ] Writing through the remapped UART at 0x1FFF_1000" ++ ) ++ .unwrap(); ++ + println!("Echoing input now"); + loop { + let c = bsp::console().read_char(); + +diff -uNr 10_privilege_level/src/memory.rs 11_virtual_memory/src/memory.rs +--- 10_privilege_level/src/memory.rs ++++ 11_virtual_memory/src/memory.rs +@@ -0,0 +1,149 @@ ++// SPDX-License-Identifier: MIT ++// ++// Copyright (c) 2018-2019 Andre Richter ++ ++//! Memory Management. ++ ++use core::{fmt, ops::RangeInclusive}; ++ ++#[derive(Copy, Clone)] ++pub enum Translation { ++ Identity, ++ Offset(usize), ++} ++ ++#[derive(Copy, Clone)] ++pub enum MemAttributes { ++ CacheableDRAM, ++ Device, ++} ++ ++#[derive(Copy, Clone)] ++pub enum AccessPermissions { ++ ReadOnly, ++ ReadWrite, ++} ++ ++#[derive(Copy, Clone)] ++pub struct AttributeFields { ++ pub mem_attributes: MemAttributes, ++ pub acc_perms: AccessPermissions, ++ pub execute_never: bool, ++} ++ ++impl Default for AttributeFields { ++ fn default() -> AttributeFields { ++ AttributeFields { ++ mem_attributes: MemAttributes::CacheableDRAM, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ } ++ } ++} ++ ++/// An architecture agnostic descriptor for a memory range. ++pub struct RangeDescriptor { ++ pub name: &'static str, ++ pub virtual_range: fn() -> RangeInclusive, ++ pub translation: Translation, ++ pub attribute_fields: AttributeFields, ++} ++ ++/// Human-readable output of a RangeDescriptor. ++impl fmt::Display for RangeDescriptor { ++ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { ++ // Call the function to which self.range points, and dereference the result, which causes ++ // Rust to copy the value. ++ let start = *(self.virtual_range)().start(); ++ let end = *(self.virtual_range)().end(); ++ let size = end - start + 1; ++ ++ // log2(1024). ++ const KIB_RSHIFT: u32 = 10; ++ ++ // log2(1024 * 1024). ++ const MIB_RSHIFT: u32 = 20; ++ ++ let (size, unit) = if (size >> MIB_RSHIFT) > 0 { ++ (size >> MIB_RSHIFT, "MiB") ++ } else if (size >> KIB_RSHIFT) > 0 { ++ (size >> KIB_RSHIFT, "KiB") ++ } else { ++ (size, "Byte") ++ }; ++ ++ let attr = match self.attribute_fields.mem_attributes { ++ MemAttributes::CacheableDRAM => "C", ++ MemAttributes::Device => "Dev", ++ }; ++ ++ let acc_p = match self.attribute_fields.acc_perms { ++ AccessPermissions::ReadOnly => "RO", ++ AccessPermissions::ReadWrite => "RW", ++ }; ++ ++ let xn = if self.attribute_fields.execute_never { ++ "PXN" ++ } else { ++ "PX" ++ }; ++ ++ write!( ++ f, ++ " {:#010X} - {:#010X} | {: >3} {} | {: <3} {} {: <3} | {}", ++ start, end, size, unit, attr, acc_p, xn, self.name ++ ) ++ } ++} ++ ++/// Type for expressing the kernel's virtual memory layout. ++pub struct KernelVirtualLayout { ++ max_virt_addr_inclusive: usize, ++ inner: [RangeDescriptor; NUM_SPECIAL_RANGES], ++} ++ ++impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { ++ pub const fn new(max: usize, layout: [RangeDescriptor; NUM_SPECIAL_RANGES]) -> Self { ++ Self { ++ max_virt_addr_inclusive: max, ++ inner: layout, ++ } ++ } ++ ++ /// For a virtual address, find and return the output address and corresponding attributes. ++ /// ++ /// If the address is not found in `inner`, return an identity mapped default with normal ++ /// cacheable DRAM attributes. ++ pub fn get_virt_addr_properties( ++ &self, ++ virt_addr: usize, ++ ) -> Result<(usize, AttributeFields), &'static str> { ++ if virt_addr > self.max_virt_addr_inclusive { ++ return Err("Address out of range"); ++ } ++ ++ for i in self.inner.iter() { ++ if (i.virtual_range)().contains(&virt_addr) { ++ let output_addr = match i.translation { ++ Translation::Identity => virt_addr, ++ Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), ++ }; ++ ++ return Ok((output_addr, i.attribute_fields)); ++ } ++ } ++ ++ Ok((virt_addr, AttributeFields::default())) ++ } ++ ++ /// Print the memory layout. ++ pub fn print_layout(&self) { ++ use crate::println; ++ ++ println!("Special memory regions:"); ++ ++ for i in self.inner.iter() { ++ println!("{}", i); ++ } ++ } ++} + +``` diff --git a/11_virtual_memory/kernel b/11_virtual_memory/kernel new file mode 100755 index 00000000..f5c63313 Binary files /dev/null and b/11_virtual_memory/kernel differ diff --git a/11_virtual_memory/kernel8.img b/11_virtual_memory/kernel8.img new file mode 100755 index 00000000..24e4e5f5 Binary files /dev/null and b/11_virtual_memory/kernel8.img differ diff --git a/11_virtual_memory/src/arch.rs b/11_virtual_memory/src/arch.rs new file mode 100644 index 00000000..b1f035c5 --- /dev/null +++ b/11_virtual_memory/src/arch.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Conditional exporting of processor architecture code. + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod aarch64; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use aarch64::*; diff --git a/11_virtual_memory/src/arch/aarch64.rs b/11_virtual_memory/src/arch/aarch64.rs new file mode 100644 index 00000000..97f45fad --- /dev/null +++ b/11_virtual_memory/src/arch/aarch64.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! AArch64. + +mod exception; +pub mod mmu; +pub mod sync; +mod time; + +use crate::{bsp, interface}; +use cortex_a::{asm, regs::*}; + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function at `0x80_000`. +#[no_mangle] +pub unsafe extern "C" fn _start() -> ! { + const CORE_MASK: u64 = 0x3; + + // Expect the boot core to start in EL2. + if (bsp::BOOT_CORE_ID == MPIDR_EL1.get() & CORE_MASK) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + wait_forever() + } +} + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with ´runtime_init::init()`. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status, where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to init(). + ELR_EL2.set(crate::runtime_init::init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::BOOT_CORE_STACK_START); + + // Use `eret` to "return" to EL1. This will result in execution of `reset()` in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static TIMER: time::Timer = time::Timer; + +//-------------------------------------------------------------------------------------------------- +// Implementation of the kernel's architecture abstraction code +//-------------------------------------------------------------------------------------------------- + +pub use asm::nop; + +/// Spin for `n` cycles. +pub fn spin_for_cycles(n: usize) { + for _ in 0..n { + asm::nop(); + } +} + +/// Return a reference to a `interface::time::TimeKeeper` implementation. +pub fn timer() -> &'static impl interface::time::Timer { + &TIMER +} + +/// Pause execution on the calling CPU core. +#[inline(always)] +pub fn wait_forever() -> ! { + loop { + asm::wfe() + } +} + +/// Information about the HW state. +pub mod state { + use cortex_a::regs::*; + + /// The current privilege level. + pub fn current_privilege_level() -> &'static str { + let el = CurrentEL.read_as_enum(CurrentEL::EL); + match el { + Some(CurrentEL::EL::Value::EL2) => "EL2", + Some(CurrentEL::EL::Value::EL1) => "EL1", + _ => "Unknown", + } + } + + #[rustfmt::skip] + pub fn print_exception_state() { + use super::{ + exception, + exception::{Debug, SError, FIQ, IRQ}, + }; + use crate::println; + + let to_mask_str = |x: bool| -> &'static str { + if x { "Masked" } else { "Unmasked" } + }; + + println!(" Debug: {}", to_mask_str(exception::is_masked::())); + println!(" SError: {}", to_mask_str(exception::is_masked::())); + println!(" IRQ: {}", to_mask_str(exception::is_masked::())); + println!(" FIQ: {}", to_mask_str(exception::is_masked::())); + } +} diff --git a/11_virtual_memory/src/arch/aarch64/exception.rs b/11_virtual_memory/src/arch/aarch64/exception.rs new file mode 100644 index 00000000..27939b84 --- /dev/null +++ b/11_virtual_memory/src/arch/aarch64/exception.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Exception handling. + +use cortex_a::regs::*; + +pub trait DaifField { + fn daif_field() -> register::Field; +} + +pub struct Debug; +pub struct SError; +pub struct IRQ; +pub struct FIQ; + +impl DaifField for Debug { + fn daif_field() -> register::Field { + DAIF::D + } +} + +impl DaifField for SError { + fn daif_field() -> register::Field { + DAIF::A + } +} + +impl DaifField for IRQ { + fn daif_field() -> register::Field { + DAIF::I + } +} + +impl DaifField for FIQ { + fn daif_field() -> register::Field { + DAIF::F + } +} + +pub fn is_masked() -> bool { + DAIF.is_set(T::daif_field()) +} diff --git a/11_virtual_memory/src/arch/aarch64/mmu.rs b/11_virtual_memory/src/arch/aarch64/mmu.rs new file mode 100644 index 00000000..441eba76 --- /dev/null +++ b/11_virtual_memory/src/arch/aarch64/mmu.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Memory Management Unit. +//! +//! Static page tables, compiled on boot; Everything 64 KiB granule. + +use crate::{ + bsp, + memory::{AccessPermissions, AttributeFields, MemAttributes}, +}; +use core::convert; +use cortex_a::{barrier, regs::*}; +use register::register_bitfields; + +// A table descriptor, as per AArch64 Reference Manual Figure D4-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next page table. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per AArch64 Reference Manual Figure D4-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Privileged execute-never + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next page table (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// Two newtypes for added type safety, so that you cannot accidentally place a TableDescriptor into +// a PageDescriptor slot in `struct PageTables`, and vice versa. +#[derive(Copy, Clone)] +#[repr(transparent)] +struct RawTableDescriptor(u64); + +#[derive(Copy, Clone)] +#[repr(transparent)] +struct RawPageDescriptor(u64); + +const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) +const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) + +/// Big monolithic struct for storing the page tables. Individual levels must be 64 KiB aligned, +/// hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +struct PageTables { + // Page descriptors, covering 64 KiB windows per entry. + lvl3: [[RawPageDescriptor; 8192]; N], + // Table descriptors, covering 512 MiB windows. + lvl2: [RawTableDescriptor; N], +} + +/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. +const ENTRIES_512_MIB: usize = bsp::addr_space_size() >> FIVETWELVE_MIB_SHIFT; + +/// The page tables. +/// +/// Supposed to land in `.bss`. Therefore, ensure that they boil down to all "0" entries. +static mut TABLES: PageTables<{ ENTRIES_512_MIB }> = PageTables { + lvl3: [[RawPageDescriptor(0); 8192]; ENTRIES_512_MIB], + lvl2: [RawTableDescriptor(0); ENTRIES_512_MIB], +}; + +trait BaseAddr { + fn base_addr_u64(&self) -> u64; + fn base_addr_usize(&self) -> usize; +} + +impl BaseAddr for [T; N] { + fn base_addr_u64(&self) -> u64 { + self as *const T as u64 + } + + fn base_addr_usize(&self) -> usize { + self as *const T as usize + } +} + +/// A descriptor pointing to the next page table. +struct TableDescriptor(register::FieldValue); + +impl TableDescriptor { + fn new(next_lvl_table_addr: usize) -> TableDescriptor { + let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; + + TableDescriptor( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ) + } +} + +impl convert::From for RawTableDescriptor { + fn from(desc: TableDescriptor) -> Self { + RawTableDescriptor(desc.0.value) + } +} + +/// Convert the kernel's generic memory range attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + desc + } +} + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +struct PageDescriptor(register::FieldValue); + +impl PageDescriptor { + fn new(output_addr: usize, attribute_fields: AttributeFields) -> PageDescriptor { + let shifted = output_addr >> SIXTYFOUR_KIB_SHIFT; + + PageDescriptor( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted as u64), + ) + } +} + +impl convert::From for RawPageDescriptor { + fn from(desc: PageDescriptor) -> Self { + RawPageDescriptor(desc.0.value) + } +} + +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; +} + +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM + MAIR_EL1::Attr1_HIGH::Memory_OuterWriteBack_NonTransient_ReadAlloc_WriteAlloc + + MAIR_EL1::Attr1_LOW_MEMORY::InnerWriteBack_NonTransient_ReadAlloc_WriteAlloc + + // Attribute 0 - Device + + MAIR_EL1::Attr0_HIGH::Device + + MAIR_EL1::Attr0_LOW_DEVICE::Device_nGnRE, + ); +} + +/// Compile the page tables from the `BSP`-supplied `virt_mem_layout()`. +/// +/// # Safety +/// +/// - User must ensure that the hardware supports the paremeters being set here. +pub unsafe fn init() -> Result<(), &'static str> { + // Fail early if translation granule is not supported. Both RPis support it, though. + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { + return Err("MMU does not support 64 KiB translation granule"); + } + + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Iterate over all page table entries and fill them at once. + for (l2_nr, l2_entry) in TABLES.lvl2.iter_mut().enumerate() { + *l2_entry = TableDescriptor::new(TABLES.lvl3[l2_nr].base_addr_usize()).into(); + + for (l3_nr, l3_entry) in TABLES.lvl3[l2_nr].iter_mut().enumerate() { + let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); + + let (output_addr, attribute_fields) = + match bsp::virt_mem_layout().get_virt_addr_properties(virt_addr) { + Err(string) => return Err(string), + Ok((a, b)) => (a, b), + }; + + *l3_entry = PageDescriptor::new(output_addr, attribute_fields).into(); + } + } + + // Set the "Translation Table Base Register". + TTBR0_EL1.set_baddr(TABLES.lvl2.base_addr_u64()); + + // Configure various settings of stage 1 of the EL1 translation regime. + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(32), // TTBR0 spans 4 GiB total. + ); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction + barrier::isb(barrier::SY); + + Ok(()) +} diff --git a/11_virtual_memory/src/arch/aarch64/sync.rs b/11_virtual_memory/src/arch/aarch64/sync.rs new file mode 100644 index 00000000..dfebc0e1 --- /dev/null +++ b/11_virtual_memory/src/arch/aarch64/sync.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Synchronization primitives. + +use crate::interface; +use core::cell::UnsafeCell; + +/// A pseudo-lock for teaching purposes. +/// +/// Used to introduce [interior mutability]. +/// +/// In contrast to a real Mutex implementation, does not protect against concurrent access to the +/// contained data. This part is preserved for later lessons. +/// +/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is +/// executing single-threaded, aka only running on a single core with interrupts disabled. +/// +/// [interior mutability]: https://doc.rust-lang.org/std/cell/index.html +pub struct NullLock { + data: UnsafeCell, +} + +unsafe impl Send for NullLock {} +unsafe impl Sync for NullLock {} + +impl NullLock { + pub const fn new(data: T) -> NullLock { + NullLock { + data: UnsafeCell::new(data), + } + } +} + +impl interface::sync::Mutex for &NullLock { + type Data = T; + + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + // In a real lock, there would be code encapsulating this line that ensures that this + // mutable reference will ever only be given out once at a time. + f(unsafe { &mut *self.data.get() }) + } +} diff --git a/11_virtual_memory/src/arch/aarch64/time.rs b/11_virtual_memory/src/arch/aarch64/time.rs new file mode 100644 index 00000000..a6c9d50c --- /dev/null +++ b/11_virtual_memory/src/arch/aarch64/time.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Timer primitives. + +use crate::{interface, warn}; +use core::time::Duration; +use cortex_a::regs::*; + +const NS_PER_S: u64 = 1_000_000_000; + +pub struct Timer; + +impl interface::time::Timer for Timer { + fn resolution(&self) -> Duration { + Duration::from_nanos(NS_PER_S / (CNTFRQ_EL0.get() as u64)) + } + + fn uptime(&self) -> Duration { + let frq: u64 = CNTFRQ_EL0.get() as u64; + let current_count: u64 = CNTPCT_EL0.get() * NS_PER_S; + + Duration::from_nanos(current_count / frq) + } + + fn spin_for(&self, duration: Duration) { + // Instantly return on zero. + if duration.as_nanos() == 0 { + return; + } + + // Calculate the register compare value. + let frq = CNTFRQ_EL0.get() as u64; + let x = match frq.checked_mul(duration.as_nanos() as u64) { + None => { + warn!("Spin duration too long, skipping"); + return; + } + Some(val) => val, + }; + let tval = x / NS_PER_S; + + // Check if it is within supported bounds. + let warn: Option<&str> = if tval == 0 { + Some("smaller") + } else if tval > u32::max_value().into() { + Some("bigger") + } else { + None + }; + + if let Some(w) = warn { + warn!( + "Spin duration {} than architecturally supported, skipping", + w + ); + return; + } + + // Set the compare value register. + CNTP_TVAL_EL0.set(tval as u32); + + // Kick off the counting. // Disable timer interrupt. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); + + loop { + // ISTATUS will be '1' when cval ticks have passed. Busy-check it. + if CNTP_CTL_EL0.is_set(CNTP_CTL_EL0::ISTATUS) { + break; + } + } + + // Disable counting again. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::CLEAR); + } +} diff --git a/11_virtual_memory/src/bsp.rs b/11_virtual_memory/src/bsp.rs new file mode 100644 index 00000000..5c5656d7 --- /dev/null +++ b/11_virtual_memory/src/bsp.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Conditional exporting of Board Support Packages. + +pub mod driver; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod rpi; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use rpi::*; diff --git a/11_virtual_memory/src/bsp/driver.rs b/11_virtual_memory/src/bsp/driver.rs new file mode 100644 index 00000000..c910274e --- /dev/null +++ b/11_virtual_memory/src/bsp/driver.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Drivers. + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod bcm; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use bcm::*; diff --git a/11_virtual_memory/src/bsp/driver/bcm.rs b/11_virtual_memory/src/bsp/driver/bcm.rs new file mode 100644 index 00000000..15283aea --- /dev/null +++ b/11_virtual_memory/src/bsp/driver/bcm.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! BCM driver top level. + +mod bcm2xxx_gpio; +mod bcm2xxx_pl011_uart; + +pub use bcm2xxx_gpio::GPIO; +pub use bcm2xxx_pl011_uart::PL011Uart; diff --git a/11_virtual_memory/src/bsp/driver/bcm/bcm2xxx_gpio.rs b/11_virtual_memory/src/bsp/driver/bcm/bcm2xxx_gpio.rs new file mode 100644 index 00000000..a9ceda61 --- /dev/null +++ b/11_virtual_memory/src/bsp/driver/bcm/bcm2xxx_gpio.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! GPIO driver. + +use crate::{arch, arch::sync::NullLock, interface}; +use core::ops; +use register::{mmio::ReadWrite, register_bitfields}; + +// GPIO registers. +// +// Descriptions taken from +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +register_bitfields! { + u32, + + /// GPIO Function Select 1 + GPFSEL1 [ + /// Pin 15 + FSEL15 OFFSET(15) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART RX + + ], + + /// Pin 14 + FSEL14 OFFSET(12) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART TX + ] + ], + + /// GPIO Pull-up/down Clock Register 0 + GPPUDCLK0 [ + /// Pin 15 + PUDCLK15 OFFSET(15) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ], + + /// Pin 14 + PUDCLK14 OFFSET(14) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ] + ] +} + +#[allow(non_snake_case)] +#[repr(C)] +pub struct RegisterBlock { + pub GPFSEL0: ReadWrite, // 0x00 + pub GPFSEL1: ReadWrite, // 0x04 + pub GPFSEL2: ReadWrite, // 0x08 + pub GPFSEL3: ReadWrite, // 0x0C + pub GPFSEL4: ReadWrite, // 0x10 + pub GPFSEL5: ReadWrite, // 0x14 + __reserved_0: u32, // 0x18 + GPSET0: ReadWrite, // 0x1C + GPSET1: ReadWrite, // 0x20 + __reserved_1: u32, // + GPCLR0: ReadWrite, // 0x28 + __reserved_2: [u32; 2], // + GPLEV0: ReadWrite, // 0x34 + GPLEV1: ReadWrite, // 0x38 + __reserved_3: u32, // + GPEDS0: ReadWrite, // 0x40 + GPEDS1: ReadWrite, // 0x44 + __reserved_4: [u32; 7], // + GPHEN0: ReadWrite, // 0x64 + GPHEN1: ReadWrite, // 0x68 + __reserved_5: [u32; 10], // + pub GPPUD: ReadWrite, // 0x94 + pub GPPUDCLK0: ReadWrite, // 0x98 + pub GPPUDCLK1: ReadWrite, // 0x9C +} + +/// The driver's private data. +struct GPIOInner { + base_addr: usize, +} + +/// Deref to RegisterBlock. +impl ops::Deref for GPIOInner { + type Target = RegisterBlock; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.ptr() } + } +} + +impl GPIOInner { + const fn new(base_addr: usize) -> GPIOInner { + GPIOInner { base_addr } + } + + /// Return a pointer to the register block. + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ + } +} + +//-------------------------------------------------------------------------------------------------- +// BSP-public +//-------------------------------------------------------------------------------------------------- +use interface::sync::Mutex; + +/// The driver's main struct. +pub struct GPIO { + inner: NullLock, +} + +impl GPIO { + pub const unsafe fn new(base_addr: usize) -> GPIO { + GPIO { + inner: NullLock::new(GPIOInner::new(base_addr)), + } + } + + /// Map PL011 UART as standard output. + /// + /// TX to pin 14 + /// RX to pin 15 + pub fn map_pl011_uart(&self) { + let mut r = &self.inner; + r.lock(|inner| { + // Map to pins. + inner + .GPFSEL1 + .modify(GPFSEL1::FSEL14::AltFunc0 + GPFSEL1::FSEL15::AltFunc0); + + // Enable pins 14 and 15. + inner.GPPUD.set(0); + arch::spin_for_cycles(150); + + inner + .GPPUDCLK0 + .write(GPPUDCLK0::PUDCLK14::AssertClock + GPPUDCLK0::PUDCLK15::AssertClock); + arch::spin_for_cycles(150); + + inner.GPPUDCLK0.set(0); + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// OS interface implementations +//-------------------------------------------------------------------------------------------------- + +impl interface::driver::DeviceDriver for GPIO { + fn compatible(&self) -> &str { + "GPIO" + } +} diff --git a/11_virtual_memory/src/bsp/driver/bcm/bcm2xxx_pl011_uart.rs b/11_virtual_memory/src/bsp/driver/bcm/bcm2xxx_pl011_uart.rs new file mode 100644 index 00000000..78303c49 --- /dev/null +++ b/11_virtual_memory/src/bsp/driver/bcm/bcm2xxx_pl011_uart.rs @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! PL011 UART driver. + +use crate::{arch, arch::sync::NullLock, interface}; +use core::{fmt, ops}; +use register::{mmio::*, register_bitfields}; + +// PL011 UART registers. +// +// Descriptions taken from +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +register_bitfields! { + u32, + + /// Flag Register + FR [ + /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// Line Control Register, UARTLCR_ LCRH. + /// + /// If the FIFO is disabled, this bit is set when the transmit holding register is empty. If + /// the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty. This bit does + /// not indicate if there is data in the transmit shift register. + TXFE OFFSET(7) NUMBITS(1) [], + + /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the + /// UARTLCR_ LCRH Register. + /// + /// If the FIFO is disabled, this bit is set when the transmit holding register is full. If + /// the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full. + TXFF OFFSET(5) NUMBITS(1) [], + + /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// UARTLCR_H Register. + /// + /// If the FIFO is disabled, this bit is set when the receive holding register is empty. If + /// the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty. + RXFE OFFSET(4) NUMBITS(1) [] + ], + + /// Integer Baud rate divisor + IBRD [ + /// Integer Baud rate divisor + IBRD OFFSET(0) NUMBITS(16) [] + ], + + /// Fractional Baud rate divisor + FBRD [ + /// Fractional Baud rate divisor + FBRD OFFSET(0) NUMBITS(6) [] + ], + + /// Line Control register + LCRH [ + /// Word length. These bits indicate the number of data bits transmitted or received in a + /// frame. + WLEN OFFSET(5) NUMBITS(2) [ + FiveBit = 0b00, + SixBit = 0b01, + SevenBit = 0b10, + EightBit = 0b11 + ], + + /// Enable FIFOs: + /// + /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding + /// registers + /// + /// 1 = transmit and receive FIFO buffers are enabled (FIFO mode). + FEN OFFSET(4) NUMBITS(1) [ + FifosDisabled = 0, + FifosEnabled = 1 + ] + ], + + /// Control Register + CR [ + /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled. + /// Data reception occurs for UART signals. When the UART is disabled in the middle of + /// reception, it completes the current character before stopping. + RXE OFFSET(9) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled. + /// Data transmission occurs for UART signals. When the UART is disabled in the middle of + /// transmission, it completes the current character before stopping. + TXE OFFSET(8) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// UART enable + UARTEN OFFSET(0) NUMBITS(1) [ + /// If the UART is disabled in the middle of transmission or reception, it completes the + /// current character before stopping. + Disabled = 0, + Enabled = 1 + ] + ], + + /// Interrupt Clear Register + ICR [ + /// Meta field for all pending interrupts + ALL OFFSET(0) NUMBITS(11) [] + ] +} + +#[allow(non_snake_case)] +#[repr(C)] +pub struct RegisterBlock { + DR: ReadWrite, // 0x00 + __reserved_0: [u32; 5], // 0x04 + FR: ReadOnly, // 0x18 + __reserved_1: [u32; 2], // 0x1c + IBRD: WriteOnly, // 0x24 + FBRD: WriteOnly, // 0x28 + LCRH: WriteOnly, // 0x2C + CR: WriteOnly, // 0x30 + __reserved_2: [u32; 4], // 0x34 + ICR: WriteOnly, // 0x44 +} + +/// The driver's mutex protected part. +struct PL011UartInner { + base_addr: usize, + chars_written: usize, +} + +/// Deref to RegisterBlock. +/// +/// Allows writing +/// ``` +/// self.DR.read() +/// ``` +/// instead of something along the lines of +/// ``` +/// unsafe { (*PL011UartInner::ptr()).DR.read() } +/// ``` +impl ops::Deref for PL011UartInner { + type Target = RegisterBlock; + + fn deref(&self) -> &Self::Target { + unsafe { &*self.ptr() } + } +} + +impl PL011UartInner { + const fn new(base_addr: usize) -> PL011UartInner { + PL011UartInner { + base_addr, + chars_written: 0, + } + } + + /// Return a pointer to the register block. + fn ptr(&self) -> *const RegisterBlock { + self.base_addr as *const _ + } + + /// Send a character. + fn write_char(&mut self, c: char) { + // Wait until we can send. + loop { + if !self.FR.is_set(FR::TXFF) { + break; + } + + arch::nop(); + } + + // Write the character to the buffer. + self.DR.set(c as u32); + } +} + +/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are +/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`, +/// we get `write_fmt()` automatically. +/// +/// The function takes an `&mut self`, so it must be implemented for the inner struct. +/// +/// See [`src/print.rs`]. +/// +/// [`src/print.rs`]: ../../print/index.html +impl fmt::Write for PL011UartInner { + fn write_str(&mut self, s: &str) -> fmt::Result { + for c in s.chars() { + // Convert newline to carrige return + newline. + if c == '\n' { + self.write_char('\r') + } + + self.write_char(c); + } + + self.chars_written += s.len(); + + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// BSP-public +//-------------------------------------------------------------------------------------------------- + +/// The driver's main struct. +pub struct PL011Uart { + inner: NullLock, +} + +impl PL011Uart { + /// # Safety + /// + /// The user must ensure to provide the correct `base_addr`. + pub const unsafe fn new(base_addr: usize) -> PL011Uart { + PL011Uart { + inner: NullLock::new(PL011UartInner::new(base_addr)), + } + } +} + +//-------------------------------------------------------------------------------------------------- +// OS interface implementations +//-------------------------------------------------------------------------------------------------- +use interface::sync::Mutex; + +impl interface::driver::DeviceDriver for PL011Uart { + fn compatible(&self) -> &str { + "PL011Uart" + } + + /// Set up baud rate and characteristics + /// + /// Results in 8N1 and 115200 baud (if the clk has been previously set to 4 MHz by the + /// firmware). + fn init(&self) -> interface::driver::Result { + let mut r = &self.inner; + r.lock(|inner| { + // Turn it off temporarily. + inner.CR.set(0); + + inner.ICR.write(ICR::ALL::CLEAR); + inner.IBRD.write(IBRD::IBRD.val(26)); // Results in 115200 baud for UART Clk of 48 MHz. + inner.FBRD.write(FBRD::FBRD.val(3)); + inner + .LCRH + .write(LCRH::WLEN::EightBit + LCRH::FEN::FifosEnabled); // 8N1 + Fifo on + inner + .CR + .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled); + }); + + Ok(()) + } +} + +impl interface::console::Write for PL011Uart { + /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to + /// serialize access. + fn write_char(&self, c: char) { + let mut r = &self.inner; + r.lock(|inner| inner.write_char(c)); + } + + fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result { + // Fully qualified syntax for the call to `core::fmt::Write::write:fmt()` to increase + // readability. + let mut r = &self.inner; + r.lock(|inner| fmt::Write::write_fmt(inner, args)) + } + + fn flush(&self) { + let mut r = &self.inner; + // Spin until the TX FIFO empty flag is set. + r.lock(|inner| loop { + if inner.FR.is_set(FR::TXFE) { + break; + } + + arch::nop(); + }); + } +} + +impl interface::console::Read for PL011Uart { + fn read_char(&self) -> char { + let mut r = &self.inner; + r.lock(|inner| { + // Wait until buffer is filled. + loop { + if !inner.FR.is_set(FR::RXFE) { + break; + } + + arch::nop(); + } + + // Read one character. + let mut ret = inner.DR.get() as u8 as char; + + // Convert carrige return to newline. + if ret == '\r' { + ret = '\n' + } + + ret + }) + } + + fn clear(&self) { + let mut r = &self.inner; + r.lock(|inner| loop { + // Read from the RX FIFO until the empty bit is '1'. + if !inner.FR.is_set(FR::RXFE) { + inner.DR.get(); + } else { + break; + } + }) + } +} + +impl interface::console::Statistics for PL011Uart { + fn chars_written(&self) -> usize { + let mut r = &self.inner; + r.lock(|inner| inner.chars_written) + } +} diff --git a/11_virtual_memory/src/bsp/rpi.rs b/11_virtual_memory/src/bsp/rpi.rs new file mode 100644 index 00000000..a09554f2 --- /dev/null +++ b/11_virtual_memory/src/bsp/rpi.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Board Support Package for the Raspberry Pi. + +mod memory_map; +mod virt_mem_layout; + +use super::driver; +use crate::{interface, memory::KernelVirtualLayout}; + +pub const BOOT_CORE_ID: u64 = 0; +pub const BOOT_CORE_STACK_START: u64 = 0x80_000; + +//-------------------------------------------------------------------------------------------------- +// Global BSP driver instances +//-------------------------------------------------------------------------------------------------- + +static GPIO: driver::GPIO = unsafe { driver::GPIO::new(memory_map::mmio::GPIO_BASE) }; +static PL011_UART: driver::PL011Uart = + unsafe { driver::PL011Uart::new(memory_map::mmio::PL011_UART_BASE) }; + +//-------------------------------------------------------------------------------------------------- +// Implementation of the kernel's BSP calls +//-------------------------------------------------------------------------------------------------- + +/// Board identification. +pub fn board_name() -> &'static str { + #[cfg(feature = "bsp_rpi3")] + { + "Raspberry Pi 3" + } + + #[cfg(feature = "bsp_rpi4")] + { + "Raspberry Pi 4" + } +} + +/// Return a reference to a `console::All` implementation. +pub fn console() -> &'static impl interface::console::All { + &PL011_UART +} + +/// Return an array of references to all `DeviceDriver` compatible `BSP` drivers. +/// +/// # Safety +/// +/// The order of devices is the order in which `DeviceDriver::init()` is called. +pub fn device_drivers() -> [&'static dyn interface::driver::DeviceDriver; 2] { + [&GPIO, &PL011_UART] +} + +/// BSP initialization code that runs after driver init. +pub fn post_driver_init() { + // Configure PL011Uart's output pins. + GPIO.map_pl011_uart(); +} + +/// Return the address space size in bytes. +pub const fn addr_space_size() -> usize { + memory_map::END_INCLUSIVE + 1 +} + +/// Return a reference to the virtual memory layout. +pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ virt_mem_layout::NUM_MEM_RANGES }> { + &virt_mem_layout::LAYOUT +} diff --git a/11_virtual_memory/src/bsp/rpi/link.ld b/11_virtual_memory/src/bsp/rpi/link.ld new file mode 100644 index 00000000..4ed0f152 --- /dev/null +++ b/11_virtual_memory/src/bsp/rpi/link.ld @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: MIT + * + * Copyright (c) 2018-2019 Andre Richter + */ + +SECTIONS +{ + /* Set current address to the value from which the RPi starts execution */ + . = 0x80000; + + __ro_start = .; + .text : + { + *(.text._start) *(.text*) + } + + .rodata : + { + *(.rodata*) + } + . = ALIGN(65536); /* Fill up to 64 KiB */ + __ro_end = .; + + .data : + { + *(.data*) + } + + /* Align to 8 byte boundary */ + .bss ALIGN(8): + { + __bss_start = .; + *(.bss*); + __bss_end = .; + } + + /DISCARD/ : { *(.comment*) } +} diff --git a/11_virtual_memory/src/bsp/rpi/memory_map.rs b/11_virtual_memory/src/bsp/rpi/memory_map.rs new file mode 100644 index 00000000..b223b26a --- /dev/null +++ b/11_virtual_memory/src/bsp/rpi/memory_map.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! The board's memory map. + +#[cfg(feature = "bsp_rpi3")] +#[rustfmt::skip] +pub const END_INCLUSIVE: usize = 0x3FFF_FFFF; + +#[cfg(feature = "bsp_rpi4")] +#[rustfmt::skip] +pub const END_INCLUSIVE: usize = 0xFFFF_FFFF; + +/// Physical devices. +#[rustfmt::skip] +pub mod mmio { + #[cfg(feature = "bsp_rpi3")] + pub const BASE: usize = 0x3F00_0000; + + #[cfg(feature = "bsp_rpi4")] + pub const BASE: usize = 0xFE00_0000; + + pub const GPIO_BASE: usize = BASE + 0x0020_0000; + pub const PL011_UART_BASE: usize = BASE + 0x0020_1000; + pub const END_INCLUSIVE: usize = super::END_INCLUSIVE; +} diff --git a/11_virtual_memory/src/bsp/rpi/virt_mem_layout.rs b/11_virtual_memory/src/bsp/rpi/virt_mem_layout.rs new file mode 100644 index 00000000..3b1b2014 --- /dev/null +++ b/11_virtual_memory/src/bsp/rpi/virt_mem_layout.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! The virtual memory layout. +//! +//! The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. +//! It is agnostic of the paging granularity that the architecture's MMU will use. + +use super::memory_map; +use crate::memory::*; +use core::ops::RangeInclusive; + +pub const NUM_MEM_RANGES: usize = 3; + +pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( + memory_map::END_INCLUSIVE, + [ + RangeDescriptor { + name: "Kernel code and RO data", + virtual_range: || { + // Using the linker script, we ensure that the RO area is consecutive and 4 KiB + // aligned, and we export the boundaries via symbols: + // + // [__ro_start, __ro_end) + extern "C" { + // The inclusive start of the read-only area, aka the address of the first + // byte of the area. + static __ro_start: u64; + + // The exclusive end of the read-only area, aka the address of the first + // byte _after_ the RO area. + static __ro_end: u64; + } + + unsafe { + // Notice the subtraction to turn the exclusive end into an inclusive end. + #[allow(clippy::range_minus_one)] + RangeInclusive::new( + &__ro_start as *const _ as usize, + &__ro_end as *const _ as usize - 1, + ) + } + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + }, + RangeDescriptor { + name: "Remapped Device MMIO", + virtual_range: || { + // The last 64 KiB slot in the first 512 MiB + RangeInclusive::new(0x1FFF_0000, 0x1FFF_FFFF) + }, + translation: Translation::Offset(memory_map::mmio::BASE + 0x20_0000), + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + RangeDescriptor { + name: "Device MMIO", + virtual_range: || { + RangeInclusive::new(memory_map::mmio::BASE, memory_map::mmio::END_INCLUSIVE) + }, + translation: Translation::Identity, + attribute_fields: AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + }, + ], +); diff --git a/11_virtual_memory/src/interface.rs b/11_virtual_memory/src/interface.rs new file mode 100644 index 00000000..55df89cb --- /dev/null +++ b/11_virtual_memory/src/interface.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Trait definitions for coupling `kernel` and `BSP` code. +//! +//! ``` +//! +-------------------+ +//! | Interface (Trait) | +//! | | +//! +--+-------------+--+ +//! ^ ^ +//! | | +//! | | +//! +----------+--+ +--+----------+ +//! | Kernel code | | BSP Code | +//! | | | | +//! +-------------+ +-------------+ +//! ``` + +/// System console operations. +pub mod console { + use core::fmt; + + /// Console write functions. + pub trait Write { + fn write_char(&self, c: char); + fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; + + /// Block execution until the last character has been physically put on the TX wire + /// (draining TX buffers/FIFOs, if any). + fn flush(&self); + } + + /// Console read functions. + pub trait Read { + fn read_char(&self) -> char { + ' ' + } + + /// Clear RX buffers, if any. + fn clear(&self); + } + + /// Console statistics. + pub trait Statistics { + /// Return the number of characters written. + fn chars_written(&self) -> usize { + 0 + } + + /// Return the number of characters read. + fn chars_read(&self) -> usize { + 0 + } + } + + /// Trait alias for a full-fledged console. + pub trait All = Write + Read + Statistics; +} + +/// Synchronization primitives. +pub mod sync { + /// Any object implementing this trait guarantees exclusive access to the data contained within + /// the mutex for the duration of the lock. + /// + /// The trait follows the [Rust embedded WG's + /// proposal](https://github.com/korken89/wg/blob/master/rfcs/0377-mutex-trait.md) and therefore + /// provides some goodness such as [deadlock + /// prevention](https://github.com/korken89/wg/blob/master/rfcs/0377-mutex-trait.md#design-decisions-and-compatibility). + /// + /// # Example + /// + /// Since the lock function takes an `&mut self` to enable deadlock-prevention, the trait is + /// best implemented **for a reference to a container struct**, and has a usage pattern that + /// might feel strange at first: + /// + /// ``` + /// static MUT: Mutex> = Mutex::new(RefCell::new(0)); + /// + /// fn foo() { + /// let mut r = &MUT; // Note that r is mutable + /// r.lock(|data| *data += 1); + /// } + /// ``` + pub trait Mutex { + /// Type of data encapsulated by the mutex. + type Data; + + /// Creates a critical section and grants temporary mutable access to the encapsulated data. + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + } +} + +/// Driver interfaces. +pub mod driver { + /// Driver result type, e.g. for indicating successful driver init. + pub type Result = core::result::Result<(), ()>; + + /// Device Driver functions. + pub trait DeviceDriver { + /// Return a compatibility string for identifying the driver. + fn compatible(&self) -> &str; + + /// Called by the kernel to bring up the device. + fn init(&self) -> Result { + Ok(()) + } + } +} + +/// Timekeeping interfaces. +pub mod time { + use core::time::Duration; + + /// Timer functions. + pub trait Timer { + /// The timer's resolution. + fn resolution(&self) -> Duration; + + /// The uptime since power-on of the device. + /// + /// This includes time consumed by firmware and bootloaders. + fn uptime(&self) -> Duration; + + /// Spin for a given duration. + fn spin_for(&self, duration: Duration); + } +} diff --git a/11_virtual_memory/src/main.rs b/11_virtual_memory/src/main.rs new file mode 100644 index 00000000..00638f0a --- /dev/null +++ b/11_virtual_memory/src/main.rs @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc(html_logo_url = "https://git.io/JeGIp")] + +//! The `kernel` +//! +//! The `kernel` is composed by glueing together code from +//! +//! - [Hardware-specific Board Support Packages] (`BSPs`). +//! - [Architecture-specific code]. +//! - HW- and architecture-agnostic `kernel` code. +//! +//! using the [`kernel::interface`] traits. +//! +//! [Hardware-specific Board Support Packages]: bsp/index.html +//! [Architecture-specific code]: arch/index.html +//! [`kernel::interface`]: interface/index.html + +#![allow(incomplete_features)] +#![feature(const_generics)] +#![feature(format_args_nl)] +#![feature(panic_info_message)] +#![feature(trait_alias)] +#![no_main] +#![no_std] + +// Conditionally includes the selected `architecture` code, which provides the `_start()` function, +// the first function to run. +mod arch; + +// `_start()` then calls `runtime_init::init()`, which on completion, jumps to `kernel_init()`. +mod runtime_init; + +// Conditionally includes the selected `BSP` code. +mod bsp; + +mod interface; +mod memory; +mod panic_wait; +mod print; + +/// Early init code. +/// +/// Concerned with with initializing `BSP` and `arch` parts. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +/// - The init calls in this function must appear in the correct order. +unsafe fn kernel_init() -> ! { + // Bring up device drivers first, so that eventual MMU errors can be printed. + for i in bsp::device_drivers().iter() { + if let Err(()) = i.init() { + // This message will only be readable if, at the time of failure, the return value of + // `bsp::console()` is already in functioning state. + panic!("Error loading driver: {}", i.compatible()) + } + } + + bsp::post_driver_init(); + + println!("Booting on: {}", bsp::board_name()); + + if let Err(string) = arch::mmu::init() { + panic!("MMU: {}", string); + } + println!("MMU online"); + + // Transition from unsafe to safe. + kernel_main() +} + +/// The main function running after the early init. +fn kernel_main() -> ! { + use core::time::Duration; + use interface::{console::All, time::Timer}; + + bsp::virt_mem_layout().print_layout(); + + println!( + "Current privilege level: {}", + arch::state::current_privilege_level() + ); + println!("Exception handling state:"); + arch::state::print_exception_state(); + + println!( + "Architectural timer resolution: {} ns", + arch::timer().resolution().as_nanos() + ); + + println!("Drivers loaded:"); + for (i, driver) in bsp::device_drivers().iter().enumerate() { + println!(" {}. {}", i + 1, driver.compatible()); + } + + println!("Timer test, spinning for 1 second"); + arch::timer().spin_for(Duration::from_secs(1)); + + let remapped_uart = unsafe { bsp::driver::PL011Uart::new(0x1FFF_1000) }; + writeln!( + remapped_uart, + "[ !!! ] Writing through the remapped UART at 0x1FFF_1000" + ) + .unwrap(); + + println!("Echoing input now"); + loop { + let c = bsp::console().read_char(); + bsp::console().write_char(c); + } +} diff --git a/11_virtual_memory/src/memory.rs b/11_virtual_memory/src/memory.rs new file mode 100644 index 00000000..1fba62d5 --- /dev/null +++ b/11_virtual_memory/src/memory.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Memory Management. + +use core::{fmt, ops::RangeInclusive}; + +#[derive(Copy, Clone)] +pub enum Translation { + Identity, + Offset(usize), +} + +#[derive(Copy, Clone)] +pub enum MemAttributes { + CacheableDRAM, + Device, +} + +#[derive(Copy, Clone)] +pub enum AccessPermissions { + ReadOnly, + ReadWrite, +} + +#[derive(Copy, Clone)] +pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, +} + +impl Default for AttributeFields { + fn default() -> AttributeFields { + AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + } + } +} + +/// An architecture agnostic descriptor for a memory range. +pub struct RangeDescriptor { + pub name: &'static str, + pub virtual_range: fn() -> RangeInclusive, + pub translation: Translation, + pub attribute_fields: AttributeFields, +} + +/// Human-readable output of a RangeDescriptor. +impl fmt::Display for RangeDescriptor { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Call the function to which self.range points, and dereference the result, which causes + // Rust to copy the value. + let start = *(self.virtual_range)().start(); + let end = *(self.virtual_range)().end(); + let size = end - start + 1; + + // log2(1024). + const KIB_RSHIFT: u32 = 10; + + // log2(1024 * 1024). + const MIB_RSHIFT: u32 = 20; + + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; + + let attr = match self.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::Device => "Dev", + }; + + let acc_p = match self.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if self.attribute_fields.execute_never { + "PXN" + } else { + "PX" + }; + + write!( + f, + " {:#010X} - {:#010X} | {: >3} {} | {: <3} {} {: <3} | {}", + start, end, size, unit, attr, acc_p, xn, self.name + ) + } +} + +/// Type for expressing the kernel's virtual memory layout. +pub struct KernelVirtualLayout { + max_virt_addr_inclusive: usize, + inner: [RangeDescriptor; NUM_SPECIAL_RANGES], +} + +impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { + pub const fn new(max: usize, layout: [RangeDescriptor; NUM_SPECIAL_RANGES]) -> Self { + Self { + max_virt_addr_inclusive: max, + inner: layout, + } + } + + /// For a virtual address, find and return the output address and corresponding attributes. + /// + /// If the address is not found in `inner`, return an identity mapped default with normal + /// cacheable DRAM attributes. + pub fn get_virt_addr_properties( + &self, + virt_addr: usize, + ) -> Result<(usize, AttributeFields), &'static str> { + if virt_addr > self.max_virt_addr_inclusive { + return Err("Address out of range"); + } + + for i in self.inner.iter() { + if (i.virtual_range)().contains(&virt_addr) { + let output_addr = match i.translation { + Translation::Identity => virt_addr, + Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), + }; + + return Ok((output_addr, i.attribute_fields)); + } + } + + Ok((virt_addr, AttributeFields::default())) + } + + /// Print the memory layout. + pub fn print_layout(&self) { + use crate::println; + + println!("Special memory regions:"); + + for i in self.inner.iter() { + println!("{}", i); + } + } +} diff --git a/11_virtual_memory/src/panic_wait.rs b/11_virtual_memory/src/panic_wait.rs new file mode 100644 index 00000000..5e6d3fa5 --- /dev/null +++ b/11_virtual_memory/src/panic_wait.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! A panic handler that infinitely waits. + +use crate::{arch, println}; +use core::panic::PanicInfo; + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + if let Some(args) = info.message() { + println!("Kernel panic: {}", args); + } else { + println!("Kernel panic!"); + } + + arch::wait_forever() +} diff --git a/11_virtual_memory/src/print.rs b/11_virtual_memory/src/print.rs new file mode 100644 index 00000000..86ab59b3 --- /dev/null +++ b/11_virtual_memory/src/print.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Printing facilities. + +use crate::{bsp, interface}; +use core::fmt; + +/// Prints without a newline. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! print { + ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*))); +} + +/// Prints with a newline. +#[macro_export] +macro_rules! println { + () => ($crate::print!("\n")); + ($string:expr) => ({ + #[allow(unused_imports)] + use crate::interface::time::Timer; + + let timestamp = $crate::arch::timer().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:03}{:03}] ", $string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000 + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + #[allow(unused_imports)] + use crate::interface::time::Timer; + + let timestamp = $crate::arch::timer().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:03}{:03}] ", $format_string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000, + $($arg)* + )); + }) +} + +/// Prints a warning, with newline. +#[macro_export] +macro_rules! warn { + ($string:expr) => ({ + #[allow(unused_imports)] + use crate::interface::time::Timer; + + let timestamp = $crate::arch::timer().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:03}{:03}] ", $string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000 + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + #[allow(unused_imports)] + use crate::interface::time::Timer; + + let timestamp = $crate::arch::timer().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:03}{:03}] ", $format_string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000, + $($arg)* + )); + }) +} + +pub fn _print(args: fmt::Arguments) { + use interface::console::Write; + + bsp::console().write_fmt(args).unwrap(); +} diff --git a/11_virtual_memory/src/runtime_init.rs b/11_virtual_memory/src/runtime_init.rs new file mode 100644 index 00000000..4cd0415a --- /dev/null +++ b/11_virtual_memory/src/runtime_init.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT +// +// Copyright (c) 2018-2019 Andre Richter + +//! Rust runtime initialization code. + +/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel +/// init code. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +pub unsafe fn init() -> ! { + extern "C" { + // Boundaries of the .bss section, provided by the linker script. + static mut __bss_start: u64; + static mut __bss_end: u64; + } + + // Zero out the .bss section. + r0::zero_bss(&mut __bss_start, &mut __bss_end); + + crate::kernel_init() +} diff --git a/README.md b/README.md index f84b736e..d1bfcab0 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,7 @@ You can find the original version of the tutorials [here](https://github.com/rust-embedded/rust-raspi3-OS-tutorials/tree/original_version). They are worth checking out for some advanced features that are not yet -rewritten, like _virtual memory_ or _exception handling_. They will be ported -over soon, though (if time permits). +rewritten, like _exception handling_. They will be ported over soon, though. Some info on the rewrite and in general: - I will first add @@ -30,8 +29,8 @@ _Cheers, [Andre](https://github.com/andre-richter)_ - [Visual Studio Code]: https://code.visualstudio.com - [Rust Language Server]: https://github.com/rust-lang/rls +[Visual Studio Code]: https://code.visualstudio.com +[Rust Language Server]: https://github.com/rust-lang/rls ## Introduction diff --git a/doc/page_tables_64KiB.png b/doc/page_tables_64KiB.png new file mode 100644 index 00000000..facf7866 Binary files /dev/null and b/doc/page_tables_64KiB.png differ diff --git a/doc/page_tables_64KiB.svg b/doc/page_tables_64KiB.svg new file mode 100644 index 00000000..354b5d31 --- /dev/null +++ b/doc/page_tables_64KiB.svg @@ -0,0 +1,1513 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + ... + 0 + + Stack + Stack + + + + + ... + Code + RO data + Code + RO data + + + + + ... + Data and BSS + Data and BSS + + + + ... + Remapped UART + 8191 + + + + 0 + ... + + 8191 + + + REMAPPED_DEVICE_MEM + UART_Offset =0x1FFF_0000 + 0x1000 = 0x1FFF_1000 =0b0000000000000_1111111111111_0001000000000000 + + TTBR_EL1 + + LVL2 tablebase address + LVL3 Table base addr + static mut TABLES.lvl2 + static mut TABLES.lvl3 + } + + Select entry 0 ofLVL2 table + + } + Virtual Address: + + 0b1_1111_1111_1111 = 8191Select entry 8191 ofLVL3 table + + APAccessPermissions + 53 + 16 + 47 + 9 + 10 + 8 + 6 + 7 + 4 + 1 + 2 + 0 + + + MAIR_EL1index + + TYPE + VALID + + + + SHShareability + AFAccessFlag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PXNPrivilegedexecutenever + 63 + + + + + + Final 48 Bit Physical Address: 0x3F201000 + + + 0x3F20 + + 0x1000 + + 0x3F20 + } + + + 0 + 15 + 47 + 16 + Entry contains baseaddress of follow-upLVL3 Table. + The LVL3 PT entry pointsto the start address ofa 64 KiB frame. The last16 Bit of the VA are theindex into the frame. + output address + 1 + +