From 4caa0188c7e5ee971545dacb65a73e8c8e37f3a3 Mon Sep 17 00:00:00 2001 From: Andre Richter Date: Sun, 4 Oct 2020 21:59:38 +0200 Subject: [PATCH] Add tutorial 15: MMIO Remap --- .../.vscode/settings.json | 0 .../Cargo.lock | 0 .../Cargo.toml | 0 .../Makefile | 0 .../README.md | 44 +- .../build.rs | 0 .../src/_arch/aarch64/cpu.rs | 0 .../src/_arch/aarch64/cpu/smp.rs | 0 .../src/_arch/aarch64/exception.rs | 0 .../_arch/aarch64/exception/asynchronous.rs | 0 .../src/_arch/aarch64/memory/mmu.rs | 0 .../src/_arch/aarch64/time.rs | 0 .../src/bsp.rs | 0 .../src/bsp/device_driver.rs | 0 .../src/bsp/device_driver/bcm.rs | 0 .../src/bsp/device_driver/bcm/bcm2xxx_gpio.rs | 0 .../device_driver/bcm/bcm2xxx_pl011_uart.rs | 0 .../src/bsp/device_driver/common.rs | 0 .../src/bsp/raspberrypi.rs | 0 .../src/bsp/raspberrypi/console.rs | 0 .../src/bsp/raspberrypi/cpu.rs | 0 .../src/bsp/raspberrypi/driver.rs | 0 .../src/bsp/raspberrypi/link.ld | 0 .../src/bsp/raspberrypi/memory.rs | 0 .../src/bsp/raspberrypi/memory/mmu.rs | 0 .../src/console.rs | 0 .../src/cpu.rs | 0 .../src/cpu/smp.rs | 0 .../src/driver.rs | 0 .../src/exception.rs | 0 .../src/exception/asynchronous.rs | 0 .../src/main.rs | 0 .../src/memory.rs | 0 .../src/memory/mmu.rs | 0 .../src/panic_wait.rs | 0 .../src/print.rs | 0 .../src/runtime_init.rs | 0 .../src/synchronization.rs | 0 .../src/time.rs | 0 12_exceptions_part1_groundwork/README.md | 28 +- .../.cargo/config | 2 + .../.vscode/settings.json | 7 + 15_virtual_memory_part2_mmio_remap/Cargo.lock | 91 + 15_virtual_memory_part2_mmio_remap/Cargo.toml | 45 + 15_virtual_memory_part2_mmio_remap/Makefile | 173 + 15_virtual_memory_part2_mmio_remap/README.md | 3075 +++++++++++++++++ 15_virtual_memory_part2_mmio_remap/build.rs | 7 + .../src/_arch/aarch64/cpu.rs | 114 + .../src/_arch/aarch64/cpu/smp.rs | 22 + .../src/_arch/aarch64/exception.S | 138 + .../src/_arch/aarch64/exception.rs | 261 ++ .../_arch/aarch64/exception/asynchronous.rs | 140 + .../src/_arch/aarch64/memory/mmu.rs | 553 +++ .../src/_arch/aarch64/time.rs | 97 + 15_virtual_memory_part2_mmio_remap/src/bsp.rs | 13 + .../src/bsp/device_driver.rs | 16 + .../src/bsp/device_driver/arm.rs | 9 + .../src/bsp/device_driver/arm/gicv2.rs | 254 ++ .../src/bsp/device_driver/arm/gicv2/gicc.rs | 155 + .../src/bsp/device_driver/arm/gicv2/gicd.rs | 211 ++ .../src/bsp/device_driver/bcm.rs | 15 + .../src/bsp/device_driver/bcm/bcm2xxx_gpio.rs | 204 ++ .../bcm/bcm2xxx_interrupt_controller.rs | 138 + .../peripheral_ic.rs | 197 ++ .../device_driver/bcm/bcm2xxx_pl011_uart.rs | 477 +++ .../src/bsp/device_driver/common.rs | 38 + .../src/bsp/raspberrypi.rs | 62 + .../src/bsp/raspberrypi/console.rs | 61 + .../src/bsp/raspberrypi/cpu.rs | 12 + .../src/bsp/raspberrypi/driver.rs | 61 + .../src/bsp/raspberrypi/exception.rs | 7 + .../bsp/raspberrypi/exception/asynchronous.rs | 36 + .../src/bsp/raspberrypi/link.ld | 49 + .../src/bsp/raspberrypi/memory.rs | 193 ++ .../src/bsp/raspberrypi/memory/mmu.rs | 171 + .../src/common.rs | 21 + .../src/console.rs | 54 + 15_virtual_memory_part2_mmio_remap/src/cpu.rs | 12 + .../src/cpu/smp.rs | 10 + .../src/driver.rs | 62 + .../src/exception.rs | 44 + .../src/exception/asynchronous.rs | 145 + 15_virtual_memory_part2_mmio_remap/src/lib.rs | 179 + .../src/main.rs | 110 + .../src/memory.rs | 52 + .../src/memory/mmu.rs | 250 ++ .../src/memory/mmu/mapping_record.rs | 224 ++ .../src/memory/mmu/types.rs | 283 ++ .../src/panic_wait.rs | 69 + .../src/print.rs | 106 + .../src/runtime_init.rs | 63 + .../src/state.rs | 92 + .../src/synchronization.rs | 151 + .../src/time.rs | 35 + .../test-macros/Cargo.toml | 14 + .../test-macros/src/lib.rs | 29 + .../test-types/Cargo.toml | 5 + .../test-types/src/lib.rs | 16 + .../tests/00_console_sanity.rb | 50 + .../tests/00_console_sanity.rs | 36 + .../tests/01_timer_sanity.rs | 50 + .../tests/02_exception_sync_page_fault.rs | 56 + .../tests/03_exception_irq_sanity.rs | 68 + .../tests/panic_exit_failure/mod.rs | 9 + .../tests/panic_exit_success/mod.rs | 9 + .../tests/runner.rb | 143 + ...kernel_user_address_space_partitioning.png | Bin 0 -> 75332 bytes 107 files changed, 9588 insertions(+), 35 deletions(-) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/.vscode/settings.json (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/Cargo.lock (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/Cargo.toml (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/Makefile (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/README.md (96%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/build.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/_arch/aarch64/cpu.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/_arch/aarch64/cpu/smp.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/_arch/aarch64/exception.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/_arch/aarch64/exception/asynchronous.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/_arch/aarch64/memory/mmu.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/_arch/aarch64/time.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/device_driver.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/device_driver/bcm.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/device_driver/common.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi/console.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi/cpu.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi/driver.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi/link.ld (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi/memory.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/bsp/raspberrypi/memory/mmu.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/console.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/cpu.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/cpu/smp.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/driver.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/exception.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/exception/asynchronous.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/main.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/memory.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/memory/mmu.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/panic_wait.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/print.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/runtime_init.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/synchronization.rs (100%) rename {11_virtual_memory => 11_virtual_memory_part1_identity_mapping}/src/time.rs (100%) create mode 100644 15_virtual_memory_part2_mmio_remap/.cargo/config create mode 100644 15_virtual_memory_part2_mmio_remap/.vscode/settings.json create mode 100644 15_virtual_memory_part2_mmio_remap/Cargo.lock create mode 100644 15_virtual_memory_part2_mmio_remap/Cargo.toml create mode 100644 15_virtual_memory_part2_mmio_remap/Makefile create mode 100644 15_virtual_memory_part2_mmio_remap/README.md create mode 100644 15_virtual_memory_part2_mmio_remap/build.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.S create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/time.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicd.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/common.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/console.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/driver.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception/asynchronous.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/link.ld create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/common.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/console.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/cpu.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/cpu/smp.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/driver.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/exception.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/exception/asynchronous.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/lib.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/main.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/memory.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/memory/mmu.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/memory/mmu/mapping_record.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/memory/mmu/types.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/panic_wait.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/print.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/runtime_init.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/state.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/synchronization.rs create mode 100644 15_virtual_memory_part2_mmio_remap/src/time.rs create mode 100644 15_virtual_memory_part2_mmio_remap/test-macros/Cargo.toml create mode 100644 15_virtual_memory_part2_mmio_remap/test-macros/src/lib.rs create mode 100644 15_virtual_memory_part2_mmio_remap/test-types/Cargo.toml create mode 100644 15_virtual_memory_part2_mmio_remap/test-types/src/lib.rs create mode 100644 15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rb create mode 100644 15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rs create mode 100644 15_virtual_memory_part2_mmio_remap/tests/01_timer_sanity.rs create mode 100644 15_virtual_memory_part2_mmio_remap/tests/02_exception_sync_page_fault.rs create mode 100644 15_virtual_memory_part2_mmio_remap/tests/03_exception_irq_sanity.rs create mode 100644 15_virtual_memory_part2_mmio_remap/tests/panic_exit_failure/mod.rs create mode 100644 15_virtual_memory_part2_mmio_remap/tests/panic_exit_success/mod.rs create mode 100755 15_virtual_memory_part2_mmio_remap/tests/runner.rb create mode 100644 doc/15_kernel_user_address_space_partitioning.png diff --git a/11_virtual_memory/.vscode/settings.json b/11_virtual_memory_part1_identity_mapping/.vscode/settings.json similarity index 100% rename from 11_virtual_memory/.vscode/settings.json rename to 11_virtual_memory_part1_identity_mapping/.vscode/settings.json diff --git a/11_virtual_memory/Cargo.lock b/11_virtual_memory_part1_identity_mapping/Cargo.lock similarity index 100% rename from 11_virtual_memory/Cargo.lock rename to 11_virtual_memory_part1_identity_mapping/Cargo.lock diff --git a/11_virtual_memory/Cargo.toml b/11_virtual_memory_part1_identity_mapping/Cargo.toml similarity index 100% rename from 11_virtual_memory/Cargo.toml rename to 11_virtual_memory_part1_identity_mapping/Cargo.toml diff --git a/11_virtual_memory/Makefile b/11_virtual_memory_part1_identity_mapping/Makefile similarity index 100% rename from 11_virtual_memory/Makefile rename to 11_virtual_memory_part1_identity_mapping/Makefile diff --git a/11_virtual_memory/README.md b/11_virtual_memory_part1_identity_mapping/README.md similarity index 96% rename from 11_virtual_memory/README.md rename to 11_virtual_memory_part1_identity_mapping/README.md index fc828c73..a018f7a5 100644 --- a/11_virtual_memory/README.md +++ b/11_virtual_memory_part1_identity_mapping/README.md @@ -1,9 +1,10 @@ -# Tutorial 11 - Virtual Memory +# Tutorial 11 - Virtual Memory Part 1: Identity Map All The Things! ## tl;dr -The `MMU` is turned on; A simple scheme is used: static `64 KiB` translation tables; For educational -purposes, we write to a remapped `UART`. +- The `MMU` is turned on. +- A simple scheme is used: static `64 KiB` translation tables. +- For educational purposes, we write to a remapped `UART`, and `identity map` everything else. ## Table of Contents @@ -23,8 +24,9 @@ purposes, we write to a remapped `UART`. ## Introduction Virtual memory is an immensely complex, but important and powerful topic. In this tutorial, we start -slow and easy by switching on the `MMU`, using static translation tables and mapping everything at -once. +slow and easy by switching on the `MMU`, using static translation tables and `identity-map` +everything at once (except for the `UART`, which we remap for educational purposes; This will be +gone again in the next tutorial). ## MMU and paging theory @@ -297,9 +299,9 @@ Minipush 1.0 ## Diff to previous ```diff -diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs +diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs --- 10_privilege_level/src/_arch/aarch64/memory/mmu.rs -+++ 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs ++++ 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// @@ -635,9 +637,9 @@ diff -uNr 10_privilege_level/src/_arch/aarch64/memory/mmu.rs 11_virtual_memory/s + } +} -diff -uNr 10_privilege_level/src/bsp/raspberrypi/link.ld 11_virtual_memory/src/bsp/raspberrypi/link.ld +diff -uNr 10_privilege_level/src/bsp/raspberrypi/link.ld 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/link.ld --- 10_privilege_level/src/bsp/raspberrypi/link.ld -+++ 11_virtual_memory/src/bsp/raspberrypi/link.ld ++++ 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/link.ld @@ -8,6 +8,7 @@ /* Set current address to the value from which the RPi starts execution */ . = 0x80000; @@ -656,9 +658,9 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/link.ld 11_virtual_memory/src/b .data : { -diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs +diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs --- 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs -+++ 11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs ++++ 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// @@ -749,9 +751,9 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory/mmu.rs 11_virtual_memory + &LAYOUT +} -diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_memory/src/bsp/raspberrypi/memory.rs +diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory.rs --- 10_privilege_level/src/bsp/raspberrypi/memory.rs -+++ 11_virtual_memory/src/bsp/raspberrypi/memory.rs ++++ 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory.rs @@ -4,6 +4,8 @@ //! BSP Memory Management. @@ -824,9 +826,9 @@ diff -uNr 10_privilege_level/src/bsp/raspberrypi/memory.rs 11_virtual_memory/src //-------------------------------------------------------------------------------------------------- -diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_memory/src/bsp.rs +diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_memory_part1_identity_mapping/src/bsp.rs --- 10_privilege_level/src/bsp.rs -+++ 11_virtual_memory/src/bsp.rs ++++ 11_virtual_memory_part1_identity_mapping/src/bsp.rs @@ -4,7 +4,7 @@ //! Conditional re-exporting of Board Support Packages. @@ -837,9 +839,9 @@ diff -uNr 10_privilege_level/src/bsp.rs 11_virtual_memory/src/bsp.rs #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; -diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs +diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory_part1_identity_mapping/src/main.rs --- 10_privilege_level/src/main.rs -+++ 11_virtual_memory/src/main.rs ++++ 11_virtual_memory_part1_identity_mapping/src/main.rs @@ -11,10 +11,12 @@ //! //! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. @@ -908,9 +910,9 @@ diff -uNr 10_privilege_level/src/main.rs 11_virtual_memory/src/main.rs loop { let c = bsp::console::console().read_char(); -diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_memory/src/memory/mmu.rs +diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_memory_part1_identity_mapping/src/memory/mmu.rs --- 10_privilege_level/src/memory/mmu.rs -+++ 11_virtual_memory/src/memory/mmu.rs ++++ 11_virtual_memory_part1_identity_mapping/src/memory/mmu.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// @@ -1112,9 +1114,9 @@ diff -uNr 10_privilege_level/src/memory/mmu.rs 11_virtual_memory/src/memory/mmu. + } +} -diff -uNr 10_privilege_level/src/memory.rs 11_virtual_memory/src/memory.rs +diff -uNr 10_privilege_level/src/memory.rs 11_virtual_memory_part1_identity_mapping/src/memory.rs --- 10_privilege_level/src/memory.rs -+++ 11_virtual_memory/src/memory.rs ++++ 11_virtual_memory_part1_identity_mapping/src/memory.rs @@ -4,6 +4,8 @@ //! Memory Management. diff --git a/11_virtual_memory/build.rs b/11_virtual_memory_part1_identity_mapping/build.rs similarity index 100% rename from 11_virtual_memory/build.rs rename to 11_virtual_memory_part1_identity_mapping/build.rs diff --git a/11_virtual_memory/src/_arch/aarch64/cpu.rs b/11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/cpu.rs similarity index 100% rename from 11_virtual_memory/src/_arch/aarch64/cpu.rs rename to 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/cpu.rs diff --git a/11_virtual_memory/src/_arch/aarch64/cpu/smp.rs b/11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs similarity index 100% rename from 11_virtual_memory/src/_arch/aarch64/cpu/smp.rs rename to 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/cpu/smp.rs diff --git a/11_virtual_memory/src/_arch/aarch64/exception.rs b/11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception.rs similarity index 100% rename from 11_virtual_memory/src/_arch/aarch64/exception.rs rename to 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception.rs diff --git a/11_virtual_memory/src/_arch/aarch64/exception/asynchronous.rs b/11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs similarity index 100% rename from 11_virtual_memory/src/_arch/aarch64/exception/asynchronous.rs rename to 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception/asynchronous.rs diff --git a/11_virtual_memory/src/_arch/aarch64/memory/mmu.rs b/11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs similarity index 100% rename from 11_virtual_memory/src/_arch/aarch64/memory/mmu.rs rename to 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/memory/mmu.rs diff --git a/11_virtual_memory/src/_arch/aarch64/time.rs b/11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/time.rs similarity index 100% rename from 11_virtual_memory/src/_arch/aarch64/time.rs rename to 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/time.rs diff --git a/11_virtual_memory/src/bsp.rs b/11_virtual_memory_part1_identity_mapping/src/bsp.rs similarity index 100% rename from 11_virtual_memory/src/bsp.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp.rs diff --git a/11_virtual_memory/src/bsp/device_driver.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/device_driver.rs similarity index 100% rename from 11_virtual_memory/src/bsp/device_driver.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/device_driver.rs diff --git a/11_virtual_memory/src/bsp/device_driver/bcm.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/bcm.rs similarity index 100% rename from 11_virtual_memory/src/bsp/device_driver/bcm.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/bcm.rs diff --git a/11_virtual_memory/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs similarity index 100% rename from 11_virtual_memory/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs diff --git a/11_virtual_memory/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs similarity index 100% rename from 11_virtual_memory/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs diff --git a/11_virtual_memory/src/bsp/device_driver/common.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/common.rs similarity index 100% rename from 11_virtual_memory/src/bsp/device_driver/common.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/device_driver/common.rs diff --git a/11_virtual_memory/src/bsp/raspberrypi.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi.rs similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi.rs diff --git a/11_virtual_memory/src/bsp/raspberrypi/console.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/console.rs similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi/console.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/console.rs diff --git a/11_virtual_memory/src/bsp/raspberrypi/cpu.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi/cpu.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/cpu.rs diff --git a/11_virtual_memory/src/bsp/raspberrypi/driver.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/driver.rs similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi/driver.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/driver.rs diff --git a/11_virtual_memory/src/bsp/raspberrypi/link.ld b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/link.ld similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi/link.ld rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/link.ld diff --git a/11_virtual_memory/src/bsp/raspberrypi/memory.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory.rs similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi/memory.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory.rs diff --git a/11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs b/11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs similarity index 100% rename from 11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs rename to 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs diff --git a/11_virtual_memory/src/console.rs b/11_virtual_memory_part1_identity_mapping/src/console.rs similarity index 100% rename from 11_virtual_memory/src/console.rs rename to 11_virtual_memory_part1_identity_mapping/src/console.rs diff --git a/11_virtual_memory/src/cpu.rs b/11_virtual_memory_part1_identity_mapping/src/cpu.rs similarity index 100% rename from 11_virtual_memory/src/cpu.rs rename to 11_virtual_memory_part1_identity_mapping/src/cpu.rs diff --git a/11_virtual_memory/src/cpu/smp.rs b/11_virtual_memory_part1_identity_mapping/src/cpu/smp.rs similarity index 100% rename from 11_virtual_memory/src/cpu/smp.rs rename to 11_virtual_memory_part1_identity_mapping/src/cpu/smp.rs diff --git a/11_virtual_memory/src/driver.rs b/11_virtual_memory_part1_identity_mapping/src/driver.rs similarity index 100% rename from 11_virtual_memory/src/driver.rs rename to 11_virtual_memory_part1_identity_mapping/src/driver.rs diff --git a/11_virtual_memory/src/exception.rs b/11_virtual_memory_part1_identity_mapping/src/exception.rs similarity index 100% rename from 11_virtual_memory/src/exception.rs rename to 11_virtual_memory_part1_identity_mapping/src/exception.rs diff --git a/11_virtual_memory/src/exception/asynchronous.rs b/11_virtual_memory_part1_identity_mapping/src/exception/asynchronous.rs similarity index 100% rename from 11_virtual_memory/src/exception/asynchronous.rs rename to 11_virtual_memory_part1_identity_mapping/src/exception/asynchronous.rs diff --git a/11_virtual_memory/src/main.rs b/11_virtual_memory_part1_identity_mapping/src/main.rs similarity index 100% rename from 11_virtual_memory/src/main.rs rename to 11_virtual_memory_part1_identity_mapping/src/main.rs diff --git a/11_virtual_memory/src/memory.rs b/11_virtual_memory_part1_identity_mapping/src/memory.rs similarity index 100% rename from 11_virtual_memory/src/memory.rs rename to 11_virtual_memory_part1_identity_mapping/src/memory.rs diff --git a/11_virtual_memory/src/memory/mmu.rs b/11_virtual_memory_part1_identity_mapping/src/memory/mmu.rs similarity index 100% rename from 11_virtual_memory/src/memory/mmu.rs rename to 11_virtual_memory_part1_identity_mapping/src/memory/mmu.rs diff --git a/11_virtual_memory/src/panic_wait.rs b/11_virtual_memory_part1_identity_mapping/src/panic_wait.rs similarity index 100% rename from 11_virtual_memory/src/panic_wait.rs rename to 11_virtual_memory_part1_identity_mapping/src/panic_wait.rs diff --git a/11_virtual_memory/src/print.rs b/11_virtual_memory_part1_identity_mapping/src/print.rs similarity index 100% rename from 11_virtual_memory/src/print.rs rename to 11_virtual_memory_part1_identity_mapping/src/print.rs diff --git a/11_virtual_memory/src/runtime_init.rs b/11_virtual_memory_part1_identity_mapping/src/runtime_init.rs similarity index 100% rename from 11_virtual_memory/src/runtime_init.rs rename to 11_virtual_memory_part1_identity_mapping/src/runtime_init.rs diff --git a/11_virtual_memory/src/synchronization.rs b/11_virtual_memory_part1_identity_mapping/src/synchronization.rs similarity index 100% rename from 11_virtual_memory/src/synchronization.rs rename to 11_virtual_memory_part1_identity_mapping/src/synchronization.rs diff --git a/11_virtual_memory/src/time.rs b/11_virtual_memory_part1_identity_mapping/src/time.rs similarity index 100% rename from 11_virtual_memory/src/time.rs rename to 11_virtual_memory_part1_identity_mapping/src/time.rs diff --git a/12_exceptions_part1_groundwork/README.md b/12_exceptions_part1_groundwork/README.md index 660601bd..50b84d47 100644 --- a/12_exceptions_part1_groundwork/README.md +++ b/12_exceptions_part1_groundwork/README.md @@ -479,8 +479,8 @@ General purpose register: ## Diff to previous ```diff -diff -uNr 11_virtual_memory/src/_arch/aarch64/exception.rs 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs ---- 11_virtual_memory/src/_arch/aarch64/exception.rs +diff -uNr 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception.rs 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs +--- 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception.rs +++ 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.rs @@ -4,7 +4,230 @@ @@ -740,8 +740,8 @@ diff -uNr 11_virtual_memory/src/_arch/aarch64/exception.rs 12_exceptions_part1_g + barrier::isb(barrier::SY); +} -diff -uNr 11_virtual_memory/src/_arch/aarch64/exception.S 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S ---- 11_virtual_memory/src/_arch/aarch64/exception.S +diff -uNr 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception.S 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S +--- 11_virtual_memory_part1_identity_mapping/src/_arch/aarch64/exception.S +++ 12_exceptions_part1_groundwork/src/_arch/aarch64/exception.S @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 @@ -883,8 +883,8 @@ diff -uNr 11_virtual_memory/src/_arch/aarch64/exception.S 12_exceptions_part1_gr + + eret -diff -uNr 11_virtual_memory/src/bsp/raspberrypi/link.ld 12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld ---- 11_virtual_memory/src/bsp/raspberrypi/link.ld +diff -uNr 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/link.ld 12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld +--- 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/link.ld +++ 12_exceptions_part1_groundwork/src/bsp/raspberrypi/link.ld @@ -14,6 +14,11 @@ *(.text._start) *(.text*) @@ -899,8 +899,8 @@ diff -uNr 11_virtual_memory/src/bsp/raspberrypi/link.ld 12_exceptions_part1_grou { *(.rodata*) -diff -uNr 11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs ---- 11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs +diff -uNr 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs +--- 11_virtual_memory_part1_identity_mapping/src/bsp/raspberrypi/memory/mmu.rs +++ 12_exceptions_part1_groundwork/src/bsp/raspberrypi/memory/mmu.rs @@ -12,7 +12,7 @@ // Public Definitions @@ -941,8 +941,8 @@ diff -uNr 11_virtual_memory/src/bsp/raspberrypi/memory/mmu.rs 12_exceptions_part RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE) } -diff -uNr 11_virtual_memory/src/bsp.rs 12_exceptions_part1_groundwork/src/bsp.rs ---- 11_virtual_memory/src/bsp.rs +diff -uNr 11_virtual_memory_part1_identity_mapping/src/bsp.rs 12_exceptions_part1_groundwork/src/bsp.rs +--- 11_virtual_memory_part1_identity_mapping/src/bsp.rs +++ 12_exceptions_part1_groundwork/src/bsp.rs @@ -4,7 +4,7 @@ @@ -954,8 +954,8 @@ diff -uNr 11_virtual_memory/src/bsp.rs 12_exceptions_part1_groundwork/src/bsp.rs #[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] mod raspberrypi; -diff -uNr 11_virtual_memory/src/main.rs 12_exceptions_part1_groundwork/src/main.rs ---- 11_virtual_memory/src/main.rs +diff -uNr 11_virtual_memory_part1_identity_mapping/src/main.rs 12_exceptions_part1_groundwork/src/main.rs +--- 11_virtual_memory_part1_identity_mapping/src/main.rs +++ 12_exceptions_part1_groundwork/src/main.rs @@ -108,6 +108,7 @@ #![feature(const_generics)] @@ -1010,8 +1010,8 @@ diff -uNr 11_virtual_memory/src/main.rs 12_exceptions_part1_groundwork/src/main. loop { let c = bsp::console::console().read_char(); -diff -uNr 11_virtual_memory/src/memory/mmu.rs 12_exceptions_part1_groundwork/src/memory/mmu.rs ---- 11_virtual_memory/src/memory/mmu.rs +diff -uNr 11_virtual_memory_part1_identity_mapping/src/memory/mmu.rs 12_exceptions_part1_groundwork/src/memory/mmu.rs +--- 11_virtual_memory_part1_identity_mapping/src/memory/mmu.rs +++ 12_exceptions_part1_groundwork/src/memory/mmu.rs @@ -42,6 +42,7 @@ diff --git a/15_virtual_memory_part2_mmio_remap/.cargo/config b/15_virtual_memory_part2_mmio_remap/.cargo/config new file mode 100644 index 00000000..e3476485 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/.cargo/config @@ -0,0 +1,2 @@ +[target.'cfg(target_os = "none")'] +runner = "target/kernel_test_runner.sh" diff --git a/15_virtual_memory_part2_mmio_remap/.vscode/settings.json b/15_virtual_memory_part2_mmio_remap/.vscode/settings.json new file mode 100644 index 00000000..a0d6a920 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "editor.formatOnSave": true, + "editor.rulers": [100], + "rust-analyzer.checkOnSave.overrideCommand": ["make", "check"], + "rust-analyzer.cargo.target": "aarch64-unknown-none-softfloat", + "rust-analyzer.cargo.features": ["bsp_rpi3"] +} diff --git a/15_virtual_memory_part2_mmio_remap/Cargo.lock b/15_virtual_memory_part2_mmio_remap/Cargo.lock new file mode 100644 index 00000000..c0df2028 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/Cargo.lock @@ -0,0 +1,91 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "cortex-a" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6922a40af4d1a2deac8c963b9f3e57311b8912490740234f1ad182425c547f80" +dependencies = [ + "register", +] + +[[package]] +name = "kernel" +version = "0.1.0" +dependencies = [ + "cortex-a", + "qemu-exit", + "register", + "test-macros", + "test-types", +] + +[[package]] +name = "proc-macro2" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ef7cd2518ead700af67bf9d1a658d90b6037d77110fd9c0445429d0ba1c6c9" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "qemu-exit" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73ae13954572c7ca0ec48ba9fe6a59c0392066eba62f8cb384ffd5addf538c5" + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "register" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deaba5b0e477d21f61a57504bb5cef4a1e86de30300b457d38971c1cfc98b815" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "syn" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c51d92969d209b54a98397e1b91c8ae82d8c87a7bb87df0b29aa2ad81454228" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "test-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "test-types", +] + +[[package]] +name = "test-types" +version = "0.1.0" + +[[package]] +name = "tock-registers" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70323afdb8082186c0986da0e10f6e4ed103d681c921c00597e98d9806dac20f" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" diff --git a/15_virtual_memory_part2_mmio_remap/Cargo.toml b/15_virtual_memory_part2_mmio_remap/Cargo.toml new file mode 100644 index 00000000..07292a2e --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "kernel" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" + +# The features section is used to select the target board. +[features] +default = [] +bsp_rpi3 = ["cortex-a", "register"] +bsp_rpi4 = ["cortex-a", "register"] + +[dependencies] +qemu-exit = "1.0.x" +test-types = { path = "test-types" } + +# Optional dependencies +cortex-a = { version = "3.0.x", optional = true } +register = { version = "0.5.x", features = ["no_std_unit_tests"], optional = true } + +##-------------------------------------------------------------------------------------------------- +## Testing +##-------------------------------------------------------------------------------------------------- + +[dev-dependencies] +test-macros = { path = "test-macros" } + +# Unit tests are done in the library part of the kernel. +[lib] +name = "libkernel" +test = true + +# Disable unit tests for the kernel binary. +[[bin]] +name = "kernel" +test = false + +# List of tests without harness. +[[test]] +name = "00_console_sanity" +harness = false + +[[test]] +name = "02_exception_sync_page_fault" +harness = false diff --git a/15_virtual_memory_part2_mmio_remap/Makefile b/15_virtual_memory_part2_mmio_remap/Makefile new file mode 100644 index 00000000..cab93584 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/Makefile @@ -0,0 +1,173 @@ +## SPDX-License-Identifier: MIT OR Apache-2.0 +## +## Copyright (c) 2018-2020 Andre Richter + +# Default to the RPi3 +BSP ?= rpi3 + +# Default to a serial device name that is common in Linux. +DEV_SERIAL ?= /dev/ttyUSB0 + +# Query the host system's kernel name +UNAME_S = $(shell uname -s) + +# BSP-specific arguments +ifeq ($(BSP),rpi3) + TARGET = aarch64-unknown-none-softfloat + KERNEL_BIN = kernel8.img + QEMU_BINARY = qemu-system-aarch64 + QEMU_MACHINE_TYPE = raspi3 + QEMU_RELEASE_ARGS = -serial stdio -display none + QEMU_TEST_ARGS = $(QEMU_RELEASE_ARGS) -semihosting + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi3.cfg + JTAG_BOOT_IMAGE = ../X1_JTAG_boot/jtag_boot_rpi3.img + LINKER_FILE = src/bsp/raspberrypi/link.ld + RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 +else ifeq ($(BSP),rpi4) + TARGET = aarch64-unknown-none-softfloat + KERNEL_BIN = kernel8.img + QEMU_BINARY = qemu-system-aarch64 + QEMU_MACHINE_TYPE = + QEMU_RELEASE_ARGS = -serial stdio -display none + QEMU_TEST_ARGS = $(QEMU_RELEASE_ARGS) -semihosting + OPENOCD_ARG = -f /openocd/tcl/interface/ftdi/olimex-arm-usb-tiny-h.cfg -f /openocd/rpi4.cfg + JTAG_BOOT_IMAGE = ../X1_JTAG_boot/jtag_boot_rpi4.img + LINKER_FILE = src/bsp/raspberrypi/link.ld + RUSTC_MISC_ARGS = -C target-cpu=cortex-a72 +endif + +# Export for build.rs +export LINKER_FILE + +# Testing-specific arguments +ifdef TEST + ifeq ($(TEST),unit) + TEST_ARG = --lib + else + TEST_ARG = --test $(TEST) + endif +endif + +QEMU_MISSING_STRING = "This board is not yet supported for QEMU." + +RUSTFLAGS = -C link-arg=-T$(LINKER_FILE) $(RUSTC_MISC_ARGS) +RUSTFLAGS_PEDANTIC = $(RUSTFLAGS) -D warnings -D missing_docs + +FEATURES = bsp_$(BSP) +COMPILER_ARGS = --target=$(TARGET) \ + --features $(FEATURES) \ + --release + +RUSTC_CMD = cargo rustc $(COMPILER_ARGS) +DOC_CMD = cargo doc $(COMPILER_ARGS) +CLIPPY_CMD = cargo clippy $(COMPILER_ARGS) +CHECK_CMD = cargo check $(COMPILER_ARGS) +TEST_CMD = cargo test $(COMPILER_ARGS) +OBJCOPY_CMD = rust-objcopy \ + --strip-all \ + -O binary + +KERNEL_ELF = target/$(TARGET)/release/kernel + +DOCKER_IMAGE = rustembedded/osdev-utils +DOCKER_CMD_TEST = docker run -i --rm -v $(shell pwd):/work/tutorial -w /work/tutorial +DOCKER_CMD_USER = $(DOCKER_CMD_TEST) -t +DOCKER_ARG_DIR_UTILS = -v $(shell pwd)/../utils:/work/utils +DOCKER_ARG_DIR_JTAG = -v $(shell pwd)/../X1_JTAG_boot:/work/X1_JTAG_boot +DOCKER_ARG_DEV = --privileged -v /dev:/dev +DOCKER_ARG_NET = --network host + +DOCKER_QEMU = $(DOCKER_CMD_USER) $(DOCKER_IMAGE) +DOCKER_GDB = $(DOCKER_CMD_USER) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) +DOCKER_TEST = $(DOCKER_CMD_TEST) $(DOCKER_IMAGE) + +# Dockerize commands that require USB device passthrough only on Linux +ifeq ($(UNAME_S),Linux) + DOCKER_CMD_DEV = $(DOCKER_CMD_USER) $(DOCKER_ARG_DEV) + + DOCKER_CHAINBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_UTILS) $(DOCKER_IMAGE) + DOCKER_JTAGBOOT = $(DOCKER_CMD_DEV) $(DOCKER_ARG_DIR_UTILS) $(DOCKER_ARG_DIR_JTAG) $(DOCKER_IMAGE) + DOCKER_OPENOCD = $(DOCKER_CMD_DEV) $(DOCKER_ARG_NET) $(DOCKER_IMAGE) +else + DOCKER_OPENOCD = echo "Not yet supported on non-Linux systems."; \# +endif + +EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) +EXEC_MINIPUSH = ruby ../utils/minipush.rb + +.PHONY: all $(KERNEL_ELF) $(KERNEL_BIN) doc qemu test chainboot jtagboot openocd gdb gdb-opt0 \ + clippy clean readelf objdump nm check + +all: $(KERNEL_BIN) + +$(KERNEL_ELF): + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(RUSTC_CMD) + +$(KERNEL_BIN): $(KERNEL_ELF) + @$(OBJCOPY_CMD) $(KERNEL_ELF) $(KERNEL_BIN) + +doc: + $(DOC_CMD) --document-private-items --open + +ifeq ($(QEMU_MACHINE_TYPE),) +qemu test: + @echo $(QEMU_MISSING_STRING) +else +qemu: $(KERNEL_BIN) + @$(DOCKER_QEMU) $(EXEC_QEMU) $(QEMU_RELEASE_ARGS) -kernel $(KERNEL_BIN) + +define KERNEL_TEST_RUNNER + #!/usr/bin/env bash + + $(OBJCOPY_CMD) $$1 $$1.img + TEST_BINARY=$$(echo $$1.img | sed -e 's/.*target/target/g') + $(DOCKER_TEST) ruby tests/runner.rb $(EXEC_QEMU) $(QEMU_TEST_ARGS) -kernel $$TEST_BINARY +endef + +export KERNEL_TEST_RUNNER +test: + @mkdir -p target + @echo "$$KERNEL_TEST_RUNNER" > target/kernel_test_runner.sh + @chmod +x target/kernel_test_runner.sh + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(TEST_CMD) $(TEST_ARG) +endif + +chainboot: $(KERNEL_BIN) + @$(DOCKER_CHAINBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(KERNEL_BIN) + +jtagboot: + @$(DOCKER_JTAGBOOT) $(EXEC_MINIPUSH) $(DEV_SERIAL) $(JTAG_BOOT_IMAGE) + +openocd: + @$(DOCKER_OPENOCD) openocd $(OPENOCD_ARG) + +define gen_gdb + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC) $1" $(RUSTC_CMD) + @$(DOCKER_GDB) gdb-multiarch -q $(KERNEL_ELF) +endef + +gdb: + $(call gen_gdb,-C debuginfo=2) + +gdb-opt0: + $(call gen_gdb,-C debuginfo=2 -C opt-level=0) + +clippy: + RUSTFLAGS="$(RUSTFLAGS_PEDANTIC)" $(CLIPPY_CMD) + +clean: + rm -rf target $(KERNEL_BIN) + +readelf: $(KERNEL_ELF) + readelf --headers $(KERNEL_ELF) + +objdump: $(KERNEL_ELF) + rust-objdump --arch-name aarch64 --disassemble --demangle --no-show-raw-insn \ + --print-imm-hex $(KERNEL_ELF) + +nm: $(KERNEL_ELF) + rust-nm --demangle --print-size $(KERNEL_ELF) | sort + +# For rust-analyzer +check: + @RUSTFLAGS="$(RUSTFLAGS)" $(CHECK_CMD) --message-format=json diff --git a/15_virtual_memory_part2_mmio_remap/README.md b/15_virtual_memory_part2_mmio_remap/README.md new file mode 100644 index 00000000..3587c349 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/README.md @@ -0,0 +1,3075 @@ +# Tutorial 15 - Virtual Memory Part 2: MMIO Remap + +## tl;dr + +- We introduce a first set of changes which we eventually need for separating `kernel` and `user` + address spaces. +- The memory mapping strategy gets more sophisticated as we do away with `identity mapping` the + whole of the board's address space. +- Instead, only ranges that are actually needed are mapped: + - The `kernel binary` stays `identity mapped` for now. + - Device `MMIO regions` are remapped lazily to a special virtual address region at the top of + the virtual address space during the device driver's `init()`. + +## Table of Contents + +- [Introduction](#introduction) +- [Implementation](#implementation) + - [A New Mapping API in `src/memory/mmu.rs`](#a-new-mapping-api-in-srcmemorymmurs) + - [Using the new API in `bsp` code and drivers](#using-the-new-api-in-bsp-code-and-drivers) + - [Additional Changes](#additional-changes) +- [Test it](#test-it) +- [Diff to previous](#diff-to-previous) + +## Introduction + +This tutorial is a first step of many needed for enabling `userspace applications` (which we +hopefully will have some day in the very distant future). + +For this, one of the features we want is a clean separation of `kernel` and `user` address spaces. +Fortunately, `ARMv8` has convenient architecture support to realize this. The following text and +pictue gives some more motivation and technical information. It is quoted from the _[ARM Cortex-A +Series Programmer’s Guide for ARMv8-A], Chapter 12.2, Separation of kernel and application Virtual +Address spaces_: + +> Operating systems typically have a number of applications or tasks running concurrently. Each of +> these has its own unique set of translation tables and the kernel switches from one to another as +> part of the process of switching context between one task and another. However, much of the memory +> system is used only by the kernel and has fixed virtual to Physical Address mappings where the +> translation table entries rarely change. The ARMv8 architecture provides a number of features to +> efficiently handle this requirement. +> +> The table base addresses are specified in the Translation Table Base Registers `TTBR0_EL1` and +> `TTBR1_EL1`. The translation table pointed to by `TTBR0` is selected when the upper bits of the VA +> are all 0. `TTBR1` is selected when the upper bits of the VA are all set to 1. [...] +> +> Figure 12-4 shows how the kernel space can be mapped to the most significant area of memory and +> the Virtual Address space associated with each application mapped to the least significant area of +> memory. However, both of these are mapped to a much smaller Physical Address space. + +

+ +

+ +This approach is also sometimes called a "[higher half kernel]". To eventually achieve this +separation, this tutorial makes a start by changing the following things: + +1. Instead of bulk-`identity mapping` the whole of the board's address space, only the particular + parts that are needed will be mapped. +1. For now, the `kernel binary` stays identity mapped. This will be changed in the next tutorial as + it is a quite difficult and peculiar exercise to remap the kernel. +1. Device `MMIO regions` are lazily remapped during the device driver's `init()`. + 1. The remappings will populate the top of the virtual address space. In the `AArch64 MMU + Driver`, we provide the top `256 MiB` for it. + 1. It is possible to define the size of the virtual address space at compile time. We chose `8 + GiB` for now, which means remapped MMIO virtual addresses will start at `7936 MiB` + (`0x1F0000000`). +1. We keep using `TTBR0` for the kernel page tables for now. This will be changed when we remap the + `kernel binary` in the next tutorial. + +[ARM Cortex-A Series Programmer’s Guide for ARMv8-A]: https://developer.arm.com/documentation/den0024/latest/ +[higher half kernel]: https://wiki.osdev.org/Higher_Half_Kernel + +## Implementation + +Until now, the whole address space of the board was identity mapped at once. The **architecture** +(`src/_arch/_/memory/**`) and **bsp** (`src/bsp/_/memory/**`) parts of the kernel worked +together directly while setting up the translation tables, without any indirection through **generic +kernel code** (`src/memory/**`). + +The way it worked was that the `architectural MMU driver` would query the `bsp code` about the start +and end of the physical address space, and any special regions in this space that need a mapping +that _is not_ normal chacheable DRAM. It would then go ahead and map the whole address space at once +and never touch the page tables again during runtime. + +Changing in this tutorial, **architecture** and **bsp** code will no longer talk to each other +directly. Instead, this is decoupled now through the kernel's **generic MMU subsystem code**. + +### A New Mapping API in `src/memory/mmu.rs` + +First, we define an interface for operating on `translation tables`: + +```rust +/// Translation table operations. +pub trait TranslationTable { + /// Anything that needs to run before any of the other provided functions can be used. + unsafe fn init(&mut self); + + /// The translation table's base address to be used for programming the MMU. + fn phys_base_address(&self) -> Address; + + /// Map the given physical pages to the given virtual pages. + unsafe fn map_pages_at( + &mut self, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str>; + + /// Obtain a free virtual page slice in the MMIO region. + /// + /// The "MMIO region" is a distinct region of the implementor's choice, which allows + /// differentiating MMIO addresses from others. This can speed up debugging efforts. + /// Ideally, those MMIO addresses are also standing out visually so that a human eye can + /// identify them. For example, by allocating them from near the end of the virtual address + /// space. + fn next_mmio_virt_page_slice( + &mut self, + num_pages: usize, + ) -> Result, &'static str>; + + /// Check if a virtual page splice is in the "MMIO region". + fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; +} +``` + +The MMU driver (`src/_arch/_/memory/mmu.rs`) has one global instance for the kernel tables which +implements this interface, and which can be accessed by calling +`arch_mmu::kernel_translation_tables()` in the generic kernel code (`src/memory/mmu.rs`). From +there, we provice a couple of memory mapping functions that wrap around this interface , and which +are exported for the rest of the kernel to use: + +```rust +/// Raw mapping of virtual to physical pages in the kernel translation tables. +/// +/// Prevents mapping into the MMIO range of the tables. +pub unsafe fn kernel_map_pages_at( + name: &'static str, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, +) -> Result<(), &'static str>; + +/// MMIO remapping in the kernel translation tables. +/// +/// Typically used by device drivers. +pub unsafe fn kernel_map_mmio( + name: &'static str, + phys_mmio_descriptor: &MMIODescriptor, +) -> Result, &'static str>; + +/// Map the kernel's binary and enable the MMU. +pub unsafe fn kernel_map_binary_and_enable_mmu() -> Result<(), &'static str> ; +``` + +### Using the new API in `bsp` code and drivers + +For now, there are two places where the new API is used. First, in `src/bsp/_/memory/mmu.rs`, which +provides a dedicated call to **map the kernel binary** (because it is the `BSP` that provides the +`linker script`, which in turn defines the final layout of the kernel in memory): + +```rust +/// Map the kernel binary. +pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { + kernel_mmu::kernel_map_pages_at( + "Kernel boot-core stack", + &phys_stack_page_desc(), + &virt_stack_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + kernel_mmu::kernel_map_pages_at( + "Kernel code and RO data", + // omitted for brevity. + )?; + + kernel_mmu::kernel_map_pages_at( + "Kernel data and bss", + // omitted for brevity. + )?; + + Ok(()) +} +``` + +Second, in device drivers, which now expect an `MMIODescriptor` type instead of a raw address. The +following is an example for the `UART`: + +```rust +impl PL011Uart { + /// Create an instance. + pub const unsafe fn new( + phys_mmio_descriptor: memory::mmu::MMIODescriptor, + irq_number: bsp::device_driver::IRQNumber, + ) -> Self { + Self { + // omitted for brevity. + } + } +} +``` + +When the kernel calls the driver's implementation of `driver::interface::DeviceDriver::init()` +during kernel boot, the MMIO Descriptor is used to remap the MMIO region on demand: + +```rust +unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = + memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)?; + + let mut r = &self.inner; + r.lock(|inner| inner.init(Some(virt_addr.into_usize())))?; + + // omitted for brevity. + + Ok(()) +} +``` + +### Supporting Changes + +There's a couple of changes not covered in this tutorial text, but the reader should ideally skim +through them: + +- [`src/memory/mmu/types.rs`](src/memory/mmu/types.rs) introduces a couple of supporting types, like + `Address`, which is used to differentiate between `Physical` and `Virtual` addresses. +- [`src/memory/mmu/mapping_record.rs`](src/memory/mmu/mapping_record.rs) provides the generic kernel + code's way of tracking previous memory mappings for use cases such as reusing existing mappings + (in case of drivers that have their MMIO ranges in the same `64 KiB` page) or printing mappings + statistics. + +## Test it + +When you load the kernel, you can now see that the driver's MMIO virtual addresses start at +`0x1F0000000`: + +Raspberry Pi 3: + +```console +$ make chainboot +[...] +Minipush 1.0 + +[MP] ⏳ Waiting for /dev/ttyUSB0 +[MP] ✅ Connected + __ __ _ _ _ _ +| \/ (_)_ _ (_) | ___ __ _ __| | +| |\/| | | ' \| | |__/ _ \/ _` / _` | +|_| |_|_|_||_|_|____\___/\__,_\__,_| + + Raspberry Pi 3 + +[ML] Requesting binary +[MP] ⏩ Pushing 67 KiB ========================================🦀 100% 33 KiB/s Time: 00:00:02 +[ML] Loaded! Executing the payload now + +[ 3.041355] Booting on: Raspberry Pi 3 +[ 3.042438] MMU online: +[ 3.043609] ----------------------------------------------------------------------------------------------------------------- +[ 3.049466] Virtual Physical Size Attr Entity +[ 3.055323] ----------------------------------------------------------------------------------------------------------------- +[ 3.061183] 0x000070000..0x00007FFFF --> 0x000070000..0x00007FFFF | 64 KiB | C RW XN | Kernel boot-core stack +[ 3.066476] 0x000080000..0x00008FFFF --> 0x000080000..0x00008FFFF | 64 KiB | C RO X | Kernel code and RO data +[ 3.071812] 0x000090000..0x0001AFFFF --> 0x000090000..0x0001AFFFF | 1 MiB | C RW XN | Kernel data and bss +[ 3.076975] 0x1F0000000..0x1F000FFFF --> 0x03F200000..0x03F20FFFF | 64 KiB | Dev RW XN | BCM GPIO +[ 3.081658] | BCM PL011 UART +[ 3.086606] 0x1F0010000..0x1F001FFFF --> 0x03F000000..0x03F00FFFF | 64 KiB | Dev RW XN | BCM Peripheral Interrupt Controller +[ 3.092462] ----------------------------------------------------------------------------------------------------------------- +``` + +Raspberry Pi 4: + +```console +$ BSP=rpi4 make chainboot +[...] +Minipush 1.0 + +[MP] ⏳ Waiting for /dev/ttyUSB0 +[MP] ✅ Connected + __ __ _ _ _ _ +| \/ (_)_ _ (_) | ___ __ _ __| | +| |\/| | | ' \| | |__/ _ \/ _` / _` | +|_| |_|_|_||_|_|____\___/\__,_\__,_| + + Raspberry Pi 4 + +[ML] Requesting binary +[MP] ⏩ Pushing 74 KiB ========================================🦀 100% 24 KiB/s Time: 00:00:03 +[ML] Loaded! Executing the payload now + +[ 3.376642] Booting on: Raspberry Pi 4 +[ 3.377030] MMU online: +[ 3.378202] ----------------------------------------------------------------------------------------------------------------- +[ 3.384059] Virtual Physical Size Attr Entity +[ 3.389916] ----------------------------------------------------------------------------------------------------------------- +[ 3.395775] 0x000070000..0x00007FFFF --> 0x000070000..0x00007FFFF | 64 KiB | C RW XN | Kernel boot-core stack +[ 3.401069] 0x000080000..0x00008FFFF --> 0x000080000..0x00008FFFF | 64 KiB | C RO X | Kernel code and RO data +[ 3.406404] 0x000090000..0x0001AFFFF --> 0x000090000..0x0001AFFFF | 1 MiB | C RW XN | Kernel data and bss +[ 3.411566] 0x1F0000000..0x1F000FFFF --> 0x0FE200000..0x0FE20FFFF | 64 KiB | Dev RW XN | BCM GPIO +[ 3.416251] | BCM PL011 UART +[ 3.421198] 0x1F0010000..0x1F001FFFF --> 0x0FF840000..0x0FF84FFFF | 64 KiB | Dev RW XN | GICD +[ 3.425709] | GICC +[ 3.430221] ----------------------------------------------------------------------------------------------------------------- +``` + +## Diff to previous +```diff + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu.rs +--- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/cpu.rs ++++ 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu.rs +@@ -68,7 +68,7 @@ + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. +- SP_EL1.set(bsp::memory::boot_core_stack_end() as u64); ++ SP_EL1.set(bsp::memory::phys_boot_core_stack_end().into_usize() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs +--- 14_exceptions_part2_peripheral_IRQs/src/_arch/aarch64/memory/mmu.rs ++++ 15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs +@@ -4,10 +4,19 @@ + + //! Memory Management Unit Driver. + //! +-//! Static translation tables, compiled on boot; Everything 64 KiB granule. ++//! Only 64 KiB granule is supported. + +-use super::{AccessPermissions, AttributeFields, MemAttributes}; +-use crate::{bsp, memory}; ++use crate::{ ++ bsp, ++ memory::{ ++ mmu, ++ mmu::{ ++ AccessPermissions, Address, AddressType, AttributeFields, MemAttributes, Page, ++ PageSliceDescriptor, Physical, Virtual, ++ }, ++ }, ++ synchronization::InitStateLock, ++}; + use core::convert; + use cortex_a::{barrier, regs::*}; + use register::{register_bitfields, InMemoryRegister}; +@@ -15,6 +24,7 @@ + //-------------------------------------------------------------------------------------------------- + // Private Definitions + //-------------------------------------------------------------------------------------------------- ++use mmu::interface::TranslationGranule; + + // A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. + register_bitfields! {u64, +@@ -81,9 +91,6 @@ + ] + } + +-const SIXTYFOUR_KIB_SHIFT: usize = 16; // log2(64 * 1024) +-const FIVETWELVE_MIB_SHIFT: usize = 29; // log2(512 * 1024 * 1024) +- + /// A table descriptor for 64 KiB aperture. + /// + /// The output points to the next table. +@@ -98,36 +105,65 @@ + #[repr(transparent)] + struct PageDescriptor(InMemoryRegister); + ++#[derive(Copy, Clone)] ++enum Granule512MiB {} ++ ++trait BaseAddr { ++ fn phys_base_addr(&self) -> Address; ++} ++ ++/// Constants for indexing the MAIR_EL1. ++#[allow(dead_code)] ++mod mair { ++ pub const DEVICE: u64 = 0; ++ pub const NORMAL: u64 = 1; ++} ++ ++/// Memory Management Unit type. ++struct MemoryManagementUnit; ++ ++/// This constant is the power-of-two exponent that defines the virtual address space size. ++/// ++/// Values tested and known to be working: ++/// - 30 (1 GiB) ++/// - 31 (2 GiB) ++/// - 32 (4 GiB) ++/// - 33 (8 GiB) ++const ADDR_SPACE_SIZE_EXPONENT: usize = 33; ++ ++const NUM_LVL2_TABLES: usize = (1 << ADDR_SPACE_SIZE_EXPONENT) >> Granule512MiB::SHIFT; ++const T0SZ: u64 = (64 - ADDR_SPACE_SIZE_EXPONENT) as u64; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++ + /// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB + /// aligned, hence the "reverse" order of appearance. + #[repr(C)] + #[repr(align(65536))] +-struct FixedSizeTranslationTable { ++pub(in crate::memory::mmu) struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], +-} + +-/// Usually evaluates to 1 GiB for RPi3 and 4 GiB for RPi 4. +-const NUM_LVL2_TABLES: usize = bsp::memory::mmu::addr_space_size() >> FIVETWELVE_MIB_SHIFT; +-type ArchTranslationTable = FixedSizeTranslationTable; ++ /// Index of the next free MMIO page. ++ cur_l3_mmio_index: usize, + +-trait BaseAddr { +- fn base_addr_u64(&self) -> u64; +- fn base_addr_usize(&self) -> usize; ++ /// Have the tables been initialized? ++ initialized: bool, + } + +-/// Constants for indexing the MAIR_EL1. +-#[allow(dead_code)] +-mod mair { +- pub const DEVICE: u64 = 0; +- pub const NORMAL: u64 = 1; +-} ++pub(in crate::memory::mmu) type ArchTranslationTable = FixedSizeTranslationTable; + +-/// Memory Management Unit type. +-struct MemoryManagementUnit; ++// Supported translation granules are exported below, so that BSP code can pick between the options. ++// This driver only supports 64 KiB at the moment. ++ ++#[derive(Copy, Clone)] ++/// 64 KiB translation granule. ++pub enum Granule64KiB {} + + //-------------------------------------------------------------------------------------------------- + // Global instances +@@ -138,7 +174,8 @@ + /// # Safety + /// + /// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". +-static mut TABLES: ArchTranslationTable = ArchTranslationTable::new(); ++static KERNEL_TABLES: InitStateLock = ++ InitStateLock::new(ArchTranslationTable::new()); + + static MMU: MemoryManagementUnit = MemoryManagementUnit; + +@@ -146,13 +183,15 @@ + // Private Code + //-------------------------------------------------------------------------------------------------- + +-impl BaseAddr for [T; N] { +- fn base_addr_u64(&self) -> u64 { +- self as *const T as u64 +- } ++impl mmu::interface::TranslationGranule for Granule512MiB { ++ const SIZE: usize = 512 * 1024 * 1024; ++ const SHIFT: usize = 29; // log2(SIZE) ++} + +- fn base_addr_usize(&self) -> usize { +- self as *const _ as usize ++impl BaseAddr for [T; N] { ++ fn phys_base_addr(&self) -> Address { ++ // The binary is still identity mapped, so we don't need to convert here. ++ Address::new(self as *const _ as usize) + } + } + +@@ -160,7 +199,7 @@ + fn from(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + +- let shifted = next_lvl_table_addr >> SIXTYFOUR_KIB_SHIFT; ++ let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table +@@ -207,23 +246,32 @@ + + impl PageDescriptor { + /// Create an instance. +- fn new(output_addr: usize, attribute_fields: AttributeFields) -> Self { ++ fn new(output_addr: *const Page, attribute_fields: &AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + +- let shifted = output_addr as u64 >> SIXTYFOUR_KIB_SHIFT; ++ let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True +- + attribute_fields.into() ++ + attribute_fields.clone().into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self(val) + } ++ ++ /// Returns the valid bit. ++ fn is_valid(&self) -> bool { ++ self.0.is_set(STAGE1_PAGE_DESCRIPTOR::VALID) ++ } + } + + impl FixedSizeTranslationTable<{ NUM_TABLES }> { ++ // Reserve the last 256 MiB of the address space for MMIO mappings. ++ const L2_MMIO_START_INDEX: usize = NUM_TABLES - 1; ++ const L3_MMIO_START_INDEX: usize = 8192 / 2; ++ + /// Create an instance. + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); +@@ -231,7 +279,55 @@ + Self { + lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], + lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], ++ cur_l3_mmio_index: 0, ++ initialized: false, ++ } ++ } ++ ++ /// The start address of the table's MMIO range. ++ #[inline(always)] ++ fn mmio_start_addr(&self) -> Address { ++ Address::new( ++ (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) ++ | (Self::L3_MMIO_START_INDEX << Granule64KiB::SHIFT), ++ ) ++ } ++ ++ /// The inclusive end address of the table's MMIO range. ++ #[inline(always)] ++ fn mmio_end_addr_inclusive(&self) -> Address { ++ Address::new( ++ (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) ++ | (8191 << Granule64KiB::SHIFT) ++ | (Granule64KiB::SIZE - 1), ++ ) ++ } ++ ++ /// Helper to calculate the lvl2 and lvl3 indices from an address. ++ #[inline(always)] ++ fn lvl2_lvl3_index_from( ++ &self, ++ addr: *const Page, ++ ) -> Result<(usize, usize), &'static str> { ++ let lvl2_index = addr as usize >> Granule512MiB::SHIFT; ++ let lvl3_index = (addr as usize & Granule512MiB::MASK) >> Granule64KiB::SHIFT; ++ ++ if lvl2_index > (NUM_TABLES - 1) { ++ return Err("Virtual page is out of bounds of translation table"); + } ++ ++ Ok((lvl2_index, lvl3_index)) ++ } ++ ++ /// Returns the PageDescriptor corresponding to the supplied Page. ++ #[inline(always)] ++ fn page_descriptor_from( ++ &mut self, ++ addr: *const Page, ++ ) -> Result<&mut PageDescriptor, &'static str> { ++ let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; ++ ++ Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + } + } + +@@ -248,28 +344,6 @@ + ); + } + +-/// Iterates over all static translation table entries and fills them at once. +-/// +-/// # Safety +-/// +-/// - Modifies a `static mut`. Ensure it only happens from here. +-unsafe fn populate_tt_entries() -> Result<(), &'static str> { +- for (l2_nr, l2_entry) in TABLES.lvl2.iter_mut().enumerate() { +- *l2_entry = TABLES.lvl3[l2_nr].base_addr_usize().into(); +- +- for (l3_nr, l3_entry) in TABLES.lvl3[l2_nr].iter_mut().enumerate() { +- let virt_addr = (l2_nr << FIVETWELVE_MIB_SHIFT) + (l3_nr << SIXTYFOUR_KIB_SHIFT); +- +- let (output_addr, attribute_fields) = +- bsp::memory::mmu::virt_mem_layout().virt_addr_properties(virt_addr)?; +- +- *l3_entry = PageDescriptor::new(output_addr, attribute_fields); +- } +- } +- +- Ok(()) +-} +- + /// Configure various settings of stage 1 of the EL1 translation regime. + fn configure_translation_control() { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); +@@ -282,7 +356,7 @@ + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks +- + TCR_EL1::T0SZ.val(32), // TTBR0 spans 4 GiB total. ++ + TCR_EL1::T0SZ.val(T0SZ), + ); + } + +@@ -290,17 +364,126 @@ + // Public Code + //-------------------------------------------------------------------------------------------------- + ++/// Return a guarded reference to the kernel's translation tables. ++pub(in crate::memory::mmu) fn kernel_translation_tables( ++) -> &'static InitStateLock { ++ &KERNEL_TABLES ++} ++ + /// Return a reference to the MMU instance. +-pub fn mmu() -> &'static impl memory::mmu::interface::MMU { ++pub(in crate::memory::mmu) fn mmu() -> &'static impl mmu::interface::MMU { + &MMU + } + + //------------------------------------------------------------------------------ + // OS Interface Code + //------------------------------------------------------------------------------ ++impl mmu::interface::TranslationGranule for Granule64KiB { ++ const SIZE: usize = 64 * 1024; ++ const SHIFT: usize = 16; // log2(SIZE) ++} ++ ++impl mmu::interface::TranslationTable ++ for FixedSizeTranslationTable<{ NUM_TABLES }> ++{ ++ unsafe fn init(&mut self) { ++ if self.initialized { ++ return; ++ } ++ ++ // Populate the l2 entries. ++ for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { ++ *lvl2_entry = self.lvl3[lvl2_nr].phys_base_addr().into_usize().into(); ++ } ++ ++ self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; ++ self.initialized = true; ++ } ++ ++ fn phys_base_address(&self) -> Address { ++ self.lvl2.phys_base_addr() ++ } ++ ++ unsafe fn map_pages_at( ++ &mut self, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++ ) -> Result<(), &'static str> { ++ assert_eq!(self.initialized, true, "Translation tables not initialized"); ++ ++ let p = phys_pages.as_slice(); ++ let v = virt_pages.as_slice(); ++ ++ if p.len() != v.len() { ++ return Err("Tried to map page slices with unequal sizes"); ++ } ++ ++ // No work to do for empty slices. ++ if p.is_empty() { ++ return Ok(()); ++ } + +-impl memory::mmu::interface::MMU for MemoryManagementUnit { +- unsafe fn init(&self) -> Result<(), &'static str> { ++ if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() { ++ return Err("Tried to map outside of physical address space"); ++ } ++ ++ let iter = p.iter().zip(v.iter()); ++ for (phys_page, virt_page) in iter { ++ let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; ++ if page_descriptor.is_valid() { ++ return Err("Virtual page is already mapped"); ++ } ++ ++ *page_descriptor = PageDescriptor::new(phys_page.as_ptr(), &attr); ++ } ++ ++ Ok(()) ++ } ++ ++ fn next_mmio_virt_page_slice( ++ &mut self, ++ num_pages: usize, ++ ) -> Result, &'static str> { ++ assert_eq!(self.initialized, true, "Translation tables not initialized"); ++ ++ if num_pages == 0 { ++ return Err("num_pages == 0"); ++ } ++ ++ if (self.cur_l3_mmio_index + num_pages) > 8191 { ++ return Err("Not enough MMIO space left"); ++ } ++ ++ let addr = (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) ++ | (self.cur_l3_mmio_index << Granule64KiB::SHIFT); ++ self.cur_l3_mmio_index += num_pages; ++ ++ Ok(PageSliceDescriptor::from_addr( ++ Address::new(addr), ++ num_pages, ++ )) ++ } ++ ++ fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool { ++ let start_addr = virt_pages.start_addr(); ++ let end_addr_inclusive = virt_pages.end_addr_inclusive(); ++ ++ for i in [start_addr, end_addr_inclusive].iter() { ++ if (*i >= self.mmio_start_addr()) && (*i <= self.mmio_end_addr_inclusive()) { ++ return true; ++ } ++ } ++ ++ false ++ } ++} ++ ++impl mmu::interface::MMU for MemoryManagementUnit { ++ unsafe fn enable( ++ &self, ++ phys_kernel_table_base_addr: Address, ++ ) -> Result<(), &'static str> { + // Fail early if translation granule is not supported. Both RPis support it, though. + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { + return Err("Translation granule not supported in HW"); +@@ -309,11 +492,8 @@ + // Prepare the memory attribute indirection register. + set_up_mair(); + +- // Populate translation tables. +- populate_tt_entries()?; +- + // Set the "Translation Table Base Register". +- TTBR0_EL1.set_baddr(TABLES.lvl2.base_addr_u64()); ++ TTBR0_EL1.set_baddr(phys_kernel_table_base_addr.into_usize() as u64); + + configure_translation_control(); + +@@ -331,3 +511,43 @@ + Ok(()) + } + } ++ ++//-------------------------------------------------------------------------------------------------- ++// Testing ++//-------------------------------------------------------------------------------------------------- ++ ++#[cfg(test)] ++pub(in crate::memory::mmu) type MinSizeArchTranslationTable = FixedSizeTranslationTable<1>; ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use test_macros::kernel_test; ++ ++ /// Check if the size of `struct TableDescriptor` is as expected. ++ #[kernel_test] ++ fn size_of_tabledescriptor_equals_64_bit() { ++ assert_eq!( ++ core::mem::size_of::(), ++ core::mem::size_of::() ++ ); ++ } ++ ++ /// Check if the size of `struct PageDescriptor` is as expected. ++ #[kernel_test] ++ fn size_of_pagedescriptor_equals_64_bit() { ++ assert_eq!( ++ core::mem::size_of::(), ++ core::mem::size_of::() ++ ); ++ } ++ ++ /// Check if KERNEL_TABLES is in .bss. ++ #[kernel_test] ++ fn kernel_tables_in_bss() { ++ let bss_range = bsp::memory::bss_range(); ++ let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; ++ ++ assert!(bss_range.contains(&kernel_tables_addr)); ++ } ++} + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicc.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs +@@ -4,7 +4,9 @@ + + //! GICC Driver - GIC CPU interface. + +-use crate::{bsp::device_driver::common::MMIODerefWrapper, exception}; ++use crate::{ ++ bsp::device_driver::common::MMIODerefWrapper, exception, synchronization::InitStateLock, ++}; + use register::{mmio::*, register_bitfields, register_structs}; + + //-------------------------------------------------------------------------------------------------- +@@ -56,12 +58,13 @@ + + /// Representation of the GIC CPU interface. + pub struct GICC { +- registers: Registers, ++ registers: InitStateLock, + } + + //-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- ++use crate::synchronization::interface::ReadWriteEx; + + impl GICC { + /// Create an instance. +@@ -71,10 +74,15 @@ + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { +- registers: Registers::new(mmio_start_addr), ++ registers: InitStateLock::new(Registers::new(mmio_start_addr)), + } + } + ++ pub unsafe fn set_mmio(&self, new_mmio_start_addr: usize) { ++ let mut r = &self.registers; ++ r.write(|regs| *regs = Registers::new(new_mmio_start_addr)); ++ } ++ + /// Accept interrupts of any priority. + /// + /// Quoting the GICv2 Architecture Specification: +@@ -87,7 +95,10 @@ + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn priority_accept_all(&self) { +- self.registers.PMR.write(PMR::Priority.val(255)); // Comment in arch spec. ++ let mut r = &self.registers; ++ r.read(|regs| { ++ regs.PMR.write(PMR::Priority.val(255)); // Comment in arch spec. ++ }); + } + + /// Enable the interface - start accepting IRQs. +@@ -97,7 +108,10 @@ + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn enable(&self) { +- self.registers.CTLR.write(CTLR::Enable::SET); ++ let mut r = &self.registers; ++ r.read(|regs| { ++ regs.CTLR.write(CTLR::Enable::SET); ++ }); + } + + /// Extract the number of the highest-priority pending IRQ. +@@ -113,7 +127,8 @@ + &self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) -> usize { +- self.registers.IAR.read(IAR::InterruptID) as usize ++ let mut r = &self.registers; ++ r.read(|regs| regs.IAR.read(IAR::InterruptID) as usize) + } + + /// Complete handling of the currently active IRQ. +@@ -132,6 +147,9 @@ + irq_number: u32, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { +- self.registers.EOIR.write(EOIR::EOIINTID.val(irq_number)); ++ let mut r = &self.registers; ++ r.read(|regs| { ++ regs.EOIR.write(EOIR::EOIINTID.val(irq_number)); ++ }); + } + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicd.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2/gicd.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicd.rs +@@ -8,8 +8,9 @@ + //! - SPI - Shared Peripheral Interrupt. + + use crate::{ +- bsp::device_driver::common::MMIODerefWrapper, state, synchronization, +- synchronization::IRQSafeNullLock, ++ bsp::device_driver::common::MMIODerefWrapper, ++ state, synchronization, ++ synchronization::{IRQSafeNullLock, InitStateLock}, + }; + use register::{mmio::*, register_bitfields, register_structs}; + +@@ -79,7 +80,7 @@ + shared_registers: IRQSafeNullLock, + + /// Access to banked registers is unguarded. +- banked_registers: BankedRegisters, ++ banked_registers: InitStateLock, + } + + //-------------------------------------------------------------------------------------------------- +@@ -116,6 +117,7 @@ + //-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- ++use crate::synchronization::interface::ReadWriteEx; + use synchronization::interface::Mutex; + + impl GICD { +@@ -127,10 +129,18 @@ + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)), +- banked_registers: BankedRegisters::new(mmio_start_addr), ++ banked_registers: InitStateLock::new(BankedRegisters::new(mmio_start_addr)), + } + } + ++ pub unsafe fn set_mmio(&self, new_mmio_start_addr: usize) { ++ let mut r = &self.shared_registers; ++ r.lock(|regs| *regs = SharedRegisters::new(new_mmio_start_addr)); ++ ++ let mut r = &self.banked_registers; ++ r.write(|regs| *regs = BankedRegisters::new(new_mmio_start_addr)); ++ } ++ + /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask. + /// + /// Quoting the GICv2 Architecture Specification: +@@ -138,7 +148,8 @@ + /// "GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that + /// corresponds only to the processor reading the register." + fn local_gic_target_mask(&self) -> u32 { +- self.banked_registers.ITARGETSR[0].read(ITARGETSR::Offset0) ++ let mut r = &self.banked_registers; ++ r.read(|regs| regs.ITARGETSR[0].read(ITARGETSR::Offset0)) + } + + /// Route all SPIs to the boot core and enable the distributor. +@@ -179,8 +190,11 @@ + match irq_num { + // Private. + 0..=31 => { +- let enable_reg = &self.banked_registers.ISENABLER; +- enable_reg.set(enable_reg.get() | enable_bit); ++ let mut r = &self.banked_registers; ++ r.read(|regs| { ++ let enable_reg = ®s.ISENABLER; ++ enable_reg.set(enable_reg.get() | enable_bit); ++ }) + } + // Shared. + _ => { + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/arm/gicv2.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs +@@ -79,7 +79,11 @@ + mod gicc; + mod gicd; + +-use crate::{bsp, cpu, driver, exception, synchronization, synchronization::InitStateLock}; ++use crate::{ ++ bsp, cpu, driver, exception, memory, memory::mmu::Physical, synchronization, ++ synchronization::InitStateLock, ++}; ++use core::sync::atomic::{AtomicBool, Ordering}; + + //-------------------------------------------------------------------------------------------------- + // Private Definitions +@@ -96,12 +100,18 @@ + + /// Representation of the GIC. + pub struct GICv2 { ++ gicd_phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ gicc_phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ + /// The Distributor. + gicd: gicd::GICD, + + /// The CPU Interface. + gicc: gicc::GICC, + ++ /// Have the MMIO regions been remapped yet? ++ is_mmio_remapped: AtomicBool, ++ + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, + } +@@ -118,11 +128,17 @@ + /// + /// # Safety + /// +- /// - The user must ensure to provide a correct MMIO start address. +- pub const unsafe fn new(gicd_mmio_start_addr: usize, gicc_mmio_start_addr: usize) -> Self { ++ /// - The user must ensure to provide correct MMIO descriptors. ++ pub const unsafe fn new( ++ gicd_phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ gicc_phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ ) -> Self { + Self { +- gicd: gicd::GICD::new(gicd_mmio_start_addr), +- gicc: gicc::GICC::new(gicc_mmio_start_addr), ++ gicd_phys_mmio_descriptor, ++ gicc_phys_mmio_descriptor, ++ gicd: gicd::GICD::new(gicd_phys_mmio_descriptor.start_addr().into_usize()), ++ gicc: gicc::GICC::new(gicc_phys_mmio_descriptor.start_addr().into_usize()), ++ is_mmio_remapped: AtomicBool::new(false), + handler_table: InitStateLock::new([None; Self::NUM_IRQS]), + } + } +@@ -139,6 +155,22 @@ + } + + unsafe fn init(&self) -> Result<(), &'static str> { ++ let remapped = self.is_mmio_remapped.load(Ordering::Relaxed); ++ if !remapped { ++ let mut virt_addr; ++ ++ // GICD ++ virt_addr = memory::mmu::kernel_map_mmio("GICD", &self.gicd_phys_mmio_descriptor)?; ++ self.gicd.set_mmio(virt_addr.into_usize()); ++ ++ // GICC ++ virt_addr = memory::mmu::kernel_map_mmio("GICC", &self.gicc_phys_mmio_descriptor)?; ++ self.gicc.set_mmio(virt_addr.into_usize()); ++ ++ // Conclude remapping. ++ self.is_mmio_remapped.store(true, Ordering::Relaxed); ++ } ++ + if cpu::smp::core_id::() == bsp::cpu::BOOT_CORE_ID { + self.gicd.boot_core_init(); + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs +@@ -5,9 +5,10 @@ + //! GPIO Driver. + + use crate::{ +- bsp::device_driver::common::MMIODerefWrapper, cpu, driver, synchronization, +- synchronization::IRQSafeNullLock, ++ bsp::device_driver::common::MMIODerefWrapper, cpu, driver, memory, memory::mmu::Physical, ++ synchronization, synchronization::IRQSafeNullLock, + }; ++use core::sync::atomic::{AtomicUsize, Ordering}; + use register::{mmio::*, register_bitfields, register_structs}; + + //-------------------------------------------------------------------------------------------------- +@@ -88,6 +89,8 @@ + + /// Representation of the GPIO HW. + pub struct GPIO { ++ phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ virt_mmio_start_addr: AtomicUsize, + inner: IRQSafeNullLock, + } + +@@ -107,6 +110,19 @@ + } + } + ++ /// Init code. ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide a correct MMIO start address. ++ pub unsafe fn init(&mut self, new_mmio_start_addr: Option) -> Result<(), &'static str> { ++ if let Some(addr) = new_mmio_start_addr { ++ self.registers = Registers::new(addr); ++ } ++ ++ Ok(()) ++ } ++ + /// Map PL011 UART as standard output. + /// + /// TX to pin 14 +@@ -135,10 +151,14 @@ + /// + /// # Safety + /// +- /// - The user must ensure to provide a correct MMIO start address. +- pub const unsafe fn new(mmio_start_addr: usize) -> Self { ++ /// - The user must ensure to provide correct MMIO descriptors. ++ pub const unsafe fn new(phys_mmio_descriptor: memory::mmu::MMIODescriptor) -> Self { + Self { +- inner: IRQSafeNullLock::new(GPIOInner::new(mmio_start_addr)), ++ phys_mmio_descriptor, ++ virt_mmio_start_addr: AtomicUsize::new(0), ++ inner: IRQSafeNullLock::new(GPIOInner::new( ++ phys_mmio_descriptor.start_addr().into_usize(), ++ )), + } + } + +@@ -158,4 +178,27 @@ + fn compatible(&self) -> &'static str { + "BCM GPIO" + } ++ ++ unsafe fn init(&self) -> Result<(), &'static str> { ++ let virt_addr = ++ memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)?; ++ ++ let mut r = &self.inner; ++ r.lock(|inner| inner.init(Some(virt_addr.into_usize())))?; ++ ++ self.virt_mmio_start_addr ++ .store(virt_addr.into_usize(), Ordering::Relaxed); ++ ++ Ok(()) ++ } ++ ++ fn virt_mmio_start_addr(&self) -> Option { ++ let addr = self.virt_mmio_start_addr.load(Ordering::Relaxed); ++ ++ if addr == 0 { ++ return None; ++ } ++ ++ Some(addr) ++ } + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs +@@ -2,12 +2,14 @@ + // + // Copyright (c) 2020 Andre Richter + +-//! Peripheral Interrupt regsler Driver. ++//! Peripheral Interrupt Controller Driver. + + use super::{InterruptController, PendingIRQs, PeripheralIRQ}; + use crate::{ + bsp::device_driver::common::MMIODerefWrapper, +- exception, synchronization, ++ driver, exception, memory, ++ memory::mmu::Physical, ++ synchronization, + synchronization::{IRQSafeNullLock, InitStateLock}, + }; + use register::{mmio::*, register_structs}; +@@ -51,11 +53,13 @@ + + /// Representation of the peripheral interrupt controller. + pub struct PeripheralIC { ++ phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ + /// Access to write registers is guarded with a lock. + wo_registers: IRQSafeNullLock, + + /// Register read access is unguarded. +- ro_registers: ReadOnlyRegisters, ++ ro_registers: InitStateLock, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +@@ -70,21 +74,27 @@ + /// + /// # Safety + /// +- /// - The user must ensure to provide a correct MMIO start address. +- pub const unsafe fn new(mmio_start_addr: usize) -> Self { ++ /// - The user must ensure to provide correct MMIO descriptors. ++ pub const unsafe fn new(phys_mmio_descriptor: memory::mmu::MMIODescriptor) -> Self { ++ let addr = phys_mmio_descriptor.start_addr().into_usize(); ++ + Self { +- wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(mmio_start_addr)), +- ro_registers: ReadOnlyRegisters::new(mmio_start_addr), ++ phys_mmio_descriptor, ++ wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(addr)), ++ ro_registers: InitStateLock::new(ReadOnlyRegisters::new(addr)), + handler_table: InitStateLock::new([None; InterruptController::NUM_PERIPHERAL_IRQS]), + } + } + + /// Query the list of pending IRQs. + fn pending_irqs(&self) -> PendingIRQs { +- let pending_mask: u64 = (u64::from(self.ro_registers.PENDING_2.get()) << 32) +- | u64::from(self.ro_registers.PENDING_1.get()); ++ let mut r = &self.ro_registers; ++ r.read(|regs| { ++ let pending_mask: u64 = ++ (u64::from(regs.PENDING_2.get()) << 32) | u64::from(regs.PENDING_1.get()); + +- PendingIRQs::new(pending_mask) ++ PendingIRQs::new(pending_mask) ++ }) + } + } + +@@ -93,6 +103,26 @@ + //------------------------------------------------------------------------------ + use synchronization::interface::{Mutex, ReadWriteEx}; + ++impl driver::interface::DeviceDriver for PeripheralIC { ++ fn compatible(&self) -> &'static str { ++ "BCM Peripheral Interrupt Controller" ++ } ++ ++ unsafe fn init(&self) -> Result<(), &'static str> { ++ let virt_addr = ++ memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)? ++ .into_usize(); ++ ++ let mut r = &self.wo_registers; ++ r.lock(|regs| *regs = WriteOnlyRegisters::new(virt_addr)); ++ ++ let mut r = &self.ro_registers; ++ r.write(|regs| *regs = ReadOnlyRegisters::new(virt_addr)); ++ ++ Ok(()) ++ } ++} ++ + impl exception::asynchronous::interface::IRQManager for PeripheralIC { + type IRQNumberType = PeripheralIRQ; + + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs +@@ -6,7 +6,7 @@ + + mod peripheral_ic; + +-use crate::{driver, exception}; ++use crate::{driver, exception, memory, memory::mmu::Physical}; + + //-------------------------------------------------------------------------------------------------- + // Private Definitions +@@ -78,10 +78,13 @@ + /// + /// # Safety + /// +- /// - The user must ensure to provide a correct MMIO start address. +- pub const unsafe fn new(_local_mmio_start_addr: usize, periph_mmio_start_addr: usize) -> Self { ++ /// - The user must ensure to provide correct MMIO descriptors. ++ pub const unsafe fn new( ++ _phys_local_mmio_descriptor: memory::mmu::MMIODescriptor, ++ phys_periph_mmio_descriptor: memory::mmu::MMIODescriptor, ++ ) -> Self { + Self { +- periph: peripheral_ic::PeripheralIC::new(periph_mmio_start_addr), ++ periph: peripheral_ic::PeripheralIC::new(phys_periph_mmio_descriptor), + } + } + } +@@ -94,6 +97,10 @@ + fn compatible(&self) -> &'static str { + "BCM Interrupt Controller" + } ++ ++ unsafe fn init(&self) -> Result<(), &'static str> { ++ self.periph.init() ++ } + } + + impl exception::asynchronous::interface::IRQManager for InterruptController { + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs +@@ -5,10 +5,13 @@ + //! PL011 UART driver. + + use crate::{ +- bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, +- synchronization, synchronization::IRQSafeNullLock, ++ bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, memory, ++ memory::mmu::Physical, synchronization, synchronization::IRQSafeNullLock, ++}; ++use core::{ ++ fmt, ++ sync::atomic::{AtomicUsize, Ordering}, + }; +-use core::fmt; + use register::{mmio::*, register_bitfields, register_structs}; + + //-------------------------------------------------------------------------------------------------- +@@ -202,6 +205,8 @@ + + /// Representation of the UART. + pub struct PL011Uart { ++ phys_mmio_descriptor: memory::mmu::MMIODescriptor, ++ virt_mmio_start_addr: AtomicUsize, + inner: IRQSafeNullLock, + irq_number: bsp::device_driver::IRQNumber, + } +@@ -232,7 +237,15 @@ + /// approximation we can get. A 5 modulo error margin is acceptable for UART and we're now at 0,01 modulo. + /// + /// This results in 8N1 and 230400 baud (we set the clock to 48 MHz in config.txt). +- pub fn init(&mut self) { ++ /// ++ /// # Safety ++ /// ++ /// - The user must ensure to provide a correct MMIO start address. ++ pub unsafe fn init(&mut self, new_mmio_start_addr: Option) -> Result<(), &'static str> { ++ if let Some(addr) = new_mmio_start_addr { ++ self.registers = Registers::new(addr); ++ } ++ + // Turn it off temporarily. + self.registers.CR.set(0); + +@@ -249,6 +262,8 @@ + self.registers + .CR + .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled); ++ ++ Ok(()) + } + + /// Send a character. +@@ -318,13 +333,18 @@ + /// + /// # Safety + /// +- /// - The user must ensure to provide a correct MMIO start address. ++ /// - The user must ensure to provide correct MMIO descriptors. ++ /// - The user must ensure to provide correct IRQ numbers. + pub const unsafe fn new( +- mmio_start_addr: usize, ++ phys_mmio_descriptor: memory::mmu::MMIODescriptor, + irq_number: bsp::device_driver::IRQNumber, + ) -> Self { + Self { +- inner: IRQSafeNullLock::new(PL011UartInner::new(mmio_start_addr)), ++ phys_mmio_descriptor, ++ virt_mmio_start_addr: AtomicUsize::new(0), ++ inner: IRQSafeNullLock::new(PL011UartInner::new( ++ phys_mmio_descriptor.start_addr().into_usize(), ++ )), + irq_number, + } + } +@@ -341,8 +361,14 @@ + } + + unsafe fn init(&self) -> Result<(), &'static str> { ++ let virt_addr = ++ memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)?; ++ + let mut r = &self.inner; +- r.lock(|inner| inner.init()); ++ r.lock(|inner| inner.init(Some(virt_addr.into_usize())))?; ++ ++ self.virt_mmio_start_addr ++ .store(virt_addr.into_usize(), Ordering::Relaxed); + + Ok(()) + } +@@ -361,6 +387,16 @@ + + Ok(()) + } ++ ++ fn virt_mmio_start_addr(&self) -> Option { ++ let addr = self.virt_mmio_start_addr.load(Ordering::Relaxed); ++ ++ if addr == 0 { ++ return None; ++ } ++ ++ Some(addr) ++ } + } + + impl console::interface::Write for PL011Uart { + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/console.rs 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/console.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/console.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/console.rs +@@ -5,7 +5,7 @@ + //! BSP console facilities. + + use super::memory; +-use crate::{bsp::device_driver, console}; ++use crate::{bsp::device_driver, console, cpu}; + use core::fmt; + + //-------------------------------------------------------------------------------------------------- +@@ -23,11 +23,25 @@ + /// + /// - Use only for printing during a panic. + pub unsafe fn panic_console_out() -> impl fmt::Write { +- let mut panic_gpio = device_driver::PanicGPIO::new(memory::map::mmio::GPIO_START); +- let mut panic_uart = device_driver::PanicUart::new(memory::map::mmio::PL011_UART_START); ++ use crate::driver::interface::DeviceDriver; + ++ let mut panic_gpio = device_driver::PanicGPIO::new(memory::map::mmio::GPIO_START.into_usize()); ++ let mut panic_uart = ++ device_driver::PanicUart::new(memory::map::mmio::PL011_UART_START.into_usize()); ++ ++ // If remapping of the driver's MMIO already happened, take the remapped start address. ++ // Otherwise, take a chance with the default physical address. ++ let maybe_gpio_mmio_start_addr = super::GPIO.virt_mmio_start_addr(); ++ let maybe_uart_mmio_start_addr = super::PL011_UART.virt_mmio_start_addr(); ++ ++ panic_gpio ++ .init(maybe_gpio_mmio_start_addr) ++ .unwrap_or_else(|_| cpu::wait_forever()); + panic_gpio.map_pl011_uart(); +- panic_uart.init(); ++ panic_uart ++ .init(maybe_uart_mmio_start_addr) ++ .unwrap_or_else(|_| cpu::wait_forever()); ++ + panic_uart + } + + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/driver.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/driver.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/driver.rs +@@ -46,7 +46,15 @@ + &self.device_drivers[..] + } + +- fn post_device_driver_init(&self) { ++ fn early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { ++ &self.device_drivers[0..=1] ++ } ++ ++ fn non_early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { ++ &self.device_drivers[2..] ++ } ++ ++ fn post_early_print_device_driver_init(&self) { + // Configure PL011Uart's output pins. + super::GPIO.map_pl011_uart(); + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/link.ld +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/link.ld ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/link.ld +@@ -39,6 +39,11 @@ + . = ALIGN(8); + __bss_end = .; + } ++ . = ALIGN(65536); ++ __data_end = .; ++ ++ __ro_size = __ro_end - __ro_start; ++ __data_size = __data_end - __ro_end; + + /DISCARD/ : { *(.comment*) } + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory/mmu.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs +@@ -4,72 +4,128 @@ + + //! BSP Memory Management Unit. + +-use super::map as memory_map; +-use crate::memory::mmu::*; +-use core::ops::RangeInclusive; ++use crate::{ ++ common, ++ memory::{ ++ mmu as kernel_mmu, ++ mmu::{ ++ interface, AccessPermissions, AttributeFields, Granule64KiB, MemAttributes, Page, ++ PageSliceDescriptor, Physical, Virtual, ++ }, ++ }, ++}; + + //-------------------------------------------------------------------------------------------------- + // Public Definitions + //-------------------------------------------------------------------------------------------------- + +-const NUM_MEM_RANGES: usize = 2; +- +-/// The virtual memory layout. +-/// +-/// The layout must contain only special ranges, aka anything that is _not_ normal cacheable DRAM. +-/// It is agnostic of the paging granularity that the architecture's MMU will use. +-pub static LAYOUT: KernelVirtualLayout<{ NUM_MEM_RANGES }> = KernelVirtualLayout::new( +- memory_map::END_INCLUSIVE, +- [ +- TranslationDescriptor { +- name: "Kernel code and RO data", +- virtual_range: ro_range_inclusive, +- physical_range_translation: Translation::Identity, +- attribute_fields: AttributeFields { +- mem_attributes: MemAttributes::CacheableDRAM, +- acc_perms: AccessPermissions::ReadOnly, +- execute_never: false, +- }, +- }, +- TranslationDescriptor { +- name: "Device MMIO", +- virtual_range: mmio_range_inclusive, +- physical_range_translation: Translation::Identity, +- attribute_fields: AttributeFields { +- mem_attributes: MemAttributes::Device, +- acc_perms: AccessPermissions::ReadWrite, +- execute_never: true, +- }, +- }, +- ], +-); ++/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to ++/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. ++pub type KernelGranule = Granule64KiB; + + //-------------------------------------------------------------------------------------------------- + // Private Code + //-------------------------------------------------------------------------------------------------- ++use interface::TranslationGranule; ++ ++/// Helper function for calculating the number of pages the given parameter spans. ++const fn size_to_num_pages(size: usize) -> usize { ++ assert!(size > 0); ++ assert!(size modulo KernelGranule::SIZE == 0); ++ ++ size >> KernelGranule::SHIFT ++} ++ ++/// The boot core's stack. ++fn virt_stack_page_desc() -> PageSliceDescriptor { ++ let num_pages = size_to_num_pages(super::boot_core_stack_size()); ++ ++ PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) ++} ++ ++/// The Read-Only (RO) pages of the kernel binary. ++fn virt_ro_page_desc() -> PageSliceDescriptor { ++ let num_pages = size_to_num_pages(super::ro_size()); ++ ++ PageSliceDescriptor::from_addr(super::virt_ro_start(), num_pages) ++} ++ ++/// The data pages of the kernel binary. ++fn virt_data_page_desc() -> PageSliceDescriptor { ++ let num_pages = size_to_num_pages(super::data_size()); + +-fn ro_range_inclusive() -> RangeInclusive { +- // Notice the subtraction to turn the exclusive end into an inclusive end. +- #[allow(clippy::range_minus_one)] +- RangeInclusive::new(super::ro_start(), super::ro_end() - 1) ++ PageSliceDescriptor::from_addr(super::virt_data_start(), num_pages) + } + +-fn mmio_range_inclusive() -> RangeInclusive { +- RangeInclusive::new(memory_map::mmio::START, memory_map::mmio::END_INCLUSIVE) ++// The binary is still identity mapped, so we don't need to convert in the following. ++ ++/// The boot core's stack. ++fn phys_stack_page_desc() -> PageSliceDescriptor { ++ virt_stack_page_desc().into() ++} ++ ++/// The Read-Only (RO) pages of the kernel binary. ++fn phys_ro_page_desc() -> PageSliceDescriptor { ++ virt_ro_page_desc().into() ++} ++ ++/// The data pages of the kernel binary. ++fn phys_data_page_desc() -> PageSliceDescriptor { ++ virt_data_page_desc().into() + } + + //-------------------------------------------------------------------------------------------------- + // Public Code + //-------------------------------------------------------------------------------------------------- + +-/// Return the address space size in bytes. +-pub const fn addr_space_size() -> usize { +- memory_map::END_INCLUSIVE + 1 ++/// Pointer to the last page of the physical address space. ++pub fn phys_addr_space_end_page() -> *const Page { ++ common::align_down( ++ super::phys_addr_space_end().into_usize(), ++ KernelGranule::SIZE, ++ ) as *const Page<_> + } + +-/// Return a reference to the virtual memory layout. +-pub fn virt_mem_layout() -> &'static KernelVirtualLayout<{ NUM_MEM_RANGES }> { +- &LAYOUT ++/// Map the kernel binary. ++/// ++/// # Safety ++/// ++/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. ++pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { ++ kernel_mmu::kernel_map_pages_at( ++ "Kernel boot-core stack", ++ &phys_stack_page_desc(), ++ &virt_stack_page_desc(), ++ &AttributeFields { ++ mem_attributes: MemAttributes::CacheableDRAM, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ }, ++ )?; ++ ++ kernel_mmu::kernel_map_pages_at( ++ "Kernel code and RO data", ++ &phys_ro_page_desc(), ++ &virt_ro_page_desc(), ++ &AttributeFields { ++ mem_attributes: MemAttributes::CacheableDRAM, ++ acc_perms: AccessPermissions::ReadOnly, ++ execute_never: false, ++ }, ++ )?; ++ ++ kernel_mmu::kernel_map_pages_at( ++ "Kernel data and bss", ++ &phys_data_page_desc(), ++ &virt_data_page_desc(), ++ &AttributeFields { ++ mem_attributes: MemAttributes::CacheableDRAM, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ }, ++ )?; ++ ++ Ok(()) + } + + //-------------------------------------------------------------------------------------------------- +@@ -84,14 +140,12 @@ + /// Check alignment of the kernel's virtual memory layout sections. + #[kernel_test] + fn virt_mem_layout_sections_are_64KiB_aligned() { +- const SIXTYFOUR_KIB: usize = 65536; +- +- for i in LAYOUT.inner().iter() { +- let start: usize = *(i.virtual_range)().start(); +- let end: usize = *(i.virtual_range)().end() + 1; ++ for i in [virt_stack_page_desc, virt_ro_page_desc, virt_data_page_desc].iter() { ++ let start: usize = i().start_addr().into_usize(); ++ let end: usize = i().end_addr().into_usize(); + +- assert_eq!(start modulo SIXTYFOUR_KIB, 0); +- assert_eq!(end modulo SIXTYFOUR_KIB, 0); ++ assert_eq!(start modulo KernelGranule::SIZE, 0); ++ assert_eq!(end modulo KernelGranule::SIZE, 0); + assert!(end >= start); + } + } +@@ -99,17 +153,18 @@ + /// Ensure the kernel's virtual memory layout is free of overlaps. + #[kernel_test] + fn virt_mem_layout_has_no_overlaps() { +- let layout = virt_mem_layout().inner(); +- +- for (i, first) in layout.iter().enumerate() { +- for second in layout.iter().skip(i + 1) { +- let first_range = first.virtual_range; +- let second_range = second.virtual_range; +- +- assert!(!first_range().contains(second_range().start())); +- assert!(!first_range().contains(second_range().end())); +- assert!(!second_range().contains(first_range().start())); +- assert!(!second_range().contains(first_range().end())); ++ let layout = [ ++ virt_stack_page_desc().into_usize_range_inclusive(), ++ virt_ro_page_desc().into_usize_range_inclusive(), ++ virt_data_page_desc().into_usize_range_inclusive(), ++ ]; ++ ++ for (i, first_range) in layout.iter().enumerate() { ++ for second_range in layout.iter().skip(i + 1) { ++ assert!(!first_range.contains(second_range.start())); ++ assert!(!first_range.contains(second_range.end())); ++ assert!(!second_range.contains(first_range.start())); ++ assert!(!second_range.contains(first_range.end())); + } + } + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi/memory.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory.rs +@@ -3,9 +3,41 @@ + // Copyright (c) 2018-2020 Andre Richter + + //! BSP Memory Management. ++//! ++//! The physical memory layout after the kernel has been loaded by the Raspberry's firmware, which ++//! copies the binary to 0x8_0000: ++//! ++//! +---------------------------------------------+ ++//! | | 0x0 ++//! | Unmapped | ++//! | | 0x6_FFFF ++//! +---------------------------------------------+ ++//! | BOOT_CORE_STACK_START | 0x7_0000 ++//! | | ^ ++//! | ... | | Stack growth direction ++//! | | | ++//! | BOOT_CORE_STACK_END_INCLUSIVE | 0x7_FFFF ++//! +---------------------------------------------+ ++//! | RO_START == BOOT_CORE_STACK_END | 0x8_0000 ++//! | | ++//! | | ++//! | .text | ++//! | .exception_vectors | ++//! | .rodata | ++//! | | ++//! | RO_END_INCLUSIVE | 0x8_0000 + __ro_size - 1 ++//! +---------------------------------------------+ ++//! | RO_END == DATA_START | 0x8_0000 + __ro_size ++//! | | ++//! | .data | ++//! | .bss | ++//! | | ++//! | DATA_END_INCLUSIVE | 0x8_0000 + __ro_size + __data_size - 1 ++//! +---------------------------------------------+ + + pub mod mmu; + ++use crate::memory::mmu::{Address, Physical, Virtual}; + use core::ops::Range; + + //-------------------------------------------------------------------------------------------------- +@@ -15,36 +47,41 @@ + // Symbols from the linker script. + extern "C" { + static __ro_start: usize; +- static __ro_end: usize; ++ static __ro_size: usize; + static __bss_start: usize; + static __bss_end: usize; ++ static __data_size: usize; + } + + //-------------------------------------------------------------------------------------------------- + // Public Definitions + //-------------------------------------------------------------------------------------------------- + +-/// The board's memory map. ++/// The board's physical memory map. + #[rustfmt::skip] + pub(super) mod map { +- pub const END_INCLUSIVE: usize = 0xFFFF_FFFF; ++ use super::*; + +- pub const BOOT_CORE_STACK_END: usize = 0x8_0000; +- +- pub const GPIO_OFFSET: usize = 0x0020_0000; +- pub const UART_OFFSET: usize = 0x0020_1000; ++ pub const BOOT_CORE_STACK_SIZE: usize = 0x1_0000; + + /// Physical devices. + #[cfg(feature = "bsp_rpi3")] + pub mod mmio { + use super::*; + +- pub const START: usize = 0x3F00_0000; +- pub const PERIPHERAL_INTERRUPT_CONTROLLER_START: usize = START + 0x0000_B200; +- pub const GPIO_START: usize = START + GPIO_OFFSET; +- pub const PL011_UART_START: usize = START + UART_OFFSET; +- pub const LOCAL_INTERRUPT_CONTROLLER_START: usize = 0x4000_0000; +- pub const END_INCLUSIVE: usize = 0x4000_FFFF; ++ pub const PERIPHERAL_IC_START: Address = Address::new(0x3F00_B200); ++ pub const PERIPHERAL_IC_SIZE: usize = 0x24; ++ ++ pub const GPIO_START: Address = Address::new(0x3F20_0000); ++ pub const GPIO_SIZE: usize = 0xA0; ++ ++ pub const PL011_UART_START: Address = Address::new(0x3F20_1000); ++ pub const PL011_UART_SIZE: usize = 0x48; ++ ++ pub const LOCAL_IC_START: Address = Address::new(0x4000_0000); ++ pub const LOCAL_IC_SIZE: usize = 0x100; ++ ++ pub const END: Address = Address::new(0x4001_0000); + } + + /// Physical devices. +@@ -52,13 +89,22 @@ + pub mod mmio { + use super::*; + +- pub const START: usize = 0xFE00_0000; +- pub const GPIO_START: usize = START + GPIO_OFFSET; +- pub const PL011_UART_START: usize = START + UART_OFFSET; +- pub const GICD_START: usize = 0xFF84_1000; +- pub const GICC_START: usize = 0xFF84_2000; +- pub const END_INCLUSIVE: usize = 0xFF84_FFFF; ++ pub const GPIO_START: Address = Address::new(0xFE20_0000); ++ pub const GPIO_SIZE: usize = 0xA0; ++ ++ pub const PL011_UART_START: Address = Address::new(0xFE20_1000); ++ pub const PL011_UART_SIZE: usize = 0x48; ++ ++ pub const GICD_START: Address = Address::new(0xFF84_1000); ++ pub const GICD_SIZE: usize = 0x824; ++ ++ pub const GICC_START: Address = Address::new(0xFF84_2000); ++ pub const GICC_SIZE: usize = 0x14; ++ ++ pub const END: Address = Address::new(0xFF85_0000); + } ++ ++ pub const END: Address = mmio::END; + } + + //-------------------------------------------------------------------------------------------------- +@@ -71,8 +117,8 @@ + /// + /// - Value is provided by the linker script and must be trusted as-is. + #[inline(always)] +-fn ro_start() -> usize { +- unsafe { &__ro_start as *const _ as usize } ++fn virt_ro_start() -> Address { ++ Address::new(unsafe { &__ro_start as *const _ as usize }) + } + + /// Size of the Read-Only (RO) range of the kernel binary. +@@ -81,8 +127,42 @@ + /// + /// - Value is provided by the linker script and must be trusted as-is. + #[inline(always)] +-fn ro_end() -> usize { +- unsafe { &__ro_end as *const _ as usize } ++fn ro_size() -> usize { ++ unsafe { &__ro_size as *const _ as usize } ++} ++ ++/// Start address of the data range. ++#[inline(always)] ++fn virt_data_start() -> Address { ++ virt_ro_start() + ro_size() ++} ++ ++/// Size of the data range. ++/// ++/// # Safety ++/// ++/// - Value is provided by the linker script and must be trusted as-is. ++#[inline(always)] ++fn data_size() -> usize { ++ unsafe { &__data_size as *const _ as usize } ++} ++ ++/// Start address of the boot core's stack. ++#[inline(always)] ++fn virt_boot_core_stack_start() -> Address { ++ virt_ro_start() - map::BOOT_CORE_STACK_SIZE ++} ++ ++/// Size of the boot core's stack. ++#[inline(always)] ++fn boot_core_stack_size() -> usize { ++ map::BOOT_CORE_STACK_SIZE ++} ++ ++/// Exclusive end address of the physical address space. ++#[inline(always)] ++fn phys_addr_space_end() -> Address { ++ map::END + } + + //-------------------------------------------------------------------------------------------------- +@@ -91,8 +171,10 @@ + + /// Exclusive end address of the boot core's stack. + #[inline(always)] +-pub fn boot_core_stack_end() -> usize { +- map::BOOT_CORE_STACK_END ++pub fn phys_boot_core_stack_end() -> Address { ++ // The binary is still identity mapped, so we don't need to convert here. ++ let end = virt_boot_core_stack_start().into_usize() + boot_core_stack_size(); ++ Address::new(end) + } + + /// Return the range spanning the .bss section. + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi.rs +--- 14_exceptions_part2_peripheral_IRQs/src/bsp/raspberrypi.rs ++++ 15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi.rs +@@ -10,17 +10,20 @@ + pub mod exception; + pub mod memory; + ++use super::device_driver; ++use crate::memory::mmu::MMIODescriptor; ++use memory::map::mmio; ++ + //-------------------------------------------------------------------------------------------------- + // Global instances + //-------------------------------------------------------------------------------------------------- +-use super::device_driver; + + static GPIO: device_driver::GPIO = +- unsafe { device_driver::GPIO::new(memory::map::mmio::GPIO_START) }; ++ unsafe { device_driver::GPIO::new(MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE)) }; + + static PL011_UART: device_driver::PL011Uart = unsafe { + device_driver::PL011Uart::new( +- memory::map::mmio::PL011_UART_START, ++ MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE), + exception::asynchronous::irq_map::PL011_UART, + ) + }; +@@ -28,14 +31,17 @@ + #[cfg(feature = "bsp_rpi3")] + static INTERRUPT_CONTROLLER: device_driver::InterruptController = unsafe { + device_driver::InterruptController::new( +- memory::map::mmio::LOCAL_INTERRUPT_CONTROLLER_START, +- memory::map::mmio::PERIPHERAL_INTERRUPT_CONTROLLER_START, ++ MMIODescriptor::new(mmio::LOCAL_IC_START, mmio::LOCAL_IC_SIZE), ++ MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE), + ) + }; + + #[cfg(feature = "bsp_rpi4")] + static INTERRUPT_CONTROLLER: device_driver::GICv2 = unsafe { +- device_driver::GICv2::new(memory::map::mmio::GICD_START, memory::map::mmio::GICC_START) ++ device_driver::GICv2::new( ++ MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE), ++ MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE), ++ ) + }; + + //-------------------------------------------------------------------------------------------------- + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/common.rs 15_virtual_memory_part2_mmio_remap/src/common.rs +--- 14_exceptions_part2_peripheral_IRQs/src/common.rs ++++ 15_virtual_memory_part2_mmio_remap/src/common.rs +@@ -0,0 +1,21 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! General purpose code. ++ ++/// Check if a value is aligned to a given size. ++#[inline(always)] ++pub const fn is_aligned(value: usize, alignment: usize) -> bool { ++ assert!(alignment.is_power_of_two()); ++ ++ (value & (alignment - 1)) == 0 ++} ++ ++/// Align down. ++#[inline(always)] ++pub const fn align_down(value: usize, alignment: usize) -> usize { ++ assert!(alignment.is_power_of_two()); ++ ++ value & !(alignment - 1) ++} + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/driver.rs 15_virtual_memory_part2_mmio_remap/src/driver.rs +--- 14_exceptions_part2_peripheral_IRQs/src/driver.rs ++++ 15_virtual_memory_part2_mmio_remap/src/driver.rs +@@ -31,6 +31,14 @@ + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + Ok(()) + } ++ ++ /// After MMIO remapping, returns the new virtual start address. ++ /// ++ /// This API assumes a driver has only a single, contiguous MMIO aperture, which will not be ++ /// the case for more complex devices. This API will likely change in future tutorials. ++ fn virt_mmio_start_addr(&self) -> Option { ++ None ++ } + } + + /// Device driver management functions. +@@ -38,15 +46,17 @@ + /// The `BSP` is supposed to supply one global instance. + pub trait DriverManager { + /// Return a slice of references to all `BSP`-instantiated drivers. +- /// +- /// # Safety +- /// +- /// - The order of devices is the order in which `DeviceDriver::init()` is called. + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + +- /// Initialization code that runs after driver init. ++ /// Return only those drivers needed for the BSP's early printing functionality. + /// +- /// For example, device driver code that depends on other drivers already being online. +- fn post_device_driver_init(&self); ++ /// For example, the default UART. ++ fn early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; ++ ++ /// Return all drivers minus early-print drivers. ++ fn non_early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; ++ ++ /// Initialization code that runs after the early print driver init. ++ fn post_early_print_device_driver_init(&self); + } + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/lib.rs 15_virtual_memory_part2_mmio_remap/src/lib.rs +--- 14_exceptions_part2_peripheral_IRQs/src/lib.rs ++++ 15_virtual_memory_part2_mmio_remap/src/lib.rs +@@ -113,6 +113,7 @@ + + #![allow(incomplete_features)] + #![feature(asm)] ++#![feature(const_fn)] + #![feature(const_generics)] + #![feature(const_panic)] + #![feature(core_intrinsics)] +@@ -137,6 +138,7 @@ + mod synchronization; + + pub mod bsp; ++pub mod common; + pub mod console; + pub mod cpu; + pub mod driver; + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/main.rs 15_virtual_memory_part2_mmio_remap/src/main.rs +--- 14_exceptions_part2_peripheral_IRQs/src/main.rs ++++ 15_virtual_memory_part2_mmio_remap/src/main.rs +@@ -26,21 +26,34 @@ + #[no_mangle] + unsafe fn kernel_init() -> ! { + use driver::interface::DriverManager; +- use memory::mmu::interface::MMU; + + exception::handling_init(); + +- if let Err(string) = memory::mmu::mmu().init() { +- panic!("MMU: {}", string); ++ if let Err(string) = memory::mmu::kernel_map_binary_and_enable_mmu() { ++ panic!("Enabling MMU failed: {}", string); + } ++ // Printing will silently fail fail from here on, because the driver's MMIO is not remapped yet. + +- for i in bsp::driver::driver_manager().all_device_drivers().iter() { ++ // Bring up the drivers needed for printing first. ++ for i in bsp::driver::driver_manager() ++ .early_print_device_drivers() ++ .iter() ++ { ++ // Any encountered errors cannot be printed yet, obviously, so just safely park the CPU. ++ i.init().unwrap_or_else(|_| cpu::wait_forever()); ++ } ++ bsp::driver::driver_manager().post_early_print_device_driver_init(); ++ // Printing available again from here on. ++ ++ // Now bring up the remaining drivers. ++ for i in bsp::driver::driver_manager() ++ .non_early_print_device_drivers() ++ .iter() ++ { + if let Err(x) = i.init() { + panic!("Error loading driver: {}: {}", i.compatible(), x); + } + } +- bsp::driver::driver_manager().post_device_driver_init(); +- // println! is usable from here on. + + // Let device drivers register and enable their handlers with the interrupt controller. + for i in bsp::driver::driver_manager().all_device_drivers() { +@@ -66,8 +79,8 @@ + + info!("Booting on: {}", bsp::board_name()); + +- info!("MMU online. Special regions:"); +- bsp::memory::mmu::virt_mem_layout().print_layout(); ++ info!("MMU online:"); ++ memory::mmu::kernel_print_mappings(); + + let (_, privilege_level) = exception::current_privilege_level(); + info!("Current privilege level: {}", privilege_level); + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/mapping_record.rs 15_virtual_memory_part2_mmio_remap/src/memory/mmu/mapping_record.rs +--- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/mapping_record.rs ++++ 15_virtual_memory_part2_mmio_remap/src/memory/mmu/mapping_record.rs +@@ -0,0 +1,224 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! A record of mapped pages. ++ ++use super::{ ++ AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, ++ PageSliceDescriptor, Physical, Virtual, ++}; ++use crate::{info, synchronization, synchronization::InitStateLock, warn}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Definitions ++//-------------------------------------------------------------------------------------------------- ++ ++/// Type describing a virtual memory mapping. ++#[allow(missing_docs)] ++#[derive(Copy, Clone)] ++struct MappingRecordEntry { ++ pub users: [Option<&'static str>; 5], ++ pub phys_pages: PageSliceDescriptor, ++ pub virt_start_addr: Address, ++ pub attribute_fields: AttributeFields, ++} ++ ++struct MappingRecord { ++ inner: [Option; 12], ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Global instances ++//-------------------------------------------------------------------------------------------------- ++ ++static KERNEL_MAPPING_RECORD: InitStateLock = ++ InitStateLock::new(MappingRecord::new()); ++ ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl MappingRecordEntry { ++ pub fn new( ++ name: &'static str, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++ ) -> Self { ++ Self { ++ users: [Some(name), None, None, None, None], ++ phys_pages: *phys_pages, ++ virt_start_addr: virt_pages.start_addr(), ++ attribute_fields: *attr, ++ } ++ } ++ ++ fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> { ++ if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) { ++ return Ok(x); ++ }; ++ ++ Err("Storage for user info exhausted") ++ } ++ ++ pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> { ++ let x = self.find_next_free_user()?; ++ *x = Some(user); ++ Ok(()) ++ } ++} ++ ++impl MappingRecord { ++ pub const fn new() -> Self { ++ Self { inner: [None; 12] } ++ } ++ ++ fn find_next_free(&mut self) -> Result<&mut Option, &'static str> { ++ if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) { ++ return Ok(x); ++ } ++ ++ Err("Storage for mapping info exhausted") ++ } ++ ++ fn find_duplicate( ++ &mut self, ++ phys_pages: &PageSliceDescriptor, ++ ) -> Option<&mut MappingRecordEntry> { ++ self.inner ++ .iter_mut() ++ .filter(|x| x.is_some()) ++ .map(|x| x.as_mut().unwrap()) ++ .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device) ++ .find(|x| x.phys_pages == *phys_pages) ++ } ++ ++ pub fn add( ++ &mut self, ++ name: &'static str, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++ ) -> Result<(), &'static str> { ++ let x = self.find_next_free()?; ++ ++ *x = Some(MappingRecordEntry::new(name, phys_pages, virt_pages, attr)); ++ Ok(()) ++ } ++ ++ pub fn print(&self) { ++ const KIB_RSHIFT: u32 = 10; // log2(1024). ++ const MIB_RSHIFT: u32 = 20; // log2(1024 * 1024). ++ ++ info!(" -----------------------------------------------------------------------------------------------------------------"); ++ info!( ++ " {:^24} {:^24} {:^7} {:^9} {:^35}", ++ "Virtual", "Physical", "Size", "Attr", "Entity" ++ ); ++ info!(" -----------------------------------------------------------------------------------------------------------------"); ++ ++ for i in self ++ .inner ++ .iter() ++ .filter(|x| x.is_some()) ++ .map(|x| x.unwrap()) ++ { ++ let virt_start = i.virt_start_addr.into_usize(); ++ let virt_end_inclusive = virt_start + i.phys_pages.size() - 1; ++ let phys_start = i.phys_pages.start_addr().into_usize(); ++ let phys_end_inclusive = i.phys_pages.end_addr_inclusive().into_usize(); ++ let size = i.phys_pages.size(); ++ ++ let (size, unit) = if (size >> MIB_RSHIFT) > 0 { ++ (size >> MIB_RSHIFT, "MiB") ++ } else if (size >> KIB_RSHIFT) > 0 { ++ (size >> KIB_RSHIFT, "KiB") ++ } else { ++ (size, "Byte") ++ }; ++ ++ let attr = match i.attribute_fields.mem_attributes { ++ MemAttributes::CacheableDRAM => "C", ++ MemAttributes::Device => "Dev", ++ }; ++ ++ let acc_p = match i.attribute_fields.acc_perms { ++ AccessPermissions::ReadOnly => "RO", ++ AccessPermissions::ReadWrite => "RW", ++ }; ++ ++ let xn = if i.attribute_fields.execute_never { ++ "XN" ++ } else { ++ "X" ++ }; ++ ++ info!( ++ " {:#011X}..{:#011X} --> {:#011X}..{:#011X} | \ ++ {: >3} {} | {: <3} {} {: <2} | {}", ++ virt_start, ++ virt_end_inclusive, ++ phys_start, ++ phys_end_inclusive, ++ size, ++ unit, ++ attr, ++ acc_p, ++ xn, ++ i.users[0].unwrap() ++ ); ++ ++ for k in i.users[1..].iter() { ++ if let Some(additional_user) = *k { ++ info!( ++ " | {}", ++ additional_user ++ ); ++ } ++ } ++ } ++ ++ info!(" -----------------------------------------------------------------------------------------------------------------"); ++ } ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++use synchronization::interface::ReadWriteEx; ++ ++/// Add an entry to the mapping info record. ++pub fn kernel_add( ++ name: &'static str, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++) -> Result<(), &'static str> { ++ let mut m = &KERNEL_MAPPING_RECORD; ++ m.write(|mr| mr.add(name, phys_pages, virt_pages, attr)) ++} ++ ++pub fn kernel_find_and_insert_mmio_duplicate( ++ phys_mmio_descriptor: &MMIODescriptor, ++ new_user: &'static str, ++) -> Option> { ++ let phys_pages: PageSliceDescriptor = phys_mmio_descriptor.clone().into(); ++ ++ let mut m = &KERNEL_MAPPING_RECORD; ++ m.write(|mr| { ++ let dup = mr.find_duplicate(&phys_pages)?; ++ ++ if let Err(x) = dup.add_user(new_user) { ++ warn!("{}", x); ++ } ++ ++ Some(dup.virt_start_addr) ++ }) ++} ++ ++/// Human-readable print of all recorded kernel mappings. ++pub fn kernel_print() { ++ let mut m = &KERNEL_MAPPING_RECORD; ++ m.read(|mr| mr.print()); ++} + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs 15_virtual_memory_part2_mmio_remap/src/memory/mmu/types.rs +--- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu/types.rs ++++ 15_virtual_memory_part2_mmio_remap/src/memory/mmu/types.rs +@@ -0,0 +1,283 @@ ++// SPDX-License-Identifier: MIT OR Apache-2.0 ++// ++// Copyright (c) 2020 Andre Richter ++ ++//! Memory Management Unit Types. ++ ++use crate::{bsp, common}; ++use core::{convert::From, marker::PhantomData, ops::RangeInclusive}; ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Definitions ++//-------------------------------------------------------------------------------------------------- ++use super::interface::TranslationGranule; ++ ++/// Metadata trait for marking the type of an address. ++pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} ++ ++/// Zero-sized type to mark a physical address. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub enum Physical {} ++ ++/// Zero-sized type to mark a virtual address. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub enum Virtual {} ++ ++/// Generic address type. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub struct Address { ++ value: usize, ++ _address_type: PhantomData, ++} ++ ++/// Generic page type. ++#[repr(C)] ++pub struct Page { ++ inner: [u8; bsp::memory::mmu::KernelGranule::SIZE], ++ _address_type: PhantomData, ++} ++ ++/// Type describing a slice of pages. ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub struct PageSliceDescriptor { ++ start: Address, ++ num_pages: usize, ++} ++ ++/// Architecture agnostic memory attributes. ++#[allow(missing_docs)] ++#[derive(Copy, Clone, PartialOrd, PartialEq)] ++pub enum MemAttributes { ++ CacheableDRAM, ++ Device, ++} ++ ++/// Architecture agnostic access permissions. ++#[allow(missing_docs)] ++#[derive(Copy, Clone)] ++pub enum AccessPermissions { ++ ReadOnly, ++ ReadWrite, ++} ++ ++/// Collection of memory attributes. ++#[allow(missing_docs)] ++#[derive(Copy, Clone)] ++pub struct AttributeFields { ++ pub mem_attributes: MemAttributes, ++ pub acc_perms: AccessPermissions, ++ pub execute_never: bool, ++} ++ ++/// An MMIO descriptor for use in device drivers. ++#[derive(Copy, Clone)] ++pub struct MMIODescriptor { ++ start_addr: Address, ++ size: usize, ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++ ++impl AddressType for Physical {} ++impl AddressType for Virtual {} ++ ++//------------------------------------------------------------------------------ ++// Address ++//------------------------------------------------------------------------------ ++ ++impl Address { ++ /// Create an instance. ++ pub const fn new(value: usize) -> Self { ++ Self { ++ value, ++ _address_type: PhantomData, ++ } ++ } ++ ++ /// Align down. ++ pub const fn align_down(self, alignment: usize) -> Self { ++ let aligned = common::align_down(self.value, alignment); ++ ++ Self { ++ value: aligned, ++ _address_type: PhantomData, ++ } ++ } ++ ++ /// Converts `Address` into an usize. ++ pub const fn into_usize(self) -> usize { ++ self.value ++ } ++} ++ ++impl core::ops::Add for Address { ++ type Output = Self; ++ ++ fn add(self, other: usize) -> Self { ++ Self { ++ value: self.value + other, ++ _address_type: PhantomData, ++ } ++ } ++} ++ ++impl core::ops::Sub for Address { ++ type Output = Self; ++ ++ fn sub(self, other: usize) -> Self { ++ Self { ++ value: self.value - other, ++ _address_type: PhantomData, ++ } ++ } ++} ++ ++//------------------------------------------------------------------------------ ++// Page ++//------------------------------------------------------------------------------ ++ ++impl Page { ++ /// Get a pointer to the instance. ++ pub const fn as_ptr(&self) -> *const Page { ++ self as *const _ ++ } ++} ++ ++//------------------------------------------------------------------------------ ++// PageSliceDescriptor ++//------------------------------------------------------------------------------ ++ ++impl PageSliceDescriptor { ++ /// Create an instance. ++ pub const fn from_addr(start: Address, num_pages: usize) -> Self { ++ assert!(common::is_aligned( ++ start.into_usize(), ++ bsp::memory::mmu::KernelGranule::SIZE ++ )); ++ assert!(num_pages > 0); ++ ++ Self { start, num_pages } ++ } ++ ++ /// Return a pointer to the first page of the described slice. ++ const fn first_page_ptr(&self) -> *const Page { ++ self.start.into_usize() as *const _ ++ } ++ ++ /// Return the number of Pages the slice describes. ++ pub const fn num_pages(&self) -> usize { ++ self.num_pages ++ } ++ ++ /// Return the memory size this descriptor spans. ++ pub const fn size(&self) -> usize { ++ self.num_pages * bsp::memory::mmu::KernelGranule::SIZE ++ } ++ ++ /// Return the start address. ++ pub const fn start_addr(&self) -> Address { ++ self.start ++ } ++ ++ /// Return the exclusive end address. ++ pub fn end_addr(&self) -> Address { ++ self.start + self.size() ++ } ++ ++ /// Return the inclusive end address. ++ pub fn end_addr_inclusive(&self) -> Address { ++ self.start + (self.size() - 1) ++ } ++ ++ /// Return a non-mutable slice of Pages. ++ /// ++ /// # Safety ++ /// ++ /// - Same as applies for `core::slice::from_raw_parts`. ++ pub unsafe fn as_slice(&self) -> &[Page] { ++ core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) ++ } ++ ++ /// Return the inclusive address range of the slice. ++ pub fn into_usize_range_inclusive(self) -> RangeInclusive { ++ RangeInclusive::new( ++ self.start_addr().into_usize(), ++ self.end_addr_inclusive().into_usize(), ++ ) ++ } ++} ++ ++impl From> for PageSliceDescriptor { ++ fn from(desc: PageSliceDescriptor) -> Self { ++ Self { ++ start: Address::new(desc.start.into_usize()), ++ num_pages: desc.num_pages, ++ } ++ } ++} ++ ++impl From> for PageSliceDescriptor { ++ fn from(desc: MMIODescriptor) -> Self { ++ let start_page_addr = desc ++ .start_addr ++ .align_down(bsp::memory::mmu::KernelGranule::SIZE); ++ ++ let len = ((desc.end_addr_inclusive().into_usize() - start_page_addr.into_usize()) ++ >> bsp::memory::mmu::KernelGranule::SHIFT) ++ + 1; ++ ++ Self { ++ start: start_page_addr, ++ num_pages: len, ++ } ++ } ++} ++ ++//------------------------------------------------------------------------------ ++// MMIODescriptor ++//------------------------------------------------------------------------------ ++ ++impl MMIODescriptor { ++ /// Create an instance. ++ pub const fn new(start_addr: Address, size: usize) -> Self { ++ assert!(size > 0); ++ ++ Self { start_addr, size } ++ } ++ ++ /// Return the start address. ++ pub const fn start_addr(&self) -> Address { ++ self.start_addr ++ } ++ ++ /// Return the inclusive end address. ++ pub fn end_addr_inclusive(&self) -> Address { ++ self.start_addr + (self.size - 1) ++ } ++ ++ /// Return the size. ++ pub const fn size(&self) -> usize { ++ self.size ++ } ++} ++ ++//-------------------------------------------------------------------------------------------------- ++// Testing ++//-------------------------------------------------------------------------------------------------- ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use test_macros::kernel_test; ++ ++ /// Check if the size of `struct Page` is as expected. ++ #[kernel_test] ++ fn size_of_page_equals_granule_size() { ++ assert_eq!( ++ core::mem::size_of::>(), ++ bsp::memory::mmu::KernelGranule::SIZE ++ ); ++ } ++} + +diff -uNr 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs 15_virtual_memory_part2_mmio_remap/src/memory/mmu.rs +--- 14_exceptions_part2_peripheral_IRQs/src/memory/mmu.rs ++++ 15_virtual_memory_part2_mmio_remap/src/memory/mmu.rs +@@ -3,23 +3,18 @@ + // Copyright (c) 2020 Andre Richter + + //! Memory Management Unit. +-//! +-//! In order to decouple `BSP` and `arch` parts of the MMU code (to keep them pluggable), this file +-//! provides types for composing an architecture-agnostic description of the kernel 's virtual +-//! memory layout. +-//! +-//! The `BSP` provides such a description through the `bsp::memory::mmu::virt_mem_layout()` +-//! function. +-//! +-//! The `MMU` driver of the `arch` code uses `bsp::memory::mmu::virt_mem_layout()` to compile and +-//! install respective translation tables. + + #[cfg(target_arch = "aarch64")] + #[path = "../_arch/aarch64/memory/mmu.rs"] + mod arch_mmu; + pub use arch_mmu::*; + +-use core::{fmt, ops::RangeInclusive}; ++mod mapping_record; ++mod types; ++ ++use crate::{bsp, synchronization, warn}; ++ ++pub use types::*; + + //-------------------------------------------------------------------------------------------------- + // Public Definitions +@@ -27,178 +22,229 @@ + + /// Memory Management interfaces. + pub mod interface { ++ use super::*; ++ ++ /// Describes the characteristics of a translation granule. ++ #[allow(missing_docs)] ++ pub trait TranslationGranule { ++ const SIZE: usize; ++ const MASK: usize = Self::SIZE - 1; ++ const SHIFT: usize; ++ } ++ ++ /// Translation table operations. ++ pub trait TranslationTable { ++ /// Anything that needs to run before any of the other provided functions can be used. ++ /// ++ /// # Safety ++ /// ++ /// - Implementor must ensure that this function can run only once or is harmless if invoked ++ /// multiple times. ++ unsafe fn init(&mut self); ++ ++ /// The translation table's base address to be used for programming the MMU. ++ fn phys_base_address(&self) -> Address; ++ ++ /// Map the given physical pages to the given virtual pages. ++ /// ++ /// # Safety ++ /// ++ /// - Using wrong attributes can cause multiple issues of different nature in the system. ++ /// - It is not required that the architectural implementation prevents aliasing. That is, ++ /// mapping to the same physical memory using multiple virtual addresses, which would ++ /// break Rust's ownership assumptions. This should be protected against in this module ++ /// (the kernel's generic MMU code). ++ unsafe fn map_pages_at( ++ &mut self, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++ ) -> Result<(), &'static str>; ++ ++ /// Obtain a free virtual page slice in the MMIO region. ++ /// ++ /// The "MMIO region" is a distinct region of the implementor's choice, which allows ++ /// differentiating MMIO addresses from others. This can speed up debugging efforts. ++ /// Ideally, those MMIO addresses are also standing out visually so that a human eye can ++ /// identify them. For example, by allocating them from near the end of the virtual address ++ /// space. ++ fn next_mmio_virt_page_slice( ++ &mut self, ++ num_pages: usize, ++ ) -> Result, &'static str>; ++ ++ /// Check if a virtual page splice is in the "MMIO region". ++ fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; ++ } + + /// MMU functions. + pub trait MMU { +- /// Called by the kernel during early init. Supposed to take the translation tables from the +- /// `BSP`-supplied `virt_mem_layout()` and install/activate them for the respective MMU. ++ /// Turns on the MMU. + /// + /// # Safety + /// ++ /// - Must only be called after the kernel translation tables have been init()'ed. + /// - Changes the HW's global state. +- unsafe fn init(&self) -> Result<(), &'static str>; ++ unsafe fn enable( ++ &self, ++ phys_kernel_table_base_addr: Address, ++ ) -> Result<(), &'static str>; + } + } + +-/// Architecture agnostic translation types. +-#[allow(missing_docs)] +-#[derive(Copy, Clone)] +-pub enum Translation { +- Identity, +- Offset(usize), +-} +- +-/// Architecture agnostic memory attributes. +-#[allow(missing_docs)] +-#[derive(Copy, Clone)] +-pub enum MemAttributes { +- CacheableDRAM, +- Device, +-} ++//-------------------------------------------------------------------------------------------------- ++// Private Code ++//-------------------------------------------------------------------------------------------------- ++use interface::{TranslationTable, MMU}; ++use synchronization::interface::ReadWriteEx; + +-/// Architecture agnostic access permissions. +-#[allow(missing_docs)] +-#[derive(Copy, Clone)] +-pub enum AccessPermissions { +- ReadOnly, +- ReadWrite, +-} ++/// Map pages in the kernel's translation tables. ++/// ++/// No input checks done, input is passed through to the architectural implementation. ++/// ++/// # Safety ++/// ++/// - See `map_pages_at()`. ++/// - Does not prevent aliasing. ++unsafe fn kernel_map_pages_at_unchecked( ++ name: &'static str, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++) -> Result<(), &'static str> { ++ arch_mmu::kernel_translation_tables() ++ .write(|tables| tables.map_pages_at(phys_pages, virt_pages, attr))?; + +-/// Collection of memory attributes. +-#[allow(missing_docs)] +-#[derive(Copy, Clone)] +-pub struct AttributeFields { +- pub mem_attributes: MemAttributes, +- pub acc_perms: AccessPermissions, +- pub execute_never: bool, +-} ++ if let Err(x) = mapping_record::kernel_add(name, phys_pages, virt_pages, attr) { ++ warn!("{}", x); ++ } + +-/// Architecture agnostic descriptor for a memory range. +-#[allow(missing_docs)] +-pub struct TranslationDescriptor { +- pub name: &'static str, +- pub virtual_range: fn() -> RangeInclusive, +- pub physical_range_translation: Translation, +- pub attribute_fields: AttributeFields, ++ Ok(()) + } + +-/// Type for expressing the kernel's virtual memory layout. +-pub struct KernelVirtualLayout { +- /// The last (inclusive) address of the address space. +- max_virt_addr_inclusive: usize, ++//-------------------------------------------------------------------------------------------------- ++// Public Code ++//-------------------------------------------------------------------------------------------------- ++use interface::TranslationGranule; + +- /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. +- inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], ++/// Raw mapping of virtual to physical pages in the kernel translation tables. ++/// ++/// Prevents mapping into the MMIO range of the tables. ++/// ++/// # Safety ++/// ++/// - See `kernel_map_pages_at_unchecked()`. ++/// - Does not prevent aliasing. Currently, we have to trust the callers. ++pub unsafe fn kernel_map_pages_at( ++ name: &'static str, ++ phys_pages: &PageSliceDescriptor, ++ virt_pages: &PageSliceDescriptor, ++ attr: &AttributeFields, ++) -> Result<(), &'static str> { ++ let is_mmio = arch_mmu::kernel_translation_tables() ++ .read(|tables| tables.is_virt_page_slice_mmio(virt_pages)); ++ if is_mmio { ++ return Err("Attempt to manually map into MMIO region"); ++ } ++ ++ kernel_map_pages_at_unchecked(name, phys_pages, virt_pages, attr)?; ++ ++ Ok(()) ++} ++ ++/// MMIO remapping in the kernel translation tables. ++/// ++/// Typically used by device drivers. ++/// ++/// # Safety ++/// ++/// - Same as `kernel_map_pages_at_unchecked()`, minus the aliasing part. ++pub unsafe fn kernel_map_mmio( ++ name: &'static str, ++ phys_mmio_descriptor: &MMIODescriptor, ++) -> Result, &'static str> { ++ let phys_pages: PageSliceDescriptor = phys_mmio_descriptor.clone().into(); ++ let offset_into_start_page = ++ phys_mmio_descriptor.start_addr().into_usize() & bsp::memory::mmu::KernelGranule::MASK; ++ ++ // Check if an identical page slice has been mapped for another driver. If so, reuse it. ++ let virt_addr = if let Some(addr) = ++ mapping_record::kernel_find_and_insert_mmio_duplicate(phys_mmio_descriptor, name) ++ { ++ addr ++ // Otherwise, allocate a new virtual page slice and map it. ++ } else { ++ let virt_pages: PageSliceDescriptor = arch_mmu::kernel_translation_tables() ++ .write(|tables| tables.next_mmio_virt_page_slice(phys_pages.num_pages()))?; ++ ++ kernel_map_pages_at_unchecked( ++ name, ++ &phys_pages, ++ &virt_pages, ++ &AttributeFields { ++ mem_attributes: MemAttributes::Device, ++ acc_perms: AccessPermissions::ReadWrite, ++ execute_never: true, ++ }, ++ )?; ++ ++ virt_pages.start_addr() ++ }; ++ ++ Ok(virt_addr + offset_into_start_page) ++} ++ ++/// Map the kernel's binary and enable the MMU. ++/// ++/// # Safety ++/// ++/// - Crucial function during kernel init. Changes the the complete memory view of the processor. ++pub unsafe fn kernel_map_binary_and_enable_mmu() -> Result<(), &'static str> { ++ let phys_base_addr = arch_mmu::kernel_translation_tables().write(|tables| { ++ tables.init(); ++ tables.phys_base_address() ++ }); ++ ++ bsp::memory::mmu::kernel_map_binary()?; ++ arch_mmu::mmu().enable(phys_base_addr) ++} ++ ++/// Human-readable print of all recorded kernel mappings. ++pub fn kernel_print_mappings() { ++ mapping_record::kernel_print() + } + + //-------------------------------------------------------------------------------------------------- +-// Public Code ++// Testing + //-------------------------------------------------------------------------------------------------- + +-impl Default for AttributeFields { +- fn default() -> AttributeFields { +- AttributeFields { +- mem_attributes: MemAttributes::CacheableDRAM, +- acc_perms: AccessPermissions::ReadWrite, +- execute_never: true, +- } +- } +-} +- +-/// Human-readable output of a TranslationDescriptor. +-impl fmt::Display for TranslationDescriptor { +- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { +- // Call the function to which self.range points, and dereference the result, which causes +- // Rust to copy the value. +- let start = *(self.virtual_range)().start(); +- let end = *(self.virtual_range)().end(); +- let size = end - start + 1; +- +- // log2(1024). +- const KIB_RSHIFT: u32 = 10; +- +- // log2(1024 * 1024). +- const MIB_RSHIFT: u32 = 20; +- +- let (size, unit) = if (size >> MIB_RSHIFT) > 0 { +- (size >> MIB_RSHIFT, "MiB") +- } else if (size >> KIB_RSHIFT) > 0 { +- (size >> KIB_RSHIFT, "KiB") +- } else { +- (size, "Byte") +- }; +- +- let attr = match self.attribute_fields.mem_attributes { +- MemAttributes::CacheableDRAM => "C", +- MemAttributes::Device => "Dev", +- }; +- +- let acc_p = match self.attribute_fields.acc_perms { +- AccessPermissions::ReadOnly => "RO", +- AccessPermissions::ReadWrite => "RW", +- }; +- +- let xn = if self.attribute_fields.execute_never { +- "PXN" +- } else { +- "PX" +- }; +- +- write!( +- f, +- " {:#010x} - {:#010x} | {: >3} {} | {: <3} {} {: <3} | {}", +- start, end, size, unit, attr, acc_p, xn, self.name +- ) +- } +-} +- +-impl KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { +- /// Create a new instance. +- pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self { +- Self { +- max_virt_addr_inclusive: max, +- inner: layout, +- } +- } +- +- /// For a virtual address, find and return the physical output address and corresponding +- /// attributes. +- /// +- /// If the address is not found in `inner`, return an identity mapped default with normal +- /// cacheable DRAM attributes. +- pub fn virt_addr_properties( +- &self, +- virt_addr: usize, +- ) -> Result<(usize, AttributeFields), &'static str> { +- if virt_addr > self.max_virt_addr_inclusive { +- return Err("Address out of range"); +- } +- +- for i in self.inner.iter() { +- if (i.virtual_range)().contains(&virt_addr) { +- let output_addr = match i.physical_range_translation { +- Translation::Identity => virt_addr, +- Translation::Offset(a) => a + (virt_addr - (i.virtual_range)().start()), +- }; +- +- return Ok((output_addr, i.attribute_fields)); +- } +- } +- +- Ok((virt_addr, AttributeFields::default())) +- } +- +- /// Print the memory layout. +- pub fn print_layout(&self) { +- use crate::info; +- +- for i in self.inner.iter() { +- info!("{}", i); +- } +- } +- +- #[cfg(test)] +- pub fn inner(&self) -> &[TranslationDescriptor; NUM_SPECIAL_RANGES] { +- &self.inner ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use test_macros::kernel_test; ++ ++ /// Sanity checks for the kernel TranslationTable implementation. ++ #[kernel_test] ++ fn translationtable_implementation_sanity() { ++ // Need to take care that `tables` fits into the stack. ++ let mut tables = MinSizeArchTranslationTable::new(); ++ ++ unsafe { tables.init() }; ++ ++ let x = tables.next_mmio_virt_page_slice(0); ++ assert!(x.is_err()); ++ ++ let x = tables.next_mmio_virt_page_slice(1_0000_0000); ++ assert!(x.is_err()); ++ ++ let x = tables.next_mmio_virt_page_slice(2).unwrap(); ++ assert_eq!(x.size(), bsp::memory::mmu::KernelGranule::SIZE * 2); ++ ++ assert_eq!(tables.is_virt_page_slice_mmio(&x), true); ++ ++ assert_eq!( ++ tables.is_virt_page_slice_mmio(&PageSliceDescriptor::from_addr(Address::new(0), 1)), ++ false ++ ); + } + } + +diff -uNr 14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs 15_virtual_memory_part2_mmio_remap/tests/02_exception_sync_page_fault.rs +--- 14_exceptions_part2_peripheral_IRQs/tests/02_exception_sync_page_fault.rs ++++ 15_virtual_memory_part2_mmio_remap/tests/02_exception_sync_page_fault.rs +@@ -21,7 +21,7 @@ + + #[no_mangle] + unsafe fn kernel_init() -> ! { +- use memory::mmu::interface::MMU; ++ use libkernel::driver::interface::DriverManager; + + bsp::console::qemu_bring_up_console(); + +@@ -30,10 +30,22 @@ + + exception::handling_init(); + +- if let Err(string) = memory::mmu::mmu().init() { +- println!("MMU: {}", string); ++ if let Err(string) = memory::mmu::kernel_map_binary_and_enable_mmu() { ++ println!("Enabling MMU failed: {}", string); + cpu::qemu_exit_failure() + } ++ // Printing will silently fail fail from here on, because the driver's MMIO is not remapped yet. ++ ++ // Bring up the drivers needed for printing first. ++ for i in bsp::driver::driver_manager() ++ .early_print_device_drivers() ++ .iter() ++ { ++ // Any encountered errors cannot be printed yet, obviously, so just safely park the CPU. ++ i.init().unwrap_or_else(|_| cpu::qemu_exit_failure()); ++ } ++ bsp::driver::driver_manager().post_early_print_device_driver_init(); ++ // Printing available again from here on. + + println!("Writing beyond mapped area to address 9 GiB..."); + let big_addr: u64 = 9 * 1024 * 1024 * 1024; + +``` diff --git a/15_virtual_memory_part2_mmio_remap/build.rs b/15_virtual_memory_part2_mmio_remap/build.rs new file mode 100644 index 00000000..40ee0284 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/build.rs @@ -0,0 +1,7 @@ +use std::env; + +fn main() { + let linker_file = env::var("LINKER_FILE").unwrap(); + + println!("cargo:rerun-if-changed={}", linker_file); +} diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu.rs b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu.rs new file mode 100644 index 00000000..6d41d19d --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural processor code. + +use crate::{bsp, cpu}; +use cortex_a::{asm, regs::*}; + +//-------------------------------------------------------------------------------------------------- +// Boot Code +//-------------------------------------------------------------------------------------------------- + +/// The entry of the `kernel` binary. +/// +/// The function must be named `_start`, because the linker is looking for this exact name. +/// +/// # Safety +/// +/// - Linker script must ensure to place this function at `0x80_000`. +#[naked] +#[no_mangle] +pub unsafe extern "C" fn _start() -> ! { + // Expect the boot core to start in EL2. + if (bsp::cpu::BOOT_CORE_ID == cpu::smp::core_id()) + && (CurrentEL.get() == CurrentEL::EL::EL2.value) + { + el2_to_el1_transition() + } else { + // If not core0, infinitely wait for events. + wait_forever() + } +} + +/// Transition from EL2 to EL1. +/// +/// # Safety +/// +/// - The HW state of EL1 must be prepared in a sound way. +/// - Exception return from EL2 must must continue execution in EL1 with +/// `runtime_init::runtime_init()`. +#[inline(always)] +unsafe fn el2_to_el1_transition() -> ! { + use crate::runtime_init; + + // Enable timer counter registers for EL1. + CNTHCTL_EL2.write(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + + // No offset for reading the counters. + CNTVOFF_EL2.set(0); + + // Set EL1 execution state to AArch64. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Set up a simulated exception return. + // + // First, fake a saved program status where all interrupts were masked and SP_EL1 was used as a + // stack pointer. + SPSR_EL2.write( + SPSR_EL2::D::Masked + + SPSR_EL2::A::Masked + + SPSR_EL2::I::Masked + + SPSR_EL2::F::Masked + + SPSR_EL2::M::EL1h, + ); + + // Second, let the link register point to runtime_init(). + ELR_EL2.set(runtime_init::runtime_init as *const () as u64); + + // Set up SP_EL1 (stack pointer), which will be used by EL1 once we "return" to it. + SP_EL1.set(bsp::memory::phys_boot_core_stack_end().into_usize() as u64); + + // Use `eret` to "return" to EL1. This results in execution of runtime_init() in EL1. + asm::eret() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +pub use asm::nop; + +/// Spin for `n` cycles. +#[inline(always)] +pub fn spin_for_cycles(n: usize) { + for _ in 0..n { + asm::nop(); + } +} + +/// Pause execution on the core. +#[inline(always)] +pub fn wait_forever() -> ! { + loop { + asm::wfe() + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- +use qemu_exit::QEMUExit; + +const QEMU_EXIT_HANDLE: qemu_exit::AArch64 = qemu_exit::AArch64::new(); + +/// Make the host QEMU binary execute `exit(1)`. +pub fn qemu_exit_failure() -> ! { + QEMU_EXIT_HANDLE.exit_failure() +} + +/// Make the host QEMU binary execute `exit(0)`. +pub fn qemu_exit_success() -> ! { + QEMU_EXIT_HANDLE.exit_success() +} diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs new file mode 100644 index 00000000..8429e1d2 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/cpu/smp.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural symmetric multiprocessing. + +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return the executing core's id. +#[inline(always)] +pub fn core_id() -> T +where + T: From, +{ + const CORE_MASK: u64 = 0b11; + + T::from((MPIDR_EL1.get() & CORE_MASK) as u8) +} diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.S b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.S new file mode 100644 index 00000000..70817be4 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.S @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +/// Call the function provided by parameter `\handler` after saving the exception context. Provide +/// the context as the first parameter to '\handler'. +.macro CALL_WITH_CONTEXT handler + // Make room on the stack for the exception context. + sub sp, sp, #16 * 17 + + // Store all general purpose registers on the stack. + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + // Add the exception link register (ELR_EL1) and the saved program status (SPSR_EL1). + mrs x1, ELR_EL1 + mrs x2, SPSR_EL1 + + stp lr, x1, [sp, #16 * 15] + str w2, [sp, #16 * 16] + + // x0 is the first argument for the function called through `\handler`. + mov x0, sp + + // Call `\handler`. + bl \handler + + // After returning from exception handling code, replay the saved context and return via `eret`. + b __exception_restore_context +.endm + +.macro FIQ_SUSPEND +1: wfe + b 1b +.endm + +//-------------------------------------------------------------------------------------------------- +// The exception vector table. +//-------------------------------------------------------------------------------------------------- +.section .exception_vectors, "ax", @progbits + +// Align by 2^11 bytes, as demanded by ARMv8-A. Same as ALIGN(2048) in an ld script. +.align 11 + +// Export a symbol for the Rust code to use. +__exception_vector_start: + +// Current exception level with SP_EL0. +// +// .org sets the offset relative to section start. +// +// # Safety +// +// - It must be ensured that `CALL_WITH_CONTEXT` <= 0x80 bytes. +.org 0x000 + CALL_WITH_CONTEXT current_el0_synchronous +.org 0x080 + CALL_WITH_CONTEXT current_el0_irq +.org 0x100 + FIQ_SUSPEND +.org 0x180 + CALL_WITH_CONTEXT current_el0_serror + +// Current exception level with SP_ELx, x > 0. +.org 0x200 + CALL_WITH_CONTEXT current_elx_synchronous +.org 0x280 + CALL_WITH_CONTEXT current_elx_irq +.org 0x300 + FIQ_SUSPEND +.org 0x380 + CALL_WITH_CONTEXT current_elx_serror + +// Lower exception level, AArch64 +.org 0x400 + CALL_WITH_CONTEXT lower_aarch64_synchronous +.org 0x480 + CALL_WITH_CONTEXT lower_aarch64_irq +.org 0x500 + FIQ_SUSPEND +.org 0x580 + CALL_WITH_CONTEXT lower_aarch64_serror + +// Lower exception level, AArch32 +.org 0x600 + CALL_WITH_CONTEXT lower_aarch32_synchronous +.org 0x680 + CALL_WITH_CONTEXT lower_aarch32_irq +.org 0x700 + FIQ_SUSPEND +.org 0x780 + CALL_WITH_CONTEXT lower_aarch32_serror +.org 0x800 + +//-------------------------------------------------------------------------------------------------- +// Helper functions +//-------------------------------------------------------------------------------------------------- +.section .text + +__exception_restore_context: + ldr w19, [sp, #16 * 16] + ldp lr, x20, [sp, #16 * 15] + + msr SPSR_EL1, x19 + msr ELR_EL1, x20 + + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + + add sp, sp, #16 * 17 + + eret diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.rs b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.rs new file mode 100644 index 00000000..c57784a4 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural synchronous and asynchronous exception handling. + +use crate::{bsp, exception}; +use core::fmt; +use cortex_a::{barrier, regs::*}; +use register::InMemoryRegister; + +// Assembly counterpart to this file. +global_asm!(include_str!("exception.S")); + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Wrapper struct for memory copy of SPSR_EL1. +#[repr(transparent)] +struct SpsrEL1(InMemoryRegister); + +/// The exception context as it is stored on the stack on exception entry. +#[repr(C)] +struct ExceptionContext { + /// General Purpose Registers. + gpr: [u64; 30], + + /// The link register, aka x30. + lr: u64, + + /// Exception link register. The program counter at the time the exception happened. + elr_el1: u64, + + /// Saved program status. + spsr_el1: SpsrEL1, +} + +/// Wrapper struct for pretty printing ESR_EL1. +struct EsrEL1; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Prints verbose information about the exception and then panics. +fn default_exception_handler(e: &ExceptionContext) { + panic!( + "\n\nCPU Exception!\n\ + FAR_EL1: {:#018x}\n\ + {}\n\ + {}", + FAR_EL1.get(), + EsrEL1 {}, + e + ); +} + +//------------------------------------------------------------------------------ +// Current, EL0 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn current_el0_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_el0_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_el0_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Current, ELx +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn current_elx_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn current_elx_irq(_e: &mut ExceptionContext) { + use exception::asynchronous::interface::IRQManager; + + let token = &exception::asynchronous::IRQContext::new(); + bsp::exception::asynchronous::irq_manager().handle_pending_irqs(token); +} + +#[no_mangle] +unsafe extern "C" fn current_elx_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch64 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch64_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Lower, AArch32 +//------------------------------------------------------------------------------ + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_synchronous(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_irq(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +#[no_mangle] +unsafe extern "C" fn lower_aarch32_serror(e: &mut ExceptionContext) { + default_exception_handler(e); +} + +//------------------------------------------------------------------------------ +// Pretty printing +//------------------------------------------------------------------------------ + +/// Human readable ESR_EL1. +#[rustfmt::skip] +impl fmt::Display for EsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let esr_el1 = ESR_EL1.extract(); + + // Raw print of whole register. + writeln!(f, "ESR_EL1: {:#010x}", esr_el1.get())?; + + // Raw print of exception class. + write!(f, " Exception Class (EC) : {:#x}", esr_el1.read(ESR_EL1::EC))?; + + // Exception class, translation. + let ec_translation = match esr_el1.read_as_enum(ESR_EL1::EC) { + Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => "Data Abort, current EL", + _ => "N/A", + }; + writeln!(f, " - {}", ec_translation)?; + + // Raw print of instruction specific syndrome. + write!(f, " Instr Specific Syndrome (ISS): {:#x}", esr_el1.read(ESR_EL1::ISS))?; + + Ok(()) + } +} + +/// Human readable SPSR_EL1. +#[rustfmt::skip] +impl fmt::Display for SpsrEL1 { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Raw value. + writeln!(f, "SPSR_EL1: {:#010x}", self.0.get())?; + + let to_flag_str = |x| -> _ { + if x { "Set" } else { "Not set" } + }; + + writeln!(f, " Flags:")?; + writeln!(f, " Negative (N): {}", to_flag_str(self.0.is_set(SPSR_EL1::N)))?; + writeln!(f, " Zero (Z): {}", to_flag_str(self.0.is_set(SPSR_EL1::Z)))?; + writeln!(f, " Carry (C): {}", to_flag_str(self.0.is_set(SPSR_EL1::C)))?; + writeln!(f, " Overflow (V): {}", to_flag_str(self.0.is_set(SPSR_EL1::V)))?; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + writeln!(f, " Exception handling state:")?; + writeln!(f, " Debug (D): {}", to_mask_str(self.0.is_set(SPSR_EL1::D)))?; + writeln!(f, " SError (A): {}", to_mask_str(self.0.is_set(SPSR_EL1::A)))?; + writeln!(f, " IRQ (I): {}", to_mask_str(self.0.is_set(SPSR_EL1::I)))?; + writeln!(f, " FIQ (F): {}", to_mask_str(self.0.is_set(SPSR_EL1::F)))?; + + write!(f, " Illegal Execution State (IL): {}", + to_flag_str(self.0.is_set(SPSR_EL1::IL)) + )?; + + Ok(()) + } +} + +/// Human readable print of the exception context. +impl fmt::Display for ExceptionContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "ELR_EL1: {:#018x}", self.elr_el1)?; + writeln!(f, "{}", self.spsr_el1)?; + writeln!(f)?; + writeln!(f, "General purpose register:")?; + + #[rustfmt::skip] + let alternating = |x| -> _ { + if x % 2 == 0 { " " } else { "\n" } + }; + + // Print two registers per line. + for (i, reg) in self.gpr.iter().enumerate() { + write!(f, " x{: <2}: {: >#018x}{}", i, reg, alternating(i))?; + } + write!(f, " lr : {:#018x}", self.lr)?; + + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::exception::PrivilegeLevel; + +/// The processing element's current privilege level. +pub fn current_privilege_level() -> (PrivilegeLevel, &'static str) { + let el = CurrentEL.read_as_enum(CurrentEL::EL); + match el { + Some(CurrentEL::EL::Value::EL2) => (PrivilegeLevel::Hypervisor, "EL2"), + Some(CurrentEL::EL::Value::EL1) => (PrivilegeLevel::Kernel, "EL1"), + Some(CurrentEL::EL::Value::EL0) => (PrivilegeLevel::User, "EL0"), + _ => (PrivilegeLevel::Unknown, "Unknown"), + } +} + +/// Init exception handling by setting the exception vector base address register. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +/// - The vector table and the symbol `__exception_vector_table_start` from the linker script must +/// adhere to the alignment and size constraints demanded by the ARMv8-A Architecture Reference +/// Manual. +pub unsafe fn handling_init() { + // Provided by exception.S. + extern "C" { + static mut __exception_vector_start: u64; + } + let addr: u64 = &__exception_vector_start as *const _ as u64; + + VBAR_EL1.set(addr); + + // Force VBAR update to complete before next instruction. + barrier::isb(barrier::SY); +} diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs new file mode 100644 index 00000000..f09a92e6 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/exception/asynchronous.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural asynchronous exception handling. + +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +mod daif_bits { + pub const IRQ: u8 = 0b0010; +} + +trait DaifField { + fn daif_field() -> register::Field; +} + +struct Debug; +struct SError; +struct IRQ; +struct FIQ; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl DaifField for Debug { + fn daif_field() -> register::Field { + DAIF::D + } +} + +impl DaifField for SError { + fn daif_field() -> register::Field { + DAIF::A + } +} + +impl DaifField for IRQ { + fn daif_field() -> register::Field { + DAIF::I + } +} + +impl DaifField for FIQ { + fn daif_field() -> register::Field { + DAIF::F + } +} + +fn is_masked() -> bool { + DAIF.is_set(T::daif_field()) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Returns whether IRQs are masked on the executing core. +pub fn is_local_irq_masked() -> bool { + !is_masked::() +} + +/// Unmask IRQs on the executing core. +/// +/// It is not needed to place an explicit instruction synchronization barrier after the `msr`. +/// Quoting the Architecture Reference Manual for ARMv8-A, section C5.1.3: +/// +/// "Writes to PSTATE.{PAN, D, A, I, F} occur in program order without the need for additional +/// synchronization." +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_unmask() { + #[rustfmt::skip] + asm!( + "msr DAIFClr, {arg}", + arg = const daif_bits::IRQ, + options(nomem, nostack, preserves_flags) + ); +} + +/// Mask IRQs on the executing core. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_mask() { + #[rustfmt::skip] + asm!( + "msr DAIFSet, {arg}", + arg = const daif_bits::IRQ, + options(nomem, nostack, preserves_flags) + ); +} + +/// Mask IRQs on the executing core and return the previously saved interrupt mask bits (DAIF). +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +#[inline(always)] +pub unsafe fn local_irq_mask_save() -> u32 { + let saved = DAIF.get(); + local_irq_mask(); + + saved +} + +/// Restore the interrupt mask bits (DAIF) using the callee's argument. +/// +/// # Safety +/// +/// - Changes the HW state of the executing core. +/// - No sanity checks on the input. +#[inline(always)] +pub unsafe fn local_irq_restore(saved: u32) { + DAIF.set(saved); +} + +/// Print the AArch64 exceptions status. +#[rustfmt::skip] +pub fn print_state() { + use crate::info; + + let to_mask_str = |x| -> _ { + if x { "Masked" } else { "Unmasked" } + }; + + info!(" Debug: {}", to_mask_str(is_masked::())); + info!(" SError: {}", to_mask_str(is_masked::())); + info!(" IRQ: {}", to_mask_str(is_masked::())); + info!(" FIQ: {}", to_mask_str(is_masked::())); +} diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs new file mode 100644 index 00000000..0d443e60 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/memory/mmu.rs @@ -0,0 +1,553 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Memory Management Unit Driver. +//! +//! Only 64 KiB granule is supported. + +use crate::{ + bsp, + memory::{ + mmu, + mmu::{ + AccessPermissions, Address, AddressType, AttributeFields, MemAttributes, Page, + PageSliceDescriptor, Physical, Virtual, + }, + }, + synchronization::InitStateLock, +}; +use core::convert; +use cortex_a::{barrier, regs::*}; +use register::{register_bitfields, InMemoryRegister}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- +use mmu::interface::TranslationGranule; + +// A table descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-15. +register_bitfields! {u64, + STAGE1_TABLE_DESCRIPTOR [ + /// Physical address of the next descriptor. + NEXT_LEVEL_TABLE_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +// A level 3 page descriptor, as per ARMv8-A Architecture Reference Manual Figure D5-17. +register_bitfields! {u64, + STAGE1_PAGE_DESCRIPTOR [ + /// Privileged execute-never. + PXN OFFSET(53) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Physical address of the next table descriptor (lvl2) or the page descriptor (lvl3). + OUTPUT_ADDR_64KiB OFFSET(16) NUMBITS(32) [], // [47:16] + + /// Access flag. + AF OFFSET(10) NUMBITS(1) [ + False = 0, + True = 1 + ], + + /// Shareability field. + SH OFFSET(8) NUMBITS(2) [ + OuterShareable = 0b10, + InnerShareable = 0b11 + ], + + /// Access Permissions. + AP OFFSET(6) NUMBITS(2) [ + RW_EL1 = 0b00, + RW_EL1_EL0 = 0b01, + RO_EL1 = 0b10, + RO_EL1_EL0 = 0b11 + ], + + /// Memory attributes index into the MAIR_EL1 register. + AttrIndx OFFSET(2) NUMBITS(3) [], + + TYPE OFFSET(1) NUMBITS(1) [ + Block = 0, + Table = 1 + ], + + VALID OFFSET(0) NUMBITS(1) [ + False = 0, + True = 1 + ] + ] +} + +/// A table descriptor for 64 KiB aperture. +/// +/// The output points to the next table. +#[derive(Copy, Clone)] +#[repr(transparent)] +struct TableDescriptor(InMemoryRegister); + +/// A page descriptor with 64 KiB aperture. +/// +/// The output points to physical memory. +#[derive(Copy, Clone)] +#[repr(transparent)] +struct PageDescriptor(InMemoryRegister); + +#[derive(Copy, Clone)] +enum Granule512MiB {} + +trait BaseAddr { + fn phys_base_addr(&self) -> Address; +} + +/// Constants for indexing the MAIR_EL1. +#[allow(dead_code)] +mod mair { + pub const DEVICE: u64 = 0; + pub const NORMAL: u64 = 1; +} + +/// Memory Management Unit type. +struct MemoryManagementUnit; + +/// This constant is the power-of-two exponent that defines the virtual address space size. +/// +/// Values tested and known to be working: +/// - 30 (1 GiB) +/// - 31 (2 GiB) +/// - 32 (4 GiB) +/// - 33 (8 GiB) +const ADDR_SPACE_SIZE_EXPONENT: usize = 33; + +const NUM_LVL2_TABLES: usize = (1 << ADDR_SPACE_SIZE_EXPONENT) >> Granule512MiB::SHIFT; +const T0SZ: u64 = (64 - ADDR_SPACE_SIZE_EXPONENT) as u64; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Big monolithic struct for storing the translation tables. Individual levels must be 64 KiB +/// aligned, hence the "reverse" order of appearance. +#[repr(C)] +#[repr(align(65536))] +pub(in crate::memory::mmu) struct FixedSizeTranslationTable { + /// Page descriptors, covering 64 KiB windows per entry. + lvl3: [[PageDescriptor; 8192]; NUM_TABLES], + + /// Table descriptors, covering 512 MiB windows. + lvl2: [TableDescriptor; NUM_TABLES], + + /// Index of the next free MMIO page. + cur_l3_mmio_index: usize, + + /// Have the tables been initialized? + initialized: bool, +} + +pub(in crate::memory::mmu) type ArchTranslationTable = FixedSizeTranslationTable; + +// Supported translation granules are exported below, so that BSP code can pick between the options. +// This driver only supports 64 KiB at the moment. + +#[derive(Copy, Clone)] +/// 64 KiB translation granule. +pub enum Granule64KiB {} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +/// The translation tables. +/// +/// # Safety +/// +/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". +static KERNEL_TABLES: InitStateLock = + InitStateLock::new(ArchTranslationTable::new()); + +static MMU: MemoryManagementUnit = MemoryManagementUnit; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl mmu::interface::TranslationGranule for Granule512MiB { + const SIZE: usize = 512 * 1024 * 1024; + const SHIFT: usize = 29; // log2(SIZE) +} + +impl BaseAddr for [T; N] { + fn phys_base_addr(&self) -> Address { + // The binary is still identity mapped, so we don't need to convert here. + Address::new(self as *const _ as usize) + } +} + +impl convert::From for TableDescriptor { + fn from(next_lvl_table_addr: usize) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = next_lvl_table_addr >> Granule64KiB::SHIFT; + val.write( + STAGE1_TABLE_DESCRIPTOR::VALID::True + + STAGE1_TABLE_DESCRIPTOR::TYPE::Table + + STAGE1_TABLE_DESCRIPTOR::NEXT_LEVEL_TABLE_ADDR_64KiB.val(shifted as u64), + ); + + TableDescriptor(val) + } +} + +/// Convert the kernel's generic memory attributes to HW-specific attributes of the MMU. +impl convert::From + for register::FieldValue +{ + fn from(attribute_fields: AttributeFields) -> Self { + // Memory attributes. + let mut desc = match attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => { + STAGE1_PAGE_DESCRIPTOR::SH::InnerShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::NORMAL) + } + MemAttributes::Device => { + STAGE1_PAGE_DESCRIPTOR::SH::OuterShareable + + STAGE1_PAGE_DESCRIPTOR::AttrIndx.val(mair::DEVICE) + } + }; + + // Access Permissions. + desc += match attribute_fields.acc_perms { + AccessPermissions::ReadOnly => STAGE1_PAGE_DESCRIPTOR::AP::RO_EL1, + AccessPermissions::ReadWrite => STAGE1_PAGE_DESCRIPTOR::AP::RW_EL1, + }; + + // Execute Never. + desc += if attribute_fields.execute_never { + STAGE1_PAGE_DESCRIPTOR::PXN::True + } else { + STAGE1_PAGE_DESCRIPTOR::PXN::False + }; + + desc + } +} + +impl PageDescriptor { + /// Create an instance. + fn new(output_addr: *const Page, attribute_fields: &AttributeFields) -> Self { + let val = InMemoryRegister::::new(0); + + let shifted = output_addr as u64 >> Granule64KiB::SHIFT; + val.write( + STAGE1_PAGE_DESCRIPTOR::VALID::True + + STAGE1_PAGE_DESCRIPTOR::AF::True + + attribute_fields.clone().into() + + STAGE1_PAGE_DESCRIPTOR::TYPE::Table + + STAGE1_PAGE_DESCRIPTOR::OUTPUT_ADDR_64KiB.val(shifted), + ); + + Self(val) + } + + /// Returns the valid bit. + fn is_valid(&self) -> bool { + self.0.is_set(STAGE1_PAGE_DESCRIPTOR::VALID) + } +} + +impl FixedSizeTranslationTable<{ NUM_TABLES }> { + // Reserve the last 256 MiB of the address space for MMIO mappings. + const L2_MMIO_START_INDEX: usize = NUM_TABLES - 1; + const L3_MMIO_START_INDEX: usize = 8192 / 2; + + /// Create an instance. + pub const fn new() -> Self { + assert!(NUM_TABLES > 0); + + Self { + lvl3: [[PageDescriptor(InMemoryRegister::new(0)); 8192]; NUM_TABLES], + lvl2: [TableDescriptor(InMemoryRegister::new(0)); NUM_TABLES], + cur_l3_mmio_index: 0, + initialized: false, + } + } + + /// The start address of the table's MMIO range. + #[inline(always)] + fn mmio_start_addr(&self) -> Address { + Address::new( + (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (Self::L3_MMIO_START_INDEX << Granule64KiB::SHIFT), + ) + } + + /// The inclusive end address of the table's MMIO range. + #[inline(always)] + fn mmio_end_addr_inclusive(&self) -> Address { + Address::new( + (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (8191 << Granule64KiB::SHIFT) + | (Granule64KiB::SIZE - 1), + ) + } + + /// Helper to calculate the lvl2 and lvl3 indices from an address. + #[inline(always)] + fn lvl2_lvl3_index_from( + &self, + addr: *const Page, + ) -> Result<(usize, usize), &'static str> { + let lvl2_index = addr as usize >> Granule512MiB::SHIFT; + let lvl3_index = (addr as usize & Granule512MiB::MASK) >> Granule64KiB::SHIFT; + + if lvl2_index > (NUM_TABLES - 1) { + return Err("Virtual page is out of bounds of translation table"); + } + + Ok((lvl2_index, lvl3_index)) + } + + /// Returns the PageDescriptor corresponding to the supplied Page. + #[inline(always)] + fn page_descriptor_from( + &mut self, + addr: *const Page, + ) -> Result<&mut PageDescriptor, &'static str> { + let (lvl2_index, lvl3_index) = self.lvl2_lvl3_index_from(addr)?; + + Ok(&mut self.lvl3[lvl2_index][lvl3_index]) + } +} + +/// Setup function for the MAIR_EL1 register. +fn set_up_mair() { + // Define the memory types being mapped. + MAIR_EL1.write( + // Attribute 1 - Cacheable normal DRAM. + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + + // Attribute 0 - Device. + MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, + ); +} + +/// Configure various settings of stage 1 of the EL1 translation regime. +fn configure_translation_control() { + let ips = ID_AA64MMFR0_EL1.read(ID_AA64MMFR0_EL1::PARange); + + TCR_EL1.write( + TCR_EL1::TBI0::Ignored + + TCR_EL1::IPS.val(ips) + + TCR_EL1::TG0::KiB_64 + + TCR_EL1::SH0::Inner + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL1::EPD0::EnableTTBR0Walks + + TCR_EL1::T0SZ.val(T0SZ), + ); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a guarded reference to the kernel's translation tables. +pub(in crate::memory::mmu) fn kernel_translation_tables( +) -> &'static InitStateLock { + &KERNEL_TABLES +} + +/// Return a reference to the MMU instance. +pub(in crate::memory::mmu) fn mmu() -> &'static impl mmu::interface::MMU { + &MMU +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +impl mmu::interface::TranslationGranule for Granule64KiB { + const SIZE: usize = 64 * 1024; + const SHIFT: usize = 16; // log2(SIZE) +} + +impl mmu::interface::TranslationTable + for FixedSizeTranslationTable<{ NUM_TABLES }> +{ + unsafe fn init(&mut self) { + if self.initialized { + return; + } + + // Populate the l2 entries. + for (lvl2_nr, lvl2_entry) in self.lvl2.iter_mut().enumerate() { + *lvl2_entry = self.lvl3[lvl2_nr].phys_base_addr().into_usize().into(); + } + + self.cur_l3_mmio_index = Self::L3_MMIO_START_INDEX; + self.initialized = true; + } + + fn phys_base_address(&self) -> Address { + self.lvl2.phys_base_addr() + } + + unsafe fn map_pages_at( + &mut self, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + assert_eq!(self.initialized, true, "Translation tables not initialized"); + + let p = phys_pages.as_slice(); + let v = virt_pages.as_slice(); + + if p.len() != v.len() { + return Err("Tried to map page slices with unequal sizes"); + } + + // No work to do for empty slices. + if p.is_empty() { + return Ok(()); + } + + if p.last().unwrap().as_ptr() >= bsp::memory::mmu::phys_addr_space_end_page() { + return Err("Tried to map outside of physical address space"); + } + + let iter = p.iter().zip(v.iter()); + for (phys_page, virt_page) in iter { + let page_descriptor = self.page_descriptor_from(virt_page.as_ptr())?; + if page_descriptor.is_valid() { + return Err("Virtual page is already mapped"); + } + + *page_descriptor = PageDescriptor::new(phys_page.as_ptr(), &attr); + } + + Ok(()) + } + + fn next_mmio_virt_page_slice( + &mut self, + num_pages: usize, + ) -> Result, &'static str> { + assert_eq!(self.initialized, true, "Translation tables not initialized"); + + if num_pages == 0 { + return Err("num_pages == 0"); + } + + if (self.cur_l3_mmio_index + num_pages) > 8191 { + return Err("Not enough MMIO space left"); + } + + let addr = (Self::L2_MMIO_START_INDEX << Granule512MiB::SHIFT) + | (self.cur_l3_mmio_index << Granule64KiB::SHIFT); + self.cur_l3_mmio_index += num_pages; + + Ok(PageSliceDescriptor::from_addr( + Address::new(addr), + num_pages, + )) + } + + fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool { + let start_addr = virt_pages.start_addr(); + let end_addr_inclusive = virt_pages.end_addr_inclusive(); + + for i in [start_addr, end_addr_inclusive].iter() { + if (*i >= self.mmio_start_addr()) && (*i <= self.mmio_end_addr_inclusive()) { + return true; + } + } + + false + } +} + +impl mmu::interface::MMU for MemoryManagementUnit { + unsafe fn enable( + &self, + phys_kernel_table_base_addr: Address, + ) -> Result<(), &'static str> { + // Fail early if translation granule is not supported. Both RPis support it, though. + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { + return Err("Translation granule not supported in HW"); + } + + // Prepare the memory attribute indirection register. + set_up_mair(); + + // Set the "Translation Table Base Register". + TTBR0_EL1.set_baddr(phys_kernel_table_base_addr.into_usize() as u64); + + configure_translation_control(); + + // Switch the MMU on. + // + // First, force all previous changes to be seen before the MMU is enabled. + barrier::isb(barrier::SY); + + // Enable the MMU and turn on data and instruction caching. + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); + + // Force MMU init to complete before next instruction. + barrier::isb(barrier::SY); + + Ok(()) + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +pub(in crate::memory::mmu) type MinSizeArchTranslationTable = FixedSizeTranslationTable<1>; + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check if the size of `struct TableDescriptor` is as expected. + #[kernel_test] + fn size_of_tabledescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if the size of `struct PageDescriptor` is as expected. + #[kernel_test] + fn size_of_pagedescriptor_equals_64_bit() { + assert_eq!( + core::mem::size_of::(), + core::mem::size_of::() + ); + } + + /// Check if KERNEL_TABLES is in .bss. + #[kernel_test] + fn kernel_tables_in_bss() { + let bss_range = bsp::memory::bss_range(); + let kernel_tables_addr = &KERNEL_TABLES as *const _ as usize as *mut u64; + + assert!(bss_range.contains(&kernel_tables_addr)); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/time.rs b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/time.rs new file mode 100644 index 00000000..d23864f3 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/_arch/aarch64/time.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Architectural timer primitives. + +use crate::{time, warn}; +use core::time::Duration; +use cortex_a::regs::*; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +const NS_PER_S: u64 = 1_000_000_000; + +/// ARMv8 Generic Timer. +struct GenericTimer; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static TIME_MANAGER: GenericTimer = GenericTimer; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the time manager. +pub fn time_manager() -> &'static impl time::interface::TimeManager { + &TIME_MANAGER +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl time::interface::TimeManager for GenericTimer { + fn resolution(&self) -> Duration { + Duration::from_nanos(NS_PER_S / (CNTFRQ_EL0.get() as u64)) + } + + fn uptime(&self) -> Duration { + let frq: u64 = CNTFRQ_EL0.get() as u64; + let current_count: u64 = CNTPCT_EL0.get() * NS_PER_S; + + Duration::from_nanos(current_count / frq) + } + + fn spin_for(&self, duration: Duration) { + // Instantly return on zero. + if duration.as_nanos() == 0 { + return; + } + + // Calculate the register compare value. + let frq = CNTFRQ_EL0.get() as u64; + let x = match frq.checked_mul(duration.as_nanos() as u64) { + None => { + warn!("Spin duration too long, skipping"); + return; + } + Some(val) => val, + }; + let tval = x / NS_PER_S; + + // Check if it is within supported bounds. + let warn: Option<&str> = if tval == 0 { + Some("smaller") + } else if tval > u32::max_value().into() { + Some("bigger") + } else { + None + }; + + if let Some(w) = warn { + warn!( + "Spin duration {} than architecturally supported, skipping", + w + ); + return; + } + + // Set the compare value register. + CNTP_TVAL_EL0.set(tval as u32); + + // Kick off the counting. // Disable timer interrupt. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::SET); + + // ISTATUS will be '1' when cval ticks have passed. Busy-check it. + while !CNTP_CTL_EL0.matches_all(CNTP_CTL_EL0::ISTATUS::SET) {} + + // Disable counting again. + CNTP_CTL_EL0.modify(CNTP_CTL_EL0::ENABLE::CLEAR); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp.rs b/15_virtual_memory_part2_mmio_remap/src/bsp.rs new file mode 100644 index 00000000..3a5657ad --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Conditional re-exporting of Board Support Packages. + +mod device_driver; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod raspberrypi; + +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use raspberrypi::*; diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver.rs new file mode 100644 index 00000000..3fe1fa55 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Device driver. + +#[cfg(feature = "bsp_rpi4")] +mod arm; +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +mod bcm; +mod common; + +#[cfg(feature = "bsp_rpi4")] +pub use arm::*; +#[cfg(any(feature = "bsp_rpi3", feature = "bsp_rpi4"))] +pub use bcm::*; diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm.rs new file mode 100644 index 00000000..6d9d3bdf --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! ARM driver top level. + +pub mod gicv2; + +pub use gicv2::*; diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs new file mode 100644 index 00000000..6b725bec --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! GICv2 Driver - ARM Generic Interrupt Controller v2. +//! +//! The following is a collection of excerpts with useful information from +//! - `Programmer's Guide for ARMv8-A` +//! - `ARM Generic Interrupt Controller Architecture Specification` +//! +//! # Programmer's Guide - 10.6.1 Configuration +//! +//! The GIC is accessed as a memory-mapped peripheral. +//! +//! All cores can access the common Distributor, but the CPU interface is banked, that is, each core +//! uses the same address to access its own private CPU interface. +//! +//! It is not possible for a core to access the CPU interface of another core. +//! +//! # Architecture Specification - 10.6.2 Initialization +//! +//! Both the Distributor and the CPU interfaces are disabled at reset. The GIC must be initialized +//! after reset before it can deliver interrupts to the core. +//! +//! In the Distributor, software must configure the priority, target, security and enable individual +//! interrupts. The Distributor must subsequently be enabled through its control register +//! (GICD_CTLR). For each CPU interface, software must program the priority mask and preemption +//! settings. +//! +//! Each CPU interface block itself must be enabled through its control register (GICD_CTLR). This +//! prepares the GIC to deliver interrupts to the core. +//! +//! Before interrupts are expected in the core, software prepares the core to take interrupts by +//! setting a valid interrupt vector in the vector table, and clearing interrupt mask bits in +//! PSTATE, and setting the routing controls. +//! +//! The entire interrupt mechanism in the system can be disabled by disabling the Distributor. +//! Interrupt delivery to an individual core can be disabled by disabling its CPU interface. +//! Individual interrupts can also be disabled (or enabled) in the distributor. +//! +//! For an interrupt to reach the core, the individual interrupt, Distributor and CPU interface must +//! all be enabled. The interrupt also needs to be of sufficient priority, that is, higher than the +//! core's priority mask. +//! +//! # Architecture Specification - 1.4.2 Interrupt types +//! +//! - Peripheral interrupt +//! - Private Peripheral Interrupt (PPI) +//! - This is a peripheral interrupt that is specific to a single processor. +//! - Shared Peripheral Interrupt (SPI) +//! - This is a peripheral interrupt that the Distributor can route to any of a specified +//! combination of processors. +//! +//! - Software-generated interrupt (SGI) +//! - This is an interrupt generated by software writing to a GICD_SGIR register in the GIC. The +//! system uses SGIs for interprocessor communication. +//! - An SGI has edge-triggered properties. The software triggering of the interrupt is +//! equivalent to the edge transition of the interrupt request signal. +//! - When an SGI occurs in a multiprocessor implementation, the CPUID field in the Interrupt +//! Acknowledge Register, GICC_IAR, or the Aliased Interrupt Acknowledge Register, GICC_AIAR, +//! identifies the processor that requested the interrupt. +//! +//! # Architecture Specification - 2.2.1 Interrupt IDs +//! +//! Interrupts from sources are identified using ID numbers. Each CPU interface can see up to 1020 +//! interrupts. The banking of SPIs and PPIs increases the total number of interrupts supported by +//! the Distributor. +//! +//! The GIC assigns interrupt ID numbers ID0-ID1019 as follows: +//! - Interrupt numbers 32..1019 are used for SPIs. +//! - Interrupt numbers 0..31 are used for interrupts that are private to a CPU interface. These +//! interrupts are banked in the Distributor. +//! - A banked interrupt is one where the Distributor can have multiple interrupts with the +//! same ID. A banked interrupt is identified uniquely by its ID number and its associated +//! CPU interface number. Of the banked interrupt IDs: +//! - 00..15 SGIs +//! - 16..31 PPIs + +mod gicc; +mod gicd; + +use crate::{ + bsp, cpu, driver, exception, memory, memory::mmu::Physical, synchronization, + synchronization::InitStateLock, +}; +use core::sync::atomic::{AtomicBool, Ordering}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +type HandlerTable = [Option; GICv2::NUM_IRQS]; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +pub type IRQNumber = exception::asynchronous::IRQNumber<{ GICv2::MAX_IRQ_NUMBER }>; + +/// Representation of the GIC. +pub struct GICv2 { + gicd_phys_mmio_descriptor: memory::mmu::MMIODescriptor, + gicc_phys_mmio_descriptor: memory::mmu::MMIODescriptor, + + /// The Distributor. + gicd: gicd::GICD, + + /// The CPU Interface. + gicc: gicc::GICC, + + /// Have the MMIO regions been remapped yet? + is_mmio_remapped: AtomicBool, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl GICv2 { + const MAX_IRQ_NUMBER: usize = 300; // Normally 1019, but keep it lower to save some space. + const NUM_IRQS: usize = Self::MAX_IRQ_NUMBER + 1; + + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new( + gicd_phys_mmio_descriptor: memory::mmu::MMIODescriptor, + gicc_phys_mmio_descriptor: memory::mmu::MMIODescriptor, + ) -> Self { + Self { + gicd_phys_mmio_descriptor, + gicc_phys_mmio_descriptor, + gicd: gicd::GICD::new(gicd_phys_mmio_descriptor.start_addr().into_usize()), + gicc: gicc::GICC::new(gicc_phys_mmio_descriptor.start_addr().into_usize()), + is_mmio_remapped: AtomicBool::new(false), + handler_table: InitStateLock::new([None; Self::NUM_IRQS]), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::ReadWriteEx; + +impl driver::interface::DeviceDriver for GICv2 { + fn compatible(&self) -> &'static str { + "GICv2 (ARM Generic Interrupt Controller v2)" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let remapped = self.is_mmio_remapped.load(Ordering::Relaxed); + if !remapped { + let mut virt_addr; + + // GICD + virt_addr = memory::mmu::kernel_map_mmio("GICD", &self.gicd_phys_mmio_descriptor)?; + self.gicd.set_mmio(virt_addr.into_usize()); + + // GICC + virt_addr = memory::mmu::kernel_map_mmio("GICC", &self.gicc_phys_mmio_descriptor)?; + self.gicc.set_mmio(virt_addr.into_usize()); + + // Conclude remapping. + self.is_mmio_remapped.store(true, Ordering::Relaxed); + } + + if cpu::smp::core_id::() == bsp::cpu::BOOT_CORE_ID { + self.gicd.boot_core_init(); + } + + self.gicc.priority_accept_all(); + self.gicc.enable(); + + Ok(()) + } +} + +impl exception::asynchronous::interface::IRQManager for GICv2 { + type IRQNumberType = IRQNumber; + + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + let mut r = &self.handler_table; + r.write(|table| { + let irq_number = irq_number.get(); + + if table[irq_number].is_some() { + return Err("IRQ handler already registered"); + } + + table[irq_number] = Some(descriptor); + + Ok(()) + }) + } + + fn enable(&self, irq_number: Self::IRQNumberType) { + self.gicd.enable(irq_number); + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + // Extract the highest priority pending IRQ number from the Interrupt Acknowledge Register + // (IAR). + let irq_number = self.gicc.pending_irq_number(ic); + + // Guard against spurious interrupts. + if irq_number > GICv2::MAX_IRQ_NUMBER { + return; + } + + // Call the IRQ handler. Panic if there is none. + let mut r = &self.handler_table; + r.read(|table| { + match table[irq_number] { + None => panic!("No handler registered for IRQ {}", irq_number), + Some(descriptor) => { + // Call the IRQ handler. Panics on failure. + descriptor.handler.handle().expect("Error handling IRQ"); + } + } + }); + + // Signal completion of handling. + self.gicc.mark_comleted(irq_number as u32, ic); + } + + fn print_handler(&self) { + use crate::info; + + info!(" Peripheral handler:"); + + let mut r = &self.handler_table; + r.read(|table| { + for (i, opt) in table.iter().skip(32).enumerate() { + if let Some(handler) = opt { + info!(" {: >3}. {}", i + 32, handler.name); + } + } + }); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs new file mode 100644 index 00000000..06aafffa --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicc.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! GICC Driver - GIC CPU interface. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, exception, synchronization::InitStateLock, +}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_bitfields! { + u32, + + /// CPU Interface Control Register + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + + /// Interrupt Priority Mask Register + PMR [ + Priority OFFSET(0) NUMBITS(8) [] + ], + + /// Interrupt Acknowledge Register + IAR [ + InterruptID OFFSET(0) NUMBITS(10) [] + ], + + /// End of Interrupt Register + EOIR [ + EOIINTID OFFSET(0) NUMBITS(10) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + pub RegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => PMR: ReadWrite), + (0x008 => _reserved1), + (0x00C => IAR: ReadWrite), + (0x010 => EOIR: ReadWrite), + (0x014 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Registers = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the GIC CPU interface. +pub struct GICC { + registers: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::synchronization::interface::ReadWriteEx; + +impl GICC { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + registers: InitStateLock::new(Registers::new(mmio_start_addr)), + } + } + + pub unsafe fn set_mmio(&self, new_mmio_start_addr: usize) { + let mut r = &self.registers; + r.write(|regs| *regs = Registers::new(new_mmio_start_addr)); + } + + /// Accept interrupts of any priority. + /// + /// Quoting the GICv2 Architecture Specification: + /// + /// "Writing 255 to the GICC_PMR always sets it to the largest supported priority field + /// value." + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn priority_accept_all(&self) { + let mut r = &self.registers; + r.read(|regs| { + regs.PMR.write(PMR::Priority.val(255)); // Comment in arch spec. + }); + } + + /// Enable the interface - start accepting IRQs. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + pub fn enable(&self) { + let mut r = &self.registers; + r.read(|regs| { + regs.CTLR.write(CTLR::Enable::SET); + }); + } + + /// Extract the number of the highest-priority pending IRQ. + /// + /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + #[allow(clippy::trivially_copy_pass_by_ref)] + pub fn pending_irq_number<'irq_context>( + &self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) -> usize { + let mut r = &self.registers; + r.read(|regs| regs.IAR.read(IAR::InterruptID) as usize) + } + + /// Complete handling of the currently active IRQ. + /// + /// Can only be called from IRQ context, which is ensured by taking an `IRQContext` token. + /// + /// To be called after `pending_irq_number()`. + /// + /// # Safety + /// + /// - GICC MMIO registers are banked per CPU core. It is therefore safe to have `&self` instead + /// of `&mut self`. + #[allow(clippy::trivially_copy_pass_by_ref)] + pub fn mark_comleted<'irq_context>( + &self, + irq_number: u32, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + let mut r = &self.registers; + r.read(|regs| { + regs.EOIR.write(EOIR::EOIINTID.val(irq_number)); + }); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicd.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicd.rs new file mode 100644 index 00000000..05379114 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/arm/gicv2/gicd.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! GICD Driver - GIC Distributor. +//! +//! # Glossary +//! - SPI - Shared Peripheral Interrupt. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, + state, synchronization, + synchronization::{IRQSafeNullLock, InitStateLock}, +}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_bitfields! { + u32, + + /// Distributor Control Register + CTLR [ + Enable OFFSET(0) NUMBITS(1) [] + ], + + /// Interrupt Controller Type Register + TYPER [ + ITLinesNumber OFFSET(0) NUMBITS(5) [] + ], + + /// Interrupt Processor Targets Registers + ITARGETSR [ + Offset3 OFFSET(24) NUMBITS(8) [], + Offset2 OFFSET(16) NUMBITS(8) [], + Offset1 OFFSET(8) NUMBITS(8) [], + Offset0 OFFSET(0) NUMBITS(8) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + SharedRegisterBlock { + (0x000 => CTLR: ReadWrite), + (0x004 => TYPER: ReadOnly), + (0x008 => _reserved1), + (0x104 => ISENABLER: [ReadWrite; 31]), + (0x108 => _reserved2), + (0x820 => ITARGETSR: [ReadWrite; 248]), + (0x824 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + BankedRegisterBlock { + (0x000 => _reserved1), + (0x100 => ISENABLER: ReadWrite), + (0x104 => _reserved2), + (0x800 => ITARGETSR: [ReadOnly; 8]), + (0x804 => @END), + } +} + +/// Abstraction for the non-banked parts of the associated MMIO registers. +type SharedRegisters = MMIODerefWrapper; + +/// Abstraction for the banked parts of the associated MMIO registers. +type BankedRegisters = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the GIC Distributor. +pub struct GICD { + /// Access to shared registers is guarded with a lock. + shared_registers: IRQSafeNullLock, + + /// Access to banked registers is unguarded. + banked_registers: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl SharedRegisters { + /// Return the number of IRQs that this HW implements. + #[inline(always)] + fn num_irqs(&mut self) -> usize { + // Query number of implemented IRQs. + // + // Refer to GICv2 Architecture Specification, Section 4.3.2. + ((self.TYPER.read(TYPER::ITLinesNumber) as usize) + 1) * 32 + } + + /// Return a slice of the implemented ITARGETSR. + #[inline(always)] + fn implemented_itargets_slice(&mut self) -> &[ReadWrite] { + assert!(self.num_irqs() >= 36); + + // Calculate the max index of the shared ITARGETSR array. + // + // The first 32 IRQs are private, so not included in `shared_registers`. Each ITARGETS + // register has four entries, so shift right by two. Subtract one because we start + // counting at zero. + let spi_itargetsr_max_index = ((self.num_irqs() - 32) >> 2) - 1; + + // Rust automatically inserts slice range sanity check, i.e. max >= min. + &self.ITARGETSR[0..spi_itargetsr_max_index] + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use crate::synchronization::interface::ReadWriteEx; +use synchronization::interface::Mutex; + +impl GICD { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + shared_registers: IRQSafeNullLock::new(SharedRegisters::new(mmio_start_addr)), + banked_registers: InitStateLock::new(BankedRegisters::new(mmio_start_addr)), + } + } + + pub unsafe fn set_mmio(&self, new_mmio_start_addr: usize) { + let mut r = &self.shared_registers; + r.lock(|regs| *regs = SharedRegisters::new(new_mmio_start_addr)); + + let mut r = &self.banked_registers; + r.write(|regs| *regs = BankedRegisters::new(new_mmio_start_addr)); + } + + /// Use a banked ITARGETSR to retrieve the executing core's GIC target mask. + /// + /// Quoting the GICv2 Architecture Specification: + /// + /// "GICD_ITARGETSR0 to GICD_ITARGETSR7 are read-only, and each field returns a value that + /// corresponds only to the processor reading the register." + fn local_gic_target_mask(&self) -> u32 { + let mut r = &self.banked_registers; + r.read(|regs| regs.ITARGETSR[0].read(ITARGETSR::Offset0)) + } + + /// Route all SPIs to the boot core and enable the distributor. + pub fn boot_core_init(&self) { + assert!( + state::state_manager().is_init(), + "Only allowed during kernel init phase" + ); + + // Target all SPIs to the boot core only. + let mask = self.local_gic_target_mask(); + + let mut r = &self.shared_registers; + r.lock(|regs| { + for i in regs.implemented_itargets_slice().iter() { + i.write( + ITARGETSR::Offset3.val(mask) + + ITARGETSR::Offset2.val(mask) + + ITARGETSR::Offset1.val(mask) + + ITARGETSR::Offset0.val(mask), + ); + } + + regs.CTLR.write(CTLR::Enable::SET); + }); + } + + /// Enable an interrupt. + pub fn enable(&self, irq_num: super::IRQNumber) { + let irq_num = irq_num.get(); + + // Each bit in the u32 enable register corresponds to one IRQ number. Shift right by 5 + // (division by 32) and arrive at the index for the respective ISENABLER[i]. + let enable_reg_index = irq_num >> 5; + let enable_bit: u32 = 1u32 << (irq_num % 32); + + // Check if we are handling a private or shared IRQ. + match irq_num { + // Private. + 0..=31 => { + let mut r = &self.banked_registers; + r.read(|regs| { + let enable_reg = ®s.ISENABLER; + enable_reg.set(enable_reg.get() | enable_bit); + }) + } + // Shared. + _ => { + let enable_reg_index_shared = enable_reg_index - 1; + + let mut r = &self.shared_registers; + r.lock(|regs| { + let enable_reg = ®s.ISENABLER[enable_reg_index_shared]; + enable_reg.set(enable_reg.get() | enable_bit); + }); + } + } + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm.rs new file mode 100644 index 00000000..5e4c9e70 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BCM driver top level. + +mod bcm2xxx_gpio; +#[cfg(feature = "bsp_rpi3")] +mod bcm2xxx_interrupt_controller; +mod bcm2xxx_pl011_uart; + +pub use bcm2xxx_gpio::*; +#[cfg(feature = "bsp_rpi3")] +pub use bcm2xxx_interrupt_controller::*; +pub use bcm2xxx_pl011_uart::*; diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs new file mode 100644 index 00000000..b9fc3c8b --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_gpio.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! GPIO Driver. + +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, cpu, driver, memory, memory::mmu::Physical, + synchronization, synchronization::IRQSafeNullLock, +}; +use core::sync::atomic::{AtomicUsize, Ordering}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// GPIO registers. +// +// Descriptions taken from +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +register_bitfields! { + u32, + + /// GPIO Function Select 1 + GPFSEL1 [ + /// Pin 15 + FSEL15 OFFSET(15) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART RX + + ], + + /// Pin 14 + FSEL14 OFFSET(12) NUMBITS(3) [ + Input = 0b000, + Output = 0b001, + AltFunc0 = 0b100 // PL011 UART TX + ] + ], + + /// GPIO Pull-up/down Clock Register 0 + GPPUDCLK0 [ + /// Pin 15 + PUDCLK15 OFFSET(15) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ], + + /// Pin 14 + PUDCLK14 OFFSET(14) NUMBITS(1) [ + NoEffect = 0, + AssertClock = 1 + ] + ] +} + +register_structs! { + #[allow(non_snake_case)] + RegisterBlock { + (0x00 => GPFSEL0: ReadWrite), + (0x04 => GPFSEL1: ReadWrite), + (0x08 => GPFSEL2: ReadWrite), + (0x0C => GPFSEL3: ReadWrite), + (0x10 => GPFSEL4: ReadWrite), + (0x14 => GPFSEL5: ReadWrite), + (0x18 => _reserved1), + (0x94 => GPPUD: ReadWrite), + (0x98 => GPPUDCLK0: ReadWrite), + (0x9C => GPPUDCLK1: ReadWrite), + (0xA0 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Registers = MMIODerefWrapper; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub struct GPIOInner { + registers: Registers, +} + +// Export the inner struct so that BSPs can use it for the panic handler. +pub use GPIOInner as PanicGPIO; + +/// Representation of the GPIO HW. +pub struct GPIO { + phys_mmio_descriptor: memory::mmu::MMIODescriptor, + virt_mmio_start_addr: AtomicUsize, + inner: IRQSafeNullLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl GPIOInner { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + registers: Registers::new(mmio_start_addr), + } + } + + /// Init code. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub unsafe fn init(&mut self, new_mmio_start_addr: Option) -> Result<(), &'static str> { + if let Some(addr) = new_mmio_start_addr { + self.registers = Registers::new(addr); + } + + Ok(()) + } + + /// Map PL011 UART as standard output. + /// + /// TX to pin 14 + /// RX to pin 15 + pub fn map_pl011_uart(&mut self) { + // Map to pins. + self.registers + .GPFSEL1 + .modify(GPFSEL1::FSEL14::AltFunc0 + GPFSEL1::FSEL15::AltFunc0); + + // Enable pins 14 and 15. + self.registers.GPPUD.set(0); + cpu::spin_for_cycles(150); + + self.registers + .GPPUDCLK0 + .write(GPPUDCLK0::PUDCLK14::AssertClock + GPPUDCLK0::PUDCLK15::AssertClock); + cpu::spin_for_cycles(150); + + self.registers.GPPUDCLK0.set(0); + } +} + +impl GPIO { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new(phys_mmio_descriptor: memory::mmu::MMIODescriptor) -> Self { + Self { + phys_mmio_descriptor, + virt_mmio_start_addr: AtomicUsize::new(0), + inner: IRQSafeNullLock::new(GPIOInner::new( + phys_mmio_descriptor.start_addr().into_usize(), + )), + } + } + + /// Concurrency safe version of `GPIOInner.map_pl011_uart()` + pub fn map_pl011_uart(&self) { + let mut r = &self.inner; + r.lock(|inner| inner.map_pl011_uart()) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::Mutex; + +impl driver::interface::DeviceDriver for GPIO { + fn compatible(&self) -> &'static str { + "BCM GPIO" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = + memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)?; + + let mut r = &self.inner; + r.lock(|inner| inner.init(Some(virt_addr.into_usize())))?; + + self.virt_mmio_start_addr + .store(virt_addr.into_usize(), Ordering::Relaxed); + + Ok(()) + } + + fn virt_mmio_start_addr(&self) -> Option { + let addr = self.virt_mmio_start_addr.load(Ordering::Relaxed); + + if addr == 0 { + return None; + } + + Some(addr) + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs new file mode 100644 index 00000000..d6ecf677 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Interrupt Controller Driver. + +mod peripheral_ic; + +use crate::{driver, exception, memory, memory::mmu::Physical}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Wrapper struct for a bitmask indicating pending IRQ numbers. +struct PendingIRQs { + bitmask: u64, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub type LocalIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_LOCAL_IRQ_NUMBER }>; +pub type PeripheralIRQ = + exception::asynchronous::IRQNumber<{ InterruptController::MAX_PERIPHERAL_IRQ_NUMBER }>; + +/// Used for the associated type of trait [`exception::asynchronous::interface::IRQManager`]. +#[derive(Copy, Clone)] +pub enum IRQNumber { + Local(LocalIRQ), + Peripheral(PeripheralIRQ), +} + +/// Representation of the Interrupt Controller. +pub struct InterruptController { + periph: peripheral_ic::PeripheralIC, +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl PendingIRQs { + pub fn new(bitmask: u64) -> Self { + Self { bitmask } + } +} + +impl Iterator for PendingIRQs { + type Item = usize; + + fn next(&mut self) -> Option { + use core::intrinsics::cttz; + + let next = cttz(self.bitmask); + if next == 64 { + return None; + } + + self.bitmask &= !(1 << next); + + Some(next as usize) + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl InterruptController { + const MAX_LOCAL_IRQ_NUMBER: usize = 11; + const MAX_PERIPHERAL_IRQ_NUMBER: usize = 63; + const NUM_PERIPHERAL_IRQS: usize = Self::MAX_PERIPHERAL_IRQ_NUMBER + 1; + + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new( + _phys_local_mmio_descriptor: memory::mmu::MMIODescriptor, + phys_periph_mmio_descriptor: memory::mmu::MMIODescriptor, + ) -> Self { + Self { + periph: peripheral_ic::PeripheralIC::new(phys_periph_mmio_descriptor), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ + +impl driver::interface::DeviceDriver for InterruptController { + fn compatible(&self) -> &'static str { + "BCM Interrupt Controller" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + self.periph.init() + } +} + +impl exception::asynchronous::interface::IRQManager for InterruptController { + type IRQNumberType = IRQNumber; + + fn register_handler( + &self, + irq: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + match irq { + IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), + IRQNumber::Peripheral(pirq) => self.periph.register_handler(pirq, descriptor), + } + } + + fn enable(&self, irq: Self::IRQNumberType) { + match irq { + IRQNumber::Local(_) => unimplemented!("Local IRQ controller not implemented."), + IRQNumber::Peripheral(pirq) => self.periph.enable(pirq), + } + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + // It can only be a peripheral IRQ pending because enable() does not support local IRQs yet. + self.periph.handle_pending_irqs(ic) + } + + fn print_handler(&self) { + self.periph.print_handler(); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs new file mode 100644 index 00000000..f0d084c8 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_interrupt_controller/peripheral_ic.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Peripheral Interrupt Controller Driver. + +use super::{InterruptController, PendingIRQs, PeripheralIRQ}; +use crate::{ + bsp::device_driver::common::MMIODerefWrapper, + driver, exception, memory, + memory::mmu::Physical, + synchronization, + synchronization::{IRQSafeNullLock, InitStateLock}, +}; +use register::{mmio::*, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +register_structs! { + #[allow(non_snake_case)] + WORegisterBlock { + (0x00 => _reserved1), + (0x10 => ENABLE_1: WriteOnly), + (0x14 => ENABLE_2: WriteOnly), + (0x24 => @END), + } +} + +register_structs! { + #[allow(non_snake_case)] + RORegisterBlock { + (0x00 => _reserved1), + (0x04 => PENDING_1: ReadOnly), + (0x08 => PENDING_2: ReadOnly), + (0x0c => @END), + } +} + +/// Abstraction for the WriteOnly parts of the associated MMIO registers. +type WriteOnlyRegisters = MMIODerefWrapper; + +/// Abstraction for the ReadOnly parts of the associated MMIO registers. +type ReadOnlyRegisters = MMIODerefWrapper; + +type HandlerTable = + [Option; InterruptController::NUM_PERIPHERAL_IRQS]; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Representation of the peripheral interrupt controller. +pub struct PeripheralIC { + phys_mmio_descriptor: memory::mmu::MMIODescriptor, + + /// Access to write registers is guarded with a lock. + wo_registers: IRQSafeNullLock, + + /// Register read access is unguarded. + ro_registers: InitStateLock, + + /// Stores registered IRQ handlers. Writable only during kernel init. RO afterwards. + handler_table: InitStateLock, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl PeripheralIC { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + pub const unsafe fn new(phys_mmio_descriptor: memory::mmu::MMIODescriptor) -> Self { + let addr = phys_mmio_descriptor.start_addr().into_usize(); + + Self { + phys_mmio_descriptor, + wo_registers: IRQSafeNullLock::new(WriteOnlyRegisters::new(addr)), + ro_registers: InitStateLock::new(ReadOnlyRegisters::new(addr)), + handler_table: InitStateLock::new([None; InterruptController::NUM_PERIPHERAL_IRQS]), + } + } + + /// Query the list of pending IRQs. + fn pending_irqs(&self) -> PendingIRQs { + let mut r = &self.ro_registers; + r.read(|regs| { + let pending_mask: u64 = + (u64::from(regs.PENDING_2.get()) << 32) | u64::from(regs.PENDING_1.get()); + + PendingIRQs::new(pending_mask) + }) + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::{Mutex, ReadWriteEx}; + +impl driver::interface::DeviceDriver for PeripheralIC { + fn compatible(&self) -> &'static str { + "BCM Peripheral Interrupt Controller" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = + memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)? + .into_usize(); + + let mut r = &self.wo_registers; + r.lock(|regs| *regs = WriteOnlyRegisters::new(virt_addr)); + + let mut r = &self.ro_registers; + r.write(|regs| *regs = ReadOnlyRegisters::new(virt_addr)); + + Ok(()) + } +} + +impl exception::asynchronous::interface::IRQManager for PeripheralIC { + type IRQNumberType = PeripheralIRQ; + + fn register_handler( + &self, + irq: Self::IRQNumberType, + descriptor: exception::asynchronous::IRQDescriptor, + ) -> Result<(), &'static str> { + let mut r = &self.handler_table; + r.write(|table| { + let irq_number = irq.get(); + + if table[irq_number].is_some() { + return Err("IRQ handler already registered"); + } + + table[irq_number] = Some(descriptor); + + Ok(()) + }) + } + + fn enable(&self, irq: Self::IRQNumberType) { + let mut r = &self.wo_registers; + r.lock(|regs| { + let enable_reg = if irq.get() <= 31 { + ®s.ENABLE_1 + } else { + ®s.ENABLE_2 + }; + + let enable_bit: u32 = 1 << (irq.get() % 32); + + // Writing a 1 to a bit will set the corresponding IRQ enable bit. All other IRQ enable + // bits are unaffected. So we don't need read and OR'ing here. + enable_reg.set(enable_bit); + }); + } + + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + _ic: &exception::asynchronous::IRQContext<'irq_context>, + ) { + let mut r = &self.handler_table; + r.read(|table| { + for irq_number in self.pending_irqs() { + match table[irq_number] { + None => panic!("No handler registered for IRQ {}", irq_number), + Some(descriptor) => { + // Call the IRQ handler. Panics on failure. + descriptor.handler.handle().expect("Error handling IRQ"); + } + } + } + }) + } + + fn print_handler(&self) { + use crate::info; + + info!(" Peripheral handler:"); + + let mut r = &self.handler_table; + r.read(|table| { + for (i, opt) in table.iter().enumerate() { + if let Some(handler) = opt { + info!(" {: >3}. {}", i, handler.name); + } + } + }); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs new file mode 100644 index 00000000..ccc265b2 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/bcm/bcm2xxx_pl011_uart.rs @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! PL011 UART driver. + +use crate::{ + bsp, bsp::device_driver::common::MMIODerefWrapper, console, cpu, driver, exception, memory, + memory::mmu::Physical, synchronization, synchronization::IRQSafeNullLock, +}; +use core::{ + fmt, + sync::atomic::{AtomicUsize, Ordering}, +}; +use register::{mmio::*, register_bitfields, register_structs}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// PL011 UART registers. +// +// Descriptions taken from +// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf +register_bitfields! { + u32, + + /// Flag Register + FR [ + /// Transmit FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// Line Control Register, UARTLCR_ LCRH. + /// + /// If the FIFO is disabled, this bit is set when the transmit holding register is empty. If + /// the FIFO is enabled, the TXFE bit is set when the transmit FIFO is empty. This bit does + /// not indicate if there is data in the transmit shift register. + TXFE OFFSET(7) NUMBITS(1) [], + + /// Transmit FIFO full. The meaning of this bit depends on the state of the FEN bit in the + /// UARTLCR_ LCRH Register. + /// + /// If the FIFO is disabled, this bit is set when the transmit holding register is full. If + /// the FIFO is enabled, the TXFF bit is set when the transmit FIFO is full. + TXFF OFFSET(5) NUMBITS(1) [], + + /// Receive FIFO empty. The meaning of this bit depends on the state of the FEN bit in the + /// UARTLCR_H Register. + /// + /// If the FIFO is disabled, this bit is set when the receive holding register is empty. If + /// the FIFO is enabled, the RXFE bit is set when the receive FIFO is empty. + RXFE OFFSET(4) NUMBITS(1) [] + ], + + /// Integer Baud rate divisor + IBRD [ + /// Integer Baud rate divisor + IBRD OFFSET(0) NUMBITS(16) [] + ], + + /// Fractional Baud rate divisor + FBRD [ + /// Fractional Baud rate divisor + FBRD OFFSET(0) NUMBITS(6) [] + ], + + /// Line Control register + LCRH [ + /// Word length. These bits indicate the number of data bits transmitted or received in a + /// frame. + WLEN OFFSET(5) NUMBITS(2) [ + FiveBit = 0b00, + SixBit = 0b01, + SevenBit = 0b10, + EightBit = 0b11 + ], + + /// Enable FIFOs: + /// + /// 0 = FIFOs are disabled (character mode) that is, the FIFOs become 1-byte-deep holding + /// registers + /// + /// 1 = transmit and receive FIFO buffers are enabled (FIFO mode). + FEN OFFSET(4) NUMBITS(1) [ + FifosDisabled = 0, + FifosEnabled = 1 + ] + ], + + /// Control Register + CR [ + /// Receive enable. If this bit is set to 1, the receive section of the UART is enabled. + /// Data reception occurs for UART signals. When the UART is disabled in the middle of + /// reception, it completes the current character before stopping. + RXE OFFSET(9) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// Transmit enable. If this bit is set to 1, the transmit section of the UART is enabled. + /// Data transmission occurs for UART signals. When the UART is disabled in the middle of + /// transmission, it completes the current character before stopping. + TXE OFFSET(8) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// UART enable + UARTEN OFFSET(0) NUMBITS(1) [ + /// If the UART is disabled in the middle of transmission or reception, it completes the + /// current character before stopping. + Disabled = 0, + Enabled = 1 + ] + ], + + /// Interrupt FIFO Level Select Register + IFLS [ + /// Receive interrupt FIFO level select. The trigger points for the receive interrupt are as + /// follows. + RXIFLSEL OFFSET(3) NUMBITS(5) [ + OneEigth = 0b000, + OneQuarter = 0b001, + OneHalf = 0b010, + ThreeQuarters = 0b011, + SevenEights = 0b100 + ] + ], + + /// Interrupt Mask Set Clear Register + IMSC [ + /// Receive timeout interrupt mask. A read returns the current mask for the UARTRTINTR + /// interrupt. On a write of 1, the mask of the interrupt is set. A write of 0 clears the + /// mask. + RTIM OFFSET(6) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ], + + /// Receive interrupt mask. A read returns the current mask for the UARTRXINTR interrupt. On + /// a write of 1, the mask of the interrupt is set. A write of 0 clears the mask. + RXIM OFFSET(4) NUMBITS(1) [ + Disabled = 0, + Enabled = 1 + ] + ], + + /// Masked Interrupt Status Register + MIS [ + /// Receive timeout masked interrupt status. Returns the masked interrupt state of the + /// UARTRTINTR interrupt. + RTMIS OFFSET(6) NUMBITS(1) [], + + /// Receive masked interrupt status. Returns the masked interrupt state of the UARTRXINTR + /// interrupt. + RXMIS OFFSET(4) NUMBITS(1) [] + ], + + /// Interrupt Clear Register + ICR [ + /// Meta field for all pending interrupts + ALL OFFSET(0) NUMBITS(11) [] + ] +} + +register_structs! { + #[allow(non_snake_case)] + pub RegisterBlock { + (0x00 => DR: ReadWrite), + (0x04 => _reserved1), + (0x18 => FR: ReadOnly), + (0x1c => _reserved2), + (0x24 => IBRD: WriteOnly), + (0x28 => FBRD: WriteOnly), + (0x2c => LCRH: WriteOnly), + (0x30 => CR: WriteOnly), + (0x34 => IFLS: ReadWrite), + (0x38 => IMSC: ReadWrite), + (0x3C => _reserved3), + (0x40 => MIS: ReadOnly), + (0x44 => ICR: WriteOnly), + (0x48 => @END), + } +} + +/// Abstraction for the associated MMIO registers. +type Registers = MMIODerefWrapper; + +#[derive(PartialEq)] +enum BlockingMode { + Blocking, + NonBlocking, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub struct PL011UartInner { + registers: Registers, + chars_written: usize, + chars_read: usize, +} + +// Export the inner struct so that BSPs can use it for the panic handler. +pub use PL011UartInner as PanicUart; + +/// Representation of the UART. +pub struct PL011Uart { + phys_mmio_descriptor: memory::mmu::MMIODescriptor, + virt_mmio_start_addr: AtomicUsize, + inner: IRQSafeNullLock, + irq_number: bsp::device_driver::IRQNumber, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl PL011UartInner { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub const unsafe fn new(mmio_start_addr: usize) -> Self { + Self { + registers: Registers::new(mmio_start_addr), + chars_written: 0, + chars_read: 0, + } + } + + /// Set up baud rate and characteristics. + /// + /// The calculation for the BRD given a target rate of 2300400 and a clock set to 48 MHz is: + /// `(48_000_000/16)/230400 = 13,02083`. `13` goes to the `IBRD` (integer field). The `FBRD` + /// (fractional field) is only 6 bits so `0,0208*64 = 1,3312 rounded to 1` will give the best + /// approximation we can get. A 5 % error margin is acceptable for UART and we're now at 0,01 %. + /// + /// This results in 8N1 and 230400 baud (we set the clock to 48 MHz in config.txt). + /// + /// # Safety + /// + /// - The user must ensure to provide a correct MMIO start address. + pub unsafe fn init(&mut self, new_mmio_start_addr: Option) -> Result<(), &'static str> { + if let Some(addr) = new_mmio_start_addr { + self.registers = Registers::new(addr); + } + + // Turn it off temporarily. + self.registers.CR.set(0); + + self.registers.ICR.write(ICR::ALL::CLEAR); + self.registers.IBRD.write(IBRD::IBRD.val(13)); + self.registers.FBRD.write(FBRD::FBRD.val(1)); + self.registers + .LCRH + .write(LCRH::WLEN::EightBit + LCRH::FEN::FifosEnabled); // 8N1 + Fifo on + self.registers.IFLS.write(IFLS::RXIFLSEL::OneEigth); // RX FIFO fill level at 1/8 + self.registers + .IMSC + .write(IMSC::RXIM::Enabled + IMSC::RTIM::Enabled); // RX IRQ + RX timeout IRQ + self.registers + .CR + .write(CR::UARTEN::Enabled + CR::TXE::Enabled + CR::RXE::Enabled); + + Ok(()) + } + + /// Send a character. + fn write_char(&mut self, c: char) { + // Spin while TX FIFO full is set, waiting for an empty slot. + while self.registers.FR.matches_all(FR::TXFF::SET) { + cpu::nop(); + } + + // Write the character to the buffer. + self.registers.DR.set(c as u32); + + self.chars_written += 1; + } + + /// Retrieve a character. + fn read_char_converting(&mut self, blocking_mode: BlockingMode) -> Option { + // If RX FIFO is empty, + if self.registers.FR.matches_all(FR::RXFE::SET) { + // immediately return in non-blocking mode. + if blocking_mode == BlockingMode::NonBlocking { + return None; + } + + // Otherwise, wait until a char was received. + while self.registers.FR.matches_all(FR::RXFE::SET) { + cpu::nop(); + } + } + + // Read one character. + let mut ret = self.registers.DR.get() as u8 as char; + + // Convert carrige return to newline. + if ret == '\r' { + ret = '\n' + } + + // Update statistics. + self.chars_read += 1; + + Some(ret) + } +} + +/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros, which in turn are +/// used to implement the `kernel`'s `print!` and `println!` macros. By implementing `write_str()`, +/// we get `write_fmt()` automatically. +/// +/// The function takes an `&mut self`, so it must be implemented for the inner struct. +/// +/// See [`src/print.rs`]. +/// +/// [`src/print.rs`]: ../../print/index.html +impl fmt::Write for PL011UartInner { + fn write_str(&mut self, s: &str) -> fmt::Result { + for c in s.chars() { + self.write_char(c); + } + + Ok(()) + } +} + +impl PL011Uart { + /// Create an instance. + /// + /// # Safety + /// + /// - The user must ensure to provide correct MMIO descriptors. + /// - The user must ensure to provide correct IRQ numbers. + pub const unsafe fn new( + phys_mmio_descriptor: memory::mmu::MMIODescriptor, + irq_number: bsp::device_driver::IRQNumber, + ) -> Self { + Self { + phys_mmio_descriptor, + virt_mmio_start_addr: AtomicUsize::new(0), + inner: IRQSafeNullLock::new(PL011UartInner::new( + phys_mmio_descriptor.start_addr().into_usize(), + )), + irq_number, + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use synchronization::interface::Mutex; + +impl driver::interface::DeviceDriver for PL011Uart { + fn compatible(&self) -> &'static str { + "BCM PL011 UART" + } + + unsafe fn init(&self) -> Result<(), &'static str> { + let virt_addr = + memory::mmu::kernel_map_mmio(self.compatible(), &self.phys_mmio_descriptor)?; + + let mut r = &self.inner; + r.lock(|inner| inner.init(Some(virt_addr.into_usize())))?; + + self.virt_mmio_start_addr + .store(virt_addr.into_usize(), Ordering::Relaxed); + + Ok(()) + } + + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + use bsp::exception::asynchronous::irq_manager; + use exception::asynchronous::{interface::IRQManager, IRQDescriptor}; + + let descriptor = IRQDescriptor { + name: "BCM PL011 UART", + handler: self, + }; + + irq_manager().register_handler(self.irq_number, descriptor)?; + irq_manager().enable(self.irq_number); + + Ok(()) + } + + fn virt_mmio_start_addr(&self) -> Option { + let addr = self.virt_mmio_start_addr.load(Ordering::Relaxed); + + if addr == 0 { + return None; + } + + Some(addr) + } +} + +impl console::interface::Write for PL011Uart { + /// Passthrough of `args` to the `core::fmt::Write` implementation, but guarded by a Mutex to + /// serialize access. + fn write_char(&self, c: char) { + let mut r = &self.inner; + r.lock(|inner| inner.write_char(c)); + } + + fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result { + // Fully qualified syntax for the call to `core::fmt::Write::write:fmt()` to increase + // readability. + let mut r = &self.inner; + r.lock(|inner| fmt::Write::write_fmt(inner, args)) + } + + fn flush(&self) { + // Spin until TX FIFO empty is set. + let mut r = &self.inner; + r.lock(|inner| { + while !inner.registers.FR.matches_all(FR::TXFE::SET) { + cpu::nop(); + } + }); + } +} + +impl console::interface::Read for PL011Uart { + fn read_char(&self) -> char { + let mut r = &self.inner; + r.lock(|inner| inner.read_char_converting(BlockingMode::Blocking).unwrap()) + } + + fn clear(&self) { + let mut r = &self.inner; + r.lock(|inner| { + // Read from the RX FIFO until it is indicating empty. + while !inner.registers.FR.matches_all(FR::RXFE::SET) { + inner.registers.DR.get(); + } + }) + } +} + +impl console::interface::Statistics for PL011Uart { + fn chars_written(&self) -> usize { + let mut r = &self.inner; + r.lock(|inner| inner.chars_written) + } + + fn chars_read(&self) -> usize { + let mut r = &self.inner; + r.lock(|inner| inner.chars_read) + } +} + +impl exception::asynchronous::interface::IRQHandler for PL011Uart { + fn handle(&self) -> Result<(), &'static str> { + let mut r = &self.inner; + r.lock(|inner| { + let pending = inner.registers.MIS.extract(); + + // Clear all pending IRQs. + inner.registers.ICR.write(ICR::ALL::CLEAR); + + // Check for any kind of RX interrupt. + if pending.matches_any(MIS::RXMIS::SET + MIS::RTMIS::SET) { + // Echo any received characters. + while let Some(c) = inner.read_char_converting(BlockingMode::NonBlocking) { + inner.write_char(c) + } + } + }); + + Ok(()) + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/common.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/common.rs new file mode 100644 index 00000000..56afb97a --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/device_driver/common.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Common device driver code. + +use core::{marker::PhantomData, ops}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +pub struct MMIODerefWrapper { + start_addr: usize, + phantom: PhantomData, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl MMIODerefWrapper { + /// Create an instance. + pub const unsafe fn new(start_addr: usize) -> Self { + Self { + start_addr, + phantom: PhantomData, + } + } +} + +impl ops::Deref for MMIODerefWrapper { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { &*(self.start_addr as *const _) } + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi.rs new file mode 100644 index 00000000..7ffb7249 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Top-level BSP file for the Raspberry Pi 3 and 4. + +pub mod console; +pub mod cpu; +pub mod driver; +pub mod exception; +pub mod memory; + +use super::device_driver; +use crate::memory::mmu::MMIODescriptor; +use memory::map::mmio; + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static GPIO: device_driver::GPIO = + unsafe { device_driver::GPIO::new(MMIODescriptor::new(mmio::GPIO_START, mmio::GPIO_SIZE)) }; + +static PL011_UART: device_driver::PL011Uart = unsafe { + device_driver::PL011Uart::new( + MMIODescriptor::new(mmio::PL011_UART_START, mmio::PL011_UART_SIZE), + exception::asynchronous::irq_map::PL011_UART, + ) +}; + +#[cfg(feature = "bsp_rpi3")] +static INTERRUPT_CONTROLLER: device_driver::InterruptController = unsafe { + device_driver::InterruptController::new( + MMIODescriptor::new(mmio::LOCAL_IC_START, mmio::LOCAL_IC_SIZE), + MMIODescriptor::new(mmio::PERIPHERAL_IC_START, mmio::PERIPHERAL_IC_SIZE), + ) +}; + +#[cfg(feature = "bsp_rpi4")] +static INTERRUPT_CONTROLLER: device_driver::GICv2 = unsafe { + device_driver::GICv2::new( + MMIODescriptor::new(mmio::GICD_START, mmio::GICD_SIZE), + MMIODescriptor::new(mmio::GICC_START, mmio::GICC_SIZE), + ) +}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Board identification. +pub fn board_name() -> &'static str { + #[cfg(feature = "bsp_rpi3")] + { + "Raspberry Pi 3" + } + + #[cfg(feature = "bsp_rpi4")] + { + "Raspberry Pi 4" + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/console.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/console.rs new file mode 100644 index 00000000..c5d0e05a --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/console.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP console facilities. + +use super::memory; +use crate::{bsp::device_driver, console, cpu}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// In case of a panic, the panic handler uses this function to take a last shot at printing +/// something before the system is halted. +/// +/// We try to init panic-versions of the GPIO and the UART. The panic versions are not protected +/// with synchronization primitives, which increases chances that we get to print something, even +/// when the kernel's default GPIO or UART instances happen to be locked at the time of the panic. +/// +/// # Safety +/// +/// - Use only for printing during a panic. +pub unsafe fn panic_console_out() -> impl fmt::Write { + use crate::driver::interface::DeviceDriver; + + let mut panic_gpio = device_driver::PanicGPIO::new(memory::map::mmio::GPIO_START.into_usize()); + let mut panic_uart = + device_driver::PanicUart::new(memory::map::mmio::PL011_UART_START.into_usize()); + + // If remapping of the driver's MMIO already happened, take the remapped start address. + // Otherwise, take a chance with the default physical address. + let maybe_gpio_mmio_start_addr = super::GPIO.virt_mmio_start_addr(); + let maybe_uart_mmio_start_addr = super::PL011_UART.virt_mmio_start_addr(); + + panic_gpio + .init(maybe_gpio_mmio_start_addr) + .unwrap_or_else(|_| cpu::wait_forever()); + panic_gpio.map_pl011_uart(); + panic_uart + .init(maybe_uart_mmio_start_addr) + .unwrap_or_else(|_| cpu::wait_forever()); + + panic_uart +} + +/// Return a reference to the console. +pub fn console() -> &'static impl console::interface::All { + &super::PL011_UART +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// Minimal code needed to bring up the console in QEMU (for testing only). This is often less steps +/// than on real hardware due to QEMU's abstractions. +/// +/// For the RPi, nothing needs to be done. +pub fn qemu_bring_up_console() {} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs new file mode 100644 index 00000000..8410a961 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/cpu.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP Processor code. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Used by `arch` code to find the early boot core. +pub const BOOT_CORE_ID: usize = 0; diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/driver.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/driver.rs new file mode 100644 index 00000000..313b957e --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/driver.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP driver support. + +use crate::driver; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Device Driver Manager type. +struct BSPDriverManager { + device_drivers: [&'static (dyn DeviceDriver + Sync); 3], +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static BSP_DRIVER_MANAGER: BSPDriverManager = BSPDriverManager { + device_drivers: [ + &super::GPIO, + &super::PL011_UART, + &super::INTERRUPT_CONTROLLER, + ], +}; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the driver manager. +pub fn driver_manager() -> &'static impl driver::interface::DriverManager { + &BSP_DRIVER_MANAGER +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use driver::interface::DeviceDriver; + +impl driver::interface::DriverManager for BSPDriverManager { + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[..] + } + + fn early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[0..=1] + } + + fn non_early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)] { + &self.device_drivers[2..] + } + + fn post_early_print_device_driver_init(&self) { + // Configure PL011Uart's output pins. + super::GPIO.map_pl011_uart(); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception.rs new file mode 100644 index 00000000..55e82119 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! BSP synchronous and asynchronous exception handling. + +pub mod asynchronous; diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception/asynchronous.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception/asynchronous.rs new file mode 100644 index 00000000..787d5926 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/exception/asynchronous.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! BSP asynchronous exception handling. + +use crate::{bsp, exception}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +#[cfg(feature = "bsp_rpi3")] +pub(in crate::bsp) mod irq_map { + use super::bsp::device_driver::{IRQNumber, PeripheralIRQ}; + + pub const PL011_UART: IRQNumber = IRQNumber::Peripheral(PeripheralIRQ::new(57)); +} + +#[cfg(feature = "bsp_rpi4")] +pub(in crate::bsp) mod irq_map { + use super::bsp::device_driver::IRQNumber; + + pub const PL011_UART: IRQNumber = IRQNumber::new(153); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the IRQ manager. +pub fn irq_manager() -> &'static impl exception::asynchronous::interface::IRQManager< + IRQNumberType = bsp::device_driver::IRQNumber, +> { + &super::super::INTERRUPT_CONTROLLER +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/link.ld b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/link.ld new file mode 100644 index 00000000..1e97ff1f --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/link.ld @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: MIT OR Apache-2.0 + * + * Copyright (c) 2018-2020 Andre Richter + */ + +SECTIONS +{ + /* Set current address to the value from which the RPi starts execution */ + . = 0x80000; + + __ro_start = .; + .text : + { + *(.text._start) *(.text*) + } + + .exception_vectors : + { + *(.exception_vectors*) + } + + .rodata : + { + *(.rodata*) + } + . = ALIGN(65536); /* Fill up to 64 KiB */ + __ro_end = .; + + .data : + { + *(.data*) + } + + /* Section is zeroed in u64 chunks, align start and end to 8 bytes */ + .bss ALIGN(8): + { + __bss_start = .; + *(.bss*); + . = ALIGN(8); + __bss_end = .; + } + . = ALIGN(65536); + __data_end = .; + + __ro_size = __ro_end - __ro_start; + __data_size = __data_end - __ro_end; + + /DISCARD/ : { *(.comment*) } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory.rs new file mode 100644 index 00000000..1a9f7312 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP Memory Management. +//! +//! The physical memory layout after the kernel has been loaded by the Raspberry's firmware, which +//! copies the binary to 0x8_0000: +//! +//! +---------------------------------------------+ +//! | | 0x0 +//! | Unmapped | +//! | | 0x6_FFFF +//! +---------------------------------------------+ +//! | BOOT_CORE_STACK_START | 0x7_0000 +//! | | ^ +//! | ... | | Stack growth direction +//! | | | +//! | BOOT_CORE_STACK_END_INCLUSIVE | 0x7_FFFF +//! +---------------------------------------------+ +//! | RO_START == BOOT_CORE_STACK_END | 0x8_0000 +//! | | +//! | | +//! | .text | +//! | .exception_vectors | +//! | .rodata | +//! | | +//! | RO_END_INCLUSIVE | 0x8_0000 + __ro_size - 1 +//! +---------------------------------------------+ +//! | RO_END == DATA_START | 0x8_0000 + __ro_size +//! | | +//! | .data | +//! | .bss | +//! | | +//! | DATA_END_INCLUSIVE | 0x8_0000 + __ro_size + __data_size - 1 +//! +---------------------------------------------+ + +pub mod mmu; + +use crate::memory::mmu::{Address, Physical, Virtual}; +use core::ops::Range; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +// Symbols from the linker script. +extern "C" { + static __ro_start: usize; + static __ro_size: usize; + static __bss_start: usize; + static __bss_end: usize; + static __data_size: usize; +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// The board's physical memory map. +#[rustfmt::skip] +pub(super) mod map { + use super::*; + + pub const BOOT_CORE_STACK_SIZE: usize = 0x1_0000; + + /// Physical devices. + #[cfg(feature = "bsp_rpi3")] + pub mod mmio { + use super::*; + + pub const PERIPHERAL_IC_START: Address = Address::new(0x3F00_B200); + pub const PERIPHERAL_IC_SIZE: usize = 0x24; + + pub const GPIO_START: Address = Address::new(0x3F20_0000); + pub const GPIO_SIZE: usize = 0xA0; + + pub const PL011_UART_START: Address = Address::new(0x3F20_1000); + pub const PL011_UART_SIZE: usize = 0x48; + + pub const LOCAL_IC_START: Address = Address::new(0x4000_0000); + pub const LOCAL_IC_SIZE: usize = 0x100; + + pub const END: Address = Address::new(0x4001_0000); + } + + /// Physical devices. + #[cfg(feature = "bsp_rpi4")] + pub mod mmio { + use super::*; + + pub const GPIO_START: Address = Address::new(0xFE20_0000); + pub const GPIO_SIZE: usize = 0xA0; + + pub const PL011_UART_START: Address = Address::new(0xFE20_1000); + pub const PL011_UART_SIZE: usize = 0x48; + + pub const GICD_START: Address = Address::new(0xFF84_1000); + pub const GICD_SIZE: usize = 0x824; + + pub const GICC_START: Address = Address::new(0xFF84_2000); + pub const GICC_SIZE: usize = 0x14; + + pub const END: Address = Address::new(0xFF85_0000); + } + + pub const END: Address = mmio::END; +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Start address of the Read-Only (RO) range. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn virt_ro_start() -> Address { + Address::new(unsafe { &__ro_start as *const _ as usize }) +} + +/// Size of the Read-Only (RO) range of the kernel binary. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn ro_size() -> usize { + unsafe { &__ro_size as *const _ as usize } +} + +/// Start address of the data range. +#[inline(always)] +fn virt_data_start() -> Address { + virt_ro_start() + ro_size() +} + +/// Size of the data range. +/// +/// # Safety +/// +/// - Value is provided by the linker script and must be trusted as-is. +#[inline(always)] +fn data_size() -> usize { + unsafe { &__data_size as *const _ as usize } +} + +/// Start address of the boot core's stack. +#[inline(always)] +fn virt_boot_core_stack_start() -> Address { + virt_ro_start() - map::BOOT_CORE_STACK_SIZE +} + +/// Size of the boot core's stack. +#[inline(always)] +fn boot_core_stack_size() -> usize { + map::BOOT_CORE_STACK_SIZE +} + +/// Exclusive end address of the physical address space. +#[inline(always)] +fn phys_addr_space_end() -> Address { + map::END +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Exclusive end address of the boot core's stack. +#[inline(always)] +pub fn phys_boot_core_stack_end() -> Address { + // The binary is still identity mapped, so we don't need to convert here. + let end = virt_boot_core_stack_start().into_usize() + boot_core_stack_size(); + Address::new(end) +} + +/// Return the range spanning the .bss section. +/// +/// # Safety +/// +/// - Values are provided by the linker script and must be trusted as-is. +/// - The linker-provided addresses must be u64 aligned. +pub fn bss_range() -> Range<*mut u64> { + unsafe { + Range { + start: &__bss_start as *const _ as *mut u64, + end: &__bss_end as *const _ as *mut u64, + } + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs new file mode 100644 index 00000000..41f55941 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/bsp/raspberrypi/memory/mmu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! BSP Memory Management Unit. + +use crate::{ + common, + memory::{ + mmu as kernel_mmu, + mmu::{ + interface, AccessPermissions, AttributeFields, Granule64KiB, MemAttributes, Page, + PageSliceDescriptor, Physical, Virtual, + }, + }, +}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// The translation granule chosen by this BSP. This will be used everywhere else in the kernel to +/// derive respective data structures and their sizes. For example, the `crate::memory::mmu::Page`. +pub type KernelGranule = Granule64KiB; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +use interface::TranslationGranule; + +/// Helper function for calculating the number of pages the given parameter spans. +const fn size_to_num_pages(size: usize) -> usize { + assert!(size > 0); + assert!(size % KernelGranule::SIZE == 0); + + size >> KernelGranule::SHIFT +} + +/// The boot core's stack. +fn virt_stack_page_desc() -> PageSliceDescriptor { + let num_pages = size_to_num_pages(super::boot_core_stack_size()); + + PageSliceDescriptor::from_addr(super::virt_boot_core_stack_start(), num_pages) +} + +/// The Read-Only (RO) pages of the kernel binary. +fn virt_ro_page_desc() -> PageSliceDescriptor { + let num_pages = size_to_num_pages(super::ro_size()); + + PageSliceDescriptor::from_addr(super::virt_ro_start(), num_pages) +} + +/// The data pages of the kernel binary. +fn virt_data_page_desc() -> PageSliceDescriptor { + let num_pages = size_to_num_pages(super::data_size()); + + PageSliceDescriptor::from_addr(super::virt_data_start(), num_pages) +} + +// The binary is still identity mapped, so we don't need to convert in the following. + +/// The boot core's stack. +fn phys_stack_page_desc() -> PageSliceDescriptor { + virt_stack_page_desc().into() +} + +/// The Read-Only (RO) pages of the kernel binary. +fn phys_ro_page_desc() -> PageSliceDescriptor { + virt_ro_page_desc().into() +} + +/// The data pages of the kernel binary. +fn phys_data_page_desc() -> PageSliceDescriptor { + virt_data_page_desc().into() +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Pointer to the last page of the physical address space. +pub fn phys_addr_space_end_page() -> *const Page { + common::align_down( + super::phys_addr_space_end().into_usize(), + KernelGranule::SIZE, + ) as *const Page<_> +} + +/// Map the kernel binary. +/// +/// # Safety +/// +/// - Any miscalculation or attribute error will likely be fatal. Needs careful manual checking. +pub unsafe fn kernel_map_binary() -> Result<(), &'static str> { + kernel_mmu::kernel_map_pages_at( + "Kernel boot-core stack", + &phys_stack_page_desc(), + &virt_stack_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + kernel_mmu::kernel_map_pages_at( + "Kernel code and RO data", + &phys_ro_page_desc(), + &virt_ro_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadOnly, + execute_never: false, + }, + )?; + + kernel_mmu::kernel_map_pages_at( + "Kernel data and bss", + &phys_data_page_desc(), + &virt_data_page_desc(), + &AttributeFields { + mem_attributes: MemAttributes::CacheableDRAM, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check alignment of the kernel's virtual memory layout sections. + #[kernel_test] + fn virt_mem_layout_sections_are_64KiB_aligned() { + for i in [virt_stack_page_desc, virt_ro_page_desc, virt_data_page_desc].iter() { + let start: usize = i().start_addr().into_usize(); + let end: usize = i().end_addr().into_usize(); + + assert_eq!(start % KernelGranule::SIZE, 0); + assert_eq!(end % KernelGranule::SIZE, 0); + assert!(end >= start); + } + } + + /// Ensure the kernel's virtual memory layout is free of overlaps. + #[kernel_test] + fn virt_mem_layout_has_no_overlaps() { + let layout = [ + virt_stack_page_desc().into_usize_range_inclusive(), + virt_ro_page_desc().into_usize_range_inclusive(), + virt_data_page_desc().into_usize_range_inclusive(), + ]; + + for (i, first_range) in layout.iter().enumerate() { + for second_range in layout.iter().skip(i + 1) { + assert!(!first_range.contains(second_range.start())); + assert!(!first_range.contains(second_range.end())); + assert!(!second_range.contains(first_range.start())); + assert!(!second_range.contains(first_range.end())); + } + } + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/common.rs b/15_virtual_memory_part2_mmio_remap/src/common.rs new file mode 100644 index 00000000..58209804 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/common.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! General purpose code. + +/// Check if a value is aligned to a given size. +#[inline(always)] +pub const fn is_aligned(value: usize, alignment: usize) -> bool { + assert!(alignment.is_power_of_two()); + + (value & (alignment - 1)) == 0 +} + +/// Align down. +#[inline(always)] +pub const fn align_down(value: usize, alignment: usize) -> usize { + assert!(alignment.is_power_of_two()); + + value & !(alignment - 1) +} diff --git a/15_virtual_memory_part2_mmio_remap/src/console.rs b/15_virtual_memory_part2_mmio_remap/src/console.rs new file mode 100644 index 00000000..e6323a20 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/console.rs @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! System console. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Console interfaces. +pub mod interface { + use core::fmt; + + /// Console write functions. + pub trait Write { + /// Write a single character. + fn write_char(&self, c: char); + + /// Write a Rust format string. + fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result; + + /// Block execution until the last character has been physically put on the TX wire + /// (draining TX buffers/FIFOs, if any). + fn flush(&self); + } + + /// Console read functions. + pub trait Read { + /// Read a single character. + fn read_char(&self) -> char { + ' ' + } + + /// Clear RX buffers, if any. + fn clear(&self); + } + + /// Console statistics. + pub trait Statistics { + /// Return the number of characters written. + fn chars_written(&self) -> usize { + 0 + } + + /// Return the number of characters read. + fn chars_read(&self) -> usize { + 0 + } + } + + /// Trait alias for a full-fledged console. + pub trait All = Write + Read + Statistics; +} diff --git a/15_virtual_memory_part2_mmio_remap/src/cpu.rs b/15_virtual_memory_part2_mmio_remap/src/cpu.rs new file mode 100644 index 00000000..9c67c0e7 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/cpu.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Processor code. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/cpu.rs"] +mod arch_cpu; +pub use arch_cpu::*; + +pub mod smp; diff --git a/15_virtual_memory_part2_mmio_remap/src/cpu/smp.rs b/15_virtual_memory_part2_mmio_remap/src/cpu/smp.rs new file mode 100644 index 00000000..b1428884 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/cpu/smp.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Symmetric multiprocessing. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/cpu/smp.rs"] +mod arch_cpu_smp; +pub use arch_cpu_smp::*; diff --git a/15_virtual_memory_part2_mmio_remap/src/driver.rs b/15_virtual_memory_part2_mmio_remap/src/driver.rs new file mode 100644 index 00000000..b6875b7f --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/driver.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Driver support. + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Driver interfaces. +pub mod interface { + /// Device Driver functions. + pub trait DeviceDriver { + /// Return a compatibility string for identifying the driver. + fn compatible(&self) -> &'static str; + + /// Called by the kernel to bring up the device. + /// + /// # Safety + /// + /// - During init, drivers might do stuff with system-wide impact. + unsafe fn init(&self) -> Result<(), &'static str> { + Ok(()) + } + + /// Called by the kernel to register and enable the device's IRQ handlers, if any. + /// + /// Rust's type system will prevent a call to this function unless the calling instance + /// itself has static lifetime. + fn register_and_enable_irq_handler(&'static self) -> Result<(), &'static str> { + Ok(()) + } + + /// After MMIO remapping, returns the new virtual start address. + /// + /// This API assumes a driver has only a single, contiguous MMIO aperture, which will not be + /// the case for more complex devices. This API will likely change in future tutorials. + fn virt_mmio_start_addr(&self) -> Option { + None + } + } + + /// Device driver management functions. + /// + /// The `BSP` is supposed to supply one global instance. + pub trait DriverManager { + /// Return a slice of references to all `BSP`-instantiated drivers. + fn all_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Return only those drivers needed for the BSP's early printing functionality. + /// + /// For example, the default UART. + fn early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Return all drivers minus early-print drivers. + fn non_early_print_device_drivers(&self) -> &[&'static (dyn DeviceDriver + Sync)]; + + /// Initialization code that runs after the early print driver init. + fn post_early_print_device_driver_init(&self); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/exception.rs b/15_virtual_memory_part2_mmio_remap/src/exception.rs new file mode 100644 index 00000000..81ea2b26 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/exception.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Synchronous and asynchronous exception handling. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/exception.rs"] +mod arch_exception; +pub use arch_exception::*; + +pub mod asynchronous; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Kernel privilege levels. +#[allow(missing_docs)] +#[derive(PartialEq)] +pub enum PrivilegeLevel { + User, + Kernel, + Hypervisor, + Unknown, +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Libkernel unit tests must execute in kernel mode. + #[kernel_test] + fn test_runner_executes_in_kernel_mode() { + let (level, _) = current_privilege_level(); + + assert!(level == PrivilegeLevel::Kernel) + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/exception/asynchronous.rs b/15_virtual_memory_part2_mmio_remap/src/exception/asynchronous.rs new file mode 100644 index 00000000..1a3902a3 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/exception/asynchronous.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Asynchronous exception handling. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/exception/asynchronous.rs"] +mod arch_exception_async; +pub use arch_exception_async::*; + +use core::{fmt, marker::PhantomData}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Interrupt descriptor. +#[derive(Copy, Clone)] +pub struct IRQDescriptor { + /// Descriptive name. + pub name: &'static str, + + /// Reference to handler trait object. + pub handler: &'static (dyn interface::IRQHandler + Sync), +} + +/// IRQContext token. +/// +/// An instance of this type indicates that the local core is currently executing in IRQ +/// context, aka executing an interrupt vector or subcalls of it. +/// +/// Concept and implementation derived from the `CriticalSection` introduced in +/// https://github.com/rust-embedded/bare-metal +#[derive(Clone, Copy)] +pub struct IRQContext<'irq_context> { + _0: PhantomData<&'irq_context ()>, +} + +/// Asynchronous exception handling interfaces. +pub mod interface { + + /// Implemented by types that handle IRQs. + pub trait IRQHandler { + /// Called when the corresponding interrupt is asserted. + fn handle(&self) -> Result<(), &'static str>; + } + + /// IRQ management functions. + /// + /// The `BSP` is supposed to supply one global instance. Typically implemented by the + /// platform's interrupt controller. + pub trait IRQManager { + /// The IRQ number type depends on the implementation. + type IRQNumberType; + + /// Register a handler. + fn register_handler( + &self, + irq_number: Self::IRQNumberType, + descriptor: super::IRQDescriptor, + ) -> Result<(), &'static str>; + + /// Enable an interrupt in the controller. + fn enable(&self, irq_number: Self::IRQNumberType); + + /// Handle pending interrupts. + /// + /// This function is called directly from the CPU's IRQ exception vector. On AArch64, + /// this means that the respective CPU core has disabled exception handling. + /// This function can therefore not be preempted and runs start to finish. + /// + /// Takes an IRQContext token to ensure it can only be called from IRQ context. + #[allow(clippy::trivially_copy_pass_by_ref)] + fn handle_pending_irqs<'irq_context>( + &'irq_context self, + ic: &super::IRQContext<'irq_context>, + ); + + /// Print list of registered handlers. + fn print_handler(&self); + } +} + +/// A wrapper type for IRQ numbers with integrated range sanity check. +#[derive(Copy, Clone)] +pub struct IRQNumber(usize); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl<'irq_context> IRQContext<'irq_context> { + /// Creates an IRQContext token. + /// + /// # Safety + /// + /// - This must only be called when the current core is in an interrupt context and will not + /// live beyond the end of it. That is, creation is allowed in interrupt vector functions. For + /// example, in the ARMv8-A case, in `extern "C" fn current_elx_irq()`. + /// - Note that the lifetime `'irq_context` of the returned instance is unconstrained. User code + /// must not be able to influence the lifetime picked for this type, since that might cause it + /// to be inferred to `'static`. + #[inline(always)] + pub unsafe fn new() -> Self { + IRQContext { _0: PhantomData } + } +} + +impl IRQNumber<{ MAX_INCLUSIVE }> { + /// Creates a new instance if number <= MAX_INCLUSIVE. + pub const fn new(number: usize) -> Self { + assert!(number <= MAX_INCLUSIVE); + + Self { 0: number } + } + + /// Return the wrapped number. + pub fn get(self) -> usize { + self.0 + } +} + +impl fmt::Display for IRQNumber<{ MAX_INCLUSIVE }> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Executes the provided closure while IRQs are masked on the executing core. +/// +/// While the function temporarily changes the HW state of the executing core, it restores it to the +/// previous state before returning, so this is deemed safe. +#[inline(always)] +pub fn exec_with_irq_masked(f: impl FnOnce() -> T) -> T { + let ret: T; + + unsafe { + let saved = local_irq_mask_save(); + ret = f(); + local_irq_restore(saved); + } + + ret +} diff --git a/15_virtual_memory_part2_mmio_remap/src/lib.rs b/15_virtual_memory_part2_mmio_remap/src/lib.rs new file mode 100644 index 00000000..55641341 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/lib.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc(html_logo_url = "https://git.io/JeGIp")] + +//! The `kernel` library. +//! +//! Used by `main.rs` to compose the final kernel binary. +//! +//! # TL;DR - Overview of important Kernel entities +//! +//! - [`bsp::console::console()`] - Returns a reference to the kernel's [console interface]. +//! - [`bsp::driver::driver_manager()`] - Returns a reference to the kernel's [driver interface]. +//! - [`bsp::exception::asynchronous::irq_manager()`] - Returns a reference to the kernel's [IRQ +//! Handling interface]. +//! - [`memory::mmu::mmu()`] - Returns a reference to the kernel's [MMU interface]. +//! - [`state::state_manager()`] - Returns a reference to the kernel's [state management] instance. +//! - [`time::time_manager()`] - Returns a reference to the kernel's [timer interface]. +//! +//! [console interface]: ../libkernel/console/interface/index.html +//! [driver interface]: ../libkernel/driver/interface/trait.DriverManager.html +//! [IRQ Handling interface]: ../libkernel/exception/asynchronous/interface/trait.IRQManager.html +//! [MMU interface]: ../libkernel/memory/mmu/interface/trait.MMU.html +//! [state management]: ../libkernel/state/struct.StateManager.html +//! [timer interface]: ../libkernel/time/interface/trait.TimeManager.html +//! +//! # Code organization and architecture +//! +//! The code is divided into different *modules*, each representing a typical **subsystem** of the +//! `kernel`. Top-level module files of subsystems reside directly in the `src` folder. For example, +//! `src/memory.rs` contains code that is concerned with all things memory management. +//! +//! ## Visibility of processor architecture code +//! +//! Some of the `kernel`'s subsystems depend on low-level code that is specific to the target +//! processor architecture. For each supported processor architecture, there exists a subfolder in +//! `src/_arch`, for example, `src/_arch/aarch64`. +//! +//! The architecture folders mirror the subsystem modules laid out in `src`. For example, +//! architectural code that belongs to the `kernel`'s memory subsystem (`src/memory.rs`) would go +//! into `src/_arch/aarch64/memory.rs`. The latter file is directly included and re-exported in +//! `src/memory.rs`, so that the architectural code parts are transparent with respect to the code's +//! module organization. That means a public function `foo()` defined in +//! `src/_arch/aarch64/memory.rs` would be reachable as `crate::memory::foo()` only. +//! +//! The `_` in `_arch` denotes that this folder is not part of the standard module hierarchy. +//! Rather, it's contents are conditionally pulled into respective files using the `#[path = +//! "_arch/xxx/yyy.rs"]` attribute. +//! +//! ## BSP code +//! +//! `BSP` stands for Board Support Package. `BSP` code is organized under `src/bsp.rs` and contains +//! target board specific definitions and functions. These are things such as the board's memory map +//! or instances of drivers for devices that are featured on the respective board. +//! +//! Just like processor architecture code, the `BSP` code's module structure tries to mirror the +//! `kernel`'s subsystem modules, but there is no transparent re-exporting this time. That means +//! whatever is provided must be called starting from the `bsp` namespace, e.g. +//! `bsp::driver::driver_manager()`. +//! +//! ## Kernel interfaces +//! +//! Both `arch` and `bsp` contain code that is conditionally compiled depending on the actual target +//! and board for which the kernel is compiled. For example, the `interrupt controller` hardware of +//! the `Raspberry Pi 3` and the `Raspberry Pi 4` is different, but we want the rest of the `kernel` +//! code to play nicely with any of the two without much hassle. +//! +//! In order to provide a clean abstraction between `arch`, `bsp` and `generic kernel code`, +//! `interface` traits are provided *whenever possible* and *where it makes sense*. They are defined +//! in the respective subsystem module and help to enforce the idiom of *program to an interface, +//! not an implementation*. For example, there will be a common IRQ handling interface which the two +//! different interrupt controller `drivers` of both Raspberrys will implement, and only export the +//! interface to the rest of the `kernel`. +//! +//! ``` +//! +-------------------+ +//! | Interface (Trait) | +//! | | +//! +--+-------------+--+ +//! ^ ^ +//! | | +//! | | +//! +----------+--+ +--+----------+ +//! | kernel code | | bsp code | +//! | | | arch code | +//! +-------------+ +-------------+ +//! ``` +//! +//! # Summary +//! +//! For a logical `kernel` subsystem, corresponding code can be distributed over several physical +//! locations. Here is an example for the **memory** subsystem: +//! +//! - `src/memory.rs` and `src/memory/**/*` +//! - Common code that is agnostic of target processor architecture and `BSP` characteristics. +//! - Example: A function to zero a chunk of memory. +//! - Interfaces for the memory subsystem that are implemented by `arch` or `BSP` code. +//! - Example: An `MMU` interface that defines `MMU` function prototypes. +//! - `src/bsp/__board_name__/memory.rs` and `src/bsp/__board_name__/memory/**/*` +//! - `BSP` specific code. +//! - Example: The board's memory map (physical addresses of DRAM and MMIO devices). +//! - `src/_arch/__arch_name__/memory.rs` and `src/_arch/__arch_name__/memory/**/*` +//! - Processor architecture specific code. +//! - Example: Implementation of the `MMU` interface for the `__arch_name__` processor +//! architecture. +//! +//! From a namespace perspective, **memory** subsystem code lives in: +//! +//! - `crate::memory::*` +//! - `crate::bsp::memory::*` + +#![allow(incomplete_features)] +#![feature(asm)] +#![feature(const_fn)] +#![feature(const_generics)] +#![feature(const_panic)] +#![feature(core_intrinsics)] +#![feature(format_args_nl)] +#![feature(global_asm)] +#![feature(linkage)] +#![feature(naked_functions)] +#![feature(panic_info_message)] +#![feature(trait_alias)] +#![no_std] +// Testing +#![cfg_attr(test, no_main)] +#![feature(custom_test_frameworks)] +#![reexport_test_harness_main = "test_main"] +#![test_runner(crate::test_runner)] + +// `mod cpu` provides the `_start()` function, the first function to run. `_start()` then calls +// `runtime_init()`, which jumps to `kernel_init()` (defined in `main.rs`). + +mod panic_wait; +mod runtime_init; +mod synchronization; + +pub mod bsp; +pub mod common; +pub mod console; +pub mod cpu; +pub mod driver; +pub mod exception; +pub mod memory; +pub mod print; +pub mod state; +pub mod time; + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// The default runner for unit tests. +pub fn test_runner(tests: &[&test_types::UnitTest]) { + println!("Running {} tests", tests.len()); + println!("-------------------------------------------------------------------\n"); + for (i, test) in tests.iter().enumerate() { + print!("{:>3}. {:.<58}", i + 1, test.name); + + // Run the actual test. + (test.test_func)(); + + // Failed tests call panic!(). Execution reaches here only if the test has passed. + println!("[ok]") + } +} + +/// The `kernel_init()` for unit tests. Called from `runtime_init()`. +#[cfg(test)] +#[no_mangle] +unsafe fn kernel_init() -> ! { + bsp::console::qemu_bring_up_console(); + + test_main(); + + cpu::qemu_exit_success() +} diff --git a/15_virtual_memory_part2_mmio_remap/src/main.rs b/15_virtual_memory_part2_mmio_remap/src/main.rs new file mode 100644 index 00000000..d693bee6 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/main.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +// Rust embedded logo for `make doc`. +#![doc(html_logo_url = "https://git.io/JeGIp")] + +//! The `kernel` binary. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +use libkernel::{bsp, cpu, driver, exception, info, memory, state, time, warn}; + +/// Early init code. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +/// - The init calls in this function must appear in the correct order: +/// - Virtual memory must be activated before the device drivers. +/// - Without it, any atomic operations, e.g. the yet-to-be-introduced spinlocks in the device +/// drivers (which currently employ IRQSafeNullLocks instead of spinlocks), will fail to +/// work on the RPi SoCs. +#[no_mangle] +unsafe fn kernel_init() -> ! { + use driver::interface::DriverManager; + + exception::handling_init(); + + if let Err(string) = memory::mmu::kernel_map_binary_and_enable_mmu() { + panic!("Enabling MMU failed: {}", string); + } + // Printing will silently fail fail from here on, because the driver's MMIO is not remapped yet. + + // Bring up the drivers needed for printing first. + for i in bsp::driver::driver_manager() + .early_print_device_drivers() + .iter() + { + // Any encountered errors cannot be printed yet, obviously, so just safely park the CPU. + i.init().unwrap_or_else(|_| cpu::wait_forever()); + } + bsp::driver::driver_manager().post_early_print_device_driver_init(); + // Printing available again from here on. + + // Now bring up the remaining drivers. + for i in bsp::driver::driver_manager() + .non_early_print_device_drivers() + .iter() + { + if let Err(x) = i.init() { + panic!("Error loading driver: {}: {}", i.compatible(), x); + } + } + + // Let device drivers register and enable their handlers with the interrupt controller. + for i in bsp::driver::driver_manager().all_device_drivers() { + if let Err(msg) = i.register_and_enable_irq_handler() { + warn!("Error registering IRQ handler: {}", msg); + } + } + + // Unmask interrupts on the boot CPU core. + exception::asynchronous::local_irq_unmask(); + + // Announce conclusion of the kernel_init() phase. + state::state_manager().transition_to_single_core_main(); + + // Transition from unsafe to safe. + kernel_main() +} + +/// The main function running after the early init. +fn kernel_main() -> ! { + use driver::interface::DriverManager; + use exception::asynchronous::interface::IRQManager; + + info!("Booting on: {}", bsp::board_name()); + + info!("MMU online:"); + memory::mmu::kernel_print_mappings(); + + let (_, privilege_level) = exception::current_privilege_level(); + info!("Current privilege level: {}", privilege_level); + + info!("Exception handling state:"); + exception::asynchronous::print_state(); + + info!( + "Architectural timer resolution: {} ns", + time::time_manager().resolution().as_nanos() + ); + + info!("Drivers loaded:"); + for (i, driver) in bsp::driver::driver_manager() + .all_device_drivers() + .iter() + .enumerate() + { + info!(" {}. {}", i + 1, driver.compatible()); + } + + info!("Registered IRQ handlers:"); + bsp::exception::asynchronous::irq_manager().print_handler(); + + info!("Echoing input now"); + cpu::wait_forever(); +} diff --git a/15_virtual_memory_part2_mmio_remap/src/memory.rs b/15_virtual_memory_part2_mmio_remap/src/memory.rs new file mode 100644 index 00000000..65d4b92b --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/memory.rs @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Memory Management. + +pub mod mmu; + +use core::ops::Range; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Zero out a memory range. +/// +/// # Safety +/// +/// - `range.start` and `range.end` must be valid. +/// - `range.start` and `range.end` must be `T` aligned. +pub unsafe fn zero_volatile(range: Range<*mut T>) +where + T: From, +{ + let mut ptr = range.start; + + while ptr < range.end { + core::ptr::write_volatile(ptr, T::from(0)); + ptr = ptr.offset(1); + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check `zero_volatile()`. + #[kernel_test] + fn zero_volatile_works() { + let mut x: [usize; 3] = [10, 11, 12]; + let x_range = x.as_mut_ptr_range(); + + unsafe { zero_volatile(x_range) }; + + assert_eq!(x, [0, 0, 0]); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/memory/mmu.rs b/15_virtual_memory_part2_mmio_remap/src/memory/mmu.rs new file mode 100644 index 00000000..778fd646 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/memory/mmu.rs @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Memory Management Unit. + +#[cfg(target_arch = "aarch64")] +#[path = "../_arch/aarch64/memory/mmu.rs"] +mod arch_mmu; +pub use arch_mmu::*; + +mod mapping_record; +mod types; + +use crate::{bsp, synchronization, warn}; + +pub use types::*; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Memory Management interfaces. +pub mod interface { + use super::*; + + /// Describes the characteristics of a translation granule. + #[allow(missing_docs)] + pub trait TranslationGranule { + const SIZE: usize; + const MASK: usize = Self::SIZE - 1; + const SHIFT: usize; + } + + /// Translation table operations. + pub trait TranslationTable { + /// Anything that needs to run before any of the other provided functions can be used. + /// + /// # Safety + /// + /// - Implementor must ensure that this function can run only once or is harmless if invoked + /// multiple times. + unsafe fn init(&mut self); + + /// The translation table's base address to be used for programming the MMU. + fn phys_base_address(&self) -> Address; + + /// Map the given physical pages to the given virtual pages. + /// + /// # Safety + /// + /// - Using wrong attributes can cause multiple issues of different nature in the system. + /// - It is not required that the architectural implementation prevents aliasing. That is, + /// mapping to the same physical memory using multiple virtual addresses, which would + /// break Rust's ownership assumptions. This should be protected against in this module + /// (the kernel's generic MMU code). + unsafe fn map_pages_at( + &mut self, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str>; + + /// Obtain a free virtual page slice in the MMIO region. + /// + /// The "MMIO region" is a distinct region of the implementor's choice, which allows + /// differentiating MMIO addresses from others. This can speed up debugging efforts. + /// Ideally, those MMIO addresses are also standing out visually so that a human eye can + /// identify them. For example, by allocating them from near the end of the virtual address + /// space. + fn next_mmio_virt_page_slice( + &mut self, + num_pages: usize, + ) -> Result, &'static str>; + + /// Check if a virtual page splice is in the "MMIO region". + fn is_virt_page_slice_mmio(&self, virt_pages: &PageSliceDescriptor) -> bool; + } + + /// MMU functions. + pub trait MMU { + /// Turns on the MMU. + /// + /// # Safety + /// + /// - Must only be called after the kernel translation tables have been init()'ed. + /// - Changes the HW's global state. + unsafe fn enable( + &self, + phys_kernel_table_base_addr: Address, + ) -> Result<(), &'static str>; + } +} + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- +use interface::{TranslationTable, MMU}; +use synchronization::interface::ReadWriteEx; + +/// Map pages in the kernel's translation tables. +/// +/// No input checks done, input is passed through to the architectural implementation. +/// +/// # Safety +/// +/// - See `map_pages_at()`. +/// - Does not prevent aliasing. +unsafe fn kernel_map_pages_at_unchecked( + name: &'static str, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, +) -> Result<(), &'static str> { + arch_mmu::kernel_translation_tables() + .write(|tables| tables.map_pages_at(phys_pages, virt_pages, attr))?; + + if let Err(x) = mapping_record::kernel_add(name, phys_pages, virt_pages, attr) { + warn!("{}", x); + } + + Ok(()) +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use interface::TranslationGranule; + +/// Raw mapping of virtual to physical pages in the kernel translation tables. +/// +/// Prevents mapping into the MMIO range of the tables. +/// +/// # Safety +/// +/// - See `kernel_map_pages_at_unchecked()`. +/// - Does not prevent aliasing. Currently, we have to trust the callers. +pub unsafe fn kernel_map_pages_at( + name: &'static str, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, +) -> Result<(), &'static str> { + let is_mmio = arch_mmu::kernel_translation_tables() + .read(|tables| tables.is_virt_page_slice_mmio(virt_pages)); + if is_mmio { + return Err("Attempt to manually map into MMIO region"); + } + + kernel_map_pages_at_unchecked(name, phys_pages, virt_pages, attr)?; + + Ok(()) +} + +/// MMIO remapping in the kernel translation tables. +/// +/// Typically used by device drivers. +/// +/// # Safety +/// +/// - Same as `kernel_map_pages_at_unchecked()`, minus the aliasing part. +pub unsafe fn kernel_map_mmio( + name: &'static str, + phys_mmio_descriptor: &MMIODescriptor, +) -> Result, &'static str> { + let phys_pages: PageSliceDescriptor = phys_mmio_descriptor.clone().into(); + let offset_into_start_page = + phys_mmio_descriptor.start_addr().into_usize() & bsp::memory::mmu::KernelGranule::MASK; + + // Check if an identical page slice has been mapped for another driver. If so, reuse it. + let virt_addr = if let Some(addr) = + mapping_record::kernel_find_and_insert_mmio_duplicate(phys_mmio_descriptor, name) + { + addr + // Otherwise, allocate a new virtual page slice and map it. + } else { + let virt_pages: PageSliceDescriptor = arch_mmu::kernel_translation_tables() + .write(|tables| tables.next_mmio_virt_page_slice(phys_pages.num_pages()))?; + + kernel_map_pages_at_unchecked( + name, + &phys_pages, + &virt_pages, + &AttributeFields { + mem_attributes: MemAttributes::Device, + acc_perms: AccessPermissions::ReadWrite, + execute_never: true, + }, + )?; + + virt_pages.start_addr() + }; + + Ok(virt_addr + offset_into_start_page) +} + +/// Map the kernel's binary and enable the MMU. +/// +/// # Safety +/// +/// - Crucial function during kernel init. Changes the the complete memory view of the processor. +pub unsafe fn kernel_map_binary_and_enable_mmu() -> Result<(), &'static str> { + let phys_base_addr = arch_mmu::kernel_translation_tables().write(|tables| { + tables.init(); + tables.phys_base_address() + }); + + bsp::memory::mmu::kernel_map_binary()?; + arch_mmu::mmu().enable(phys_base_addr) +} + +/// Human-readable print of all recorded kernel mappings. +pub fn kernel_print_mappings() { + mapping_record::kernel_print() +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Sanity checks for the kernel TranslationTable implementation. + #[kernel_test] + fn translationtable_implementation_sanity() { + // Need to take care that `tables` fits into the stack. + let mut tables = MinSizeArchTranslationTable::new(); + + unsafe { tables.init() }; + + let x = tables.next_mmio_virt_page_slice(0); + assert!(x.is_err()); + + let x = tables.next_mmio_virt_page_slice(1_0000_0000); + assert!(x.is_err()); + + let x = tables.next_mmio_virt_page_slice(2).unwrap(); + assert_eq!(x.size(), bsp::memory::mmu::KernelGranule::SIZE * 2); + + assert_eq!(tables.is_virt_page_slice_mmio(&x), true); + + assert_eq!( + tables.is_virt_page_slice_mmio(&PageSliceDescriptor::from_addr(Address::new(0), 1)), + false + ); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/memory/mmu/mapping_record.rs b/15_virtual_memory_part2_mmio_remap/src/memory/mmu/mapping_record.rs new file mode 100644 index 00000000..9263f8c6 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/memory/mmu/mapping_record.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! A record of mapped pages. + +use super::{ + AccessPermissions, Address, AttributeFields, MMIODescriptor, MemAttributes, + PageSliceDescriptor, Physical, Virtual, +}; +use crate::{info, synchronization, synchronization::InitStateLock, warn}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Type describing a virtual memory mapping. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +struct MappingRecordEntry { + pub users: [Option<&'static str>; 5], + pub phys_pages: PageSliceDescriptor, + pub virt_start_addr: Address, + pub attribute_fields: AttributeFields, +} + +struct MappingRecord { + inner: [Option; 12], +} + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static KERNEL_MAPPING_RECORD: InitStateLock = + InitStateLock::new(MappingRecord::new()); + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +impl MappingRecordEntry { + pub fn new( + name: &'static str, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Self { + Self { + users: [Some(name), None, None, None, None], + phys_pages: *phys_pages, + virt_start_addr: virt_pages.start_addr(), + attribute_fields: *attr, + } + } + + fn find_next_free_user(&mut self) -> Result<&mut Option<&'static str>, &'static str> { + if let Some(x) = self.users.iter_mut().find(|x| x.is_none()) { + return Ok(x); + }; + + Err("Storage for user info exhausted") + } + + pub fn add_user(&mut self, user: &'static str) -> Result<(), &'static str> { + let x = self.find_next_free_user()?; + *x = Some(user); + Ok(()) + } +} + +impl MappingRecord { + pub const fn new() -> Self { + Self { inner: [None; 12] } + } + + fn find_next_free(&mut self) -> Result<&mut Option, &'static str> { + if let Some(x) = self.inner.iter_mut().find(|x| x.is_none()) { + return Ok(x); + } + + Err("Storage for mapping info exhausted") + } + + fn find_duplicate( + &mut self, + phys_pages: &PageSliceDescriptor, + ) -> Option<&mut MappingRecordEntry> { + self.inner + .iter_mut() + .filter(|x| x.is_some()) + .map(|x| x.as_mut().unwrap()) + .filter(|x| x.attribute_fields.mem_attributes == MemAttributes::Device) + .find(|x| x.phys_pages == *phys_pages) + } + + pub fn add( + &mut self, + name: &'static str, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, + ) -> Result<(), &'static str> { + let x = self.find_next_free()?; + + *x = Some(MappingRecordEntry::new(name, phys_pages, virt_pages, attr)); + Ok(()) + } + + pub fn print(&self) { + const KIB_RSHIFT: u32 = 10; // log2(1024). + const MIB_RSHIFT: u32 = 20; // log2(1024 * 1024). + + info!(" -----------------------------------------------------------------------------------------------------------------"); + info!( + " {:^24} {:^24} {:^7} {:^9} {:^35}", + "Virtual", "Physical", "Size", "Attr", "Entity" + ); + info!(" -----------------------------------------------------------------------------------------------------------------"); + + for i in self + .inner + .iter() + .filter(|x| x.is_some()) + .map(|x| x.unwrap()) + { + let virt_start = i.virt_start_addr.into_usize(); + let virt_end_inclusive = virt_start + i.phys_pages.size() - 1; + let phys_start = i.phys_pages.start_addr().into_usize(); + let phys_end_inclusive = i.phys_pages.end_addr_inclusive().into_usize(); + let size = i.phys_pages.size(); + + let (size, unit) = if (size >> MIB_RSHIFT) > 0 { + (size >> MIB_RSHIFT, "MiB") + } else if (size >> KIB_RSHIFT) > 0 { + (size >> KIB_RSHIFT, "KiB") + } else { + (size, "Byte") + }; + + let attr = match i.attribute_fields.mem_attributes { + MemAttributes::CacheableDRAM => "C", + MemAttributes::Device => "Dev", + }; + + let acc_p = match i.attribute_fields.acc_perms { + AccessPermissions::ReadOnly => "RO", + AccessPermissions::ReadWrite => "RW", + }; + + let xn = if i.attribute_fields.execute_never { + "XN" + } else { + "X" + }; + + info!( + " {:#011X}..{:#011X} --> {:#011X}..{:#011X} | \ + {: >3} {} | {: <3} {} {: <2} | {}", + virt_start, + virt_end_inclusive, + phys_start, + phys_end_inclusive, + size, + unit, + attr, + acc_p, + xn, + i.users[0].unwrap() + ); + + for k in i.users[1..].iter() { + if let Some(additional_user) = *k { + info!( + " | {}", + additional_user + ); + } + } + } + + info!(" -----------------------------------------------------------------------------------------------------------------"); + } +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- +use synchronization::interface::ReadWriteEx; + +/// Add an entry to the mapping info record. +pub fn kernel_add( + name: &'static str, + phys_pages: &PageSliceDescriptor, + virt_pages: &PageSliceDescriptor, + attr: &AttributeFields, +) -> Result<(), &'static str> { + let mut m = &KERNEL_MAPPING_RECORD; + m.write(|mr| mr.add(name, phys_pages, virt_pages, attr)) +} + +pub fn kernel_find_and_insert_mmio_duplicate( + phys_mmio_descriptor: &MMIODescriptor, + new_user: &'static str, +) -> Option> { + let phys_pages: PageSliceDescriptor = phys_mmio_descriptor.clone().into(); + + let mut m = &KERNEL_MAPPING_RECORD; + m.write(|mr| { + let dup = mr.find_duplicate(&phys_pages)?; + + if let Err(x) = dup.add_user(new_user) { + warn!("{}", x); + } + + Some(dup.virt_start_addr) + }) +} + +/// Human-readable print of all recorded kernel mappings. +pub fn kernel_print() { + let mut m = &KERNEL_MAPPING_RECORD; + m.read(|mr| mr.print()); +} diff --git a/15_virtual_memory_part2_mmio_remap/src/memory/mmu/types.rs b/15_virtual_memory_part2_mmio_remap/src/memory/mmu/types.rs new file mode 100644 index 00000000..d0098f6f --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/memory/mmu/types.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Memory Management Unit Types. + +use crate::{bsp, common}; +use core::{convert::From, marker::PhantomData, ops::RangeInclusive}; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- +use super::interface::TranslationGranule; + +/// Metadata trait for marking the type of an address. +pub trait AddressType: Copy + Clone + PartialOrd + PartialEq {} + +/// Zero-sized type to mark a physical address. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub enum Physical {} + +/// Zero-sized type to mark a virtual address. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub enum Virtual {} + +/// Generic address type. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub struct Address { + value: usize, + _address_type: PhantomData, +} + +/// Generic page type. +#[repr(C)] +pub struct Page { + inner: [u8; bsp::memory::mmu::KernelGranule::SIZE], + _address_type: PhantomData, +} + +/// Type describing a slice of pages. +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub struct PageSliceDescriptor { + start: Address, + num_pages: usize, +} + +/// Architecture agnostic memory attributes. +#[allow(missing_docs)] +#[derive(Copy, Clone, PartialOrd, PartialEq)] +pub enum MemAttributes { + CacheableDRAM, + Device, +} + +/// Architecture agnostic access permissions. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +pub enum AccessPermissions { + ReadOnly, + ReadWrite, +} + +/// Collection of memory attributes. +#[allow(missing_docs)] +#[derive(Copy, Clone)] +pub struct AttributeFields { + pub mem_attributes: MemAttributes, + pub acc_perms: AccessPermissions, + pub execute_never: bool, +} + +/// An MMIO descriptor for use in device drivers. +#[derive(Copy, Clone)] +pub struct MMIODescriptor { + start_addr: Address, + size: usize, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +impl AddressType for Physical {} +impl AddressType for Virtual {} + +//------------------------------------------------------------------------------ +// Address +//------------------------------------------------------------------------------ + +impl Address { + /// Create an instance. + pub const fn new(value: usize) -> Self { + Self { + value, + _address_type: PhantomData, + } + } + + /// Align down. + pub const fn align_down(self, alignment: usize) -> Self { + let aligned = common::align_down(self.value, alignment); + + Self { + value: aligned, + _address_type: PhantomData, + } + } + + /// Converts `Address` into an usize. + pub const fn into_usize(self) -> usize { + self.value + } +} + +impl core::ops::Add for Address { + type Output = Self; + + fn add(self, other: usize) -> Self { + Self { + value: self.value + other, + _address_type: PhantomData, + } + } +} + +impl core::ops::Sub for Address { + type Output = Self; + + fn sub(self, other: usize) -> Self { + Self { + value: self.value - other, + _address_type: PhantomData, + } + } +} + +//------------------------------------------------------------------------------ +// Page +//------------------------------------------------------------------------------ + +impl Page { + /// Get a pointer to the instance. + pub const fn as_ptr(&self) -> *const Page { + self as *const _ + } +} + +//------------------------------------------------------------------------------ +// PageSliceDescriptor +//------------------------------------------------------------------------------ + +impl PageSliceDescriptor { + /// Create an instance. + pub const fn from_addr(start: Address, num_pages: usize) -> Self { + assert!(common::is_aligned( + start.into_usize(), + bsp::memory::mmu::KernelGranule::SIZE + )); + assert!(num_pages > 0); + + Self { start, num_pages } + } + + /// Return a pointer to the first page of the described slice. + const fn first_page_ptr(&self) -> *const Page { + self.start.into_usize() as *const _ + } + + /// Return the number of Pages the slice describes. + pub const fn num_pages(&self) -> usize { + self.num_pages + } + + /// Return the memory size this descriptor spans. + pub const fn size(&self) -> usize { + self.num_pages * bsp::memory::mmu::KernelGranule::SIZE + } + + /// Return the start address. + pub const fn start_addr(&self) -> Address { + self.start + } + + /// Return the exclusive end address. + pub fn end_addr(&self) -> Address { + self.start + self.size() + } + + /// Return the inclusive end address. + pub fn end_addr_inclusive(&self) -> Address { + self.start + (self.size() - 1) + } + + /// Return a non-mutable slice of Pages. + /// + /// # Safety + /// + /// - Same as applies for `core::slice::from_raw_parts`. + pub unsafe fn as_slice(&self) -> &[Page] { + core::slice::from_raw_parts(self.first_page_ptr(), self.num_pages) + } + + /// Return the inclusive address range of the slice. + pub fn into_usize_range_inclusive(self) -> RangeInclusive { + RangeInclusive::new( + self.start_addr().into_usize(), + self.end_addr_inclusive().into_usize(), + ) + } +} + +impl From> for PageSliceDescriptor { + fn from(desc: PageSliceDescriptor) -> Self { + Self { + start: Address::new(desc.start.into_usize()), + num_pages: desc.num_pages, + } + } +} + +impl From> for PageSliceDescriptor { + fn from(desc: MMIODescriptor) -> Self { + let start_page_addr = desc + .start_addr + .align_down(bsp::memory::mmu::KernelGranule::SIZE); + + let len = ((desc.end_addr_inclusive().into_usize() - start_page_addr.into_usize()) + >> bsp::memory::mmu::KernelGranule::SHIFT) + + 1; + + Self { + start: start_page_addr, + num_pages: len, + } + } +} + +//------------------------------------------------------------------------------ +// MMIODescriptor +//------------------------------------------------------------------------------ + +impl MMIODescriptor { + /// Create an instance. + pub const fn new(start_addr: Address, size: usize) -> Self { + assert!(size > 0); + + Self { start_addr, size } + } + + /// Return the start address. + pub const fn start_addr(&self) -> Address { + self.start_addr + } + + /// Return the inclusive end address. + pub fn end_addr_inclusive(&self) -> Address { + self.start_addr + (self.size - 1) + } + + /// Return the size. + pub const fn size(&self) -> usize { + self.size + } +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check if the size of `struct Page` is as expected. + #[kernel_test] + fn size_of_page_equals_granule_size() { + assert_eq!( + core::mem::size_of::>(), + bsp::memory::mmu::KernelGranule::SIZE + ); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/panic_wait.rs b/15_virtual_memory_part2_mmio_remap/src/panic_wait.rs new file mode 100644 index 00000000..52a89d17 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/panic_wait.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! A panic handler that infinitely waits. + +use crate::{bsp, cpu, exception}; +use core::{fmt, panic::PanicInfo}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +fn _panic_print(args: fmt::Arguments) { + use fmt::Write; + + unsafe { bsp::console::panic_console_out().write_fmt(args).unwrap() }; +} + +/// The point of exit for the "standard" (non-testing) `libkernel`. +/// +/// This code will be used by the release kernel binary and the `integration tests`. It is linked +/// weakly, so that the integration tests can overload it to exit `QEMU` instead of spinning +/// forever. +/// +/// This is one possible approach to solve the problem that `cargo` can not know who the consumer of +/// the library will be: +/// - The release kernel binary that should safely park the paniced core, +/// - or an `integration test` that is executed in QEMU, which should just exit QEMU. +#[cfg(not(test))] +#[linkage = "weak"] +#[no_mangle] +fn _panic_exit() -> ! { + cpu::wait_forever() +} + +/// Prints with a newline - only use from the panic handler. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! panic_println { + ($($arg:tt)*) => ({ + _panic_print(format_args_nl!($($arg)*)); + }) +} + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + unsafe { exception::asynchronous::local_irq_mask() }; + + if let Some(args) = info.message() { + panic_println!("\nKernel panic: {}", args); + } else { + panic_println!("\nKernel panic!"); + } + + _panic_exit() +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +/// The point of exit when the library is compiled for testing. +#[cfg(test)] +#[no_mangle] +fn _panic_exit() -> ! { + cpu::qemu_exit_failure() +} diff --git a/15_virtual_memory_part2_mmio_remap/src/print.rs b/15_virtual_memory_part2_mmio_remap/src/print.rs new file mode 100644 index 00000000..8b6f3f98 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/print.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Printing facilities. + +use crate::{bsp, console}; +use core::fmt; + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +#[doc(hidden)] +pub fn _print(args: fmt::Arguments) { + use console::interface::Write; + + bsp::console::console().write_fmt(args).unwrap(); +} + +/// Prints without a newline. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! print { + ($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*))); +} + +/// Prints with a newline. +/// +/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html +#[macro_export] +macro_rules! println { + () => ($crate::print!("\n")); + ($($arg:tt)*) => ({ + $crate::print::_print(format_args_nl!($($arg)*)); + }) +} + +/// Prints an info, with a newline. +#[macro_export] +macro_rules! info { + ($string:expr) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:03}{:03}] ", $string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000 + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[ {:>3}.{:03}{:03}] ", $format_string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000, + $($arg)* + )); + }) +} + +/// Prints a warning, with a newline. +#[macro_export] +macro_rules! warn { + ($string:expr) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:03}{:03}] ", $string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000 + )); + }); + ($format_string:expr, $($arg:tt)*) => ({ + #[allow(unused_imports)] + use crate::time::interface::TimeManager; + + let timestamp = $crate::time::time_manager().uptime(); + let timestamp_subsec_us = timestamp.subsec_micros(); + + $crate::print::_print(format_args_nl!( + concat!("[W {:>3}.{:03}{:03}] ", $format_string), + timestamp.as_secs(), + timestamp_subsec_us / 1_000, + timestamp_subsec_us % 1_000, + $($arg)* + )); + }) +} diff --git a/15_virtual_memory_part2_mmio_remap/src/runtime_init.rs b/15_virtual_memory_part2_mmio_remap/src/runtime_init.rs new file mode 100644 index 00000000..a882dbe3 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/runtime_init.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2018-2020 Andre Richter + +//! Rust runtime initialization code. + +use crate::{bsp, memory}; + +//-------------------------------------------------------------------------------------------------- +// Private Code +//-------------------------------------------------------------------------------------------------- + +/// Zero out the .bss section. +/// +/// # Safety +/// +/// - Must only be called pre `kernel_init()`. +#[inline(always)] +unsafe fn zero_bss() { + memory::zero_volatile(bsp::memory::bss_range()); +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section, then jumps to kernel +/// init code. +/// +/// # Safety +/// +/// - Only a single core must be active and running this function. +pub unsafe fn runtime_init() -> ! { + extern "Rust" { + fn kernel_init() -> !; + } + + zero_bss(); + kernel_init() +} + +//-------------------------------------------------------------------------------------------------- +// Testing +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use test_macros::kernel_test; + + /// Check `bss` section layout. + #[kernel_test] + fn bss_section_is_sane() { + use core::mem; + + let start = bsp::memory::bss_range().start as *const _ as usize; + let end = bsp::memory::bss_range().end as *const _ as usize; + + assert_eq!(start % mem::size_of::(), 0); + assert_eq!(end % mem::size_of::(), 0); + assert!(end >= start); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/state.rs b/15_virtual_memory_part2_mmio_remap/src/state.rs new file mode 100644 index 00000000..af1d9348 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/state.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! State information about the kernel itself. + +use core::sync::atomic::{AtomicU8, Ordering}; + +//-------------------------------------------------------------------------------------------------- +// Private Definitions +//-------------------------------------------------------------------------------------------------- + +/// Different stages in the kernel execution. +#[derive(Copy, Clone, Eq, PartialEq)] +enum State { + /// The kernel starts booting in this state. + Init, + + /// The kernel transitions to this state when jumping to `kernel_main()` (at the end of + /// `kernel_init()`, after all init calls are done). + SingleCoreMain, + + /// The kernel transitions to this state when it boots the secondary cores, aka switches + /// exectution mode to symmetric multiprocessing (SMP). + MultiCoreMain, +} + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Maintains the kernel state and state transitions. +pub struct StateManager(AtomicU8); + +//-------------------------------------------------------------------------------------------------- +// Global instances +//-------------------------------------------------------------------------------------------------- + +static STATE_MANAGER: StateManager = StateManager::new(); + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +/// Return a reference to the global StateManager. +pub fn state_manager() -> &'static StateManager { + &STATE_MANAGER +} + +impl StateManager { + const INIT: u8 = 0; + const SINGLE_CORE_MAIN: u8 = 1; + const MULTI_CORE_MAIN: u8 = 2; + + /// Create a new instance. + pub const fn new() -> Self { + Self(AtomicU8::new(Self::INIT)) + } + + /// Return the current state. + fn state(&self) -> State { + let state = self.0.load(Ordering::Acquire); + + match state { + Self::INIT => State::Init, + Self::SINGLE_CORE_MAIN => State::SingleCoreMain, + Self::MULTI_CORE_MAIN => State::MultiCoreMain, + _ => panic!("Invalid KERNEL_STATE"), + } + } + + /// Return if the kernel is still in an init state. + pub fn is_init(&self) -> bool { + self.state() == State::Init + } + + /// Transition from Init to SingleCoreMain. + pub fn transition_to_single_core_main(&self) { + if self + .0 + .compare_exchange( + Self::INIT, + Self::SINGLE_CORE_MAIN, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_err() + { + panic!("transition_to_single_core_main() called while state != Init"); + } + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/synchronization.rs b/15_virtual_memory_part2_mmio_remap/src/synchronization.rs new file mode 100644 index 00000000..d4bb2c23 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/synchronization.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Synchronization primitives. + +use core::cell::UnsafeCell; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Synchronization interfaces. +pub mod interface { + + /// Any object implementing this trait guarantees exclusive access to the data contained within + /// the Mutex for the duration of the provided closure. + /// + /// The trait follows the [Rust embedded WG's proposal] and therefore provides some goodness + /// such as [deadlock prevention]. + /// + /// # Example + /// + /// Since the lock function takes an `&mut self` to enable deadlock-prevention, the trait is + /// best implemented **for a reference to a container struct**, and has a usage pattern that + /// might feel strange at first: + /// + /// [Rust embedded WG's proposal]: https://github.com/rust-embedded/wg/blob/master/rfcs/0377-mutex-trait.md + /// [deadlock prevention]: https://github.com/rust-embedded/wg/blob/master/rfcs/0377-mutex-trait.md#design-decisions-and-compatibility + /// + /// ``` + /// static MUT: Mutex> = Mutex::new(RefCell::new(0)); + /// + /// fn foo() { + /// let mut r = &MUT; // Note that r is mutable + /// r.lock(|data| *data += 1); + /// } + /// ``` + pub trait Mutex { + /// The type of encapsulated data. + type Data; + + /// Creates a critical section and grants temporary mutable access to the encapsulated data. + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + } + + /// A reader-writer exclusion type. + /// + /// The implementing object allows either a number of readers or at most one writer at any point + /// in time. + pub trait ReadWriteEx { + /// The type of encapsulated data. + type Data; + + /// Grants temporary mutable access to the encapsulated data. + fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R; + + /// Grants temporary immutable access to the encapsulated data. + fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R; + } +} + +/// A pseudo-lock for teaching purposes. +/// +/// Used to introduce [interior mutability]. +/// +/// In contrast to a real Mutex implementation, does not protect against concurrent access from +/// other cores to the contained data. This part is preserved for later lessons. +/// +/// The lock will only be used as long as it is safe to do so, i.e. as long as the kernel is +/// executing on a single core. +/// +/// [interior mutability]: https://doc.rust-lang.org/std/cell/index.html +pub struct IRQSafeNullLock { + data: UnsafeCell, +} + +/// A pseudo-lock that is RW during the single-core kernel init phase and RO afterwards. +/// +/// Intended to encapsulate data that is populated during kernel init when no concurrency exists. +pub struct InitStateLock { + data: UnsafeCell, +} + +//-------------------------------------------------------------------------------------------------- +// Public Code +//-------------------------------------------------------------------------------------------------- + +unsafe impl Sync for IRQSafeNullLock {} + +impl IRQSafeNullLock { + /// Wraps `data` into a new `IRQSafeNullLock`. + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), + } + } +} + +unsafe impl Sync for InitStateLock {} + +impl InitStateLock { + pub const fn new(data: T) -> Self { + Self { + data: UnsafeCell::new(data), + } + } +} + +//------------------------------------------------------------------------------ +// OS Interface Code +//------------------------------------------------------------------------------ +use crate::{exception, state}; + +impl interface::Mutex for &IRQSafeNullLock { + type Data = T; + + fn lock(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + // In a real lock, there would be code encapsulating this line that ensures that this + // mutable reference will ever only be given out once at a time. + let data = unsafe { &mut *self.data.get() }; + + // Execute the closure while IRQs are masked. + exception::asynchronous::exec_with_irq_masked(|| f(data)) + } +} + +impl interface::ReadWriteEx for &InitStateLock { + type Data = T; + + fn write(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R { + assert!( + state::state_manager().is_init(), + "InitStateLock::write called after kernel init phase" + ); + assert!( + !exception::asynchronous::is_local_irq_masked(), + "InitStateLock::write called with IRQs unmasked" + ); + + let data = unsafe { &mut *self.data.get() }; + + f(data) + } + + fn read(&mut self, f: impl FnOnce(&Self::Data) -> R) -> R { + let data = unsafe { &*self.data.get() }; + + f(data) + } +} diff --git a/15_virtual_memory_part2_mmio_remap/src/time.rs b/15_virtual_memory_part2_mmio_remap/src/time.rs new file mode 100644 index 00000000..cd3ceec3 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/src/time.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! Timer primitives. + +#[cfg(target_arch = "aarch64")] +#[path = "_arch/aarch64/time.rs"] +mod arch_time; +pub use arch_time::*; + +//-------------------------------------------------------------------------------------------------- +// Public Definitions +//-------------------------------------------------------------------------------------------------- + +/// Timekeeping interfaces. +pub mod interface { + use core::time::Duration; + + /// Time management functions. + /// + /// The `BSP` is supposed to supply one global instance. + pub trait TimeManager { + /// The timer's resolution. + fn resolution(&self) -> Duration; + + /// The uptime since power-on of the device. + /// + /// This includes time consumed by firmware and bootloaders. + fn uptime(&self) -> Duration; + + /// Spin for a given duration. + fn spin_for(&self, duration: Duration); + } +} diff --git a/15_virtual_memory_part2_mmio_remap/test-macros/Cargo.toml b/15_virtual_memory_part2_mmio_remap/test-macros/Cargo.toml new file mode 100644 index 00000000..a570f72b --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/test-macros/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "test-macros" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.x" +quote = "1.x" +syn = { version = "1.x", features = ["full"] } +test-types = { path = "../test-types" } diff --git a/15_virtual_memory_part2_mmio_remap/test-macros/src/lib.rs b/15_virtual_memory_part2_mmio_remap/test-macros/src/lib.rs new file mode 100644 index 00000000..092c4806 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/test-macros/src/lib.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +use proc_macro::TokenStream; +use proc_macro2::Span; +use quote::quote; +use syn::{parse_macro_input, Ident, ItemFn}; + +#[proc_macro_attribute] +pub fn kernel_test(_attr: TokenStream, input: TokenStream) -> TokenStream { + let f = parse_macro_input!(input as ItemFn); + + let test_name = &format!("{}", f.sig.ident.to_string()); + let test_ident = Ident::new( + &format!("{}_TEST_CONTAINER", f.sig.ident.to_string().to_uppercase()), + Span::call_site(), + ); + let test_code_block = f.block; + + quote!( + #[test_case] + const #test_ident: test_types::UnitTest = test_types::UnitTest { + name: #test_name, + test_func: || #test_code_block, + }; + ) + .into() +} diff --git a/15_virtual_memory_part2_mmio_remap/test-types/Cargo.toml b/15_virtual_memory_part2_mmio_remap/test-types/Cargo.toml new file mode 100644 index 00000000..a0be2c57 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/test-types/Cargo.toml @@ -0,0 +1,5 @@ +[package] +name = "test-types" +version = "0.1.0" +authors = ["Andre Richter "] +edition = "2018" diff --git a/15_virtual_memory_part2_mmio_remap/test-types/src/lib.rs b/15_virtual_memory_part2_mmio_remap/test-types/src/lib.rs new file mode 100644 index 00000000..371bb557 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/test-types/src/lib.rs @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Types for the `custom_test_frameworks` implementation. + +#![no_std] + +/// Unit test container. +pub struct UnitTest { + /// Name of the test. + pub name: &'static str, + + /// Function pointer to the test. + pub test_func: fn(), +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rb b/15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rb new file mode 100644 index 00000000..a6b549b2 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2019-2020 Andre Richter + +require 'expect' + +TIMEOUT_SECS = 3 + +# Verify sending and receiving works as expected. +class TxRxHandshake + def name + 'Transmit and Receive handshake' + end + + def run(qemu_out, qemu_in) + qemu_in.write_nonblock('ABC') + raise('TX/RX test failed') if qemu_out.expect('OK1234', TIMEOUT_SECS).nil? + end +end + +# Check for correct TX statistics implementation. Depends on test 1 being run first. +class TxStatistics + def name + 'Transmit statistics' + end + + def run(qemu_out, _qemu_in) + raise('chars_written reported wrong') if qemu_out.expect('6', TIMEOUT_SECS).nil? + end +end + +# Check for correct RX statistics implementation. Depends on test 1 being run first. +class RxStatistics + def name + 'Receive statistics' + end + + def run(qemu_out, _qemu_in) + raise('chars_read reported wrong') if qemu_out.expect('3', TIMEOUT_SECS).nil? + end +end + +##-------------------------------------------------------------------------------------------------- +## Test registration +##-------------------------------------------------------------------------------------------------- +def subtest_collection + [TxRxHandshake.new, TxStatistics.new, RxStatistics.new] +end diff --git a/15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rs b/15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rs new file mode 100644 index 00000000..5aa38f09 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/00_console_sanity.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Console sanity tests - RX, TX and statistics. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +mod panic_exit_failure; + +use libkernel::{bsp, console, print}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + use bsp::console::{console, qemu_bring_up_console}; + use console::interface::*; + + qemu_bring_up_console(); + + // Handshake + assert_eq!(console().read_char(), 'A'); + assert_eq!(console().read_char(), 'B'); + assert_eq!(console().read_char(), 'C'); + print!("OK1234"); + + // 6 + print!("{}", console().chars_written()); + + // 3 + print!("{}", console().chars_read()); + + // The QEMU process running this test will be closed by the I/O test harness. + loop {} +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/01_timer_sanity.rs b/15_virtual_memory_part2_mmio_remap/tests/01_timer_sanity.rs new file mode 100644 index 00000000..e0b3c162 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/01_timer_sanity.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Timer sanity tests. + +#![feature(custom_test_frameworks)] +#![no_main] +#![no_std] +#![reexport_test_harness_main = "test_main"] +#![test_runner(libkernel::test_runner)] + +mod panic_exit_failure; + +use core::time::Duration; +use libkernel::{bsp, cpu, time, time::interface::TimeManager}; +use test_macros::kernel_test; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + bsp::console::qemu_bring_up_console(); + + // Depending on CPU arch, some timer bring-up code could go here. Not needed for the RPi. + + test_main(); + + cpu::qemu_exit_success() +} + +/// Simple check that the timer is running. +#[kernel_test] +fn timer_is_counting() { + assert!(time::time_manager().uptime().as_nanos() > 0) +} + +/// Timer resolution must be sufficient. +#[kernel_test] +fn timer_resolution_is_sufficient() { + assert!(time::time_manager().resolution().as_nanos() < 100) +} + +/// Sanity check spin_for() implementation. +#[kernel_test] +fn spin_accuracy_check_1_second() { + let t1 = time::time_manager().uptime(); + time::time_manager().spin_for(Duration::from_secs(1)); + let t2 = time::time_manager().uptime(); + + assert_eq!((t2 - t1).as_secs(), 1) +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/02_exception_sync_page_fault.rs b/15_virtual_memory_part2_mmio_remap/tests/02_exception_sync_page_fault.rs new file mode 100644 index 00000000..0e219a8f --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/02_exception_sync_page_fault.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +//! Page faults must result in synchronous exceptions. + +#![feature(format_args_nl)] +#![no_main] +#![no_std] + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +/// +/// Reaching this code is a success, because it is called from the synchronous exception handler, +/// which is what this test wants to achieve. +/// +/// It also means that this integration test can not use any other code that calls panic!() directly +/// or indirectly. +mod panic_exit_success; + +use libkernel::{bsp, cpu, exception, memory, println}; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + use libkernel::driver::interface::DriverManager; + + bsp::console::qemu_bring_up_console(); + + println!("Testing synchronous exception handling by causing a page fault"); + println!("-------------------------------------------------------------------\n"); + + exception::handling_init(); + + if let Err(string) = memory::mmu::kernel_map_binary_and_enable_mmu() { + println!("Enabling MMU failed: {}", string); + cpu::qemu_exit_failure() + } + // Printing will silently fail fail from here on, because the driver's MMIO is not remapped yet. + + // Bring up the drivers needed for printing first. + for i in bsp::driver::driver_manager() + .early_print_device_drivers() + .iter() + { + // Any encountered errors cannot be printed yet, obviously, so just safely park the CPU. + i.init().unwrap_or_else(|_| cpu::qemu_exit_failure()); + } + bsp::driver::driver_manager().post_early_print_device_driver_init(); + // Printing available again from here on. + + println!("Writing beyond mapped area to address 9 GiB..."); + let big_addr: u64 = 9 * 1024 * 1024 * 1024; + core::ptr::read_volatile(big_addr as *mut u64); + + // If execution reaches here, the memory access above did not cause a page fault exception. + cpu::qemu_exit_failure() +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/03_exception_irq_sanity.rs b/15_virtual_memory_part2_mmio_remap/tests/03_exception_irq_sanity.rs new file mode 100644 index 00000000..6e2ae8f7 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/03_exception_irq_sanity.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2020 Andre Richter + +//! IRQ handling sanity tests. + +#![feature(custom_test_frameworks)] +#![no_main] +#![no_std] +#![reexport_test_harness_main = "test_main"] +#![test_runner(libkernel::test_runner)] + +mod panic_exit_failure; + +use libkernel::{bsp, cpu, exception}; +use test_macros::kernel_test; + +#[no_mangle] +unsafe fn kernel_init() -> ! { + bsp::console::qemu_bring_up_console(); + + exception::handling_init(); + exception::asynchronous::local_irq_unmask(); + + test_main(); + + cpu::qemu_exit_success() +} + +/// Check that IRQ masking works. +#[kernel_test] +fn local_irq_mask_works() { + // Precondition: IRQs are unmasked. + assert!(exception::asynchronous::is_local_irq_masked()); + + unsafe { exception::asynchronous::local_irq_mask() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + // Restore earlier state. + unsafe { exception::asynchronous::local_irq_unmask() }; +} + +/// Check that IRQ unmasking works. +#[kernel_test] +fn local_irq_unmask_works() { + // Precondition: IRQs are masked. + unsafe { exception::asynchronous::local_irq_mask() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + unsafe { exception::asynchronous::local_irq_unmask() }; + assert!(exception::asynchronous::is_local_irq_masked()); +} + +/// Check that IRQ mask save is saving "something". +#[kernel_test] +fn local_irq_mask_save_works() { + // Precondition: IRQs are unmasked. + assert!(exception::asynchronous::is_local_irq_masked()); + + let first = unsafe { exception::asynchronous::local_irq_mask_save() }; + assert!(!exception::asynchronous::is_local_irq_masked()); + + let second = unsafe { exception::asynchronous::local_irq_mask_save() }; + assert_ne!(first, second); + + unsafe { exception::asynchronous::local_irq_restore(first) }; + assert!(exception::asynchronous::is_local_irq_masked()); +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/panic_exit_failure/mod.rs b/15_virtual_memory_part2_mmio_remap/tests/panic_exit_failure/mod.rs new file mode 100644 index 00000000..b4ac73d1 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/panic_exit_failure/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +#[no_mangle] +fn _panic_exit() -> ! { + libkernel::cpu::qemu_exit_failure() +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/panic_exit_success/mod.rs b/15_virtual_memory_part2_mmio_remap/tests/panic_exit_success/mod.rs new file mode 100644 index 00000000..54bb072d --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/panic_exit_success/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +// +// Copyright (c) 2019-2020 Andre Richter + +/// Overwrites libkernel's `panic_wait::_panic_exit()` with the QEMU-exit version. +#[no_mangle] +fn _panic_exit() -> ! { + libkernel::cpu::qemu_exit_success() +} diff --git a/15_virtual_memory_part2_mmio_remap/tests/runner.rb b/15_virtual_memory_part2_mmio_remap/tests/runner.rb new file mode 100755 index 00000000..5b012304 --- /dev/null +++ b/15_virtual_memory_part2_mmio_remap/tests/runner.rb @@ -0,0 +1,143 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# SPDX-License-Identifier: MIT OR Apache-2.0 +# +# Copyright (c) 2019-2020 Andre Richter + +require 'English' +require 'pty' + +# Test base class. +class Test + INDENT = ' ' + + def print_border(status) + puts + puts "#{INDENT}-------------------------------------------------------------------" + puts status + puts "#{INDENT}-------------------------------------------------------------------\n\n\n" + end + + def print_error(error) + puts + print_border("#{INDENT}❌ Failure: #{error}: #{@test_name}") + end + + def print_success + print_border("#{INDENT}✅ Success: #{@test_name}") + end + + def print_output + puts "#{INDENT}-------------------------------------------------------------------" + print INDENT + print '🦀 ' + print @output.join('').gsub("\n", "\n#{INDENT}") + end + + def finish(error) + print_output + + exit_code = if error + print_error(error) + false + else + print_success + true + end + + exit(exit_code) + end +end + +# Executes tests with console I/O. +class ConsoleTest < Test + def initialize(binary, qemu_cmd, test_name, console_subtests) + super() + + @binary = binary + @qemu_cmd = qemu_cmd + @test_name = test_name + @console_subtests = console_subtests + @cur_subtest = 1 + @output = ["Running #{@console_subtests.length} console-based tests\n", + "-------------------------------------------------------------------\n\n"] + end + + def format_test_name(number, name) + formatted_name = "#{number.to_s.rjust(3)}. #{name}" + formatted_name.ljust(63, '.') + end + + def run_subtest(subtest, qemu_out, qemu_in) + @output << format_test_name(@cur_subtest, subtest.name) + + subtest.run(qemu_out, qemu_in) + + @output << "[ok]\n" + @cur_subtest += 1 + end + + def exec + error = false + + PTY.spawn(@qemu_cmd) do |qemu_out, qemu_in| + begin + @console_subtests.each { |t| run_subtest(t, qemu_out, qemu_in) } + rescue StandardError => e + error = e.message + end + + finish(error) + end + end +end + +# A wrapper around the bare QEMU invocation. +class RawTest < Test + MAX_WAIT_SECS = 5 + + def initialize(binary, qemu_cmd, test_name) + super() + + @binary = binary + @qemu_cmd = qemu_cmd + @test_name = test_name + @output = [] + end + + def exec + error = 'Timed out waiting for test' + io = IO.popen(@qemu_cmd) + + while IO.select([io], nil, nil, MAX_WAIT_SECS) + begin + @output << io.read_nonblock(1024) + rescue EOFError + io.close + error = $CHILD_STATUS.to_i != 0 + break + end + end + + finish(error) + end +end + +##-------------------------------------------------------------------------------------------------- +## Script entry point +##-------------------------------------------------------------------------------------------------- +binary = ARGV.last +test_name = binary.gsub(%r{.*deps/}, '').split('-')[0] +console_test_file = "tests/#{test_name}.rb" +qemu_cmd = ARGV.join(' ') + +test_runner = if File.exist?(console_test_file) + load console_test_file + # subtest_collection is provided by console_test_file + ConsoleTest.new(binary, qemu_cmd, test_name, subtest_collection) + else + RawTest.new(binary, qemu_cmd, test_name) + end + +test_runner.exec diff --git a/doc/15_kernel_user_address_space_partitioning.png b/doc/15_kernel_user_address_space_partitioning.png new file mode 100644 index 0000000000000000000000000000000000000000..ebf39fd75a455b6717e09cccc7cf7d738d330e71 GIT binary patch literal 75332 zcmagG1yoR7yDhvC2?-GtL{bS+Qo2JxKopSfmImoggAz$8Nh#^>?r!Ps?r!+k=6%02 z&OPVe@$WGhNI2@itqs-Pl+pK!9#yMj-5 ztp(r8p@Pc=^`j5?{)x>SB^z0DeH(jiD?P~Ilew85owcr&p57;GLvx!Q#9BTGdJ4T2 zel6z^vpHq2Bx^v3yvw~wCsg7?i;YWFR(zh`{CT)~ouo`OJM&Yvg5{t*%Jz^k%{D@s z+0;ZuHe&N%@Q!e!aKdx$pC|Hg_7Ji zi|gtJ|K}c`(3Tdm|J;N9(*M7|C@P>20sNhqL~=WnESUO~EV0)YqyC6USXfTx_IYwL zVj74+sIj2oe5p%^Qr6(HX^V5B2LJV{WR{%UR*0p@OqO4u29od!nVJ&C#BBDL1!KK` z{{wC5?9*gbplD#+%j($lbjI%D|3^!E z*2!j-<&I9i(rof=hmwuTbl@I_I-TufR@e~w|M>A>F^vm zRZ_D)KWIRy$bq~jnyKi6l__eJcZ@YT-M^T``kHn>V>sln7A?VsHTF->q zO-Ji@;8>vuBE;t+R+rFN``Ae2HdOwas@=8%ZK}+X6tn_@#*02BT4)shvDF5*3=Xmo zaX8>qJCz^?+2S8DR0cq8T(lOOku`mPJI);StgO)7RIZqlWnMyBr+IO&n3<(dwn00v zv4bHHjVZ~gsY&)b)0`w&SXiwWCp!m|)9vl|dU|@sNTT^%xvHwFp5poS7aH`M>RMV_ zI-Krm$Jrho9nEcQL~rHmcSmXO&elU7OPvu54W8&@ExDSt0bN~PJSK{n3Yhx(`UMwG zPEMl*dVX0VT3T8qXT5Q}LN+!wakg8Vo3q>7aa;NMx~++lv0RYH_0e0dT&Iyx4^zNZtq#U>*yr+W(sP&B)_h@qij$)0NIr)R9Jti9tJ8X9>81s%tu zeKVHkpg__dR z(P6TiPj?m4L-2c43&G4iWH#uz3vG@SqBtLKwzJ(#S38S|i+2>}R913tZf+Lvmbjc* zfe<67rbYwLX=lSDCiXYzjayP4>FUB5%8>6oR*Ilk@5|Q}g_QXj!14<5_iffutF)u* zt$=1XHqc#NUB_IVot@bJ7OqS^GKZ&v%sT~2qO^;Uqvx3IDTJvy0j$5K>O zq@bcoa=*DAYthlw-2!)O$jl?(IKDs0}_RVrjtqphrMY$O=tVyRWSJ+|MzzPQmSx3W7u z7^M70%#Q-Kb#*m0-TP{7%C9Z+bU0@M3Bo7n`zrB4cEy|GAJwx=rPGbU1{NQ;bz#J819O=(2}NwQ_s!3enA zT%Dk`uJpzSf(Xr&bKaY8wy>}W;n% zp;gFIO0aNQ>If@yIV%t^Ht31DCnY6SuIqkv^f;VeJws+{xjR~Sw(c`pt2f4zkgc7} z%fBNdt>A~a0p?Q`xKxrcA1de}mxFHh-MwIUw-GqFjyK1shrkkOF0)*)a-EKh#1Atz zA1gotfo|1$yglXTdU2dCXAjne;c(Wk$^6w-!?BsdjtseU9R04yguf-B{HC-aGQwoo zSsf}mQ{zv=O3VlI2#_&%LF6b$KT3WV_mr2MT=XYZZB{m&^yG=)+`!+zAw#Ytj_ONY)AH)pAL$iI3dS#vk;obMlK$Er9om4qX-!^c zIvt{lp4k1UB1GzE78+C!Su(_vnA!JNV8gAzBzcalUry&ulvi#ru`vB%JWCFH|Q5c zxt>Ct&dSRJ1`Q9aHl@OhrGD`|h{L_%jR+422*}zVSW%Sk*_54N7S-qWey_a6>+B|^ zuRlB#rmn7zWJb$(-~D;zmP~?Lj&)~*eO>W}JvPZTkxoAT`1tq(U*XXLqi0nnrwdmX z99@yccR;2Ei9n)cs$VztqeSy}TpF#l`Invr48p=G0W5eyf7-TNYMebZ-Ok>m$n4Bi zHzeQFq#@$B`pCaM<(zE6!%h<}@!2;xWisE{o*^S{oQmu2K!C3L)Wm4KNPJxllW(t1`vmX1t}SsMw*VFtn5%iE*3w+Fyk?08Ue)^g{^G|=dRRpBg!xt9i4j0IBSOU5)aQod<8jTMUt%ZS^W2a+vCFb{=L~;3ykUTQvsh6ulmiTR*ZOuGejxsal zM(U^W?CkWc?~66(F)@BPD}L2CBD&PE&53f|US&Kc*VHR*_C}CWN)i9b)e2@mzwJbc zX_}Qjp$SbWjX{LTM}e^N16 zY#e{(VXl;W5wY@J)OJWGcK$>q^Xs220F3&Z-bX2Rh*wYfj>fD4s8BrF0YEIJl6R}dy9cUQI@kX zTVhO&`R456*o8R%SQD(R^5IyP1pf1*BgG2(prg`1iQ_|6vGQ(ov~*q8-Bs>8)X9JT zq{~eR@XKm%B?xgPm=T`&Z&#I7aJ+6XM63yP- z>EaYPxZ>)Nh2=|>$BQ*&qPTh;k$5W0>qFGw$5d3#9GvYpa#>G*_+z4P6{pE4OAaVu zZf<33o114J!+r2PR(UjxOOu^Vr)@^({^s>-Llcw7<0f;<3N;_rRg@y@8xk#;+}%Eh zD&}h7c33Yey0{FJiXbDOgM3lm!^(1LKV)uYm64T?3K4S`HU0jqo@ZqJDo3O8YYP9T z`efg)jD~G2!lJc^P*)r$K@H!&VmMUS)^EKZcGi~gw8(6LWrP#Px6M1n3Mdj2R1%p< zzQ0(q13vutfgxQQU`+a8xw$gq5?2^WUTLTC@X!WyyT$SbV@|g3Kr_^#5!k%aW#imDcm42_J)ZkX&f zez83@>ZUgx&BHdED3Mp4`uknc^La{gN=kxz{Iy`~Q!?sf3LM|xy*EZjZv*7BtX$kk zh#fP%BW^e_x3`tu)w<#;(SH7+Z3#|N_D*Lv{eUj}93_pM2^$+hsZbyDQst-#WLVDt zL18&sN{Z)m-O(?SlhaBFq6-$jgT$dk+p;E`>KX7_SQzP}p&;3e!mt_}(b`QH4F?GE%_!arYkVm?dA$BNOqrOhSV^eA1OwfFxvdXmy*?l*h^ z@f`kO@5AOo1Mu~>;!5dHMb@Ed${ayHoI>PHbXkO3!~PkU{0l!SekwZ3yg(gSR11i6b% zV-NQ$TdBD!0PH6#Z}6cHsgkVa*7FMw`%`Cx#VBwLjf|dI)CXY){LHRJnHKe7hiv)349fmeIQc1=|s8^-D)g>*r7WonNb)~mZfmbq)ed)dJK`3G%KL~AsEdAZ7Pdgpvil7t4^IpNWW93rEpuf1QHR?<1&76L`AePhM~e8q zwKX#+K#TUU>ZV<_3(#}Gpd>a&T!8{It1&rM@(D68@0968t-~Gxxkxuckgaw^yghBO z`ebuoAEe5TBKYnd`mbL)huho;?d>n$l-b`7s@&*5Z_x3^%K7T=@88F$WoG8PrHYu4 znfY~b@sZ6M8*=z?jm44SeZ|1#GMlsKkXBzEJwE6BhUo-IJcn!S_xARDA&)ocR1KY- zbQ3YiMO}BYUcFL z)-Z~B`*a)+G3qdJ&-<^NTPy8%Kp!x7GH@*~mbDWW9$@3Xv$VXIrk=_j$K#kd-7s$f z;e@ff?=K>e@VI;j<*$PSM`L5PU_fKMuaDSU6-ke_4`xdu{sf*wnTq6#t0G1FCRrsF z0P-}4roVeoW{3y??N@sB3RL3M(E)5O2(;S>nZ0!1!kUJ`2IHUJKEGWR;5YE~4&U^;O{^37UjBBa7V3nKm3wp~Y51cJHI}vjkX8O|y0v z``@xsl2NP7xSkMuVXUJnK7I$sKCUWRu-GapB{Eo1kFIyIv2oSJ+Klj{t3(F=NPv_h zw+SNtwP>oAxwfrM(NL&Y+A1d(S3|!BdeCz;Vo&fMkm{B|snj?+8VSB6qlSV5sMOTe zS+_I#KoO5R=pt_jc`RbWp^cQFhwjsX8k;CQ`yH@2i6MfW$PhmR1=-&hWV{IvLA2r$ zu;IOZ2`~%T$w4HA^-25qtkw$a3sT)uK#4NnV5s!@GlE7{fJeZoC;a<=E-`_Bcp_^5 z`T6;80RnJf%B_F<7FXZ&mf2#Zhd3cA6csK2nsgZNZ}L9{L!79?z!*$5MMXxoq+acW z%Wa3eaI}GcJ1Xx|d)4YgC^L6==Y;-qBR~|p-I3|4N_UM1oRBvCdt*!a^`!|9}_y*GYzt^3=V2gb_m+}-LZCF-PTwKt~iW-k@+DD#$ z>x0up6f5;Y;Li!%fYJ+N{GQm_KQbjWkH5~%Nx$A64Twcy)So<|9=bY?{v63?_H8o% z@DO8^Yfv#g=jTt}Pcs}@7A|$aUvgONCW{vb5|8lGmmX(#*(sGY_ zh?Rv!_!rZ2wAQ4gnP-}{GS@;BIQ7FNSq*#NzEMz6rM>{DyQ{lfOY3Rv+laWh&!eLp zVe9J~8;uPO@8?b@O7kE1HB%3j6w{#xi&5CqzFZ>@2?>$UFwxgXBPT~J*9BR(%K34o zg{rSG8Xeu)FB@B1Z9Tmli;0t+g`nVs@!sO%;;@DpR*;s(#KdIMb9{Xn0jsze@RmZR zsp(Nq?~ymwp4hv0-b%Ee#VEcAH(+f6K9e&^#J^#EC==VthFs;06_4vawsbzT-a5`* z`f(3l=i}e+BIpHMUmk}l<%`|L$N$JmS(T~Kv=#kbaROVqeQ(3OwJ&Lez0AQ4%f+?I zGeDNi$A%{%snPw*ZFWj?Sl|Sltk^SO;V)N4W z3V8^5-_#A6*x!}>cR56Tff?A4jf0`@@fJh>GPAA?@G5IRIx}vTDl-NYBz*B{!z`{< zP^rq^!yJ`=sa_8_8tHz(Ugba)ly# zgIPlu55lmy#C_{p$XbU;>Nuolkr9^^_J^$1VBtOvcD|I`; z{2~T7cO}`F6xhDGQ9F{Uqx2K&kpw+))Sa5OJbFe1@%PD=EhF#O$&ID<^JN^vB?6x&P2Pd z@ifI;k|-9VFBP^M4WpxFFYIA(;UoUQ-<0;cG@ZkF*{~~$Q+!qq&JIzc0Egz}v(Gj6lQ00O*7!Iak!*2kf4A)SP z5^iT0J;8_)&)Ef1(atNiVL1%7SJ(iZ<8k1E-%0N!W@r0T%V+!+52Ick$t8uJGBZCs z*_mlAw_dS0-c*j^b;bjfb3@9zh`f*OIH2Hd92t4+=H|8w=wXmo_IYeEZ(pK8Fr>s| zxzAIx&T$9K4Tx8z>jk+VPo@-CmEi7BegUN0X3#n|xd;Ts@wt8ny#IXHFXq#$lNmk# z7EbHsy2(i*I+Y@i_;}LsViOcqRn^ZY9`N|E__na%NyYF49us?!v$A3W!DeH)>vdhw z_Sy9H2|xY(Y{Jams`Op3nP}_kdIIUCuh5%+bF*VPGCn*!V2P##p{*k5M{F#sn^M{R zl^%8P>}*^vm!UgT6(yg=?$Rn(Jc0V=GVyi~ZR)cpu<~?QUQbq@;6HitD^0o)z1T$l zb!vd7pr)qOmNK03(K=Hb)A+NEUCs~FHz$9<6L9~&h=s+gqN1V?h58bw7Z*vQfh1{i z=^=IP@C{;y1pP{vRq1J9sJjW>1!nk&_lwOwethKQ#!Ir&quI+?a#MG&E2jF+XY}Vf zO}iRnEi$qLufPrkd8DSIBV+#Z0vcFb%gJ5uL=jttvhuKqZfk3rz%6bpShWI$+t}pH zO&K8KZEXct`z3aFEfh=?zzTB2z`y_rkRl>8ljhEyI}aW_@Br&HhTHzWXaG@t{Z|yr zt8-hmS~p_r<*v`ArR-dGn?5AGPGFh8TOEZb?QCATAIZ(FTid{Z$|e-&^=`}j-ouq$ z`BrQ(35l-(0c>`kG{0<5jW3!^V3Fc{$kk@Aa>B)g(iT)P%~aeGxGM|FEDy{#C)|n5 zXIgX?S2;o1URRcXwK%L?;zH9CQ;i0_Vr3Q4nHL?*mL(3OsU)YPyDD^4wV0d7TwPmB zG>cwNQyU{Dr-)sE-qI>RVbwKn8aN6uT&p zVfw2at!Y(DFiy6o8iUD2Er7J7R%U@(PmchB^Vw)fd9T&3`?vV;Qw(Q3d2gR$owlflH=?xbXWv@wlh$RhbtH+tsa*M_17W z$Jxajo7K=Exw{?=s%(NvZGT)w2$?^TC|ngGLc5Jsdy!0>PgNIan3&LFD}FS9tzSz= z=bNZIsdsfXEA*6)4|`dGub`Zse#zpn&&k2D{2BE+@w9OBZ+-s`7|49iFE!Nw93n@a zy+nz;FN0>y>uRs5-E&(EXm(i>MY>w$xbZ7m2rG~xH;D-EK!4ufulsT~vwXoSBQ`U}Lt(EHyu9(TLaK1=btpq`nWoP0n+LT~ekEbNW2u-49W z_37D4yyd~N1f;B(_Gz+=vF7^R#Aag{Q?FwelUq6=s0tz37uJcXYl=+Al;j>RalbRh zA+TdpmBnsTsV>von07I9zj5Oj&goQnGFtwV6v(42t;Q8rOPhHsBVm0bBSJDVOf}IZ za@y71gpc9f1=2E5#A7v)3+HAq+?piv=HliY(*;C&!G%GO8r@(P9F&6AmX`J3?me_v zXu+JUbtf&<@BR`UO(+@9=kMVGjgE~a00ED~ai3!Uh?5P@O}N}&Sn;^+iT%Mo23EDs ze)6+|R#E>m5-bv)ZtDv~6e%)hSq$jZ?h^7mh>0ccD|j<(T-zvm9fa0P3%I6#ya3#- z3d&aKcrA#2CY$9IwHz1}4b+LmktRLxh;$EG|^hp3MkG@5*{ z$$#hO2Es5?uQPmif4^bA*$wEGe!^7`Cd`M2jTSX#4a4`=Q1=GnQ z6aaEpr>%+TH79>=I0}(?^P>dWwn;cSA=7J?rq=nd}=kB&MtwX}3@yUX+^3Dz(H ziZ|5)sAXby+;qH2eFZqUDT=v?OB$wPPd(QqQ(>fPn}Ke=8fUX2#^2##1&%*GS;BAD#*Zpj`!}NR4B@jCQ2}(X@Ge%$zuEWa z9laHkZa=_wiUe-@2b0$*oLW~nb@z^*9Xw0fS5gUoy^~wpFgd3C;m?nBnF)V&o1qtE}kU;ll>k&=Ii|&E2YR*xw%rhCGA551;ft6mc8G{mu z%0}&WGCym13&x{PkeN{5@yO4A29hHNacma<(K={LB(D(tco(|wL(tt9XqgMU}V2~t1PqavGdwTFCmCgIx)`e)% z;F%0{Ia-ER*k6-R;$!I#tB|B) z3UFL=&TiHh>Z?DL>l*Z`HwGmPAV7%c_AQIPLW@1j?ChTT`N1mdZPq8p*sF#&7n$#A z@yIzTaRG>!>j;}rIvcR7ay~&24UBC7MBppkbJR2Li$D{_;{=f-aZZv5akx&bayUbF z{vxzHupS>A6Z56=ghco98`9_l7KV`u$l{_$=(LpdJJ8dYIU~etz-LtA&uMK>@bR^P z!ivvf@iYgR1As=O1u}jls|lXl=^P52s+je=JiNWV`=r4>oc2wnMu(8?)3mo(<>&0$8kzm(*nwifgo45{+KOvp4%dx*`+)r1vmQ51Ho+14 z>{<0jQc&E|YL?>y=;c*YGcUAlV^f4XT$rE##or&-#aQvphJxmAz*0WL^UDVSgf{ZV zEQfz0x^%NSjnO(dBqU?-TDmYD9^g3vRE*VShwzq+tRp^cwY1EB2Vs6;!4njPsCF-k$N;h) z%{OBNtU1ERSij8yw3c4^W2=a%`_nY;l!H`m*(mj6#ZRJGcfNvcevpuZ^t>cb z4Ir^#5)R6|Lk8qn9WaEv3#Y9I(45hHnny`VNvk!07yxR5@lx}^jS+OaJ%+>(pwMKF z7GFRbwE?#Teeg0X9zxyF&^?+6sfZ~}{4$c-8%xHCm>O5Ey8eDV=#^OuldxzfKZBIm z>JykhVbG}Z3JADQOWTFUP!;54yBhaXF_zWnq`l%^mU@Pzn48&3?F>OaC2k9*s`EOeb z8FaWaaxa$q3;@n3tDTyM1?nIjH=(}6lRyqCMoYJgmE>^Re8j|5pfISX|2^NaQ^RJ9 z4_RP%&fhUsH9PN7YL+miw8JSnxZqohh_N6&){x#PNR&G5Od z^!T~CxvY{EDl%%{!=t1AQgcmsD}*g9o@lrhpyoI|KTia*ahbi;HJT;7{;8b^CY0`a z9BtG@WFPu{ck-^eqIyjo*3sf@W9|nsBr*hJx0_=jZJ9sF)K0GVnEh1nSw2$pxxEKu zO2csXXT!gvia}Q|J}LW(hv+^_wWOukNPG(t!IxjZd{x#pYOK(oB^W@Eh=>=^&aOR? z!x^H#bC=#v0u>r42aFv*fdX1UR1^gf8KVv$n44Yj%C~{+8zRANgn?S0oz4`zk^Wa|gb5%|?Q04`n zT8`nJ7L(k~SHgfId96bmy!3#|G4_E15WOHXV-Rya0;7$JhW4epn%C`Y>9K>8lg?^i zg2Tmeb{{s3@Wr#qs;LiSRYgOm{{S8!=p#>?t>lWN+v;RHR<362Z2y9R*!%Zi08Kfm z*4asa=RgxEkAhKWuXxF0PwvYQQxDR4eKtNOpaFanEhy;TJfL z4u)q=6P3+l&UL*vZSL-4kPPD5*$qPl^h9A`E8xGW`k`7{s32Kc!8N!0*%lN8)fVwhTIJRptt*_)xSBc&>($znJDXkMO4;11db7RcP%Kt$R|9^o<4 z%{RPm!Zb5}P|mRe7Zsd+z3vl#2Q1M7sA6I2zzd5s04`$z<*@1lk27Vsh-Q_ij~Fq) zy^nCIIqpE$Jho4QgQdksR)>pOOFn6!LSRq#xgRSsyI#y~+zsTr43p8kXE_?x-+?VH z@%=9*xFDL z2PjLIMkcWu!KR6%-m9^ zyW&g0*ruzkv{KI0kBuQfyPIP)KS(ZIKNo_f;-m~RG;p_+IUP~2kK_i{UZ0~Mtq(OG zt_=i7MN!-Ez|)TN6&~+BAaQzZxKANLz@^lf84}%546Cx1S1w~$#N1f(U|U7%0Yu@t zBB+^T_}xifzkdCSg{A58>_EEMSU!fsQgGGqb{OvfT}3Xj+MN^yWT)V`IM62Z35@V? zJv6yOu-fqZY6~~7`{tMdaFF>OC3g;ziX#tM-`J!`nXq&LgpUOMfOf+ zq#h91DGbZ801s-S=g-0Ys&|j2VCyH~&%HrhbVU`S`5_L;MvJ4+(Ycj~%7qdU6X$~p zh=|VxpIkJcDV$FA0S?ZW-@l(;Yrk&p09$r{xvZ?*i5=c8)%x+{M^y5MGy=)zFAG*R zVIAkx`;}2t`Alxl2oDyPf$8`zA`j2t!Sl(61`M$EiJrtQKu~A6`I}x6GwDoy+~Pqh zvc!h|9CdCMqrZ0-S_5IbS$~2M3I+iSDqy)ifBpK^x979JI5&rY#r{yB1HBE*>Fy{c zm@}Ad(${}(VQ()2*pB(S8de7jhDZK{XK6={6}D#nK!HSre42GI?nNTxD`gF@0I}+b zM)nT|o7woczP=9}TwFQ}E&jX5$1v;V86X0tPA9JzWjMd zdwcsUHa0K&q3>m`2YKh#*ugFNwuT;&ktc>9YM=MSPtn=L;Ji*=xW~@U{>|GPo!B#N z@M3okvE|2;ud&?rf)qH~gK5%AMTXc?H-b?T`(8AFR7Df)#Cw3dIe8K434aT3Aq9DP zbpgcOd0+;Bu%Qe5le@rLH&qF225{?=_ijmLkfw5lyziGepU_`kU0I%NtAnjm2gG>W zNIfqce<_b{CF?sta(Dn(=PPf(S%P@_2Bs0{hT4;5mWD$axYJdR;CnHQv@a?JR8nYhy8^0O-*-YQzc=J@d06Hm=4@! z=(fkg!sY9u`Q)IYxMOZ^PQ>fN! zEDRU~HTBpJ)hd)21fxi>8;S2`$6c+dUZ{4VKM9qBlCrU-1r5aUZv!m%(Xp^@W_~z_ z;?@9YKmRx79X_86M~V4N0I>MNla9}38VRNVTzAQpboT%g?lR`|6yUyCUDKKCg-EU| zQNU@U06^yzK3t1V!_3?q$z*#Es_V(2z4O?t4ycsYgG!*zK*n1CZfmkE0BFolUbe3* z0i7oUIO-1KnG#q2=kBF0TA&6+eaC*%(I@JU70k9l5{-JZ;UF~OvIf}KlC!-~luUbo9Z`kXe zwv9m$@Ln9QuVG=Gp!N0li!>FAU7k-c0$}M6tSVUt}0*X z1FFI4RiGa#mU{qD{sda@)9EcFwYo8wQ1`?z;uPxt=qM>*(h{>DG#|x;&mfHYD3{q^ zq?wuy$Akg70qdYew__3V5hfcuJxp)xPw#x8R&9sEtWPI;^2tog+&nf=F&yskcR>Jp z;1mGUnyHepUqEGYyfxtsbOtnHfA9xsZKzgz;V4+~cOXzMd}u9I3D^TP+^bLT`C;a@ zJ8JlipkPRY3V_$YveQu^08e$UDjW-Ny*~S~eI&%X$4kzA>9~9t%o+6^(lRs*%1$_& zt(P6!+TC3Nf;$B?>-XIDmBDbNsl*#-H%A5gVGZ$JsLY+8h~4~(uCm&0iUIQIiM`yN z3QZV``W}Dm-kVF4kellbT#YX5X6X%Az@Y#lEtF0b4<^)6QR_BFa{G&nWH(3ih5k_x z-iQRXB>MYPLQm=Fh!?161yod=0yu%(GyB(bCeQi;C-T>?wrtr-1>7Q|E6+(rkXdX_ zGmxO!uCYu2vtAHx01YX)w|91slp6>HwGA;nOT-^b2mao4pQeL852Ru$mR$wCp32i3 zxD&%)Nx1!~r@#y>aLB{F73;(0WY2xWvBYQ4>~1jsqaGMFk<9^U{tQ^)0dxrxqgb6N zWoQW?M)wsayWI&A1*zH+`gkjfo4;&65}4RwK|B1Ja-$j#Rs+E?ymM-KwLJq z2EKG$4h(QZfk58*78d3hcfkMnT1r=%EciEE);L$T&h>sMm9$NX^Scda$)C?9EY9}z zfRwT@qUqiY=mE>wI)u^j<;z#ma5z&uNCqZKS0{H*hNx4WtCpF(i;*OU{Y6jH&Er__ zGS`)7cnniMz!kWVdEC7;oLAIUot~PTOI3L_)0oYmR`q$CxAhql6LqT{I0OWr<}n`1 z`tnjXVi8|TqB}#+*f?h6pz|T|&!Nykz%?3Q0Bj=8|M+&}paZA`hN}RHw4}?XeyDYG zCE~QY8%9IJ*8$6UX=OOD!4p`pwWcR6BiZeVfZFBzkgAuQ;t<2_W`PXIRKdW)Ma&)8 zwAa@?ma~58TP5!k?|ioJzghCNu{Qn@zr*gM>n%ipAMQZET`O-A^!&Vbk8@CWGU5t^ z0&cD^d?#BHGfg1YZnt=AxpZxY=LK57i^suLo&4XJ3pCW|lGz%lYhmIr*we1yozH-!{ zjJ{aapM~=5KB$SE`e0Kq$ z=;?^n1^NouAHn%0EXXGy+#P>6ILW=EI*|ne+>_2uO_nv=aF#UZ1d?`7OgWIB0G7{7 z;KkMFwO#+dS)xQ6K9#e$MqOEbQ}_0T)!tvsteIV!KkwDQIG>OpLe$RXoh@Hayr?Jf zpi333FNG$|EGE;U!>G7X0Rg}L)PS~+AuB5j5T5k2hrYrv6Io(9_NA-qJxoRdFow&8 zhii>Vz*1L-!22O5CiWk62Ri{bN3{X=?saBbN^c|d1MaW~ev}}Zn!tGiFhwuzr#mCe zgQB94A=o=--?K5os8o3TxLlV;aZ5l>Zuku2;&?5{>^%hz0`zx)IVd>zrTxW9ujWGQ zlIq73emw+_0AiJfqj=mPzi?9^)E7(=pd*eTwsFPlaNpdt7r0aq92Gt3{P_6GS#D4t zGdO5K1V9faF)^`{Qe=Cq;vj$$`*CnM2{Tz8nkidL|>1t5Kb2 zv=9i+)Cs{Nns5UV8n43#MMwY~6dNxh`~uL;N5WbFMQzrZ#O>UHmVjm=5&6gm*k;1i zyEz&6$YF4^^M?)zl1tzG3^=Kt5{YUjB7ip z?bWMeWXM4=+b0#9VQ)_t4V2~5HC4QE{O*{(!grpUr0BK<`vdK^=XjElaYs7oSkd9& z`Nlj0z_e+LYrqgi_l@5%07yPSgx4MZA&ojs_1SK5-?5mD^nK^#a(`$l7T5}6!(lX&jg z?S&&9)HdhD->5glgOmvLxG+{+SJx_F)zU5bGw;r7szkWbyX^&3aA0W!Ti3H-F|Cgu zeWb~q7v{?~tyypp>HxZ5k=zlC=mAbzfF|HWWQsY{A}T6e<7YQmiVQ%fIm9;XC)+R{ zbe}5o_QoPF7*eh_#v(b@|RHyXK^|{>`_hp zGWY#Walg;I%kfbqAf$^gZM`AGHkLaH$-b8L^7MQH98B_V+fo1u*Egrch)-~FjMmlH z|EYTdHoMc=@*y8_cA!$laM_{(PfyD)W`jN;zkl=cYUmI@I%xvSD6|wfj3_Ad60?d) zK)C@TXY-!j?P*-2p zR)DbZ36v5L5&1K245ZBT3?(+a?(RZKQ!~Rf1`8YzmzZQ%5U_TAr!u4at?T^A7G&gp=QXSJ#!+)Wjm4R>uWiw0|$@bcIa<*kw;alrHz3+n#i; zKgj@~wXc9KCSc$H?CUF?z%%0?eX<9tu9Q+{V3>#=)=aAo_@jR=Ogt0+Ww`Gf#`^Dj z7xfrXKR<)$Tl~jBw!d`viw6H@OAx3q8YQj%{+NtTa-?0cym%EA>#ipGpoF2-%$Hc( zqL#Pn9(IvcB$nvDo3?Tx+eZ);^qQDOn?LfkHBdjWgLSH?!O>~#rI#$5v?}MAL08(_ ztA>6V@;py*e0M=6!$Zyy^_C6(hyUk z?f3ntJkqY;oFD>yUYOrH-lTKF=_h`GGX$Vba7hOwrV#(J@$ffo{r+jHsa%lO-d}%k z+Lk_)NK*0`Sbf?@BK+Q!|NlKUSP-sp%Z#v~$K=`G|M3FU0lfhLZ_ZXCL`2uFI937( z#%lM8udBtF!}ZrsuOAh8J^%OtMPd|7zb^&@UIO4?CJ3EiLH1A6tpE>09&FjsFJeZr zjMVY*R}rE6L_GJNZoF!$dqW8O=ELb`&_EesKLJh_fWeSvV}JJSZMUqh9L= z0u2G`2rF!iPo;mhdCCb10Q&E_Bz}zU#J!()^6jSRp}IZ~68Kd23Vo;{(%E%d=KrNr zx2H`Z0kU8+#b) z0K2j2SkZ%N*2}$XS0eo`6@;6T?%AcK=M&4T`LYFnU^XVtW-nB5EHEVF_YA4+;Qt8o zeeiD*WWEC;3#v={&Mjz8g2#ag3~ht?Zi;0b*YVZPffvs7m;BdNIVLFFc~1&kOugRR zPsnBFv?U8B5|^f?)|2!MpdKoH;riw+4$d=SK`Sw{8s#VARwe$d3eWBG7Hqf9FR%NK zyBVT4iZ+foCwEGtC${D*a(Y)RZKpmB#cAxEI43FIG*G`X=&4#F&8 zje4x97W7O=p|>r#(YpRjv-2}bVDEhMJF6ydzSy_5H_8Cu?Qa&|bOn>1xr8K3#fPgM z|72!1?DN2_xek3fV11)W;yARNPxwP+Exk3)Dc0C1^y|%Q#DtO8~o*FKR@cS{X((d0z ztn(x1BlmUl;R0va9^r)+>U2hY>9S@rWn&(VP9m9 zzc$2DU=R4;O&C))hz%>OaTS;Ob+WFB0cA#NV?#;_2#5;rLTHu$o@Z!Y?6^NA<#Cwp zaQ)(K!TI`Et2N4*^*-M28i#d8$3`|9H`GQZz)0a$ z?{f|H4L+=m->wm1WjXw2e9FlpCxr<0F?zj$v^>MFo;mj?HiH9PG}X!Ag#4RMez%l& z_v|ONH(chdZQ@Qa^x%wL$uHGBkwKhzPLxz6qV)#LIXl6BIH@qL0)7N^l4o$$Z{a+Mg zHa{^n_YUl@cX&CkevsB|#>eiq-}HcBtW+K=bDBRHm>Nm*@{0R_$?xs8ZUjMGEqSO= zR^NtZ?b?3ePUTKLuG>`N8^42Dp+Ks)B{SDYH+yG54_mqtjfpuvK4xKOU-@Ak69fFq zRj2b`s&0H_3vAMV`lQhFF60!LO%W_y$;QFU$`WpH^<=JyI9*c zkrNVC}^ z4I*_p_(Jc>54BI(PyYl0M?}wRP8$ ze-4EZYvuWRCiC@Z@5-EaYN#Ya+S`G+m|*Umm{8?H{V>UNbYfDx=#f1|-m4*h?Y+sJ zxM)qa>rS|(#VhFX7Y<(c)yc!eDb3wmMGp>4%$2YE!i)y=7rV``w6-q>-bE0&Z8M9e zYSz|t*4m#R8tnV-rBXVs)uZxPA9gKaiQ>P~D!x2TWd1uj8BdI)=H%2rkdfTqgS4x;{0N*{r?P3jaX;eJ5a81t^H=$z1nKp3A0MQ zkdM<$TkW%>{aiB&Jzur>H(|V1JNf0Vg;$r4@wL&Pq0u3hT z(?x%5#ANRZeaY+Mo|Pg+VYD+g;({dm=Y21EprnL}i7FfcmW1XXm?IvrefeJrtRl`;3UPy!cb9hxB^a z`}X&Y;C&3yyQ^E}3jy5X2GB?F4>fw>#6LVdOk_8z4ayP8H3~>6Me=+x*zqvfx!`2l zIRY#ki7@fz?#R_V;A~&ch}R4&d4_(_Qo5nj z_t}^k7>`XWw@Mr@_gby5&sJ8FNUsdnC=~v<>dX96thn>r?g^gr4={Dm@A~Zg7J~Bq z4w?b51-k72jPFVcu%Q6*d3@Ih3JprgVF0tOT%swBy%qu3%raz;f&GLkc`jxdUf zf`DWU z-=vfwwJ6NDNJan zb;Po&?77qtm=&v`SNS(I->We4YQNHRXg^*?T2qNu!7$B7#VZ{Xcc2oG_xlO=^i1Sh zp&k92O@FZQodLatN_L8q=g@xaG}t=^tIr!XYNCVmLmLBsiLd*> zSb8PjL1_=S2-opEw^2iO?r>U6^SCF+0xTDKF53V4`|C6Bkl2qSv6oyO%dA%ro%*~FhBfN;dtc;9bldIdwv!3P z%Z;i1_NlIVmbt&YbnksR5N(SQxo&_>Vfu}xiOGRJeEBDI3H6Y_vYy)w7e9`+jB8p$9<_hA+E(@*$JdxziqdhnpQgE?Q(Y+bu`>2CDZgY~a~U_1%f zz|!yb>f%(eh?V}Yqgd2_^ftApRz@=vL2VX{mS=~SJ~0(5iP2#vKO_CHYZV;`^M@GA z*#(L%V!i(`dAw6NAkDRdaQ*J3!)fjh}?hpsbL z9rq&Z4q_eIjp{l^zdt|D8%ZfR3mX=hlgRVY{==R1iqheM{Y|4Q^CEY8&AfuDY2rj# zEOR~P+ifRy85omM;Eq?0SG_8$q|;g#cyLSw^td6ZUvM4yVlJOAGDyBk&gm3`*0XQu z|AUIB+9{LGB7;>W1nHtG|6{xhXj<~kGpd9uIi;7%zlV#9aE99{4?$6dN%X^&SkVvL zY9$qTHqG7n_+U%REg#;q7Ok19!y2Rtsg>N0Qm($wM`aeyefVh@hTq5t+EYbA&5Bew6F%|X&{r}rqgpHx#9g~ncZWRk$5F0~RR&={cv;Xb zzzSR%pyxPasXr8au#|J}OIe9;Mr!v?3CaXzWoH`^+CsD}O2kY|OiqQX%)qmW`t$ih z`jNaj?df(~WqjK}L2T;?oPA%I489vn%>;vLPqHhJ>m1KI&DkRljj)8*B{8EPZ!fjj zwa19sti6+q#W1BuQVV%CMxgFXLyEA;$3D)$>UimzM`Jz@&1roq(r;`-WC3<82(CXO2{equTcyDY(rm#iC)OS`&`MZmk~ zVqCBSprV;Xnfvc|>W@?1`na_Wp9xKePq=YH$UOxXIeQv6^s7lAY|*|~D(Sb&{lZ*^ zC=sm2$MYhU- zdx*q_U*7By7C9ZlK(SS1HS6nuVGSNZGkqh#)Y7tAgtSz}QEn`~(gMWq0lrO`%L7P9 zFwH*D;+jJ8m{m=WuXF_Xf;glGw74awo+HoJ*8}#dR}kmj$;gA;ik(0F%%x>3i*NZ> z7IP(tG0hfveSyw3uATnI5zZ%&@q%U55}TNX1;I+rZ)IT96^rJBSREB4?=(BV+Al&n z6;7k{*>i2d6QG~R(pQn85GY}=q^m$_wOO|A?*$?yJ* zyjJ4n=ZAv9N`!YZ@qI^vM4tJ7*dqrlC#!~rstjNbB`AwM2Zc;khu*jT`XA=#(oy^7 zqV4F1vOA#VqCHOqKu(w*JR-b*{kmA~hY`r+w8ffzew3+r2Zqg=x@ zN6&!i%kZxZ-jDnSfn^Gu)|cG}&vKNpI_d`UX#H+DiWU7%kEEQP5SvVc{m9~}0!&U$ zUaANp`Bi#YU&6EU%DJ4rCLJpzl>-VAQ=g%Ahevw(hcEIbb)sq$F`eooHy+aB7kOteDY^9ox~{b~BsF5bQFs1fC2p#8(AMX-PvP7Vc$_FGj@*E1QK(h1 zr~$cE%lTHt?1sTrh=G7InzYUOe*5rO&kk)Bz~Te@gL9UbsfvnK(SBto*qqjZBtVqJ7v(jJT1GB++75Stl|BC zVpD>Cv~qcWs8xhFyT>`$%g!LXRaQcc@J=aF}pi(>5PWh_kDUJC9$ zrFWfamyeycPV3os?xsV#h#^Lrn+J1eYNVxQ+1>5v#YfW+V2!p-xVZBD{n=EzEiz+g zdFKBF&INqjAY18CnKEA8+|wY5a@dOyM`Z9c-f-23o1o>pd?de00eHGlPJfH)sKDZoHJt!mXum>KX3e$>vWvoFQC|d3e!jvj0%iTSB{UIYWv-OGy zuS0Nxy7KLceJW>XtOP*e8%IO@A;LxC~h%-|4_yj$^l5oc1O@F+g6iUBmd^H zkjIXlD&YBWr)+Vq_r=AtHof0;VXr>~Z9xI=I^jlgVs@mNpMB|@EUZup+{u?41(qNv zfoqe1)bvYVAo`^8x*zNW(}g?(V-7(PAnqV+P*734G9C)KO1~G2CGzsSs}Lk>c;)N4 z+C%VTU+t!QB!@+P0I&{;+H_M+u@LllRgUF!ML;&3gdW3#KSm`aB*doSx-5vw>7Sk0 zemruSRs)#(H9OHI&1B$q58$;bdbl6{a8XcW2YRW+?p46m-$Cg#{`(yPWm*0lO!`hY zfBk{oy|zBaF)-s6r=DlF;Pq}p@fWNt>KDoci6IRQ4X+qHCLaRpmDlSh2kup-6i?7r zx)&LG|2g*jBse@Ksl|^BfEJH4>DgU}tr9^JM6R3}1*>mo%G@|utP1ZT%ZatavH(#Y zFBr|2yKE`=)^3MPmDt#*ia>tL+Dn?j?O>1ko%7i_eHx@?HA{#eqhR#W{_64fdcGRv zbU1uZy9rDqSQ>eujeCVOQ$+AzV;kEJSzolW*aJX`|H;sVVq4O-)jmuqnr}U1S9@M` zFzTUAl~D7mYOT3{#iMPdw;<5hnRs0yj>C{|&QHw5v-%rpQ)$ z%lDoy1pcU)xv@48Z&n9Q2o8SSsfFoiqITgEHPg99S>Plf^#18MR(lRtt*SPck>rfN zK%G0&z2(l6AL+nX$XvudnEkKGQU*moU08pBG~o;$&Y!X5It#a%XFwdXA3)zjcEVRM znx2`H10XqSNBDAdMn)@fZ{B$R2$gnCcbfV=T3A{wIz{FT06s}ej*-6{)+8dt91hbj z0*R4vjACR24)wH-7tbYI85OY3m%o=@jfY8)Hw8uvuFtR=s-sfQJbL+p{u2>l-t119 zfCjNuZx_#hDpeFE1K?iLw*7^g8>T}(H(5dpEO(TyZIzH#=H2UHI&o&WT3PO5lQ=iw zHXx3Y<#xMzd8Ae;xW4C4M=TU_5~edE$w}7Ru?7)bfEx3it`jJ3inLo_?5_?-cxyNi z*J*B84^>G+cj(0>AI3haLD^U02E`h5k+vkk7~#3$>d?;Fn5Tc9IhxaMBNyRGSpo;A z6ogMi2O#P4k96!|$K?WhnMjM5(2L69FUrSTaA-LpF@;Na6E;_l-!xj!q2)N_ES8YL z(W3{f1Jq$cvff7n zkrkAc2?HGJ4ZT8G^)YAUH$}_hI5a7R2|!nl=5%>z6*}FsEMDrb6YZGKN_FS0k1XVL z9W8dpQ|LNTd3pJR0(Y)89RxlHer9(?QjcEJZm&5!|)7gSbL2^&X_1(LHSf3dA<#4e8;-RTeZk9L*{6} z$Oom4U&?pIs-kPLjvnQ^#7+@V60kT41V~`{MSZ(&sWFf4j@`AAe2>P@2(6N0Fr`oh zso{(z$aYi!Bfm)kUZur-p<@R81hOtruamt4&7FS7e3>J0;jR-JP?i#1C>ZDu(&z-W zjk=>}KP8k094y#}U>Lj~Kfwk#z5RSc`Ca63N70oq_P+Soh*Q9Q_@mMZU;beFzvvNI z+g6U~% zI=H&oP}*0EDe>ZJS*j__)9Lii)r(ff5jPF54;>C|MnKk;e}24g2H+fE6k-UT$}nFB z?dzNl%()F)+6JG#c>$>~xG!)^Z$<&!6Z~{6VIeWtlQ6QUS|wyph@=PX$rZi{6T%~q z;44UF#1ZRs)_%AF;OIVNm%wU)yZ6c%wHY*6TW&@U2`DV|QIC`lR_l&G<=1FhDZLgL z`b=9FMc9XlA80a62)7u)Fe&%t+fXp8fYlzh&3cXj=(QnM5%nF)<)tewG3lwor2Q%y zlDE3*2P6n9JYtZSUF>&MyHjaRI@ns^I)#Tj^B)jx>q*Wo%k)+iuFoaj15}bXRj|D* zVx^1j0;#28%WZ%8bbzqHJrSa9#d@>bEGvr^R{17Gd*y*eQ5`x&Y;BM@YQn6X{}xiE zq0H<|qsuD|;8=ygaVSX&|Ge=KCg@m=Km z2vRkMcb8ruKrdEV@7J8nz_F9SRK4AUW z?a$iBY$h!yctII3$y*Rp<)F7sewPm-bp_l$GcHxJDiqTy0cwPy-!1iKr~%apK{&QX zYQ-F>+rf237z1zvkYK56jobwfR1#QM=EUd~7=0FQ?7OXeCFigFAu>ZpRG@y)`|Tx= z*w3beNUO&oyTi5TBNE<=pz(~Kot+WD%r1aCiaLShhxs9>PVg%j&rd{3y%j07j9<7z z>#4`^f?^f;FK^e&i&UlEEIsQy-3w_rQoHqFaHS5Q3#()M86+FnlEeU_Ndi8yZi7JZ zfF^bCAxtZg(VJsyd|ZdBH+limXR5_H5y+99J;e%k+)rE>i~Mj*m@Ay7g!|`BZ`L~4 zcbdVPCQe1iYA(NUp@`0ScJN2E%J74mFBMFH7>G%26MQFfNDXsmz#f5@p2SCeYFdtp%=SWexZyRcDF3d@#XCe`p7>wNc_z+RuyahA^4k zU5KqF_(AxN1zD=%0k?$IVp%Ze$KlVZ$On4dh`WbLD+E7adC|#*c3Z`E(H)rjcO>XZ zD=2WMR|&Tv^0+3`q#i-RMo@M@2cCf)#y`NPrQHJU7&K~o)+#DKmZ>td3se6@tBVcS zHn!PT~= z$P+YH$-lqa^^AZqMG_D`Pq;G5m@0sSvaL+DD(~&;(g8>UY!3p9uvj1u0#IFNr@okR zo;RDz$(&B%BVgzkHhlBy8lY-48kC9#pEbIQbo(mOZK6gRP>GxbR1tviO&rGYw$m0J zi!3$2UI@qhNL0Abo7{nDxc~tqgTe=JGaf)G@K?zWTz&P=L&Q&q+^=saHnJM9EW$Gh z&yj82*a8|j299+y5?4R&v{4#9KJTr>HT-Fs_l-(_BdQ-Ms5|uvz>p^hY)};X_lNVX zX2y$?znm9;dE@X69E{rf^&h@4>f8oWBv{x&8%7<+g-?wlue$_Jozck9S%7CE=+f_2 zVo+FHCYTs3UqDF%)#vQ`^Ymf=iO?|HREYwF8fKrMSAxj!={U}Ktu7U}WmHyMokS%9 zkmXg*hG$aIv~j$EHd+p~3|6?3JD_&K|Ba zP%OrTM!dTy*ZFe0AAlCxUC0X443luS;VT*qk(l|OaY-CAZT$POSM!notDrGZTKDgZX{CX@W@ zm)!*q2CHwUfj+tY>(e9jM(1$XOZSXbTo@RCMX9CRFt;c18dUB1@r-TQ8Xf~{&qwUq zjjhEh4ng=3G&Wcsd#dIB4vF&qJ|8S=Omf1*Rk&-Fln$tCg8+ce7SSmc&O6=T5D+zO zqM2xuUa!>2Fqx-)+)D6^jc`f$wnFOTwJI;62{1qBKt4Lbpfr-*;*w!L<0a`Ia|B%P z1r^ATUpSd4j z2*b?|Wl0at-ibtSvg41aN~Ub>gvd8fG@8ic6f2I}wUmxbr5 zI&~&AkOO>jQj`a=oF+r!T%_W@YJE>DY;g@CIZO<#u_XNd#Y-hHWH<}^g;EKAtKC7!wHX86HV$2|&3LaW$<+83QiO*k_Qn5s;Ko zZeW9}d$=HEMI5MHyiTh``6GUj18&KzvtGiQ|LMA5o;!|8Sc6_hoQ%d*A!@msiPb-Q zvkYglbBausifo2}9t%fU05X`{Kef%kO5?PG+GpQAwd~9VFVS%ta-mshZf;iOfkmK# z;DNrGCeBSXE^LWoVqfk)zf|ijjYv8OIU!Idh_>6D70qiA{6i`WvbU;rEu%-GEQgiz z88<#Y=ydPFWAH>Vr;MZt+fB{UBjd1s1gqVG5EYz!BH&3`MX2d-TnIZQO12O$;|c8G z10x1&KpwSmxB&FhfYLoqpe)sRBB`8R{}8<|2jcmFRFgq?Y|zNcncM~eX602i78UBX z%X#SWTY#GYPGw}>2S(-6zXXc+gZ^_bj}HpR{JE4)JI2nsS^s;OST3qID_tl{2BC9(e41 zqU~_X&WiHO4sx-)-xUVE!Nw6|Sqqzq>jhW4`_R8pJGefEZZ;oGPz;}WrgI;*$-G-v zr}K~D8W|yw0OYCnp?rSV4p$&P*bwFh)Vlwd;2`dxo-=4Z@WR{HHR=m^dn zMEO9mBn+@GEL(0TYoZkWxi-x@)a44|}D_VDDm!K!%>K~Bq*S=VPO+ly0Zd~8i zD>USc@p8zsbU;jOX``Rta$clu@Z$IQ_4>Pu!`1mN&o!{(H{kqCD^U?x$C7k>CtVeS zE*W0*EPijQE<(y#OY5@o`S(;Ad*nm<_G3C0qA?ns3{joC4xKwpOP3lTI7gkNtB~u> zyVUFBdIkY_e>2H*jiVe0d6zyibeRO*W7)01H3NTOH;f=p2KBmB>GcKJjs2i1oz!7M z(1x&y6lG#v!q-V2{2lKg0g^ekZrCr8lyBo&)Fs`s3bdp&P9)=-GF2tGYE9_F#E z5&#%ag7fpPGpJJML9bbki zuk1h0;}R@zmIuHPbq04=`Ehwm(bCFUuoCoN!vIT_alfD!pXB(>kxryh0GFbCrX`tRT4n)UCG=hjn%%nZeTb>EeqE*EX~P zNBppMX~%AvW9zg2^5$ORWt>A8Jo;Zhe5uGCIL){d_~h5DTHL{1KIk?eG?;qW$;=I@h!Mlr6*AGJhzB3LRmS)6>VAm|r?h+|xu5 z@MKDk_ZJlf(c@+*StcH~W$!m;HXP->t1~smLQrN(RHg z)KNYrvD5E`4Q?Y0bJgg*>6avsl)x(6}?byq{f{~Fwl)D{g-gSB*=ao!0Vu7 zzNlvXd9Wf%(T_*JFc-3=S*dl$!YlsaE>`Xorc|Y@H4&I;6?JYJkZQAoz83=9@MBnSa^4c*1wzcy*GeL=cDb zjEAdH7ENN&=*&LLJb{`1?{a>luHU49Nzfh|A(MwZdn3BK+TJYazdXb8ygIPW@<(!W z)Vz5Sc|^dW=o$1!)zX4~$MKH%G6zGk`UJEBZMYDK*~{}|p|G+mP~i=ep6d|5&5xEK zyX6((R=d&z^gaw-Yge92-Cdm~t(Bbd9>}y+7^@wj^sY!(@p~*+a+Ifz7ZjJ4{&qHA zJ&oRtaMon`bK#w$AO;@;>At5D^UY=c94m)rv-?C04PMfXlg*yHPdsLInK&RiYB(Xp zBOEotNtU33IR;ouYXvUcb&L(|Q6LYPv###G)od9+@zoG{Q4d&KP6_(fXLuP=5}YSAo{)@je19*O3Uz4uqk{l;}7x~S@=?tHgavxiR@4GcR zUoRYTzT%seVp=eorQ$JDn`$s+>%X=T;nx4o;ELa&>5mU>paa$UORA3H;BaCR12)JC z+TnOg5ZeuDpk2kcIoUg<@?RZBge;3F^M#W&Ed)Q`@YR>AH-Gm^%zakL8>Q1|Xh?%a z)g=cg#%FrsV}m>;?2_MfP-=Ezar$sCzyv9U(0m7!)1bYEo}Ikk?eCZ03Cw4)++Mr5 zAn&(2S4=lkBI(H+ycMf80rZ6MflO-1QcmzvSKGu@ztO2wx`IPai!_zeo-GV~{B>Kf zS~W;U<;={?0J4A;4yt)wiqg=tEw?wZvQqRb?0Mnt$HDTH#oxlAT?`1&8jpayi~+_G z*mgDv&Q^mSkJ(oLH#AY&e=^?JSBY1(ne#PpVV)h_-CUDhKxv+?VV(X+>(8;1iRZh0 zkeNT)Hz!LwsBTR?vuLztUN=E`O_C3N$ssyOJK@qY&^a}pGr-O1A?d@&SD{!# zIxG9s)3fH%#elq(YS-y@uNr7Wi1OxPVT4^8!x}XV+yjc6;XmQqNd=Nd+@2-5 zZedfL?Eh-GMRiD^wg)^G+RfAZ%y#?i9`@-`9I0*Ab1GeG85wge9F5c?r+Q~Yt7D&@ zH)+NwQpf$<04>xm}WXV^QMB%fEL7XBBq> z<~rP_J-+O=kBB)X!C+x|t@XE&=UOGVRmIy!K7Jf@?%hO<<-pO*f#+I|v-!?~s1fYbXL{$rZYuNCZ&dzE=BG==&T#6Fth5DXsd*R<2P}7bJhanJ8HG*64r7$r{|>AJmm~^O=x0&)VV_rUA!XL(1xwN@;!a znrqrFOh&_*OAvbonNyO@nuqpO_c8kKlg&v*b*ZGe#i-f3de$S_aI-7?xvh9r#%C#=2^m7^To6< zN@;4*aGVI(Q}3%|(m@<~qlJ}ghMI?RTE4>FDtqbD*WOGxH9j|vk@g=jmWDy z7Hk;W5HzDx@(0)W9(pw2TyY$9-Ad2)+MS`3laLOTIbG*pjr=RCFNB)Ev2>-&`Sr5b zNQCa|^i*an%tSatu} zWjkO&Qk<^qt4i&%tfcpxD3G>ON}){2DL$vPJkl7*%7v2U}N$SCS^;_DRpC%f)N-hNVWV zuroY!J}y39?C$W>eT9Ba7n31VQ=1Yl+p6QuF@V1b@SFG!zbEO>rUFQ-UPA24_q9ev!&eIV4fC7Ef!woBqTP+Q^SKvW$>h*1?x zGj+IOZAn*w99d&O?23|+UsG)bbH(gyv$8W&ik^1S1_HM_?5Bh46{}-O_mpE4$vStc z!UD8vq=IN@>3W4mo_}*OY^oZ*s6J7<(qXgqtYet$h%ED(&hlN~A~sB^2*9WySLsnD zCUpBd?|vN2_%T>_YBJ%3gp2FfS&T2jg`7YmjvUn8= z(Dcp=M>g^h)kmF&7;`#ZM$E~}dkIFJT%P=xlMui&Kxw=A9&L@fz7Dj((PMX2VJUSGp-Z-tRME8)(|?A8|4@`aLcqeivUgy!%s3VlaVGA>o&n^9QOo1Hrk_`H z6jR-~{%tb!?-Ts;Y$a8<5WfO12?LLT^{6KZcWNT^kRS0~zc-p)1jsQN5s=#5zt+XY zeD?Mg)UfpI%liFM!2bu&N;)#lHN&9&9Z0=F$@Pn&GlzH?XN?DA5P=G%5#rZ6=(WHE zL-rLrnG64!eH}&Q+Vp)s{DqSj;HT>S@~&gAJjl)8?}S=xNEnsxNaTe|ZNnxWl(C}j z;Da5|xdFdbveMg81f9HnM91-P=#JNpo}g7q?VRq{gJLAx>;DTkQjNL^0Pq&V4Z<2|d2DJRS zb}*;bK$5P7av8{t;yhwKhQtpRF6aNdw*s0E&BZM159O^vfd^8QgxRlYPN;|I-92%m z-{JlmEv}dhoaM_ZV*$Xb<`rkm`x@A!`xJD;sRiyv692^I>ie%AS+#jKtoAxgDOR#( zI8j|{p2WW9OR5g;sC#FOh$V%v6<$^@q(!Ex1Scj)aVae3x8swsDvGQt2MaG70;y{Yj{U zjX`z?K?R5f*C4E$2L4NwDMhMr}|gckNAQA@u~0IifM!MlJli~;tLdo zR#hSHeZ{>1d1ENI9ELp<{Ol&X&zBy@i1nvfS);ksxDx+*{6~z6Icn&`*mctGMf?ui z9wPuv`6UN@pakY3y+U+Za6WJISBS!gEnb)QU;S|F%PR*Cc~@5PYrsLkMqQk1PcB*Q z_JQJ2%GRQ}SD#ALi)=hFYzGYWkvfcy)#1`>s1riXOM!C0r!)V6n4qBQn~z^O93wQU z8KBF5auNw+cB20<=uxXWZ}M*K z#cnuKmF;{xNa{bTcV41!3%2JZRc#)E+U#d$u7KIZ+BS5?=G>tt&z?2Hw%C0(dDo!* zw|l*4gmvG9gQhhsEmPG?>do~BF^qq@DY;fJ-|l*NYk+Fvowu?zJF)w3Z+)>z&^)F7Uq!UGPGfWF zJrbR-OjSf~P(;P99UZXmf{+z6e_p+$i#~?&ZJt0xGT3c@GE#ESPt-9m?%Vh7pMSQE z{rpghs&)Ds=G||7P8AM60d`vvP9qixrv=dyhP7R=;n$P1XU|GHC}7(-dPrd@ZDNkiz)8x-{@hf0 zbjNW40TseyLVgbUB65V61N2W9OJ_myns@jkG>kf zK@fKw7X-$J=(sT;CpcIVEE8X(gRaum4Ld|n$6!T+{bZ!7<9XQorXr9xkyYH`+n7Tr zwEvd$P?bv#PGGZJUA)J7F7o>%>=?a%Mu2q*k|!Hp-U7M$tzk@--dnMs_*w>d$aV-i z*iW3Mgue)|!u|L?aFrzy8+V^g##~OYYqiCk&O$Mi@|cZZFj3e zg=&GskNf-Y&Go0UOVa+iuzkmlx;|b$jl6^pAl#C4eb#xu-(jnlw_=+v!QPo!uw8RB zoXKg~k$*`{{7?FF`eWWS*5*uTr)9>w)KN2%~6&C)2zL`s(fP#aIh)iJcI@#uD&_o$iT?G?) z0Zr5>Uj6`aoLr+pK?y6M+9!X4ZeIF>2M>@uvZw*-|I{{7~52}m?eGgTjArj~7xJ&o5?M*bPj|rwaN_F+X5Q8RNMSIsK z7>1_0gSxOr5+*~OYM|8xsF(CLNH6t^oVxSTo!BNgeeHRAditxNAY-vFj1T1LO0=h^ zr!(DJY*@<+8`h<8aH;e+>;n75E&?8E_T@>VnA>6;XFAZHf1nA3fBeYZVB$EQfxl)N z{)g{Rhv^bMgetVeWU3+u!B$$=z~x^h5MEaHokd0~OZ|$0gM;HFH@6(_u0f>#`P^w3 zJ!BDjs?j4SX;2|HjfMKxeK@>{PzOZ=90(kwdRkgg0mRW@X$%|JnKmYr580Ulg^0U^ zZ5yA1YXM(y>iF?XRN0@zOYm+yTw=#fD^ zz`bLK5AztjOgdxu)eQCnUB@<${KCVNAyHN?rSrIj;u@Cr{%!jXa+OHeV0o;5)ADW| z$?z?h%p0n?F?=a2sM2)=ci7K(rQJIC9MSYSl7(2ABA!GmVRX)DH8jt@t zo{NIK303tB6=GFDSjyVkAy}A6Q2HnW&s{8gygCufqDss zE4SFXa-Jrlaj7$zi@_Aj2sregRO5=cfu=(fcK+g0jf;Y#wYbH_b>6&r6XTDS%#V+G zBx+|DUf#`;b()HJKu3!xUq^PF>cOX`aXFuR{{YIV!g5W|2nngdK`1=A52nn;>;|KD z1pSrJ8a*WyM~kh*tMlqNCX`)$er3bq($lMVdAJ7*Up<3b>@ZDz*hESY4v$IJ%IGij z`T{#E$5l0EaX98P3}e_i@lMp6g<-pXq4BPlIzStGr4PD5*MgTEsH6mSlpJ6#Fam@S z(~!*J-)Zm#e-Q|Tt8n}qB$o*KD4j$CUi|rcNiGx4_Cf$e`sJFG4=ig4f(VpGdi5%N zxi9FFm~}Ui3SFQhe`xUKJ5a-t^zq#M(0qq69CLEQ2@RYChzO0MzLE~3uj$#YsVFF< zt(QsPWixl(3f z^ONA;;c!~;?^t&sRq<}Z4v4SpJSLzMg-D_%AEiiJ)TvMMn!=c1<5Q2Ynf` zGLGK!1 z&qq+cZtH>$%lf@(fx3NMjRChn{dp=+#ZuEBkGs(1W?CQfY`l2I5_)*%XX*0huZ-Ju zdC?HV&lw(drK0D_{{3RIKjgrgTp`?9V0eG;=iX4tq zQ|^VC2hxI@ut`#aMrw+3^i`9Vbfb#ON&%Sq?CE};W2zKCj;b)xdMfgSklaA=ns>nxt~E`0-mjvEehfz25uhs>bmqS?OL@kfPpQ^cBE9=8rY%@oj53FO*HnI8@o)myrww|w3Y3b^&-KN2;H-duzv}-^EA9aYv zusGwaXt)AMf@nb1Qt5pM4gjv=fGu9N=or`pkp`PkRgWE9X)r)%Xgo6o-%YHi!UE|A z0Lp|qkoqo2)~#KrxA%4eo~fj3y4U(#L+(Ap z=-5~@KQ5AvJ_%FV)_Q1x52GLZK;0a_psj4p|%dRIjuv!-j1 zjB2Odfb<5#6gG^53t}=TM;&BZStj)->P6vJID}(wi^$P}qm)pNl=csIO6~=}Tv$6N z(VZ+3ab4dsM9hBpHq}kQo@3gSG8=b6lh=4|dH66mcX)#aSnHN5z4kh`GF0TW?iQ>G zM$>C|*4z0>>6k--yljcKO+ccm#uKoIG@jV7C$=Y0W~=!bnyC)3vL?ex-ec`~DmG&; z+lOMnL*<8k@HTFp8Mlp+D=XxXP#Z%QxCKfCL?3gm=M5W^7$YB%oD-VduXxmRK42?d*%fkqW1=Ojd16~wNx$LLOwU)Sic=3 zVjGvv_7rwq8fKf%-Q*-qt{?R^a>^E;0Pf$%yKtA&kuo3E0QwK6{^%-21 z6HkJyrk(&POO9jlc_A}lRanstj`xBRlqpw2pA)m*v&(68FlIHRX}N=tZx9;IIET`Q zs}R-1a}vPA0q(y>hPgxUu2C*(e>kco&)ls3YWB2h6wTFnCa)pJfcgHZxLXlj|U&e+oUJxMdHR5H*bZ$X!i%`}y-# z?c$?aupKb+)Z6w@pFk_pS*y(#B$qKTX?GVk45u2@?p_j{a;k+c>Y|zrOYU2>eU~l= zHng)dp5o?i1MhNpee)8GxEnxrVwF&p{p?*YMm}vzF*U z0(m5roX$FRKS85m22g>LHPr$^eTz2zWFP+GpHz1O{^6FP;Q_t;SKxrtl0FE?n!w@( zqz8DZYt`m7`ws#b-`!Mo?@%t({-DVcqLZ(b$cPOF0)6SLS6=a4Vqd z4$H7x+Z>)>c1xnUTGR{`6B%3{NfkkEH_x>3w_1^lTJox4GS!Mmr|o`fHYRkaN>jcy zwnrfPq`bOXeW4)VY;xxWVy`P(GaWJj!QV!LJ`Zc8)q+waGfz18II~82iqGcM4>$|# z#CQxRifOeA93IlMOPN!nUS3{a$YkGUvfvEUx(hGSDt5~|p|Gr8;5Z?j+GADJr%h<5 z-;GV;o!XGl#k<%iavyo)$H>2=0zB6lP|P%Z=?#4MLC4baSWa8j7$p3@RM*c^kX-+m z&PzT_upSzgN-U`w_LW&EDH_I_#@K8_JAtFs&QV z9&9|9e0d8UoyqU_2dQW-4POC=fiDCE)6}RTR6pLal6B*k4Z0o6#_fJ&Rm#Kdoc{sn zL%aUAhh^GrX0o{SB`zTE`Tctj!Z(a4zWfhXVVlesAK5tff<(LCZ`Z3rib`O>i|v*C zQZckEIm57)331%?Ip9lJX-2H`bVC*@C5#xQ-{#8?zXne#XHyY>&iDcD-w=3<7uQI% z9J~7D`SZdk+wZTJd|Dly{t|ICe;427Hu&oTmWCN8_2#B|djGx+#Chg_u{T-@uf&18 zJ^2Ail9N*dPAsm4rsyorxtyJBCuvI3|74W(LWwKzj6~!jW-L&WDhs1C1Q&7RCRK(v z#>bnk`w8}EoI);rBSWO=Pd{O=mZRVYaWc8lOc=epCY2oB9Oatdb(FB|HmZv3AraX_ zNtqYJ@?Xa;KRpe~zs0K7SkJvs@3jr!V~wU4(M!JQO=(0%5Jiuzc8!Ye zoO2^-z;Rz2MgaV?#tKX8$d@wx_2hVSn6ETNbF8&1Ilwb%?%UEhDb=6DWcIa{gpCw9 z#C63ZVvYSFm1@yfFIBA!Ia+k&mm(d$wV@TjoOQTs$U-o3KSR+$TI93wJBKy*aKv() ziIl&tuO9&n2l3Im50}9bm44mI%8SWUR#_M>bB3+pKz?}lFhq>pm5Bm*KS7u8w~_p~ zH*V*svXZEpZEoVeK(1*et-V0^yF|Soap&&c*K-QQ04c~CJncEchY@F}yk+gf1;@bB zU$teC*Iuo2d1=^sNFpYv`Dlci@o8MGP@t-6$)@#jVSeu;eC-1`+$VoI;V(fA4p;mv z3*ThH>FROp1LT<9EXpY7U!{U7*eT3Ttg+`MAFx@>Qc$~r7mQqw5Qz8HckVlY_<*n` z@`Anc)bG7WLs&&^8bF>-4f12DO|=VK6ZDH*5CsPT{J>)MIfzF663xpWff)T0iQM&(7cU-p?;pVAj@frOHWx!T|T5?s$v8q zZa=^ss8W9?8RAai|3*o_$?9151<^j$FoBPV#7)q31%gDw;MQ1vRf1kl(I0>O&b&Ex zNHn~CbZg1w4P-xQK!<`w3Bk1KifUruvn=Ns2c#lM2c)TJ>D;B%2uFgS)S`7EB%nkR zH8zC3HS{)9&6_?RBIJ6X>i!pxtj|ixDIlI_YiV;TLH`&3*^c~-$3NaVQMFD^u=zq& zXZVWhDZ%r76#)XkbEF+*mnb~mb)VMciH}wCY|24oQWj9Fsk^ zU*&{8)?o$|N764IA_9HK-9}s6N*BYJdXHskE3$HLslCvSo|O;EG{Dcz91n^A>DT8Y-Mm z+Q!s@WP85q`Wfd(_gF(MW_E%k`^w*cB%?V=Ir^>TxYIK@d^8q{iUS+J!zHU#R6B7U z46I1<_Iq8vAY7lQiWsI=l;3FI7$Bc@uSZyZZJZOAIyF%|7dx;lagyL-*3F5W76soF z2{8#4DS5iEiF3pBfpYCM>tJ+4_Kh0`b=F87CNR&=UqK4*Nv+MD;}5sANCbd}vMP0Q zUNtg)YO+w`jLIAJik;j9mw}EmUS=2;vz~!q)on!!?Flx06?f}2GBWK)=+Wyvq}tX1 zFAtAu5!*;iVWGrsXU9zR8hC;SdEy13v*QkZ9gg2lgchX&lNm!-Ae1sEEJ*ZW$tp z@H!yg4?M+-7keK4{Wo4c3xO(Qj6yhK-fo%ZZJ&hh)s4JPzedi;Rh6O9029J0)5vk=?V+6>W9Wqfc)RaG_-?IH`Sw+;4&oC znltgjZ*5f{C^RA10&t-JX~3bEG2G@KR}E}YIKp^NNl09Ck8oE4GT+6bL>5_q6tz~B zdjpb|0xs2DO#{V)_B2yB|FA_8wXqU<<75s~e&9LSA*_yQnK{Zc!P(d${io zXR>FyT}nrig~eAWJI9^?k*P^umO2WcM$HG?BZFpY@ zq^o98OD>RN0zpz?r2eJB(32*SyRgx~My~De{|-^($2!?eAc5pD8~W4-2f7C1AzeN~ zLQB%M2ngYv5CzdoSFWV#yZ7BA)CttkzXC|0yBUkU+LEoZlZlc7`(wTSZ)II-x%-&A z_&OlMWJ+Mo)qN>uN^thXPYbcW%>4Ei92rgcXbg>%XCOAakP2nH_X32z*fqilvt3N~ z8eE*>fix83-+^k&m;?v^CFO{r9axE{DEHOp*7XOZ8k0=iJ6tNWmhV*P?si?-8~n5CVx%f8!-Uwd{=UEe2$*GJb-_<3~FhruO#r_Fnaal}zf+&WK z&q@B6VA#DTH0@7g+Vb{?GkN)umzQS$e}}0#(cJnoc>^>T~=f%~p*)iu6iD zb=Q||+QoP5_?=9lc`aZ%d)s|(j;H=$HtuF;t6dGzU?83AuVzbLo-^I=#tSK8hVgUd zk;HtHq4Y-P)#{UT!!1?sCh{~Jniu{eNeixOe0kERnykJ0i2PuA%2~pl&vR%p$H2HC zqRf9_I%thmKW=%>Y%nA-6un8aZijA{`&vdndA2~<^4gD{qH*Ey&4-D2nBAWJ$>0DD zwx>Hi!&VSpj-Xt?k}6@cr;>?{Eryah>&HQ!so4u?Nzr3wIbv`Disoqi z*`oyFDT!Fl?egIVH7tU0&R=`rY$7%2$KW`%lq$bPTK~wtePdV@}r|ixqJ-*w5nAu~dVGiG?5-SwWEg;2TJ^gp7uD z-KBesk-agX*j7@;XdV_D={X-Mtz^+T?f7wEDu6p|?q_c9wBLK^!r|uUSM&q2bGBvh zJ{M@V-z=1@Gd;*n{>4`PNlVL4P`Twy(RMbrHt;R-xK8RaV8%+u;pOKjwM@qelI)&( zvc7lzX^d?Syyfy(YtBM?dY5s`+-#qwb`GZJBWXS~2*HeimLRkvevguerNO zN3F2Cod@^uiet_*FNj$rVeAK0^;64Sid>pzdPAo=IP<5xnv7TTN#SMn16(^iS4Ho! zjgYlhwBhXcp}&u^EL%!fk#D~2WLXYI&V6oJT?9g9t_(`@pti$R`o&@=6lo6Ga2-2$ z5n4V3!jBGpzNp4=N>H#jtjx>Hi_tA2Ai%w%*}Hh<0i;$bFwu!?X6%FAZ#)GJ`}zIe z1I?|}HNZB#i2NWsGql!y{`z(BS#zYiZv=U6@)*C_l$AmGP_JG-R`P^eU*l4n9?fy0 z!K#y?Oy8Prm^Ov6eXzgVt z19Ng)A%W)aGX%Bi}bt(WKRiQg>Jhl9RuYZ_{h6I9sr^@|F~DT%I*_^vXb`l zA>G>b;IluBCFV=+3xcK9?%XaqV|6UuR-xF2Uu5CCqBI{KtI!I?O(sob8jj7*1)I|OC_6ZQ(O-+z@34L23#!1ZAYv0IB8b)e1; z?jcTRL8_use6Hfl+hTn3L5a$Mj=09@r0V4Ky7*7=D@TrebXhHL&N8ZW0KBp3b5pX8 zJSX0v)dyRZ4@hmpZk!4dihq}GBq!&0=X;pv*B+0s1)Yx%Wwo=+?OF|8X5G}2Bq&bM&3^82lmq@sD_W#$`E$>@yMVm-gJ~mI{dTtIC=dBLR|=e&7Z~q4KH}O+ z`g;?$$cf>*nSb<3pXHyrV9OXC=d2+a<2`^x!sI+0HuKM~l z_~W9ORwNVR;(VYV)&63EH*lxz59m_N+X55JQi^0flf>@%Sdof-Pl9c~pXmBvn1hB| z_t@LdP1N$r$A^TYM?cLiy*>{b2^LVV=O~>B9J+5Y)mI-@LJZjd;qE=7qS~5v(MA*$ z6htx-6j5@LoK-RiNX}U#3rNl=B0+*A1r(5+lSDy+WI;eOl7l1x$*~EimV3Wv-1~lK zoZolc{;`LQy}En#Ts5nndg`f~tLFYh1O$bohSa${->xgr!}?<)p!pG)-qGwf;krX? zWMqzFB8{d~d~0r$TX`;>LBVbLvlzMwUqgD4p|lm*sg-A+2MRoCs6;=o$$EGi*{K&9 z?53M0@Z}ZJCoi$dj+QOkiHw=QX)~B2oLXnUg1Xus6_vY$l;AWpDypZpKY*8bhS=bWMA!&*T9qK_74GX>o?Xv(X;q>!-2840$LP<|amWL~SQHa$uFcCYE) zNh_8^_GUX0m-^v@-t9XYDkcxik|(Qm|AjNCVhnT1Zv-xj- zUEHMyiInP^$C}f@hVsc)KdIH=r;=ql6mK8I?<*dj>7e%~he5B+PT}(xlBbN_qnNvD zZf~%fhtdTCBKJ#1k`48|R_(oi037OS5CS9D@x5gys`rO#_r;20J2RP>e$)1ALlu|* z%((X?!c~6SG~*xcp@C*%aQ6=II`;_ayMCPD*L`G+{>(~&Iwts6!MP!n9PXwMsF&M$ z6#rzbzMOp}^_a=n4{-{IEoZ%w=}ygbZW{Nt98v#p^sD7hPb!z= zCou=B9Bg@N^OKfYAcwFA-67CH3H{4z_V)G!VlN(^aijD=Ps27?BYwmLETn#oNS^Ph zfC>1l4H`sB;-}9a@=aAK0P9$`Cka2tyeAOSN7B}>+4^b`!d_WWut*7xev;j(uVlTj zpWLzhi-H`rfsH`yk$}=K%aFZ&G1+P71LYELSA2~O`@XLsrh#W4j8VUb$awt-pg@{Q z2%L3jP>{Vs{PA-W6S?KCN-2viKExJ_tKi7y1jXXNT5Nj`($DAo=#iX|O#%tT=S`V3 z>m=q{@euy(1b?r8)cHMZo#J8~Vy3u%-(+Et4W1DIn$cjscE8gL<>7MM9K=ci{8>%S z5CB8@1b*$gzg-)^7TrA(-G5U(F?fHhA|rQ>RU{@h77z&P+`szJhP!~LrM!<`=@|A( z)Uz98)T}Q9)ov>YV!*#6WZ@%JK@Q2no5a1sHB!0>4_6)9^&??us4r6;h1i7tq80@4|I#&b2Af!y2G4tU))CNqF1` zY+hgSzQLm>3l7dzhsp)AlNQL2L{yvs@&QPe?|7GsOM(8Npv}W+P@aA7khWtvsLXm;RKI_Z z8swQ{@Js%qw{75s^?c^tGvQ!om? zRj1-ls`E-kT=zND*d<=7=}Y}}C(1v63PhzNIV$-6S73Yax`U8iV0mg^++Ket`)InXP&x|5)Yi5e?DF0S(E!kXyIG5;#I@3qS2&9QphAA$?Kd_h0E1L&EU=+{8; ze*F((O_#y_elg|kC%X!81QV;)sm2ln;Yg=hF|w$gmMS}E2tMEFkhl6J+w)U;tS9<- z6!!Qdb>)iTsVo6U(s;OE-Qu0%xWvRXD1*?;RYcuh>h19Bw`C#q&Ld_% zy9+(T@Y)b}o$OWQsy|bM#N`2d?cIlnGe8O{AeV?@L2tZG6xv?`E?hneN%&<|DhsK0 zkNjq$(j%lF{#c5%OWH(lZ*6h^O5}`6cUjZPdlDwmbuy!=aAcJU6l(6{Z?kd_^V89P z*3!Y@?2bbI1Ft&guEDNyuWv-<>j%Ym6cbW_v*z|5wO7yA>UN?)X@!^tw)C_qMt9Cf z(K*+xdC01mnA}Dj+FOtdrC6abYAmi*nj*;Y4UcfTbrfb1+vU z5BpEO!mU^Jxd^sR#zb9>d*RQDPBsF%}+)(SV}=#eB!k|(uIw#;o=hB>ciWfw}?9KD%3~sz;=D@6rQU-r}k&| zF13JC+N*LpS;qx7sH-oV3-bc+l_4*kV-oNTyeDblvL zy^zqdxlx=Pg2qek)H)jZE=$X^TxcQyMj=uzhQVcW(XTd!-Eh(gJTgX+(=x*e$0C== z<^alD*d(>^OZ9sM3o41YE2a1pVK;dEn<=sgu$66?Dh&Zb1XOo_x0Bplyb{6<6Kxlf*6KlD_zXkA_=JKZ2+A_JU{$Q*GK&6P8i zx%%V6MY~$IFE{y9ab569 zaMv^GMM%xQi)u_JYN%ig+Ns|r4-1HSoy|vMeLZ&c7Bso{m5g&~Ez&>?8`T5(R!x%I z?<>w#p94VyF6=T{R@USJS@A&XV4|Fom`m&2!gu|2UfXq6GGcZ>3gC%4X6%1l4)0>8yYbRm(b60PaBrfW-7Li)--r`ZMa@0ZTEt zD1g@a2M0I+f7eASEgGBG{{dbatk;uwKLC@?;)eQuhHSfVs={879GBboKFi3WK}3SJ z&+rE31@zwv#0;mX&?Eb+tLN0%Lf7v}*X^N;r_6&3tX(Kd>l+WGinDw8Z9U}yt|6~A$i)AD%uv8U=-s9^s zQG07b*+SLA8Hc!UEr(Vw0GY!eGPOTbS^@5|p$5>a1th^ai*rTGuN|g-Cs?odaw$z^ z4I6qt01hd08`ZF`^;GKvzh&lKE&%`4QxltIZF)l@UP)nT6w(b=((VF`P~I>j(3ug2urZ=ck9YkpmX9htaU<&`AZb0(Lo zF(2ydIJGxQQeDvRoU31y_{?#ed*`HQFl!&v{6xXn-u?%9k0#UwV_9`|!YGFNc8B#- zki-Gla|%A6>Pe3|@|NZp!Q}(j$?IvRZFlGG`MF*9EC!1-)26LUN;nb7s$)2)Apfpl zXxddO2MqknA^Qnn{ULMlYnti|FZj6R) zBsYK}^6kVbZ};uDYt#aEE6fQ?qrC6rqd;D%F8(T*DU)5@$wLA(v*e$E+l zEuHMyv7-dgg3;2W;Nc%RAr+cmV&?E(RuDs^CFuLqBzY_-P>CcM3tmATzsh*k0y$4P z2(d1l9k=C{AMIAgW}Wc6Z6v>#6D1drWqY?0x)>H7F1WICy8k;fKi{S(5!26=`{}W- zZ|Mi$mQs4U+v1A^xn3E$tzB)Bf<64RWnB#=E7~U1?;Mj!qP2|EpH1CL8$Xf zWj3tKU-(Xu(bINWO-)Vq3Kg#g&&qiA^3*12BkG3ss>afpL4ljnzAh3<%tcP&on^-2 zA>alF@x@LvS4Jy@*BlnBU)TfHdm;H9JIzYDBWfd@Ro6vE&>_I6TprgoZ}+FoNSAM3m2y)6#z8dwG1?9>FYgj_&iD zt77ZJ%C|2`!oQ1|VgW}V$j^n9n8;bSH)dV1{f&*<}Y zbr^6=vfkPzZe(0s_8Ji!I(*e*dm*es^}s}1m$Yq^$Fa9YQ|Q=kcD>4e^N!W<+N+!X zo<>8VCD10P&irThI)3r9LA;&SUM%b3V;tr@6Iydci4d>j1J3<02F-xXVz+~K0k31S z3#g^BEb&4~=r;eE#@ebB7g$wwo>I)=(*67E)7eIB&8BVaHv|MEXI|jJdCMcd(?l=` z`3imj`ZaMHhmmolQ`eAfppnIar3QBV!&h){>&rmt+4`0J8SkyTkMELy`K{3uB^k{l znpE%0fk!Rq|9M;EK2z*D)RP{{_XS4eR8-94XI{$hdd$kQGGqxz-}%k95EuC=z~Qk( zS8sHC^`_Sqy&rBnsR=ALT_iXbHsc&Bmaddh{IqkWh*jz9cQS3f< zj{7Xvz$aZ%g0A|3s@8Wea;Gz&#I3T3j0^#fJ&V^}x_jbGu_jHKwQ*e5UKMK(9yAtv zbg>=VHM;&B)i^GvN=cbP`8~%ZDtf@GahXvNHq(Uoc$|I0Yf)NSbZ}B~p92;a z78(CXTtQc>=NZ0F8@GtZil3f5wV84tYYNUL^=leZ(?(jgm41&XdPLjZbu@Jf-T<+dFu?u|E-jzRvqcQYS%e#sv<3 zy;^@#C#7X`6Ha^QGG}jN?|*v^mFrez{C&wsfLcD3OoFa1I;-%bfq#z(n?hx+MXk0bp2uFk|27Ao>r`4WeQ#hz6+2i=#*-QE?zyp^VO zR&;rt#;c5ZHJ?VRjfrL?GD+f561965XY`hUz^+S~ip)_T8f6Rn3XD<*m41UR0#tZB5GA51*Z!4zwHz5L8*Fd>oO<@j7URlOg@ zAoCp9%*xU{`g)0*URT zE7CeRv3!Tc$#|+3)_p$_lU(P?4O<%{NpqdLw6VDv2s;dQ_~NpE{d#zOe;XVTFW6=p zMBpE!rU`Y6j6{~A>Ol?*BhBw){7(n#G={suXssai=(%dt;@;*D>qS z1IUP{ZF5fk&%6rjlKgv_uk{j zh?Q^rnGpKo`EJ|9D4AI!pDe$iz}U>}0?b#MdAqs2c=lwxvW&?5<}JALC>>gF;&`2j ze2yF#T-F7i6Z_gWCDtfyG@bc0DMgc}h@%&l`fEf+D=cy3|y4}0K!!mu3uep|` zsNYh-`g(d{LGPaV1BnI&!?v! zjr{!iQALt8Rti5DR-}2qGlpQK%AE$9K0sP;N6XJoWY#&FLzJf3Qz4_IBt7LrNO1jn z1lxqvT;u08xvVGc$F@#xZjZwaS-cLeV%!t$h)heUr-U*_mD{FCMH?X@;U@g}moH!5 z0P%=YWu@IzZn+$Tv@D!{JXnJ{RiX8LEwKc6xbmWVfn9GUdWo*2p*603N%IQIj zI$jef5)$|K;+w5q*xAZ8Tt3Ns1aoay+MV(2;H3ISrQ&!Qx@!a31_EuJbhxxw1i_s@ zeq1$h&YS6aUkOdfMU&ZjFxgkMc5H6^aAl;}fqiz4us5#<-_3r%xKa6)`dxh{s;gvC zrpBBck3=67AS>ZWdt&#<5;b&>`lyB1l3*#RjPfkfmsWz#Zs2nvjGs8qmhY=n{N1Mw2 zuB3!Ou*@zTZ|yxe_E|Xko9wO)g+V=@8e?}(zov4Ui+R327pZN7R>kG4!NtW9 zuzzV^;ja{MOvaZEziiP>;LS;MF_1`@W&nOW^@W**MbHz4+JVWlrof9%E1V=Eo@HN0 zdFmTGa^r%2f*DZ3a-1SXBqityf3X!1vX8Ik(DkS2ejapwkC-> z6IIxXz0A%gD_h~99IIfFSX5*+e*E~N@0pR;nD4be6TUd8I6*Tyh?tV!!m>I3F40ln zAsYJ@4-ae4m^s^>iqp(X=x~X2!n%>h)SR44v`u!_B3J~LPyQ6;ZIva$98=!HGb#uL zsg%f%fn+Lt(ix`Ef_*vE-vE^EU=YT+s+Sq zj#>TDG;kp`;>Pd6QdXB$;TLIb-3VkD*&wfSHf6tjw zyb;&h)@6epY8k0>F3~|#3zXd%wuLs1U|qhwncbCpUp6+y;fcc0UNaggltLkf#Yi)* z(1ng$3evJtQc}6xDpAOn)Lo93X89B+qqZR|B2u3C!T@lYS|dJ<#BWDu=NEZ-(G~qS zsEn;TKD*bSkAEFiZav(+?L&E6R-&V-re+Sf(NRTz31E7HF;0X^<7Cor!)(w|#ry2V z#WGBscW32a1qCtGJH*ftFq+!g6|E7;(ABuG1W3^Kx==_+P#DKT1PGFeu!jqjElOW{ z{YMKh(~wF_u#8DU5{z7c+$daT28#f|AySH8E(-o$>TGVLg}(_bFZ^|u1jJAOUSN6u z!GjkSsyCPh{<)4DE=gG<+Bt{&I{jMw!%PGb?>umGtAcBeXyM+7U4nyyM-`t3$*>+qkOk})hPmO=hbPy6S~oZ>Zb zU+3gljzBmZKCQtOtG@yAtFNuC6uVz?({zV5vLAgF7!y^=A(DUl^c>1*X@eL5*37_i z-hu1B!@=Q&6!0fip9M@e_C=<(2StV{Q&UasH-3=o)w+xvh0b zYdtFb0|MgvPR#C-vl)I4^F}Luh3)3xaLL}mK?33!$76eZEuDl#w`0e}qPdMoN4LX1 z8u;IcW4aTz^3g6Cueg*s*7_9r5faIBh(n@fx~RjCAC5hsQuAjw6BvXglx`Ta=J+&xma_dHneDs59lO zhK8$1dgNQLn7yD)@Z^c)Mq?ZPb-p!82?=h*1?;)-9xS(|;LLqLbiRFj;%{uGemPCb z=XJHS@aES1b(`@or0j-uU!g}dGAb&h;63S+G1s+fUrwvsx)hw1}YK=v-^L2&9Z%*E}c3 zFD;F@({(4-;~7q&Cd!+V3!1VHyF0%LYV?8rX9%=W}41ypGwfB~8 z?DRUM*baRlexSU$?@q{Rwq*}cCB^$_KzV7%l;!D^dnkG7+Z5L#m*UFGZ%uY@gJhgf z(9VrXwUZPr9NgT^A2IYLJHQjvasRafHGa#C;sbctC?hMYa{yUCmA_-P4!ksnJ@aFx z6gJ)^PEH2~jqfVP`LWDTo-i_pX?8}zb~79 zA|}Q-yqK8mLm2|Qr_<4AhQ93Z2x}TX4dn*`o5OtQzy|H9;oKGx{mQ6&Jl1XRJ1IW3 zsC3xx*v-{=_MziiT-1+RUF+k!W-lAJr)oSeFnN?o>FJ&GiLW5i32Kz4I_sER( zK*3z_m@9Q2RhrVeOSalK$>x0cVb9wBKEzS5?$m+;9u#EL7YKbnL}a|qY8c>NdA|_z ztqGQGgK}=5Q2oN%D6i@W9OE3ymZ)oT_FT2fYo6D{bWXV}`vU5t^#0IAysFDn!*H(3 zn5w0hd6p7Y%&!=t86PQS0#v5b^$e#oV{wm^502zjHSeS^Xqp>Om2rc*sjqoGmS*fH z24I@Tk~vcQoe}qpF#>{MxrD0;^#;t+QTNsiU? zTul}W)4<*^fTDd_VVls{+bgQ%0F&h6O>4~6U022IaTpnM+;&E!wfrP$BR@Wj?_*HS zbgJ;4tVN@e-tQ&Myl|2mnz_9Sq+RYr>5d3vbCqsHX=lXT2~D zHUYyAsI4t{z6a` zZAv@GYa16qkoJU@lk<|YGUrI;8PVRF*aZ~gsYI$Pzcyg#Pp{@5Um0V3ckVF^p?Qf; zNul{vo|TztJS--S5F=h*$D1AdR{*3tcAcG|?NNT8Ka$ce+++sP!P zo*ePXo!VJg@K4ruo(eF~1CkGDG|fCmCoP*xf0qdhSe!U6*W0|dlM(KY^hVQuY@>>= zgJf>$FtVI$@HqPs-?TA zH_++j?)>bT+}nz?!YSy)C%j5YnP6KF2gRf*H|b=nm*QkT)Q>6oSCu7=xDWziI-XQUbWTX`{q0ad8UL^>$Lt7 z_RE)F&dz2{ABJD}wy_}-iyP%Po0{&Go_GuH-Sn_!#C093%0tP*>jzjB!07=q! zpQ?Q3L`A)V?m;+(c;YU)_^jJ(^S3%`b-JKtM~w#}D*kc>nT{IQyCfKN*RL-<=6u zfpk)>>MU^M;!NGcJA{8Uvu_v%-3B|md2>BhvF?3B!mE!UpJ9Dq5Mc-*MmSq9mbyQ2O+ zY&<+ym`_Jv)uO*}dM@x^rs^wwMEo!qhuIVTrC|L&+;F3bhlgPGF4=1^tw)gatsc6% zSJ(m~DzQIR(yDzzill(2wX>am5x|%H`W0itEQh(pS9*f|%aL~|eKgD-CKRqU~H0q9sJpQ9Vz8$Yy`P$UW{e5>yzFMLCRbu#&ZV85tg5#0W zfcqMlt)BhhI-t$AGIHNk>vtLq`qgR`-nB{wt;<}R7^pu_ z=bcisFQPIt^*)EnOwF4O&`L7{G`E!A{XOU~-$87f70JHi9rIY?@t%AN@n`QvP72;X z{ytMK%|PfHTkERKQw9g&glMn@Pz$EhBh1#zml9VPS9H-f5{= z3*_)=EJC4f-4g#E*DdfpDBEbB+bS}R81fdGVWL;cQ7x|*y(waLcAOB}wk_PSA)kDX zZ>VFhJ50Xrt#5G-6i>3^&jb zamU*nGmAi$6P0X0|E`8}V6yL3hz7{K)pI3U?e#sSkN5}4Gt3W^i zF;u!CWn^g?E=R0yWMqT_902a`RZ7ZDfMzf;UTL=1lx84T`+}<-P5U+T?^Ob@nKa)z zGC*jvoBTq&@YrVX6Tz2=q8^t^nff@e2vv;6D(#Mi`B1m)3YXRZVtM zx0XF?$A^EJ8BW)_==lD9cCc4e4#Bs|HjRLsJS^youkeJDkNOCEXRtkg@C4?%>}!`$UG`OMU>D?$5aQxqG7PCT)0zrroO!+1gE* z%DtQbQn2^`hy?%;f?g2zX6@Z!7H&&DY{$idMqrqM0bmccDJv;y{h&R^>7~UUuzjsi zkCU*gJ~z<4zThh`&qx-6{JK40s!WHzmt?Ef|BtmR_Lw=Q@+IiOppoR%9|%x;>R71SPElB!eYh0M`d7bMGQ9I;i2JtJ^-?Wtems zpd0WEf!V4beh)Iu91_|Id+nV2`ZWb;?)w=bY&eU;-@muC5O}&40b_gF%Mcrg|7*k( zooS#NYNQNB6+3MbvFHm+T7C$11PZ0qy8fO_#&Y(O0d;pbp7tz5AHZOz^&tw_aBhdc zufVELFInJglF}2gT194U)6;gahb-v)8b|J4BDd3ahC~u5px)1_xR+^$uCvxXeY3JC zPfKh2L&untOLr`aLP3T(Qsz;bhZ%62+VkE8`5{o&P%4i|Oo2PXAx-&)i;b;Yeuv=> zZHHaFH0!f6ge!>j*s8Y2MU9M&A?Srns9LP!ZFy%KU~TTKBRWqBiJg{qvhs`eWvL_3 zs91(XC<5X)K|$lVuc7+fnB@EJR>L|i$?@r}*C7&ztI$02AF@MMQE|uxM?$UP$wQj1 z-jKr^O4eEMX72@<=K|cqz{gdUC6g8wc*J(XK5mya+0XoeVQU!CqGw~F>+}50mMNjV z$xf0(Cu&f~-!(_iZfv9{JAdLEfA?Jd$&pxqt)-<(T%5WxGrP33=lKKuN28OKQxdC_ zjt>e(NPFz0`|mgtS5OK0+#H)Fap!>w5RjLemNn`*)j{Fm>H<$AX)|SlytB)^k6*fL zeYOhhaKJEH|1-ZdSy0|s@xGEe^?urTKe%^+ACcW=$#M zF^EqbE?>5WZ=u}Wpagj^{&SAgd(jGKZRQbpVo&Ecg}&yWvl_tZtOMfIGbAh zH5BG82gOR@QM~Kak6oGg0EIMs63b+?WjC1#hKv|>4_YsE+HL@st80`0`$W92uDv~u zMNBjI*Ney|v`PCk_2)wTdlKoLIA+Er(13+P!8A-(^R0PKPR{$l^jWd9siwO1QjWzE zi#(e%O5Uc7<>ei+YGYH8I9&NYKK^j63eNj!FPZrH4TH19fvJs$hv!?##AtQZsDQ2( z!}?BKLP9nbU!{OX=i3r-sG${%2y{8bRph?W#E~*|R&A_vj|AIXz?!Yv;0?h60Y_&=1qIJeef6R2lAfLpVPC7@05*gg zKd={$oFue;^4GMJ3R{p!u_Z#zbng}M$L`n-Ahxr$`w8#cu>~g+67-TjFi|787X0#^ z;Z=gj4jA_J3cACa&<9v$mJbA+k&DJ0tlLLS2y$NXhMZoJJQ#k8Y>(k?4 z#8cwT2SN6Oyiyx!O*w^WX}z0TbRlN^^ZE}N2PO`?s^gzhhIZ(gcq65L!g*TlueZLV*ft9S3$pkBP8(AFfy>epCHMXN z_rLpp`686j+LR1t=R5h}R$rv0i6n3rV#>>JKX9GplvbYN7iSqqX(0zw7GU zNcB~JfuMK(WKJ`}J2$bt1@ZOGtgLbozB1@MFqdxwwI@|k%kRCEUg5kkzL4#B+v|S> z*Hrdh4IT z9Zj|Cdf}QAj!1V2!h^b#5bfZJc!VL`GcfE9yIW$wI)t|2N6hGNsCx~5hsK^p^feOJ z$&Ze0Ljh#p?|-3e{)oGnf-kVt?|zklKzZ^X>v6Bx=}`cl=%obhSA?}1D1Q4xquPnl zjb_RZl3EPZ>vvD@>FK>tq|zld;lSQ}cGaKlL!judk4lTkvvhC!OAXJ<%E>YD+Kl6( zmVVR;SPgBoS15oZj1#e@L_xI;OFBHMKi+iZ!R3V;26!ev0=9nyQtm7Z!+Oi`{53)1 z;`RFEP&nhkLg-5DgzpBJS1`)Y#r8eZMB$<4?xXY|FNf#ifju}8NFIsspa9&CKilB; z^%M7C$G>?KB>9@};#g~Qv!#~}=N9QLEXS+t*qlp^?n~l$M^4mWbn_eA+X&`=9w4bA z3my$vXqeFk@^ruAjZ`_?$SVWCaqh~YenmR)vk^^I?2c=T#YW~-=QYJe%I{JGT zw9jz{Z;Mav{LZ8{H8lmwI?cDf+Y0gSjf}t^R~aH%QTxPZe(ZY4*(wn6&0<@ zv4s8xf)|t@;9R7H_qR4DRKfZjh;fVmqme~gxiNC*G>?psf#AQXN9@WV;(OE@Qq$&8 z?WAIFWatE*T0eQhG_E8g)6q9nF3)_Kd>j*HkM1^QIWgrz8MPN%x}AwqqBtN)1?GSy1Z z$EI|U3Bl2x1H~jzi)K6euMah8{t-Zln5Hx3KYPst$NYXU7ZAT>&i0j zw!MPaMvor{0SCfj@CgI;2qvSCG~H6?j3IZB-Q^vc`NYj zDp&s2M`A#YB7PA+;=98bJ`F7-23N_iGbg*>zRk1Jo?6CjS@(b5wYkRs+(vqc^o1Fp zdqy(hV^~;qm?*W}rBtW)%$n;)UmKmZso!&K@_q54ievw;hZ{s7PWtQy+{NI(K18KJ z=l0AgREJ}qzKe+o&FV40Kz;d=%Q${VP>=)@>*OWmW1vJZf|>&4M1G)Nx=gmimEM%)yQVp!JK3 zIaEhTpoT%OWSB!dJlJvxN|HEZplni1tjpEV12DpE{X3)V3zqV05eUOaz3yYB7fjX9lX#I0$d zRxtYt*gv8QQ1;Gl#R^%qlvl9f6;*eL#Z6QlqLh_OtXfWU<4J+SR39NBBYR=qW0*%g zJW)WYr$!pCKJuC zY+UR9PLi=S8#I?N}zo`fsYH^Emq z^nbc$oM`lZ^4Iouj{PgaK@6|?>?Xl`oY3A<`5K5%^B6Tyr=C5{A|F+!;gP3N5H(tI zR^u#VId16tKHX*ftGEYa*mZKW+FBKFM!w^*zSpk0NV(~{Sec^bxbwspsP8l3D4XE@-~Ok}%^-%y>b~3@R$Wbo%Q!6ivt#{% zJWcVDAy5V$i}hI+J^sX<>{S~^!}kd5-knI9#{lYkhNwwT?rG#bAtpz~`}-HL2+YV< z*t$G5q&=pFVD=`1PBivgpV#9{CHQ7wTX#`+fq?i~a2+in_HRJ{h+AaoeW?HY!DapS zn}$!9FQuO5?C?W(w0E%}gD2f5)mSYpEBn95w0y&9ERtO7(Kb_d1UIs9}?5LLAe| zX1P}A(j6Y&;Y*5oRGv#86&Qg#+mDGU&f&32iQ1ARL{^;T zgAuq}sM$zFKbbsH~=gPa1#QsIoj zw0isP1%>k2(UFmlQehNwV`u1fvF5h6JmMPn?FEjzD#x6Yz~vVi=DG4n1AnMoVWS;j z3l;4u5B1u-vJg^sJa+bZP0NWo@9Qv3+^Xg2copaO?=ku%B%zBv$*`kaS}qh87UGOS z1*|(+tVlx-tbp5UK*kac(_INjNHR5S*xA{M*bVfIb+xp}s72M*nOLQzrI8a3EYXMC zi=_&N(WD?MfI+yQ`}WLxl4jO)ccqAjvXwD;d3kZhXm8zeSnSTqD+7wIXo!=6Ir!n7 zXiK3>;Hs}ZA0WNwQAe||RIX%1KU{|alG(T4B>8{bgZ8HiEXj(_vI|7+5ST>tl__IJ(xz6C_1f7jyQ z6jFu!Uoi&qRsa9^qWH)P!UNiN6I6Ry;swZtMDL!XBN6atzsRKsxBN>gjAPR0q4wh?*5f6k#m!lVo=-|B?xHPrO)-!BEr^FrPihs(&x zwa(mv;UlrkHK_^alC0)26KxU7{GNZrQ1=o?)n|VHPLCx+EWBVEe<$kHW zvNNbS6nnMsq8euWtsk{CUGc6vp+On_3aYbmrJbF<#^L$L@W6Xg0#1d>Bfcdj3Dt&0 z0gyB#uEv0L?CP^)&t1xLK_Zi81fd>9vFLokfI2|?_8w^Q5P7%HsRJ35%FQh06%+_F z?DT&)@vJxg2*l=I_bBB8-tpe~l>Bjb;S7twymyqduw_2vo#nX_hF(?Tu_dnA$K6 z!couww}A{txke*ac+C?sbf`(f1;;^hyiRLpX6aQryn!XO~{ z2F5CA2NYeE!!{hF*w)?T&c^a|!sV);S5jPPWJGJL6e!yqPU+;3AE6dhRn7nP=g<3= zo~4-vOFw3qxoyU!R!a3 z0%LP?e?-far@5Z$v>e5t-G^h=8PhsZqZHsx!u}HlDXv=KYKS6LJr=6DjoxHFmkI^X zA}>u_l|4Yldv(CmiUnj8TcHFp)xj) zT^o8L{cZouwg@PUwJM32I3h)Gs<;j5Bs{( z1ST6^MRpj-FtL26FhPCq*mb{u807y=^URecc4v$b%WtuTSh2+())>l+j35{!@ zOgOu$eb~`JesaYOO1}XY$XAATo573V+9vEdi=I7v6Mx71s_ot!By^c42QOBeFn{mw zj|6Ild%JZo2Z!b<{SsTYMtzR>bZZ`dDL$yPE}#$zFGy2696XDTM3AgsOPg3+=7%~F zqY2KvQz`Z42ci)j?F8N>PX=v_lR*q0!=R0gvbR5DV$rQ5Mx?f?KHeUcZDK+|ethwz z!K=9&hHBSze1A4Z7l1|zD?C=;rnEE*|o-tgk;NOYwJyC%;TN8M`f0L5s#$z3SQhyDIu^-M0dmJtx1^y_w7SL9S8s zGk|k$0ri*n#!XgMw0Ux_`ndhUG(}Za`xG-2;FNRZuA>@0 zKG!cX>pz##HdVX$vN&Yp8{zeU2Or726MMu6AOA?lPknkOO?tH9w*XK1cN1p9E<9y( znj}$EEsWoKBi{*Lt1)m}Y2n5)eV%T?6{8e(!UEkd2cPEgT07L?9N;OV?m@>-Kx!%z zq7wWIEul8`9MR`CfWB*b_2=2f^bCx1eMy4_8R$i%6J)V0MyKYFVJHRHb<`su{=LR3 zF~^x={2@(0o;q*3pF{|N^=Zkw`M6(oG;7qP=DMs9u@+%Vsv=IX6^(a0wB^wIsZ2Rv;#Y0^$92_!d+90kwWgA&w+Z?fB@sE!&`YlRF^bMYs>AA`Hrff zDFAI6HDBS{bANx^x88(=(LYMIWj+Y`QladPrsSN~`sC&*iF$VqWMx5sWP=UWSj4DA z&e){i4=KF#jt2hHWZc{f5B&9Am&&ls}z=ASb2<439F|~PX zXGSCD9*h;{aZ^;3fSQ{4uV_=De=&BTf#9`7j3K{?vRZDqmayB>Wsv3I-V;@6A6~pH z_wKFXs{{AslxJp|ANZ@OAulDM#>J12NXzuz0Ff=K0ow9jvd<6Ojrh;pG6!w`&qM5^ zou|ob2hcH)1}h7~jm>Q(aYWXGYF=4kpeV_HN+Z#K8o`T(J?_1Mc=G@3aiT|#&9tO; zDGyAcsN1MV0m4J&Dg_UZ>K_a0l{#-zQ*%Lf$DZ$I4)rcK%uYMdcHoM4n=8^pui*=z z;^K+FZLJNwCiM|?JO&Fx`V~+d%f9}lPIpiZ0|Vz*hP(Zvqqh|E$v8!`mJYo@I1XUK9)6huj7{=K>geKm+;{*KE5%qog2Pj zI4J(`;Rocgd>>)f{QkY~CI)mazBZYd+J0$A&lPe=&h1SFGSiDxQUB4HzKaB*2Z&66 zz3UbBTwo5RBbjr*4g2Yuu*dN^*L9JuXEmU)K|0l{cWZj<=Im3lFSfJ$cQY{gDAGn6 z?TdAOA|l2>^5}<&cU2A!_tx4PPbOxC~@H^GC|9)$fCZnyBLQXMr^N#kVNGPR*yds#vIm`tZAgbSMt*QHBgO z{`EfG_k7)L<;-WPFY-!@i!ytd0V2NpsQOMhdv1`mvn$UBzigeK@|ILrGx+Mp3aU}Q zwgZO{v>EDAS7~bBg`0N`MX3k~SjX8lzF(WD?vNHt6!gAeY)n(SR%464`8NXzkT-gM zydrJchu!-`%pla%dK8;ITU7X?yKjWI+un?<@CVaKoLFeRz36L?^s$wg9+_KCRVz%w zDs!m1F-jVIu8HlZQwRf5G@J-aVb&4TJ+${{0{h_0PTSg8#5CxXH>%VU3SmLxrux7Q z4Gp!#QfK*`{FWPQ4vLKbw74B>2fh9c4Jc5{R=KaF^pTOf1H}3(a{a7Sv=;}Y1_U9k zau0f!KY}69u=i2DQ_5m2laZ2YEw**w@zi(_am{`I1c3@f<$Q{n1?7eiklwxg_MXR^ z$}DL~Agi|#ZmiYSzuL_GVdx^#(1FOHK#xoe-7H@rYL|vlcjT+wCe5&|;JqDycG$e6 zJ&N=!!8bDPtaFj3CkcB8pz)J1?p5rgkAMGP7L0{OC8e)IL6x6>gqDu(vXCoPmx@+s zaB$OCadT8Vdme+%z=O59D$A1MxSOB?t2q_GS&x1>HZnykGXW|2KvB%b$`_4O{tw%MCk6%VK1kUL1&29$3 ztJIypK71Aqy*N)i$3a#G^(X%tU4a>u`SD-eOw3s?9&Z=@{`&13#`EVlM<9^AZ8T}M zkR&9Wv+>9JXJgX&^P)EsD0TZq-R|$sV0589SDPR|oQ13vhZqXmmnlYww@LF`!rtRg zuUz-TA^s(vwf{#b7?>(lfo>_MD00<;PLJ;~H&c7q2Y>u>j)|d* z$mk|LZz56zR(Vv|GJHji&5I%JPe&`LAIZ3jaeWdiDJq|eHG*Ul5tYJuoFkv*L1PuT z*F{XixRr*us`EddiW&6(XM5dHM7Eaz>@QWsT(}Nu{vcM40{M1Y9FMAKic{-v6NqDg zxVpvbape@>@iw>IX7D-&Q24*v`|@Zg|G#gvQ>mnqP~SokLiR1~Qb_h?i0u1r$V{c> zn?w;JOWCsx*%^ZrvhUkq$i6f78D^gM)%W+ipZlEqKKD7#dH#9+c;=kWY0)*;^|{{b z>%F|(Coi9R{~l6m$jfd*2iefn)WPO7_7@Kao8!uxA`ZSzt`RXVIR^>#QNm%1@OI88 zCq&8d(Qf*9A9e%5;EUNVInD2cfu3O)TU=B`b4)au+=4DKXnbnT zr=EybEWS`*XBRd9DGtpTP+r+^@%&_rGYMLUdD)@}K}JS(zoGvW87hIkpNGTA_(RKx z#gE^Tvc7_h;U0i8k|)h=r<>*5n&RbdU<=}N;_1I*eU zfvvl=6kjhWl?F4uqd&w$QpsRCY1(*G&{kkMO*ep#*!bCciGx8e-!f(K)#J^$b|e}P zjPkLFl>2>cBW2s|sL)0fWGjO^Tf!Rh$fnzUX{x5qAn1JS&%P|UIr;gMU9P(vHC*B+T))&Wa4pc+af4DZ$G5|b!^7NhYfB#A$qQa!N{q zYGJFhi``R{A4a1n?q9jtjew6#41?wL)FndkVevIc0zMq8g-|k^{^T(sB1%I3+g-lQ zshL#GpXzKCJo4_x>Ws==cxJj5szFm1SEd9RqLdH~(pn_Y)d=sZLQ`-^kabGY^&kT> zt#nDU$>)nP=6{Hjg^wNksxjSKCJ}HvqPaN7iF9F@)jXm}2a1$FEw2FARxISeL zeE)v;qtxh1WID45`r!Wkm3cxJ=A*K&9wF>w?hcSZQJ z0w!*7?nM(RFtVPV@CCLrkKn?HVdJN72I2wsdXxRT7n+QjLl0n^sG}Cqif-&+`Ywry zAx+f`H;@I<8g;yEOzPYY{Rt6=)+Ls zdi6T8K6_uFp~?&QD8Czu4G#ZcidGUD>8cFx0+!#mY@# zF*E!9ya48HU^U5Wp98*@7i_DmtKWEHqPR)Wb@_mgbq|BQ_r@uh>|lW5G{)2PWAkle zhN<1zDQ!|5IcB3TK$MV7o&o&tO$VH#G(cL5d{NMxoZLul&0U8XQ^-l7H!+6yX0@F# z^it2s9;||fq;Gkovh||_K7s5QZpplqf5OuLSOu|9_&=|LpyA4>LAdxjx)P0WVJFi* zUiV6kor(MQUc46-aC+!wz+dsIIcNT6E~+SrXZkBEzEpVcEN0-whpf{VdB5o-?mN%& zjJfTI&C7#rM?#`WJn}_KeBx5Y3XG`cnh{0r;)z%pH_P$Se(^Xt@f63HlPBd(;(#XV?FG8Fmt7J7L<(e0qk!& z{S+3$&#m9su{zVKmGIKz)bcCaQ|z@T&MDO_RqAn- zm49x=K*%$%bOGVbwNeM$zKmB?KQpr%bG@6zV;rw!l2WWvom(j(iBHv6x;ng2o>G|MJ4Q`6R*7zR&XxV%h4M!N{@o1InT~3 zEr>@cskZiZ*=5QFzq}zfzjBXPjt*M7BnP8OVL73>)H+hjosi46qa#t5FhwVB-%Wvo_t-ylihK)MGUVH8?w^T zZob;|L4SVz1@jF-(VY0T1$DRP%k!?qE0uv#9TSzl_VvL&n0OkQLN6xO{ABMuAoZdB z{n;9i*&Y=yiXBcd<^5T1mcDQ3rNUW3y83i?`d0RX}B7r9AkwMitpwhRrj8csi&~|L; z(_VJ>N|BtGH9jcNm8Pv-A3=-b_Fci$kn6+mwM>V(b>!Z8^L~Q@W)vt6EgP@k#P4Zq#By_MiZhzNARy`BGTMYLIK21$F1NjEswO1EnUt zy}bc%h2m3oDNz((@6c1~RURlR@F4j^@?A`?%ay-& zVT>3Vu6XC$40QN$T0DA`NRi*58ZEzaJv9agj%c63AMFc*W3T>4`=pD^L{Al&Ek(K0 zwss{Yb8B6{{@f9t^Nh+^B9oNF)mgyfDWITm?e%MR?LxH!xNeuC>6Db|r-7d$r8S>oRrv_3z88}P*-tAA}fGtz$i2tY-!O}e}|4zz1mQk{SP_81F) zxo$mE5Z9Tn>MNu#-#9aNRVIjRwLc_oWWzL5u0_;ax@J++bVQ+g(KB!ucMXHexos+X zYVSp+q5yASLpD=oQ@GM0J-nm+AUX7;1mSYzkC3139r>!%$61G+S55Z^KXC!VZTJk? zu2MJ`?v$+PeJPd8|1|-7!|)H|LjEkad~%40ALF0vTYUq&Fn|e+8O%qUeal(;#z$bW z0hUhIr+>C~Ug??Zk4&TTWH%ieNOVyM4n4)hz|74A+dw}Z;f007Hv**9@4zP5L#b)n zg8G9b^exrK*iAlNK_-mZ{@|^OQK4b@#qh+AE3}YdFFT)8adM2biviJVR_7=4*oTjL zqUGs37`jC)UkF#gGcsg(>^#Q!oEiD@tHJtzh`QiOp0oUOH}G4wgTLtU>tX|9t-YOLF{o z_xyi*pa1`&|2p&kBnSUbo+%=%Qu<^*Y?$XTKc@nN0bn%m5;}l-hoR^QvXeT?H{s^nkoqw)RlCI9MimVVIeBBs z8~IKo6uW?ybGf9`;B2QWm}G=E7z$llu;4tq5^(>|?UZ_Qpu?nXGO4 z`k?Outn($$&!w7_V`{rM$@L<{zQz7xw`HUGgkVKYFwG9VMMctn%Z*ZJaO?R`XjSv+ zIKsEN&`5Esaj$%4d8fX%{Ek6ja!!C-k;ZrmIq*M8uU~DUA3~I^J6b>2a;kKJu#7JL z@Zo5_eXpj1?^*|zw#>r?Yx=h%MQ_4_@U}dgcCB)ZU)dJ9em;HgH47-}`>gd^T`LJP z-P~G+EsyFv7FJeEpKdEB)@qeWqkqnAVuv( z;+AP&kxRmS**Xfml3ts+nHdZV|Ap`@|M@;i;E;cNLiGDcSOCM5mJx&u$hlFk%qQQm zeA_cd(XVg9;#S~^hb#gmM#7zG+OTK2KK$CuP&@~|K(9_2eH0i@zOk&%&#BG|1B7ogE0hUh?(myl7BMkctw zb1`yWYU&EBAClGM=9|m|TStF|3fWX=L%(o$%o?BD>FiO;P2oWdsPuV*ywnkP+d)`DpA3xJ9HS>If4yKi zei=)9{FX-vr9|J#C5=5M|Kcx!JLGlv{(Rs3XQ*@QKLx37g*6gk;o;+ICi%8uq#2Kp zu&^Zf{%H_CWg)4aATGIN_=ODRg|Gbm_ueu(jXDi8QgAv?3bP^>(q}B>7>~+#U!z|G z1ODW+VXoy{3}uR6wGZTWT9gheXAm*TG%D6Cr%}>jJ?|Ijiko7^ZuqC^WC~2RHB{)L zUVBmJ3eyepZNps^sk7q17G6ZEsji-`+{5E;G=T^ZFxu<2aeEIjvJ2`oY-3)eow!m#NpC`C} z{}P@CB*KM`#v1ke3*hppZWB!fo=D_9U%HUywJ@j{#(rCFDJ2Ft#=FC>tlX0tfF;p= zDkbjulNtKCE8FX&nVuYTuqlCXl}tEIU`YFRh7_xT$gtw(D4PR`V2m(C!i;&q(F^hL zie|zM9a|T$bYGC?`VsP%ev`H&`WNUZ1o~_3JF2QcHu4kp-9bv?o*Lz^${Y#s7nO8 z(8D)i=MdbKZAa>RpOwik)1B$Q`HtA&%H(^Wz)J9C8kcsh^#U%M589)mNi1;CPPloX zym0x)Yc_=8zAHmN0SpajZz|Z=G>}W6J%@e3eWNm^cByo)XkhKya|Z_O%NKfb5(IR! zGJy7cawJ$u$JO=Q+5!m&#!kYz1yb(S+_(HXI8X$6v^?3p9v=238B{l2U!{A)gh!6kWkEVtsc$q@5>ekVI^KpNXizS#3 zKA^-Xn2o>j=3;$Qq<9UonA)ZETqNkVxOK~EH!z(%YZ(K7uBNoT(r!r}{Z+8?mQ%{- zBLCU5Rf);TQ64j0k-J!qee@bc*O(6i7l~nA*-|y0Kj@Nk+uRIszy8khf8aTFi92&$CvwEt?cS>RNu{`c!*Ov=Vl3KbLHW zi;N1Nuy9kuN71vR#1U2c`v_2;Pe4?J&S)Ns0G>{D#mz(+ulZPsZ}mD+&9#Zp0aG2?|K3`?)5^K@J^3Qm08POdpl$te~~Lq7n!Z&dHp^7-lJ z?>8dy;pTU(>0a^|Q23%GJiM2Wu;41{lViy*7e5#h9FL=KrHEF41X7|X$AJ=b|2TL& zVzfM3tb27&0#Cz&!(JWE#W58jSIzJ(P>_0$QszDa0Yx!) zqQ+^g9rhFA$P^20fC*U?(0!+MAS!*J3K`+9Z}RKHP?b7fgS zh{trNr;3=6L0+xflekZUJKkug>qx*t?&=ff2%({&uH(}iPBHT6wjV!#>etP6e0@+c zcGB&FY4>O49jd+X{<1g}9~X7kNa)G$I+pwGoBh+KdR#ee6{nbaR&VY{oM)8VB8I-b z{;4TOZbF|mqTT`wX6`#j^=zg$dsw+IL-4U!q88GygH?V=Pv`tvglKgjr&LqAk4%g= zg-vh&xZ*GLZyD1Hh6gxQnS8zK-Mg2|;oa5Ni}Mvd{Ru3HkU!k0-Z&FsfO&rG1;!pIzue=Tn%o`IlD$2HLjelVSmZ=8VVElSt z$!1SZt=66mE{f2HX-Ml25`Oar;c3;Ry=w1pT^O=Cj%WCw>pCs+@w> z)1R#gw}^Eyu*C%4fq6q5ZG~U8*o)*>C|Z3qwcBmP#M(*WWHp-(ihu!0kH_YwRG0oA%p$c4o!#^yKH{OFuhPzO6<(h%!4^Kn=%0W<> z(qypN=(VS`oyYu()m>w~QN`+j8Z{h{>K|INO#$m0m64J0F5K$wBGVfH z5yYSrpF8GOY&2fY>9lgf`IOkB+qZ+1Q<>fz7eQTyz-C1cEF}sfu^X;s_Q&_^x!hw2+D-aCviHf#{08D0P2}d(8 z)*P!~2hpCBpd08Nnj6?yP#4fi8at2aX6QYPkB`4-5q{x8HM}&w&#rGp$3Hm{a`w|8 zcz6iD!71>9^wXb)PGu6^^cRBazunkoJ%8KDNmMI(mv2!#P7hx-M(?7@=N%w>4DNh8TZL|qT`zHev|05!V4geLL_JixMmP5TFG)z zcKfq)v%Y7BP**T_ohTss8^W%9 z!}XLnjICfd>DI<*4S_I78M(srhPHP6i){DexvP+@W^bxRJ6t zbNrgLYjM9g&5tsbQm}<&qDkNeY9&M6fR^)}-AQ>9%x03T*L(&%l0jQ& zNXQD<%AH0uFzZ6O41B!2Q^T)#4i@!mya^62#4EY{e!iQPJ9@%(*qXmM|A9J8(~f=| z%+MD%>AJXZQNE0^O>_98ES84n zom;8_@-O#1p@p*b!So#S8e3RE)R$`&44belBF`wEs;MwRo(ckNUDZDGV1@CXSYDgo zKVBE~`Y3HJUc+^<*V2QO1hi6zq|Gzi1826W>xKEn_2PCDIyjAW|KXmO&iPOK32`v$ zvieGCt1<)&Rv|nxE~YE=-Mi1i#wDZ0j!w_bP1<>Un@P04`X^2fHeUlqS&u}Y)oFqI z@z-NFT@MPBEw!sF^|nPxIn9gG9SR4M5fz?TeSQ_SP)OO{vjG zKw5!DKHM|z{rfKcq|a;zRQ6=P!7-aeljl?;e*|ni$fp*a>bp=TbMKlFD;hn8Vf3LB-~ z6_s>O1%knb)e}1@Gj9UEy<$3YiM>vnvneF zIS$Br_2JRW{ZD^FyxCV!?uHsW@wfBrO#NiA+wt4j!}D$j5Uk;OBu@AAs0T|5e{sw^ z@8ua6wb2|u|A+f2Z63O9&)wYB}biIN(OkfVlfn(d(qOZ+cX?@J6#9Df#P^^A)$NMULEX& zmpP$#GnJ@U_H074E z?U8ha?zJ`SK9j(&Hdn{_Wqwap&Vga;9fKI9)ddpih@Me7rVJ!H3@96d6)CkLg`jhk zfd$NjcJbie8bbJFaM@uvwJf){^$47O$WpCb2YS1wAc}Kuz)rTDDerQFZI;1to6GpL zo}L~fB@vyBkB~ZzW8%O90q)>qjQOlvxw9UU?A668R^qb&Q1}ior_24UgLUX5=7BDF zv+p)CPUKz0CyYB*K;{hF08>c=17o+fu@t_-A#RxpXheZwQc_DH8#YM&wDOy0J@a5H z(!dYS3|0DvuK=!*1~wb&NMQzf)?Y$+Rr}QV@cSHho*Z>yZith1`97lk2C_$HYd_7& z_1EoTe!8xGFZ(f{_kswz3M>P7g$ZMyHb1mCWlkP|yexVg&8U%Mn!yHh;qUFwxbLmI z@!J@X@d$)sQ1uo;;_5jvbxfCB7_Ta<8<(qOem|=^4i)- zv!UNnfdvkP?4BX-XHrI)P8Wc%5CW8RxUs31oQ}>|op;2flm&|vLt_ylL~Ub!kUNtf z6Xx^NPOteBm~U||u2C;2Dxws8Ra8|U!%A=EuChJ}y5da_cw1LkVUwZ!+M6R86VS++70I6z1aIkGa z@67u5Nsj$-8sY1lQjQ&*PWXo^cHLPqHG(b7Vw8~k zKgvqZPA0F}o-Yt&p#6xE?TI`x{9W%7dvmSinpQYcA*}3c1Mt#<^Fy3GS~FN-?IV(1 zg6Ft>~lR!S<>XMG;XYz&EQDR(2ye0Ps_4yw5d{7np9E{+BzbbUP7o~ROP zZ&R#r^^9KBWt(_lj5KJoY>v%@M8a?o`uZQhy8VORF~dFlxU#hwS5fvRS zXcz9{VXnv&Rlyrsg9+D$_YVjLq$d9q`Y_be94Tkg{7q}_{1m)I~|QKjvzl* zfc&78aTBQlK+2gw!cgylOvKLFhQlzT!sciw@|s}g?WCyLb8hUGi&vEBO{81{xG@gY zW(rOQ!C2>}GvW280roJzUa7cyX=$k-@9^21lQle8tvh!D5a92`B6&1S7j<7#GlbUx z!ea`6Xt2gWBuv=Y0Akekm8o_I{$V&P?Lx-^JgPfC8B~aN{@P9}Dp=dX3t&#O!zLb8JYg(&tcHW|xuQ^Qr{X0yb

#mEZG?p=th0-<)NhojDyii}e0PnZLhM+3;@ zbSFW;gn)nj_HBNm9{_xtCF-)Cv(~?0k>v1d4i>6!*HE#dgZaEpWJu2CFsx&%@FrYt z2UpGyl)4$yK`#9I)XW3Xkc#;YHo@YNP$;eFp^%ha6Mj&Y160VhLCU7>9FbDsI57QD z3b5o5oO+yMfqdoco@G1EzU1m|DB=}Y5YgGZ)7)1$Pep4 zm7V+xOtgJ-OEFR*gmX=?iu3DQi)?rExf3Cr38<=Evk46>0{6F9Y2BHIx?rP#4v)92 zsr>P#D{=WuRpV5+%f>t+B40I=?5DsH0Dj6= z<+`#~mXW>_Vbd~p^@OZ@>dS*%QC0-gh~s-B>XQ&eOrhQh9)V9i*1nYMQculiP(N(tH0lwlsq*5t( z69Q?{kN=G*GLTgulca4gU}wM`;*e^_oiKq5e0(>7#HY=lRWTJT&h(4pC}*Rt_A7XR zqYVH>bc9Gg*+Hv6fCs zO4{^bVlI9Dd?(ZZyTW55Bfs9_M*kqUET(71j3#|{Xb$8MzYNPV5zKf^KT^zMrOysG zg_Lp?F44j589<@%_Kc@Kl$!D(Bv$|wJphF9G3<&-0%27Lw}-gjXWs}K3!xC=UXYRL zUAudAO7oCCm>8d#%tAb2PP}JqSG;v=JYpmUv2?%AOv7$bbvWJ<_o?{RnQptqp~@ii zGr#?aDMe7iHe=xk>Jke97j*M?3PU5wtz=R%EFKzlst~x&Od#@di%2y zvjL|wGqZN8hBA~StZKVB4zAP;9}aLkz%OjhToDQ*w@+??A?^gbY_4L606+U*>QW%N z&4(f6v-Iw;`MPPG9Cj@0sy-bD^m#}HO4x}jxu$eCvyf}LIEq`NFDiouqA*pNujNvF3S0++v>oRm&rQAw4 zEv)CkZrw*t9Dpv6w{$>4c8)QBgB2vm?fS0G-UlQZQ8a)j^`Q*MQZ#F4tUSGE$9s~4 zDO8aQtS%hnlx#>BoXpz2QnSmyeBxvT())6{0lIrS<8QF=!p(__-bZBD%q+gIz+q;i z9;pdH+k%atSmMU1o6T5RDVzJQb$ zY0KH03uPr>9li!WC6m{0wkPMLf=?dr90JYgFF_!3O9Q%WhE7UCq;9B@17-Q0?C(>Y zIL5@pry0Vuf>&ea3#yO#mLt{`!V5HO*Q zvNV(K>=%Frp)66D6hO4DrkPXHep33A*w8un?+rkjeMK=9P&q&@)4<}O%kIX=dgPh? zWCjN&`&p=Uj1rZ zS^%e)0u3Q55!FwjAg2vk)JlVRv$w(wW%VVkC)YW~?h?}AwW|RtmFHBzXKoq+*EhrS z2OvF&@WLt8jk%6_c{m?nUMt@HzuS;TIR57*sQ-iJqHQI&3!PWv!snKdejJq>YS*)_ Hnm+qq6M3q9 literal 0 HcmV?d00001