Add code for tutorial 07
Andre Richter
5 years ago
parent
059b77fa36
commit
228428c2ca
@ -0,0 +1,4 @@
|
||||
newline_style = "Unix"
|
||||
edition = "2018"
|
||||
format_code_in_doc_comments = true
|
||||
merge_imports = true
|
||||
@ -0,0 +1,5 @@
|
||||
{
|
||||
"editor.formatOnSave": true,
|
||||
"rust.features": ["bsp_rpi3"],
|
||||
"rust.all_targets": false,
|
||||
}
|
||||
@ -0,0 +1,42 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "cortex-a"
|
||||
version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"register 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kernel"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cortex-a 2.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"r0 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"register 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r0"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "register"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"tock-registers 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tock-registers"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[metadata]
|
||||
"checksum cortex-a 2.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cbb16c411ab74044f174746a6cbae67bcdebea126e376b5441e5986e6a6aa950"
|
||||
"checksum r0 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e2a38df5b15c8d5c7e8654189744d8e396bddc18ad48041a500ce52d6948941f"
|
||||
"checksum register 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "469bb5ddde81d67fb8bba4e14d77689b8166cfd077abe7530591cefe29d05823"
|
||||
"checksum tock-registers 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c758f5195a2e0df9d9fecf6f506506b2766ff74cf64db1e995c87e2761a5c3e2"
|
||||
@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "kernel"
|
||||
version = "0.1.0"
|
||||
authors = ["Andre Richter <andre.o.richter@gmail.com>"]
|
||||
edition = "2018"
|
||||
|
||||
[package.metadata.cargo-xbuild]
|
||||
sysroot_path = "../xbuild_sysroot"
|
||||
|
||||
# The features section is used to select the target board.
|
||||
[features]
|
||||
default = []
|
||||
bsp_rpi3 = ["cortex-a", "register"]
|
||||
|
||||
[dependencies]
|
||||
r0 = "0.2.*"
|
||||
|
||||
# Optional dependencies
|
||||
cortex-a = { version = "2.*", optional = true }
|
||||
register = { version = "0.3.*", optional = true }
|
||||
@ -0,0 +1,90 @@
|
||||
## SPDX-License-Identifier: MIT
|
||||
##
|
||||
## Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
# Default to the RPi3
|
||||
ifndef BSP
|
||||
BSP = bsp_rpi3
|
||||
endif
|
||||
|
||||
# BSP-specific arguments
|
||||
ifeq ($(BSP),bsp_rpi3)
|
||||
TARGET = aarch64-unknown-none-softfloat
|
||||
OUTPUT = kernel8.img
|
||||
QEMU_BINARY = qemu-system-aarch64
|
||||
QEMU_MACHINE_TYPE = raspi3
|
||||
QEMU_MISC_ARGS = -serial null -serial stdio
|
||||
LINKER_FILE = src/bsp/rpi3/link.ld
|
||||
RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C relocation-model=pic
|
||||
endif
|
||||
|
||||
SOURCES = $(wildcard **/*.rs) $(wildcard **/*.S) $(wildcard **/*.ld)
|
||||
|
||||
XRUSTC_CMD = cargo xrustc \
|
||||
--target=$(TARGET) \
|
||||
--features $(BSP) \
|
||||
--release \
|
||||
-- \
|
||||
-C link-arg=-T$(LINKER_FILE) \
|
||||
$(RUSTC_MISC_ARGS)
|
||||
|
||||
CARGO_OUTPUT = target/$(TARGET)/release/kernel
|
||||
|
||||
OBJCOPY_CMD = cargo objcopy \
|
||||
-- \
|
||||
--strip-all \
|
||||
-O binary
|
||||
|
||||
CONTAINER_UTILS = rustembedded/osdev-utils
|
||||
|
||||
DOCKER_CMD = docker run -it --rm
|
||||
DOCKER_ARG_CURDIR = -v $(shell pwd):/work -w /work
|
||||
DOCKER_ARG_TTY = --privileged -v /dev:/dev
|
||||
|
||||
DOCKER_EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) -kernel $(OUTPUT)
|
||||
DOCKER_EXEC_RASPBOOT = raspbootcom
|
||||
DOCKER_EXEC_RASPBOOT_DEV = /dev/ttyUSB0
|
||||
# DOCKER_EXEC_RASPBOOT_DEV = /dev/ttyACM0
|
||||
|
||||
.PHONY: all doc qemu qemuasm chainboot clippy clean readelf objdump nm
|
||||
|
||||
all: clean $(OUTPUT)
|
||||
|
||||
$(CARGO_OUTPUT): $(SOURCES)
|
||||
RUSTFLAGS="-D warnings -D missing_docs" $(XRUSTC_CMD)
|
||||
|
||||
$(OUTPUT): $(CARGO_OUTPUT)
|
||||
cp $< .
|
||||
$(OBJCOPY_CMD) $< $(OUTPUT)
|
||||
|
||||
doc:
|
||||
cargo xdoc --target=$(TARGET) --features $(BSP) --document-private-items
|
||||
xdg-open target/$(TARGET)/doc/kernel/index.html
|
||||
|
||||
qemu: all
|
||||
$(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(CONTAINER_UTILS) \
|
||||
$(DOCKER_EXEC_QEMU) $(QEMU_MISC_ARGS)
|
||||
|
||||
qemuasm: all
|
||||
$(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(CONTAINER_UTILS) \
|
||||
$(DOCKER_EXEC_QEMU) -d in_asm
|
||||
|
||||
chainboot:
|
||||
$(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(DOCKER_ARG_TTY) \
|
||||
$(CONTAINER_UTILS) $(DOCKER_EXEC_RASPBOOT) $(DOCKER_EXEC_RASPBOOT_DEV) \
|
||||
demo_payload.img
|
||||
|
||||
clippy:
|
||||
cargo xclippy --target=$(TARGET) --features $(BSP)
|
||||
|
||||
clean:
|
||||
cargo clean
|
||||
|
||||
readelf:
|
||||
readelf -a kernel
|
||||
|
||||
objdump:
|
||||
cargo objdump --target $(TARGET) -- -disassemble -print-imm-hex kernel
|
||||
|
||||
nm:
|
||||
cargo nm --target $(TARGET) -- kernel | sort
|
||||
@ -0,0 +1,423 @@
|
||||
# Tutorial 07 - UART Chainloader
|
||||
|
||||
## tl;dr
|
||||
|
||||
Running from an SD card was a nice experience, but it would be extremely tedious
|
||||
to do it for every new binary. Let's write a [chainloader] using [position
|
||||
independent code]. This will be the last binary you need to put on the SD card
|
||||
for quite some time. Each following tutorial will prove a `chainboot` target in
|
||||
the `Makefile` that lets you conveniently load the kernel over `UART`.
|
||||
|
||||
Our chainloader is called `MiniLoad` and is inspired by [raspbootin].
|
||||
|
||||
[chainloader]: https://en.wikipedia.org/wiki/Chain_loading
|
||||
[position independent code]: https://en.wikipedia.org/wiki/Position-independent_code
|
||||
[raspbootin]: https://github.com/mrvn/raspbootin
|
||||
|
||||
You can try it with this tutorial already:
|
||||
1. Copy `kernel8.img` to the SD card.
|
||||
2. Execute `make chainboot`.
|
||||
3. Now plug in the USB Serial.
|
||||
4. Let the magic happen.
|
||||
|
||||
In this tutorial, a version of the kernel from the previous tutorial is loaded
|
||||
for demo purposes. In subsequent tuts, it will be the working directory's
|
||||
kernel.
|
||||
|
||||
### Observing the jump
|
||||
|
||||
The `Makefile` in this tutorial has an additional target, `qemuasm`, that lets
|
||||
you nicely observe the jump from the loaded address (`0x80_XXX`) to the
|
||||
relocated code at (`0x3EFF_0XXX`):
|
||||
|
||||
```console
|
||||
make qemuasm
|
||||
[...]
|
||||
IN:
|
||||
0x000809fc: d0000008 adrp x8, #0x82000
|
||||
0x00080a00: 52800020 movz w0, #0x1
|
||||
0x00080a04: f9408908 ldr x8, [x8, #0x110]
|
||||
0x00080a08: d63f0100 blr x8
|
||||
|
||||
----------------
|
||||
IN:
|
||||
0x3eff0528: d0000008 adrp x8, #0x3eff2000
|
||||
0x3eff052c: d0000009 adrp x9, #0x3eff2000
|
||||
0x3eff0530: f9411508 ldr x8, [x8, #0x228]
|
||||
0x3eff0534: f9411929 ldr x9, [x9, #0x230]
|
||||
0x3eff0538: eb08013f cmp x9, x8
|
||||
0x3eff053c: 540000c2 b.hs #0x3eff0554
|
||||
[...]
|
||||
```
|
||||
|
||||
## Diff to previous
|
||||
```diff
|
||||
Binary files 06_drivers_gpio_uart/demo_payload.img and 07_uart_chainloader/demo_payload.img differ
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/Makefile 07_uart_chainloader/Makefile
|
||||
--- 06_drivers_gpio_uart/Makefile
|
||||
+++ 07_uart_chainloader/Makefile
|
||||
@@ -15,7 +15,7 @@
|
||||
QEMU_MACHINE_TYPE = raspi3
|
||||
QEMU_MISC_ARGS = -serial null -serial stdio
|
||||
LINKER_FILE = src/bsp/rpi3/link.ld
|
||||
- RUSTC_MISC_ARGS = -C target-cpu=cortex-a53
|
||||
+ RUSTC_MISC_ARGS = -C target-cpu=cortex-a53 -C relocation-model=pic
|
||||
endif
|
||||
|
||||
SOURCES = $(wildcard **/*.rs) $(wildcard **/*.S) $(wildcard **/*.ld)
|
||||
@@ -39,9 +39,14 @@
|
||||
|
||||
DOCKER_CMD = docker run -it --rm
|
||||
DOCKER_ARG_CURDIR = -v $(shell pwd):/work -w /work
|
||||
-DOCKER_EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) -kernel $(OUTPUT)
|
||||
+DOCKER_ARG_TTY = --privileged -v /dev:/dev
|
||||
|
||||
-.PHONY: all doc qemu clippy clean readelf objdump nm
|
||||
+DOCKER_EXEC_QEMU = $(QEMU_BINARY) -M $(QEMU_MACHINE_TYPE) -kernel $(OUTPUT)
|
||||
+DOCKER_EXEC_RASPBOOT = raspbootcom
|
||||
+DOCKER_EXEC_RASPBOOT_DEV = /dev/ttyUSB0
|
||||
+# DOCKER_EXEC_RASPBOOT_DEV = /dev/ttyACM0
|
||||
+
|
||||
+.PHONY: all doc qemu qemuasm chainboot clippy clean readelf objdump nm
|
||||
|
||||
all: clean $(OUTPUT)
|
||||
|
||||
@@ -60,6 +65,15 @@
|
||||
$(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(CONTAINER_UTILS) \
|
||||
$(DOCKER_EXEC_QEMU) $(QEMU_MISC_ARGS)
|
||||
|
||||
+qemuasm: all
|
||||
+ $(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(CONTAINER_UTILS) \
|
||||
+ $(DOCKER_EXEC_QEMU) -d in_asm
|
||||
+
|
||||
+chainboot:
|
||||
+ $(DOCKER_CMD) $(DOCKER_ARG_CURDIR) $(DOCKER_ARG_TTY) \
|
||||
+ $(CONTAINER_UTILS) $(DOCKER_EXEC_RASPBOOT) $(DOCKER_EXEC_RASPBOOT_DEV) \
|
||||
+ demo_payload.img
|
||||
+
|
||||
clippy:
|
||||
cargo xclippy --target=$(TARGET) --features $(BSP)
|
||||
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/arch/aarch64.rs 07_uart_chainloader/src/arch/aarch64.rs
|
||||
--- 06_drivers_gpio_uart/src/arch/aarch64.rs
|
||||
+++ 07_uart_chainloader/src/arch/aarch64.rs
|
||||
@@ -23,7 +23,7 @@
|
||||
|
||||
if bsp::BOOT_CORE_ID == MPIDR_EL1.get() & CORE_MASK {
|
||||
SP.set(bsp::BOOT_CORE_STACK_START);
|
||||
- crate::runtime_init::init()
|
||||
+ crate::relocate::relocate_self::<u64>()
|
||||
} else {
|
||||
// if not core0, infinitely wait for events
|
||||
wait_forever()
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/bsp/driver/bcm/bcm2xxx_mini_uart.rs 07_uart_chainloader/src/bsp/driver/bcm/bcm2xxx_mini_uart.rs
|
||||
--- 06_drivers_gpio_uart/src/bsp/driver/bcm/bcm2xxx_mini_uart.rs
|
||||
+++ 07_uart_chainloader/src/bsp/driver/bcm/bcm2xxx_mini_uart.rs
|
||||
@@ -251,6 +251,15 @@
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| fmt::Write::write_fmt(inner, args))
|
||||
}
|
||||
+
|
||||
+ fn flush(&self) {
|
||||
+ let mut r = &self.inner;
|
||||
+ r.lock(|inner| loop {
|
||||
+ if inner.AUX_MU_LSR.is_set(AUX_MU_LSR::TX_IDLE) {
|
||||
+ break;
|
||||
+ }
|
||||
+ });
|
||||
+ }
|
||||
}
|
||||
|
||||
impl interface::console::Read for MiniUart {
|
||||
@@ -267,14 +276,14 @@
|
||||
}
|
||||
|
||||
// Read one character.
|
||||
- let mut ret = inner.AUX_MU_IO.get() as u8 as char;
|
||||
-
|
||||
- // Convert carrige return to newline.
|
||||
- if ret == '
' {
|
||||
- ret = '
|
||||
'
|
||||
- }
|
||||
+ inner.AUX_MU_IO.get() as u8 as char
|
||||
+ })
|
||||
+ }
|
||||
|
||||
- ret
|
||||
+ fn clear(&self) {
|
||||
+ let mut r = &self.inner;
|
||||
+ r.lock(|inner| {
|
||||
+ inner.AUX_MU_IIR.write(AUX_MU_IIR::FIFO_CLEAR::All);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/bsp/rpi3/link.ld 07_uart_chainloader/src/bsp/rpi3/link.ld
|
||||
--- 06_drivers_gpio_uart/src/bsp/rpi3/link.ld
|
||||
+++ 07_uart_chainloader/src/bsp/rpi3/link.ld
|
||||
@@ -5,9 +5,10 @@
|
||||
|
||||
SECTIONS
|
||||
{
|
||||
- /* Set current address to the value from which the RPi3 starts execution */
|
||||
- . = 0x80000;
|
||||
+ /* Set the link address to the top-most 40 KiB of DRAM */
|
||||
+ . = 0x3F000000 - 0x10000;
|
||||
|
||||
+ __binary_start = .;
|
||||
.text :
|
||||
{
|
||||
*(.text._start) *(.text*)
|
||||
@@ -31,5 +32,14 @@
|
||||
__bss_end = .;
|
||||
}
|
||||
|
||||
+ .got :
|
||||
+ {
|
||||
+ *(.got*)
|
||||
+ }
|
||||
+
|
||||
+ /* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */
|
||||
+ . = ALIGN(8);
|
||||
+ __binary_end = .;
|
||||
+
|
||||
/DISCARD/ : { *(.comment*) }
|
||||
}
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/bsp/rpi3.rs 07_uart_chainloader/src/bsp/rpi3.rs
|
||||
--- 06_drivers_gpio_uart/src/bsp/rpi3.rs
|
||||
+++ 07_uart_chainloader/src/bsp/rpi3.rs
|
||||
@@ -12,6 +12,9 @@
|
||||
pub const BOOT_CORE_ID: u64 = 0;
|
||||
pub const BOOT_CORE_STACK_START: u64 = 0x80_000;
|
||||
|
||||
+/// The address on which the RPi3 firmware loads every binary by default.
|
||||
+pub const BOARD_DEFAULT_LOAD_ADDRESS: usize = 0x80_000;
|
||||
+
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Global BSP driver instances
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/interface.rs 07_uart_chainloader/src/interface.rs
|
||||
--- 06_drivers_gpio_uart/src/interface.rs
|
||||
+++ 07_uart_chainloader/src/interface.rs
|
||||
@@ -26,6 +26,10 @@
|
||||
pub trait Write {
|
||||
fn write_char(&self, c: char);
|
||||
fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;
|
||||
+
|
||||
+ /// Block execution until the last character has been physically put on
|
||||
+ /// the TX wire (draining TX buffers/FIFOs, if any).
|
||||
+ fn flush(&self);
|
||||
}
|
||||
|
||||
/// Console read functions.
|
||||
@@ -33,6 +37,9 @@
|
||||
fn read_char(&self) -> char {
|
||||
' '
|
||||
}
|
||||
+
|
||||
+ /// Clear RX buffers, if any.
|
||||
+ fn clear(&self);
|
||||
}
|
||||
|
||||
/// Console statistics.
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/main.rs 07_uart_chainloader/src/main.rs
|
||||
--- 06_drivers_gpio_uart/src/main.rs
|
||||
+++ 07_uart_chainloader/src/main.rs
|
||||
@@ -23,8 +23,11 @@
|
||||
// `_start()` function, the first function to run.
|
||||
mod arch;
|
||||
|
||||
-// `_start()` then calls `runtime_init::init()`, which on completion, jumps to
|
||||
-// `kernel_entry()`.
|
||||
+// `_start()` then calls `relocate::relocate_self()`.
|
||||
+mod relocate;
|
||||
+
|
||||
+// `relocate::relocate_self()` calls `runtime_init::init()`, which on
|
||||
+// completion, jumps to `kernel_entry()`.
|
||||
mod runtime_init;
|
||||
|
||||
// Conditionally includes the selected `BSP` code.
|
||||
@@ -41,18 +44,48 @@
|
||||
// Run the BSP's initialization code.
|
||||
bsp::init();
|
||||
|
||||
- println!("[0] Booting on: {}", bsp::board_name());
|
||||
+ println!(" __ __ _ _ _ _ ");
|
||||
+ println!("| \/ (_)_ _ (_) | ___ __ _ __| |");
|
||||
+ println!("| |\/| | | ' \| | |__/ _ \/ _` / _` |");
|
||||
+ println!("|_| |_|_|_||_|_|____\___/\__,_\__,_|");
|
||||
+ println!();
|
||||
+ println!("{:^37}", bsp::board_name());
|
||||
+ println!();
|
||||
+ println!("[ML] Requesting binary");
|
||||
+ bsp::console().flush();
|
||||
+
|
||||
+ // Clear the RX FIFOs, if any, of spurious received characters before
|
||||
+ // starting with the loader protocol.
|
||||
+ bsp::console().clear();
|
||||
+
|
||||
+ // Notify raspbootcom to send the binary.
|
||||
+ for _ in 0..3 {
|
||||
+ bsp::console().write_char(3 as char);
|
||||
+ }
|
||||
|
||||
- println!("[1] Drivers loaded:");
|
||||
- for (i, driver) in bsp::device_drivers().iter().enumerate() {
|
||||
- println!(" {}. {}", i + 1, driver.compatible());
|
||||
+ // Read the binary's size.
|
||||
+ let mut size: u32 = u32::from(bsp::console().read_char() as u8);
|
||||
+ size |= u32::from(bsp::console().read_char() as u8) << 8;
|
||||
+ size |= u32::from(bsp::console().read_char() as u8) << 16;
|
||||
+ size |= u32::from(bsp::console().read_char() as u8) << 24;
|
||||
+
|
||||
+ // Trust it's not too big.
|
||||
+ print!("OK");
|
||||
+
|
||||
+ let kernel_addr: *mut u8 = bsp::BOARD_DEFAULT_LOAD_ADDRESS as *mut u8;
|
||||
+ unsafe {
|
||||
+ // Read the kernel byte by byte.
|
||||
+ for i in 0..size {
|
||||
+ *kernel_addr.offset(i as isize) = bsp::console().read_char() as u8;
|
||||
+ }
|
||||
}
|
||||
|
||||
- println!("[2] Chars written: {}", bsp::console().chars_written());
|
||||
- println!("[3] Echoing input now");
|
||||
+ println!("[ML] Loaded! Executing the payload now
|
||||
");
|
||||
+ bsp::console().flush();
|
||||
|
||||
- loop {
|
||||
- let c = bsp::console().read_char();
|
||||
- bsp::console().write_char(c);
|
||||
- }
|
||||
+ // Use black magic to get a function pointer.
|
||||
+ let kernel: extern "C" fn() -> ! = unsafe { core::mem::transmute(kernel_addr as *const ()) };
|
||||
+
|
||||
+ // Jump to loaded kernel!
|
||||
+ kernel()
|
||||
}
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/relocate.rs 07_uart_chainloader/src/relocate.rs
|
||||
--- 06_drivers_gpio_uart/src/relocate.rs
|
||||
+++ 07_uart_chainloader/src/relocate.rs
|
||||
@@ -0,0 +1,47 @@
|
||||
+// SPDX-License-Identifier: MIT
|
||||
+//
|
||||
+// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
+
|
||||
+//! Relocation code.
|
||||
+
|
||||
+/// Relocates the own binary from `bsp::BOARD_DEFAULT_LOAD_ADDRESS` to the
|
||||
+/// `__binary_start` address from the linker script.
|
||||
+///
|
||||
+/// # Safety
|
||||
+///
|
||||
+/// - Only a single core must be active and running this function.
|
||||
+/// - Function must not use the `bss` section.
|
||||
+pub unsafe fn relocate_self<T>() -> ! {
|
||||
+ extern "C" {
|
||||
+ static __binary_start: usize;
|
||||
+ static __binary_end: usize;
|
||||
+ }
|
||||
+
|
||||
+ let binary_start_addr: usize = &__binary_start as *const _ as _;
|
||||
+ let binary_end_addr: usize = &__binary_end as *const _ as _;
|
||||
+ let binary_size_in_byte: usize = binary_end_addr - binary_start_addr;
|
||||
+
|
||||
+ // Get the relocation destination address from the linker symbol.
|
||||
+ let mut reloc_dst_addr: *mut T = binary_start_addr as *mut T;
|
||||
+
|
||||
+ // The address of where the previous firmware loaded us.
|
||||
+ let mut src_addr: *const T = crate::bsp::BOARD_DEFAULT_LOAD_ADDRESS as *const _;
|
||||
+
|
||||
+ // Copy the whole binary.
|
||||
+ //
|
||||
+ // This is essentially a `memcpy()` optimized for throughput by transferring
|
||||
+ // in chunks of T.
|
||||
+ let n = binary_size_in_byte / core::mem::size_of::<T>();
|
||||
+ for _ in 0..n {
|
||||
+ use core::ptr;
|
||||
+
|
||||
+ ptr::write_volatile::<T>(reloc_dst_addr, ptr::read_volatile::<T>(src_addr));
|
||||
+ reloc_dst_addr = reloc_dst_addr.offset(1);
|
||||
+ src_addr = src_addr.offset(1);
|
||||
+ }
|
||||
+
|
||||
+ // Call `init()` through a trait object, causing the jump to use an absolute
|
||||
+ // address to reach the relocated binary. An elaborate explanation can be
|
||||
+ // found in the runtime_init.rs source comments.
|
||||
+ crate::runtime_init::get().init()
|
||||
+}
|
||||
|
||||
diff -uNr 06_drivers_gpio_uart/src/runtime_init.rs 07_uart_chainloader/src/runtime_init.rs
|
||||
--- 06_drivers_gpio_uart/src/runtime_init.rs
|
||||
+++ 07_uart_chainloader/src/runtime_init.rs
|
||||
@@ -4,23 +4,44 @@
|
||||
|
||||
//! Rust runtime initialization code.
|
||||
|
||||
-/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section,
|
||||
-/// then calls the kernel entry.
|
||||
+/// We are outsmarting the compiler here by using a trait as a layer of
|
||||
+/// indirection. Because we are generating PIC code, a static dispatch to
|
||||
+/// `init()` would generate a relative jump from the callee to `init()`.
|
||||
+/// However, when calling `init()`, code just finished copying the binary to the
|
||||
+/// actual link-time address, and hence is still running at whatever location
|
||||
+/// the previous loader has put it. So we do not want a relative jump, because
|
||||
+/// it would not jump to the relocated code.
|
||||
///
|
||||
-/// Called from `BSP` code.
|
||||
-///
|
||||
-/// # Safety
|
||||
-///
|
||||
-/// - Only a single core must be active and running this function.
|
||||
-pub unsafe fn init() -> ! {
|
||||
- extern "C" {
|
||||
- // Boundaries of the .bss section, provided by the linker script.
|
||||
- static mut __bss_start: u64;
|
||||
- static mut __bss_end: u64;
|
||||
+/// By indirecting through a trait object, we can make use of the property that
|
||||
+/// vtables store absolute addresses. So calling `init()` this way will kick
|
||||
+/// execution to the relocated binary.
|
||||
+pub trait RunTimeInit {
|
||||
+ /// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section,
|
||||
+ /// then calls the kernel entry.
|
||||
+ ///
|
||||
+ /// Called from `BSP` code.
|
||||
+ ///
|
||||
+ /// # Safety
|
||||
+ ///
|
||||
+ /// - Only a single core must be active and running this function.
|
||||
+ unsafe fn init(&self) -> ! {
|
||||
+ extern "C" {
|
||||
+ // Boundaries of the .bss section, provided by the linker script.
|
||||
+ static mut __bss_start: u64;
|
||||
+ static mut __bss_end: u64;
|
||||
+ }
|
||||
+
|
||||
+ // Zero out the .bss section.
|
||||
+ r0::zero_bss(&mut __bss_start, &mut __bss_end);
|
||||
+
|
||||
+ crate::kernel_entry()
|
||||
}
|
||||
+}
|
||||
|
||||
- // Zero out the .bss section.
|
||||
- r0::zero_bss(&mut __bss_start, &mut __bss_end);
|
||||
+struct Traitor;
|
||||
+impl RunTimeInit for Traitor {}
|
||||
|
||||
- crate::kernel_entry()
|
||||
+/// Give the callee a `RunTimeInit` trait object.
|
||||
+pub fn get() -> &'static dyn RunTimeInit {
|
||||
+ &Traitor {}
|
||||
}
|
||||
```
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,11 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Conditional exporting of processor architecture code.
|
||||
|
||||
#[cfg(feature = "bsp_rpi3")]
|
||||
mod aarch64;
|
||||
|
||||
#[cfg(feature = "bsp_rpi3")]
|
||||
pub use aarch64::*;
|
||||
@ -0,0 +1,45 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! AArch64.
|
||||
|
||||
pub mod sync;
|
||||
|
||||
use crate::bsp;
|
||||
use cortex_a::{asm, regs::*};
|
||||
|
||||
/// The entry of the `kernel` binary.
|
||||
///
|
||||
/// The function must be named `_start`, because the linker is looking for this
|
||||
/// exact name.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - Linker script must ensure to place this function at `0x80_000`.
|
||||
#[no_mangle]
|
||||
pub unsafe extern "C" fn _start() -> ! {
|
||||
const CORE_MASK: u64 = 0x3;
|
||||
|
||||
if bsp::BOOT_CORE_ID == MPIDR_EL1.get() & CORE_MASK {
|
||||
SP.set(bsp::BOOT_CORE_STACK_START);
|
||||
crate::relocate::relocate_self::<u64>()
|
||||
} else {
|
||||
// if not core0, infinitely wait for events
|
||||
wait_forever()
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Implementation of the kernel's architecture abstraction code
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
pub use asm::nop;
|
||||
|
||||
/// Pause execution on the calling CPU core.
|
||||
#[inline(always)]
|
||||
pub fn wait_forever() -> ! {
|
||||
loop {
|
||||
asm::wfe()
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Synchronization primitives.
|
||||
|
||||
use crate::interface;
|
||||
use core::cell::UnsafeCell;
|
||||
|
||||
/// A pseudo-lock for teaching purposes.
|
||||
///
|
||||
/// Used to introduce [interior mutability].
|
||||
///
|
||||
/// In contrast to a real Mutex implementation, does not protect against
|
||||
/// concurrent access to the contained data. This part is preserved for later
|
||||
/// lessons.
|
||||
///
|
||||
/// The lock will only be used as long as it is safe to do so, i.e. as long as
|
||||
/// the kernel is executing single-threaded, aka only running on a single core
|
||||
/// with interrupts disabled.
|
||||
///
|
||||
/// [interior mutability]: https://doc.rust-lang.org/std/cell/index.html
|
||||
pub struct NullLock<T: ?Sized> {
|
||||
data: UnsafeCell<T>,
|
||||
}
|
||||
|
||||
unsafe impl<T: ?Sized + Send> Send for NullLock<T> {}
|
||||
unsafe impl<T: ?Sized + Send> Sync for NullLock<T> {}
|
||||
|
||||
impl<T> NullLock<T> {
|
||||
pub const fn new(data: T) -> NullLock<T> {
|
||||
NullLock {
|
||||
data: UnsafeCell::new(data),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> interface::sync::Mutex for &NullLock<T> {
|
||||
type Data = T;
|
||||
|
||||
fn lock<R>(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R {
|
||||
// In a real lock, there would be code encapsulating this line that
|
||||
// ensures that this mutable reference will ever only be given out once
|
||||
// at a time.
|
||||
f(unsafe { &mut *self.data.get() })
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Conditional exporting of Board Support Packages.
|
||||
|
||||
mod driver;
|
||||
|
||||
#[cfg(feature = "bsp_rpi3")]
|
||||
mod rpi3;
|
||||
|
||||
#[cfg(feature = "bsp_rpi3")]
|
||||
pub use rpi3::*;
|
||||
@ -0,0 +1,11 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Drivers.
|
||||
|
||||
#[cfg(feature = "bsp_rpi3")]
|
||||
mod bcm;
|
||||
|
||||
#[cfg(feature = "bsp_rpi3")]
|
||||
pub use bcm::*;
|
||||
@ -0,0 +1,11 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! BCM driver top level.
|
||||
|
||||
mod bcm2837_gpio;
|
||||
mod bcm2xxx_mini_uart;
|
||||
|
||||
pub use bcm2837_gpio::GPIO;
|
||||
pub use bcm2xxx_mini_uart::MiniUart;
|
||||
@ -0,0 +1,162 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! GPIO driver.
|
||||
|
||||
use crate::{arch, arch::sync::NullLock, interface};
|
||||
use core::ops;
|
||||
use register::{mmio::ReadWrite, register_bitfields};
|
||||
|
||||
// GPIO registers.
|
||||
//
|
||||
// Descriptions taken from
|
||||
// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf
|
||||
register_bitfields! {
|
||||
u32,
|
||||
|
||||
/// GPIO Function Select 1
|
||||
GPFSEL1 [
|
||||
/// Pin 15
|
||||
FSEL15 OFFSET(15) NUMBITS(3) [
|
||||
Input = 0b000,
|
||||
Output = 0b001,
|
||||
AltFunc5 = 0b010 // Mini UART RX
|
||||
|
||||
],
|
||||
|
||||
/// Pin 14
|
||||
FSEL14 OFFSET(12) NUMBITS(3) [
|
||||
Input = 0b000,
|
||||
Output = 0b001,
|
||||
AltFunc5 = 0b010 // Mini UART TX
|
||||
]
|
||||
],
|
||||
|
||||
/// GPIO Pull-up/down Clock Register 0
|
||||
GPPUDCLK0 [
|
||||
/// Pin 15
|
||||
PUDCLK15 OFFSET(15) NUMBITS(1) [
|
||||
NoEffect = 0,
|
||||
AssertClock = 1
|
||||
],
|
||||
|
||||
/// Pin 14
|
||||
PUDCLK14 OFFSET(14) NUMBITS(1) [
|
||||
NoEffect = 0,
|
||||
AssertClock = 1
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
#[allow(non_snake_case)]
|
||||
#[repr(C)]
|
||||
pub struct RegisterBlock {
|
||||
pub GPFSEL0: ReadWrite<u32>, // 0x00
|
||||
pub GPFSEL1: ReadWrite<u32, GPFSEL1::Register>, // 0x04
|
||||
pub GPFSEL2: ReadWrite<u32>, // 0x08
|
||||
pub GPFSEL3: ReadWrite<u32>, // 0x0C
|
||||
pub GPFSEL4: ReadWrite<u32>, // 0x10
|
||||
pub GPFSEL5: ReadWrite<u32>, // 0x14
|
||||
__reserved_0: u32, // 0x18
|
||||
GPSET0: ReadWrite<u32>, // 0x1C
|
||||
GPSET1: ReadWrite<u32>, // 0x20
|
||||
__reserved_1: u32, //
|
||||
GPCLR0: ReadWrite<u32>, // 0x28
|
||||
__reserved_2: [u32; 2], //
|
||||
GPLEV0: ReadWrite<u32>, // 0x34
|
||||
GPLEV1: ReadWrite<u32>, // 0x38
|
||||
__reserved_3: u32, //
|
||||
GPEDS0: ReadWrite<u32>, // 0x40
|
||||
GPEDS1: ReadWrite<u32>, // 0x44
|
||||
__reserved_4: [u32; 7], //
|
||||
GPHEN0: ReadWrite<u32>, // 0x64
|
||||
GPHEN1: ReadWrite<u32>, // 0x68
|
||||
__reserved_5: [u32; 10], //
|
||||
pub GPPUD: ReadWrite<u32>, // 0x94
|
||||
pub GPPUDCLK0: ReadWrite<u32, GPPUDCLK0::Register>, // 0x98
|
||||
pub GPPUDCLK1: ReadWrite<u32>, // 0x9C
|
||||
}
|
||||
|
||||
/// The driver's private data.
|
||||
struct GPIOInner {
|
||||
base_addr: usize,
|
||||
}
|
||||
|
||||
/// Deref to RegisterBlock.
|
||||
impl ops::Deref for GPIOInner {
|
||||
type Target = RegisterBlock;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
unsafe { &*self.ptr() }
|
||||
}
|
||||
}
|
||||
|
||||
impl GPIOInner {
|
||||
const fn new(base_addr: usize) -> GPIOInner {
|
||||
GPIOInner { base_addr }
|
||||
}
|
||||
|
||||
/// Return a pointer to the register block.
|
||||
fn ptr(&self) -> *const RegisterBlock {
|
||||
self.base_addr as *const _
|
||||
}
|
||||
|
||||
/// Map Mini UART as standard output.
|
||||
///
|
||||
/// TX to pin 14
|
||||
/// RX to pin 15
|
||||
pub fn map_mini_uart(&mut self) {
|
||||
// Map to pins.
|
||||
self.GPFSEL1
|
||||
.modify(GPFSEL1::FSEL14::AltFunc5 + GPFSEL1::FSEL15::AltFunc5);
|
||||
|
||||
// Enable pins 14 and 15.
|
||||
self.GPPUD.set(0);
|
||||
for _ in 0..150 {
|
||||
arch::nop();
|
||||
}
|
||||
|
||||
self.GPPUDCLK0
|
||||
.write(GPPUDCLK0::PUDCLK14::AssertClock + GPPUDCLK0::PUDCLK15::AssertClock);
|
||||
for _ in 0..150 {
|
||||
arch::nop();
|
||||
}
|
||||
|
||||
self.GPPUDCLK0.set(0);
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// BSP-public
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
use interface::sync::Mutex;
|
||||
|
||||
/// The driver's main struct.
|
||||
pub struct GPIO {
|
||||
inner: NullLock<GPIOInner>,
|
||||
}
|
||||
|
||||
impl GPIO {
|
||||
pub const unsafe fn new(base_addr: usize) -> GPIO {
|
||||
GPIO {
|
||||
inner: NullLock::new(GPIOInner::new(base_addr)),
|
||||
}
|
||||
}
|
||||
|
||||
// Only visible to other BSP code.
|
||||
pub fn map_mini_uart(&self) {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| inner.map_mini_uart());
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// OS interface implementations
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
impl interface::driver::DeviceDriver for GPIO {
|
||||
fn compatible(&self) -> &str {
|
||||
"GPIO"
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,296 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Mini UART driver.
|
||||
|
||||
use crate::{arch, arch::sync::NullLock, interface};
|
||||
use core::{fmt, ops};
|
||||
use register::{mmio::*, register_bitfields};
|
||||
|
||||
// Mini UART registers.
|
||||
//
|
||||
// Descriptions taken from
|
||||
// https://github.com/raspberrypi/documentation/files/1888662/BCM2837-ARM-Peripherals.-.Revised.-.V2-1.pdf
|
||||
register_bitfields! {
|
||||
u32,
|
||||
|
||||
/// Auxiliary enables
|
||||
AUX_ENABLES [
|
||||
/// If set the mini UART is enabled. The UART will immediately
|
||||
/// start receiving data, especially if the UART1_RX line is
|
||||
/// low.
|
||||
/// If clear the mini UART is disabled. That also disables any
|
||||
/// mini UART register access
|
||||
MINI_UART_ENABLE OFFSET(0) NUMBITS(1) []
|
||||
],
|
||||
|
||||
/// Mini Uart Interrupt Identify
|
||||
AUX_MU_IIR [
|
||||
/// Writing with bit 1 set will clear the receive FIFO
|
||||
/// Writing with bit 2 set will clear the transmit FIFO
|
||||
FIFO_CLEAR OFFSET(1) NUMBITS(2) [
|
||||
Rx = 0b01,
|
||||
Tx = 0b10,
|
||||
All = 0b11
|
||||
]
|
||||
],
|
||||
|
||||
/// Mini Uart Line Control
|
||||
AUX_MU_LCR [
|
||||
/// Mode the UART works in
|
||||
DATA_SIZE OFFSET(0) NUMBITS(2) [
|
||||
SevenBit = 0b00,
|
||||
EightBit = 0b11
|
||||
]
|
||||
],
|
||||
|
||||
/// Mini Uart Line Status
|
||||
AUX_MU_LSR [
|
||||
/// This bit is set if the transmit FIFO is empty and the transmitter is
|
||||
/// idle. (Finished shifting out the last bit).
|
||||
TX_IDLE OFFSET(6) NUMBITS(1) [],
|
||||
|
||||
/// This bit is set if the transmit FIFO can accept at least
|
||||
/// one byte.
|
||||
TX_EMPTY OFFSET(5) NUMBITS(1) [],
|
||||
|
||||
/// This bit is set if the receive FIFO holds at least 1
|
||||
/// symbol.
|
||||
DATA_READY OFFSET(0) NUMBITS(1) []
|
||||
],
|
||||
|
||||
/// Mini Uart Extra Control
|
||||
AUX_MU_CNTL [
|
||||
/// If this bit is set the mini UART transmitter is enabled.
|
||||
/// If this bit is clear the mini UART transmitter is disabled.
|
||||
TX_EN OFFSET(1) NUMBITS(1) [
|
||||
Disabled = 0,
|
||||
Enabled = 1
|
||||
],
|
||||
|
||||
/// If this bit is set the mini UART receiver is enabled.
|
||||
/// If this bit is clear the mini UART receiver is disabled.
|
||||
RX_EN OFFSET(0) NUMBITS(1) [
|
||||
Disabled = 0,
|
||||
Enabled = 1
|
||||
]
|
||||
],
|
||||
|
||||
/// Mini Uart Baudrate
|
||||
AUX_MU_BAUD [
|
||||
/// Mini UART baudrate counter
|
||||
RATE OFFSET(0) NUMBITS(16) []
|
||||
]
|
||||
}
|
||||
|
||||
#[allow(non_snake_case)]
|
||||
#[repr(C)]
|
||||
pub struct RegisterBlock {
|
||||
__reserved_0: u32, // 0x00
|
||||
AUX_ENABLES: ReadWrite<u32, AUX_ENABLES::Register>, // 0x04
|
||||
__reserved_1: [u32; 14], // 0x08
|
||||
AUX_MU_IO: ReadWrite<u32>, // 0x40 - Mini Uart I/O Data
|
||||
AUX_MU_IER: WriteOnly<u32>, // 0x44 - Mini Uart Interrupt Enable
|
||||
AUX_MU_IIR: WriteOnly<u32, AUX_MU_IIR::Register>, // 0x48
|
||||
AUX_MU_LCR: WriteOnly<u32, AUX_MU_LCR::Register>, // 0x4C
|
||||
AUX_MU_MCR: WriteOnly<u32>, // 0x50
|
||||
AUX_MU_LSR: ReadOnly<u32, AUX_MU_LSR::Register>, // 0x54
|
||||
__reserved_2: [u32; 2], // 0x58
|
||||
AUX_MU_CNTL: WriteOnly<u32, AUX_MU_CNTL::Register>, // 0x60
|
||||
__reserved_3: u32, // 0x64
|
||||
AUX_MU_BAUD: WriteOnly<u32, AUX_MU_BAUD::Register>, // 0x68
|
||||
}
|
||||
|
||||
/// The driver's mutex protected part.
|
||||
struct MiniUartInner {
|
||||
base_addr: usize,
|
||||
chars_written: usize,
|
||||
}
|
||||
|
||||
/// Deref to RegisterBlock.
|
||||
///
|
||||
/// Allows writing
|
||||
/// ```
|
||||
/// self.MU_IER.read()
|
||||
/// ```
|
||||
/// instead of something along the lines of
|
||||
/// ```
|
||||
/// unsafe { (*MiniUart::ptr()).MU_IER.read() }
|
||||
/// ```
|
||||
impl ops::Deref for MiniUartInner {
|
||||
type Target = RegisterBlock;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
unsafe { &*self.ptr() }
|
||||
}
|
||||
}
|
||||
|
||||
impl MiniUartInner {
|
||||
const fn new(base_addr: usize) -> MiniUartInner {
|
||||
MiniUartInner {
|
||||
base_addr,
|
||||
chars_written: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return a pointer to the register block.
|
||||
fn ptr(&self) -> *const RegisterBlock {
|
||||
self.base_addr as *const _
|
||||
}
|
||||
|
||||
/// Send a character.
|
||||
fn write_char(&mut self, c: char) {
|
||||
// Wait until we can send.
|
||||
loop {
|
||||
if self.AUX_MU_LSR.is_set(AUX_MU_LSR::TX_EMPTY) {
|
||||
break;
|
||||
}
|
||||
|
||||
arch::nop();
|
||||
}
|
||||
|
||||
// Write the character to the buffer.
|
||||
self.AUX_MU_IO.set(c as u32);
|
||||
}
|
||||
}
|
||||
|
||||
/// Implementing `core::fmt::Write` enables usage of the `format_args!` macros,
|
||||
/// which in turn are used to implement the `kernel`'s `print!` and `println!`
|
||||
/// macros. By implementing `write_str()`, we get `write_fmt()` automatically.
|
||||
///
|
||||
/// The function takes an `&mut self`, so it must be implemented for the inner
|
||||
/// struct.
|
||||
///
|
||||
/// See [`src/print.rs`].
|
||||
///
|
||||
/// [`src/print.rs`]: ../../print/index.html
|
||||
impl fmt::Write for MiniUartInner {
|
||||
fn write_str(&mut self, s: &str) -> fmt::Result {
|
||||
for c in s.chars() {
|
||||
// Convert newline to carrige return + newline.
|
||||
if c == '\n' {
|
||||
self.write_char('\r')
|
||||
}
|
||||
|
||||
self.write_char(c);
|
||||
}
|
||||
|
||||
self.chars_written += s.len();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// BSP-public
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// The driver's main struct.
|
||||
pub struct MiniUart {
|
||||
inner: NullLock<MiniUartInner>,
|
||||
}
|
||||
|
||||
impl MiniUart {
|
||||
/// # Safety
|
||||
///
|
||||
/// The user must ensure to provide the correct `base_addr`.
|
||||
pub const unsafe fn new(base_addr: usize) -> MiniUart {
|
||||
MiniUart {
|
||||
inner: NullLock::new(MiniUartInner::new(base_addr)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// OS interface implementations
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
use interface::sync::Mutex;
|
||||
|
||||
impl interface::driver::DeviceDriver for MiniUart {
|
||||
fn compatible(&self) -> &str {
|
||||
"MiniUart"
|
||||
}
|
||||
|
||||
/// Set up baud rate and characteristics (115200 8N1).
|
||||
fn init(&self) -> interface::driver::Result {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| {
|
||||
// Enable register access to the MiniUart
|
||||
inner.AUX_ENABLES.modify(AUX_ENABLES::MINI_UART_ENABLE::SET);
|
||||
inner.AUX_MU_IER.set(0); // disable RX and TX interrupts
|
||||
inner.AUX_MU_CNTL.set(0); // disable send and receive
|
||||
inner.AUX_MU_LCR.write(AUX_MU_LCR::DATA_SIZE::EightBit);
|
||||
inner.AUX_MU_BAUD.write(AUX_MU_BAUD::RATE.val(270)); // 115200 baud
|
||||
inner.AUX_MU_MCR.set(0); // set "ready to send" high
|
||||
|
||||
// Enable receive and send.
|
||||
inner
|
||||
.AUX_MU_CNTL
|
||||
.write(AUX_MU_CNTL::RX_EN::Enabled + AUX_MU_CNTL::TX_EN::Enabled);
|
||||
|
||||
// Clear FIFOs before using the device.
|
||||
inner.AUX_MU_IIR.write(AUX_MU_IIR::FIFO_CLEAR::All);
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl interface::console::Write for MiniUart {
|
||||
/// Passthrough of `args` to the `core::fmt::Write` implementation, but
|
||||
/// guarded by a Mutex to serialize access.
|
||||
fn write_char(&self, c: char) {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| inner.write_char(c));
|
||||
}
|
||||
|
||||
fn write_fmt(&self, args: core::fmt::Arguments) -> fmt::Result {
|
||||
// Fully qualified syntax for the call to
|
||||
// `core::fmt::Write::write:fmt()` to increase readability.
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| fmt::Write::write_fmt(inner, args))
|
||||
}
|
||||
|
||||
fn flush(&self) {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| loop {
|
||||
if inner.AUX_MU_LSR.is_set(AUX_MU_LSR::TX_IDLE) {
|
||||
break;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl interface::console::Read for MiniUart {
|
||||
fn read_char(&self) -> char {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| {
|
||||
// Wait until buffer is filled.
|
||||
loop {
|
||||
if inner.AUX_MU_LSR.is_set(AUX_MU_LSR::DATA_READY) {
|
||||
break;
|
||||
}
|
||||
|
||||
arch::nop();
|
||||
}
|
||||
|
||||
// Read one character.
|
||||
inner.AUX_MU_IO.get() as u8 as char
|
||||
})
|
||||
}
|
||||
|
||||
fn clear(&self) {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| {
|
||||
inner.AUX_MU_IIR.write(AUX_MU_IIR::FIFO_CLEAR::All);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl interface::console::Statistics for MiniUart {
|
||||
fn chars_written(&self) -> usize {
|
||||
let mut r = &self.inner;
|
||||
r.lock(|inner| inner.chars_written)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Board Support Package for the Raspberry Pi 3.
|
||||
|
||||
mod memory_map;
|
||||
|
||||
use super::driver;
|
||||
use crate::interface;
|
||||
|
||||
pub const BOOT_CORE_ID: u64 = 0;
|
||||
pub const BOOT_CORE_STACK_START: u64 = 0x80_000;
|
||||
|
||||
/// The address on which the RPi3 firmware loads every binary by default.
|
||||
pub const BOARD_DEFAULT_LOAD_ADDRESS: usize = 0x80_000;
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Global BSP driver instances
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static GPIO: driver::GPIO = unsafe { driver::GPIO::new(memory_map::mmio::GPIO_BASE) };
|
||||
static MINI_UART: driver::MiniUart =
|
||||
unsafe { driver::MiniUart::new(memory_map::mmio::MINI_UART_BASE) };
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
// Implementation of the kernel's BSP calls
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Board identification.
|
||||
pub fn board_name() -> &'static str {
|
||||
"Raspberry Pi 3"
|
||||
}
|
||||
|
||||
/// Return a reference to a `console::All` implementation.
|
||||
pub fn console() -> &'static impl interface::console::All {
|
||||
&MINI_UART
|
||||
}
|
||||
|
||||
/// Return an array of references to all `DeviceDriver` compatible `BSP`
|
||||
/// drivers.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// The order of devices is the order in which `DeviceDriver::init()` is called.
|
||||
pub fn device_drivers() -> [&'static dyn interface::driver::DeviceDriver; 2] {
|
||||
[&GPIO, &MINI_UART]
|
||||
}
|
||||
|
||||
/// The BSP's main initialization function.
|
||||
///
|
||||
/// Called early on kernel start.
|
||||
pub fn init() {
|
||||
for i in device_drivers().iter() {
|
||||
if let Err(()) = i.init() {
|
||||
// This message will only be readable if, at the time of failure,
|
||||
// the return value of `bsp::console()` is already in functioning
|
||||
// state.
|
||||
panic!("Error loading driver: {}", i.compatible())
|
||||
}
|
||||
}
|
||||
|
||||
// Configure MiniUart's output pins.
|
||||
GPIO.map_mini_uart();
|
||||
}
|
||||
@ -0,0 +1,45 @@
|
||||
/* SPDX-License-Identifier: MIT
|
||||
*
|
||||
* Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
*/
|
||||
|
||||
SECTIONS
|
||||
{
|
||||
/* Set the link address to the top-most 40 KiB of DRAM */
|
||||
. = 0x3F000000 - 0x10000;
|
||||
|
||||
__binary_start = .;
|
||||
.text :
|
||||
{
|
||||
*(.text._start) *(.text*)
|
||||
}
|
||||
|
||||
.rodata :
|
||||
{
|
||||
*(.rodata*)
|
||||
}
|
||||
|
||||
.data :
|
||||
{
|
||||
*(.data*)
|
||||
}
|
||||
|
||||
/* Align to 8 byte boundary */
|
||||
.bss ALIGN(8):
|
||||
{
|
||||
__bss_start = .;
|
||||
*(.bss*);
|
||||
__bss_end = .;
|
||||
}
|
||||
|
||||
.got :
|
||||
{
|
||||
*(.got*)
|
||||
}
|
||||
|
||||
/* Fill up to 8 byte, b/c relocating the binary is done in u64 chunks */
|
||||
. = ALIGN(8);
|
||||
__binary_end = .;
|
||||
|
||||
/DISCARD/ : { *(.comment*) }
|
||||
}
|
||||
@ -0,0 +1,13 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! The board's memory map.
|
||||
|
||||
/// Physical devices.
|
||||
#[rustfmt::skip]
|
||||
pub mod mmio {
|
||||
pub const BASE: usize = 0x3F00_0000;
|
||||
pub const GPIO_BASE: usize = BASE + 0x0020_0000;
|
||||
pub const MINI_UART_BASE: usize = BASE + 0x0021_5000;
|
||||
}
|
||||
@ -0,0 +1,112 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Trait definitions for coupling `kernel` and `BSP` code.
|
||||
//!
|
||||
//! ```
|
||||
//! +-------------------+
|
||||
//! | Interface (Trait) |
|
||||
//! | |
|
||||
//! +--+-------------+--+
|
||||
//! ^ ^
|
||||
//! | |
|
||||
//! | |
|
||||
//! +----------+--+ +--+----------+
|
||||
//! | Kernel code | | BSP Code |
|
||||
//! | | | |
|
||||
//! +-------------+ +-------------+
|
||||
//! ```
|
||||
|
||||
/// System console operations.
|
||||
pub mod console {
|
||||
use core::fmt;
|
||||
|
||||
/// Console write functions.
|
||||
pub trait Write {
|
||||
fn write_char(&self, c: char);
|
||||
fn write_fmt(&self, args: fmt::Arguments) -> fmt::Result;
|
||||
|
||||
/// Block execution until the last character has been physically put on
|
||||
/// the TX wire (draining TX buffers/FIFOs, if any).
|
||||
fn flush(&self);
|
||||
}
|
||||
|
||||
/// Console read functions.
|
||||
pub trait Read {
|
||||
fn read_char(&self) -> char {
|
||||
' '
|
||||
}
|
||||
|
||||
/// Clear RX buffers, if any.
|
||||
fn clear(&self);
|
||||
}
|
||||
|
||||
/// Console statistics.
|
||||
pub trait Statistics {
|
||||
/// Return the number of characters written.
|
||||
fn chars_written(&self) -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
/// Return the number of characters read.
|
||||
fn chars_read(&self) -> usize {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait alias for a full-fledged console.
|
||||
pub trait All = Write + Read + Statistics;
|
||||
}
|
||||
|
||||
/// Synchronization primitives.
|
||||
pub mod sync {
|
||||
/// Any object implementing this trait guarantees exclusive access to the
|
||||
/// data contained within the mutex for the duration of the lock.
|
||||
///
|
||||
/// The trait follows the [Rust embedded WG's
|
||||
/// proposal](https://github.com/korken89/wg/blob/master/rfcs/0377-mutex-trait.md)
|
||||
/// and therefore provides some goodness such as [deadlock
|
||||
/// prevention](https://github.com/korken89/wg/blob/master/rfcs/0377-mutex-trait.md#design-decisions-and-compatibility).
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// Since the lock function takes an `&mut self` to enable
|
||||
/// deadlock-prevention, the trait is best implemented **for a reference to
|
||||
/// a container struct**, and has a usage pattern that might feel strange at
|
||||
/// first:
|
||||
///
|
||||
/// ```
|
||||
/// static MUT: Mutex<RefCell<i32>> = Mutex::new(RefCell::new(0));
|
||||
///
|
||||
/// fn foo() {
|
||||
/// let mut r = &MUT; // Note that r is mutable
|
||||
/// r.lock(|data| *data += 1);
|
||||
/// }
|
||||
/// ```
|
||||
pub trait Mutex {
|
||||
/// Type of data encapsulated by the mutex.
|
||||
type Data;
|
||||
|
||||
/// Creates a critical section and grants temporary mutable access to
|
||||
/// the encapsulated data.
|
||||
fn lock<R>(&mut self, f: impl FnOnce(&mut Self::Data) -> R) -> R;
|
||||
}
|
||||
}
|
||||
|
||||
/// Driver interfaces.
|
||||
pub mod driver {
|
||||
/// Driver result type, e.g. for indicating successful driver init.
|
||||
pub type Result = core::result::Result<(), ()>;
|
||||
|
||||
/// Device Driver operations.
|
||||
pub trait DeviceDriver {
|
||||
/// Return a compatibility string for identifying the driver.
|
||||
fn compatible(&self) -> &str;
|
||||
|
||||
/// Called by the kernel to bring up the device.
|
||||
fn init(&self) -> Result {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
// Rust embedded logo for `make doc`.
|
||||
#![doc(html_logo_url = "https://git.io/JeGIp")]
|
||||
|
||||
//! The `kernel`
|
||||
//!
|
||||
//! The `kernel` is composed by glueing together hardware-specific Board Support
|
||||
//! Package (`BSP`) code and hardware-agnostic `kernel` code through the
|
||||
//! [`kernel::interface`] traits.
|
||||
//!
|
||||
//! [`kernel::interface`]: interface/index.html
|
||||
|
||||
#![feature(format_args_nl)]
|
||||
#![feature(panic_info_message)]
|
||||
#![feature(trait_alias)]
|
||||
#![no_main]
|
||||
#![no_std]
|
||||
|
||||
// Conditionally includes the selected `architecture` code, which provides the
|
||||
// `_start()` function, the first function to run.
|
||||
mod arch;
|
||||
|
||||
// `_start()` then calls `relocate::relocate_self()`.
|
||||
mod relocate;
|
||||
|
||||
// `relocate::relocate_self()` calls `runtime_init::init()`, which on
|
||||
// completion, jumps to `kernel_entry()`.
|
||||
mod runtime_init;
|
||||
|
||||
// Conditionally includes the selected `BSP` code.
|
||||
mod bsp;
|
||||
|
||||
mod interface;
|
||||
mod panic_wait;
|
||||
mod print;
|
||||
|
||||
/// Entrypoint of the `kernel`.
|
||||
fn kernel_entry() -> ! {
|
||||
use interface::console::All;
|
||||
|
||||
// Run the BSP's initialization code.
|
||||
bsp::init();
|
||||
|
||||
println!(" __ __ _ _ _ _ ");
|
||||
println!("| \\/ (_)_ _ (_) | ___ __ _ __| |");
|
||||
println!("| |\\/| | | ' \\| | |__/ _ \\/ _` / _` |");
|
||||
println!("|_| |_|_|_||_|_|____\\___/\\__,_\\__,_|");
|
||||
println!();
|
||||
println!("{:^37}", bsp::board_name());
|
||||
println!();
|
||||
println!("[ML] Requesting binary");
|
||||
bsp::console().flush();
|
||||
|
||||
// Clear the RX FIFOs, if any, of spurious received characters before
|
||||
// starting with the loader protocol.
|
||||
bsp::console().clear();
|
||||
|
||||
// Notify raspbootcom to send the binary.
|
||||
for _ in 0..3 {
|
||||
bsp::console().write_char(3 as char);
|
||||
}
|
||||
|
||||
// Read the binary's size.
|
||||
let mut size: u32 = u32::from(bsp::console().read_char() as u8);
|
||||
size |= u32::from(bsp::console().read_char() as u8) << 8;
|
||||
size |= u32::from(bsp::console().read_char() as u8) << 16;
|
||||
size |= u32::from(bsp::console().read_char() as u8) << 24;
|
||||
|
||||
// Trust it's not too big.
|
||||
print!("OK");
|
||||
|
||||
let kernel_addr: *mut u8 = bsp::BOARD_DEFAULT_LOAD_ADDRESS as *mut u8;
|
||||
unsafe {
|
||||
// Read the kernel byte by byte.
|
||||
for i in 0..size {
|
||||
*kernel_addr.offset(i as isize) = bsp::console().read_char() as u8;
|
||||
}
|
||||
}
|
||||
|
||||
println!("[ML] Loaded! Executing the payload now\n");
|
||||
bsp::console().flush();
|
||||
|
||||
// Use black magic to get a function pointer.
|
||||
let kernel: extern "C" fn() -> ! = unsafe { core::mem::transmute(kernel_addr as *const ()) };
|
||||
|
||||
// Jump to loaded kernel!
|
||||
kernel()
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! A panic handler that infinitely waits.
|
||||
|
||||
use crate::{arch, println};
|
||||
use core::panic::PanicInfo;
|
||||
|
||||
#[panic_handler]
|
||||
fn panic(info: &PanicInfo) -> ! {
|
||||
if let Some(args) = info.message() {
|
||||
println!("Kernel panic: {}", args);
|
||||
} else {
|
||||
println!("Kernel panic!");
|
||||
}
|
||||
|
||||
arch::wait_forever()
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Printing facilities.
|
||||
|
||||
use crate::{bsp, interface};
|
||||
use core::fmt;
|
||||
|
||||
/// Prints without a newline.
|
||||
///
|
||||
/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html
|
||||
#[macro_export]
|
||||
macro_rules! print {
|
||||
($($arg:tt)*) => ($crate::print::_print(format_args!($($arg)*)));
|
||||
}
|
||||
|
||||
/// Prints with a newline.
|
||||
///
|
||||
/// Carbon copy from https://doc.rust-lang.org/src/std/macros.rs.html
|
||||
#[macro_export]
|
||||
macro_rules! println {
|
||||
() => ($crate::print!("\n"));
|
||||
($($arg:tt)*) => ({
|
||||
$crate::print::_print(format_args_nl!($($arg)*));
|
||||
})
|
||||
}
|
||||
|
||||
pub fn _print(args: fmt::Arguments) {
|
||||
use interface::console::Write;
|
||||
|
||||
bsp::console().write_fmt(args).unwrap();
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Relocation code.
|
||||
|
||||
/// Relocates the own binary from `bsp::BOARD_DEFAULT_LOAD_ADDRESS` to the
|
||||
/// `__binary_start` address from the linker script.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - Only a single core must be active and running this function.
|
||||
/// - Function must not use the `bss` section.
|
||||
pub unsafe fn relocate_self<T>() -> ! {
|
||||
extern "C" {
|
||||
static __binary_start: usize;
|
||||
static __binary_end: usize;
|
||||
}
|
||||
|
||||
let binary_start_addr: usize = &__binary_start as *const _ as _;
|
||||
let binary_end_addr: usize = &__binary_end as *const _ as _;
|
||||
let binary_size_in_byte: usize = binary_end_addr - binary_start_addr;
|
||||
|
||||
// Get the relocation destination address from the linker symbol.
|
||||
let mut reloc_dst_addr: *mut T = binary_start_addr as *mut T;
|
||||
|
||||
// The address of where the previous firmware loaded us.
|
||||
let mut src_addr: *const T = crate::bsp::BOARD_DEFAULT_LOAD_ADDRESS as *const _;
|
||||
|
||||
// Copy the whole binary.
|
||||
//
|
||||
// This is essentially a `memcpy()` optimized for throughput by transferring
|
||||
// in chunks of T.
|
||||
let n = binary_size_in_byte / core::mem::size_of::<T>();
|
||||
for _ in 0..n {
|
||||
use core::ptr;
|
||||
|
||||
ptr::write_volatile::<T>(reloc_dst_addr, ptr::read_volatile::<T>(src_addr));
|
||||
reloc_dst_addr = reloc_dst_addr.offset(1);
|
||||
src_addr = src_addr.offset(1);
|
||||
}
|
||||
|
||||
// Call `init()` through a trait object, causing the jump to use an absolute
|
||||
// address to reach the relocated binary. An elaborate explanation can be
|
||||
// found in the runtime_init.rs source comments.
|
||||
crate::runtime_init::get().init()
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
//
|
||||
// Copyright (c) 2018-2019 Andre Richter <andre.o.richter@gmail.com>
|
||||
|
||||
//! Rust runtime initialization code.
|
||||
|
||||
/// We are outsmarting the compiler here by using a trait as a layer of
|
||||
/// indirection. Because we are generating PIC code, a static dispatch to
|
||||
/// `init()` would generate a relative jump from the callee to `init()`.
|
||||
/// However, when calling `init()`, code just finished copying the binary to the
|
||||
/// actual link-time address, and hence is still running at whatever location
|
||||
/// the previous loader has put it. So we do not want a relative jump, because
|
||||
/// it would not jump to the relocated code.
|
||||
///
|
||||
/// By indirecting through a trait object, we can make use of the property that
|
||||
/// vtables store absolute addresses. So calling `init()` this way will kick
|
||||
/// execution to the relocated binary.
|
||||
pub trait RunTimeInit {
|
||||
/// Equivalent to `crt0` or `c0` code in C/C++ world. Clears the `bss` section,
|
||||
/// then calls the kernel entry.
|
||||
///
|
||||
/// Called from `BSP` code.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
/// - Only a single core must be active and running this function.
|
||||
unsafe fn init(&self) -> ! {
|
||||
extern "C" {
|
||||
// Boundaries of the .bss section, provided by the linker script.
|
||||
static mut __bss_start: u64;
|
||||
static mut __bss_end: u64;
|
||||
}
|
||||
|
||||
// Zero out the .bss section.
|
||||
r0::zero_bss(&mut __bss_start, &mut __bss_end);
|
||||
|
||||
crate::kernel_entry()
|
||||
}
|
||||
}
|
||||
|
||||
struct Traitor;
|
||||
impl RunTimeInit for Traitor {}
|
||||
|
||||
/// Give the callee a `RunTimeInit` trait object.
|
||||
pub fn get() -> &'static dyn RunTimeInit {
|
||||
&Traitor {}
|
||||
}
|
||||
Loading…
Reference in New Issue