Merge pull request #2199 from dr7ana/oxen-logging-bump

Oxen-logging version bump

.clang-format

@@ -54,3 +54,8 @@ PointerAlignment: Left
 # when wrapping function calls/declarations, force each parameter to have its own line
 BinPackParameters: 'false'
 BinPackArguments: 'false'
+
+# TODO: uncomment me when we are reading to rearrange the header includes
+# IncludeBlocks: Regroup
+# IncludeCategories: 'llarp/'
+

.clang-tidy

@@ -1,2 +1,2 @@
 HeaderFilterRegex: 'llarp/.*'
-Checks: 'readability-else-after-return,clang-analyzer-core-*,modernize-*,-modernize-use-trailing-return-type,-modernize-use-nodiscard,bugprone-*'
+Checks: 'readability-else-after-return,clang-analyzer-core-*,modernize-*,-modernize-use-trailing-return-type,-modernize-use-nodiscard,bugprone-*,-bugprone-easily-swappable-parameters'

.drone.jsonnet

@@ -164,8 +164,6 @@ local windows_cross_pipeline(name,
         apt_get_quiet + ' update',
         apt_get_quiet + ' install -y eatmydata',
         'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y build-essential cmake git pkg-config ccache g++-mingw-w64-x86-64-posix nsis zip icoutils automake libtool librsvg2-bin bison',
-        'update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix',
-        'update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix',
         'JOBS=' + jobs + ' VERBOSE=1 ./contrib/windows.sh -DSTRIP_SYMBOLS=ON -DGUI_EXE=$${DRONE_WORKSPACE}/gui/release/Lokinet-GUI_portable.exe' +
         ci_dep_mirror(local_mirror),
       ] + extra_cmds,
@@ -383,13 +381,16 @@ local docs_pipeline(name, image, extra_cmds=[], allow_fail=false) = {
   debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable/arm32v7', arch='arm64', jobs=4),
 
   // cross compile targets
-  linux_cross_pipeline('Cross Compile (arm/arm64)', cross_targets=['arm-linux-gnueabihf', 'aarch64-linux-gnu']),
-  linux_cross_pipeline('Cross Compile (ppc64le)', cross_targets=['powerpc64le-linux-gnu']),
+  // Aug 11: these are exhibiting some dumb failures in libsodium and external deps, TOFIX later
+  //linux_cross_pipeline('Cross Compile (arm/arm64)', cross_targets=['arm-linux-gnueabihf', 'aarch64-linux-gnu']),
+  //linux_cross_pipeline('Cross Compile (ppc64le)', cross_targets=['powerpc64le-linux-gnu']),
+
   // Not currently building successfully:
   //linux_cross_pipeline('Cross Compile (mips)', cross_targets=['mips-linux-gnu', 'mipsel-linux-gnu']),
 
   // android apk builder
-  apk_builder('android apk', docker_base + 'flutter', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']),
+  // Aug 11: this is also failing in openssl, TOFIX later
+  //apk_builder('android apk', docker_base + 'flutter', extra_cmds=['UPLOAD_OS=android ./contrib/ci/drone-static-upload.sh']),
 
   // Windows builds (x64)
   windows_cross_pipeline('Windows (amd64)',

.github/workflows/clean_issues.yml

@@ -0,0 +1,22 @@
+name: Close incomplete issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/stale@v4.1.1
+        with:
+          only-labels: incomplete
+          days-before-issue-stale: 14
+          days-before-issue-close: 7
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been 'incomplete' for 14 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 7 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "external/nlohmann"]
 	path = external/nlohmann
 	url = https://github.com/nlohmann/json.git
-[submodule "external/cxxopts"]
-	path = external/cxxopts
-	url = https://github.com/jarro2783/cxxopts.git
 [submodule "external/ghc-filesystem"]
 	path = external/ghc-filesystem
 	url = https://github.com/gulrak/filesystem.git
@@ -39,3 +36,6 @@
 [submodule "gui"]
 	path = gui
 	url = https://github.com/oxen-io/lokinet-gui.git
+[submodule "external/CLI11"]
+	path = external/CLI11
+	url = https://github.com/CLIUtils/CLI11.git

CMakeLists.txt

@@ -49,8 +49,7 @@ endif()
 option(USE_AVX2 "enable avx2 code" OFF)
 option(USE_NETNS "enable networking namespace support. Linux only" OFF)
 option(NATIVE_BUILD "optimise for host system and FPU" ON)
-option(EMBEDDED_CFG "optimise for older hardware or embedded systems" OFF)
-option(BUILD_LIBLOKINET "build liblokinet.so" ON)
+option(WITH_EMBEDDED_LOKINET "build liblokinet.so for embedded lokinet" OFF)
 option(XSAN "use sanitiser, if your system has it (requires -DCMAKE_BUILD_TYPE=Debug)" OFF)
 option(USE_JEMALLOC "Link to jemalloc for memory allocations, if found" ON)
 option(TESTNET "testnet build" OFF)
@@ -149,7 +148,6 @@ endif()
 
 find_package(PkgConfig REQUIRED)
 
-
 if(NOT BUILD_STATIC_DEPS)
   pkg_check_modules(LIBUV libuv>=1.18.0 IMPORTED_TARGET)
 endif()

cmake/StaticBuild.cmake

@@ -242,6 +242,7 @@ set(openssl_system_env "")
 set(openssl_arch "")
 set(openssl_configure_command ./config)
 set(openssl_flags "CFLAGS=${deps_CFLAGS}")
+set(unbound_ldflags "")
 if(CMAKE_CROSSCOMPILING)
   if(ARCH_TRIPLET STREQUAL x86_64-w64-mingw32)
     set(openssl_arch mingw64)
@@ -258,9 +259,12 @@ if(CMAKE_CROSSCOMPILING)
     set(openssl_arch linux-mips64)
   elseif(ARCH_TRIPLET STREQUAL mips-linux-gnu)
     set(openssl_arch linux-mips32)
+  elseif(ARCH_TRIPLET STREQUAL mips-openwrt-linux)
+    set(unbound_ldflags "-latomic")
+    set(openssl_arch linux-mips32)
   elseif(ARCH_TRIPLET STREQUAL mipsel-linux-gnu)
     set(openssl_arch linux-mips)
-  elseif(ARCH_TRIPLET STREQUAL aarch64-linux-gnu)
+  elseif(ARCH_TRIPLET STREQUAL aarch64-linux-gnu OR ARCH_TRIPLET STREQUAL aarch64-openwrt-linux-musl)
     # cross compile arm64
     set(openssl_arch linux-aarch64)
   elseif(ARCH_TRIPLET MATCHES arm-linux)
@@ -319,7 +323,7 @@ build_external(unbound
   --enable-static --with-libunbound-only --with-pic
   --$<IF:$<BOOL:${WITH_LTO}>,enable,disable>-flto --with-ssl=${DEPS_DESTDIR}
   --with-libexpat=${DEPS_DESTDIR}
-  "CC=${deps_cc}" "CFLAGS=${deps_CFLAGS}"
+  "CC=${deps_cc}" "CFLAGS=${deps_CFLAGS}" "LDFLAGS=${unbound_ldflags}"
 )
 add_static_target(libunbound unbound_external libunbound.a)
 if(NOT WIN32)

cmake/check_for_std_filesystem.cmake

@@ -47,5 +47,5 @@ else()
   set(GHC_FILESYSTEM_WITH_INSTALL OFF CACHE INTERNAL "")
   add_subdirectory(external/ghc-filesystem)
   target_link_libraries(filesystem INTERFACE ghc_filesystem)
-  target_compile_definitions(filesystem INTERFACE USE_GHC_FILESYSTEM)
+  target_compile_definitions(filesystem INTERFACE USE_GHC_FILESYSTEM CLI11_HAS_FILESYSTEM=0)
 endif()

cmake/macos.cmake

@@ -150,7 +150,7 @@ if(BUILD_PACKAGE)
       "${lokinet_installer}"
       COMMAND ./seticon lokinet.icns "${lokinet_installer}.dmg"
   )
-  add_custom_target(package DEPENDS "${lokinet_installer}.dmg")
+  add_custom_target(dmg DEPENDS "${lokinet_installer}.dmg")
 endif()
 
 

contrib/android-configure.sh

@@ -33,7 +33,6 @@ for abi in $build_abis; do
         -DBUILD_PACKAGE=ON \
         -DBUILD_SHARED_LIBS=OFF \
         -DBUILD_TESTING=OFF \
-        -DBUILD_LIBLOKINET=OFF \
         -DWITH_TESTS=OFF \
         -DWITH_BOOTSTRAP=OFF \
         -DNATIVE_BUILD=OFF \

contrib/bencode-dump.py

@@ -11,6 +11,13 @@ elif len(sys.argv) != 2 or sys.argv[1].startswith('-'):
 else:
     f = open(sys.argv[1], 'rb')
 
+
+initial = f.peek(2)
+is_hex = False
+if initial.startswith(b'64') or initial.startswith(b'6c'):
+    print("Input looks like hex bencoded data; parsing as hex input", file=sys.stderr)
+    is_hex = True
+
 class HexPrinter():
     def __init__(self, data):
         self.data = data
@@ -20,7 +27,12 @@ class HexPrinter():
 
 
 def next_byte():
-    b = f.read(1)
+    if is_hex:
+        pair = f.read(2)
+        assert pair is not None and len(pair) == 2
+        b = int(pair, 16).to_bytes(1, 'big')
+    else:
+        b = f.read(1)
     assert b is not None and len(b) == 1
     return b
 
@@ -28,7 +40,7 @@ def next_byte():
 def parse_int():
     s = b''
     x = next_byte()
-    while x in b"0123456789":
+    while x in b"0123456789-":
         s += x
         x = next_byte()
     assert x == b'e' and len(s) > 0, "Invalid integer encoding"
@@ -42,7 +54,10 @@ def parse_string(s):
         x = next_byte()
     assert x == b':', "Invalid string encoding"
     s = int(s)
-    data = f.read(s)
+    if is_hex:
+        data = bytes.fromhex(f.read(2*s).decode('ascii'))
+    else:
+        data = f.read(s)
     assert len(data) == s, "Truncated string data"
     # If the string is ascii then convert to string:
     if all(0x20 <= b <= 0x7e for b in data):

contrib/format-version.sh

@@ -1,5 +1,5 @@
 
-CLANG_FORMAT_DESIRED_VERSION=14
+CLANG_FORMAT_DESIRED_VERSION=15
 
 CLANG_FORMAT=$(command -v clang-format-$CLANG_FORMAT_DESIRED_VERSION 2>/dev/null)
 if [ $? -ne 0 ]; then

contrib/mac-configure.sh

@@ -13,7 +13,6 @@ cd build-mac
 cmake \
       -G Ninja \
       -DBUILD_STATIC_DEPS=ON \
-      -DBUILD_LIBLOKINET=OFF \
       -DWITH_TESTS=OFF \
       -DWITH_BOOTSTRAP=OFF \
       -DNATIVE_BUILD=OFF \

contrib/mac.sh

@@ -19,7 +19,7 @@ fi
 
 cd build-mac
 rm -rf Lokinet\ *
-ninja -j${JOBS:-1} package
+ninja -j${JOBS:-1} dmg
 cd ..
 
 echo -e "Build complete, your app is here:\n"

contrib/omq-rpc.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+import nacl.bindings as sodium
+from nacl.public import PrivateKey
+from nacl.signing import SigningKey, VerifyKey
+import nacl.encoding
+import requests
+import zmq
+import zmq.utils.z85
+import sys
+import re
+import time
+import random
+import shutil
+
+
+context = zmq.Context()
+socket = context.socket(zmq.DEALER)
+socket.setsockopt(zmq.CONNECT_TIMEOUT, 5000)
+socket.setsockopt(zmq.HANDSHAKE_IVL, 5000)
+#socket.setsockopt(zmq.IMMEDIATE, 1)
+
+if len(sys.argv) > 1 and any(sys.argv[1].startswith(x) for x in ("ipc://", "tcp://", "curve://")):
+    remote = sys.argv[1]
+    del sys.argv[1]
+else:
+    remote = "ipc://./rpc.sock"
+
+curve_pubkey = b''
+my_privkey, my_pubkey = b'', b''
+
+# If given a curve://whatever/pubkey argument then transform it into 'tcp://whatever' and put the
+# 'pubkey' back into argv to be handled below.
+if remote.startswith("curve://"):
+    pos = remote.rfind('/')
+    pkhex = remote[pos+1:]
+    remote = "tcp://" + remote[8:pos]
+    if len(pkhex) != 64 or not all(x in "0123456789abcdefABCDEF" for x in pkhex):
+        print("curve:// addresses must be in the form curve://HOST:PORT/REMOTE_PUBKEY_HEX", file=sys.stderr)
+        sys.exit(1)
+    sys.argv[1:0] = [pkhex]
+
+if len(sys.argv) > 1 and len(sys.argv[1]) == 64 and all(x in "0123456789abcdefABCDEF" for x in sys.argv[1]):
+    curve_pubkey = bytes.fromhex(sys.argv[1])
+    del sys.argv[1]
+    socket.curve_serverkey = curve_pubkey
+    if len(sys.argv) > 1 and len(sys.argv[1]) == 64 and all(x in "0123456789abcdefABCDEF" for x in sys.argv[1]):
+        my_privkey = bytes.fromhex(sys.argv[1])
+        del sys.argv[1]
+        my_pubkey = zmq.utils.z85.decode(zmq.curve_public(zmq.utils.z85.encode(my_privkey)))
+    else:
+        my_privkey = PrivateKey.generate()
+        my_pubkey = my_privkey.public_key.encode()
+        my_privkey = my_privkey.encode()
+
+        print("No curve client privkey given; generated a random one (pubkey: {}, privkey: {})".format(
+            my_pubkey.hex(), my_privkey.hex()), file=sys.stderr)
+    socket.curve_secretkey = my_privkey
+    socket.curve_publickey = my_pubkey
+
+if not 2 <= len(sys.argv) <= 3 or any(x in y for x in ("--help", "-h") for y in sys.argv[1:]):
+    print("Usage: {} [ipc:///path/to/sock|tcp://1.2.3.4:5678] [SERVER_CURVE_PUBKEY [LOCAL_CURVE_PRIVKEY]] COMMAND ['JSON']".format(
+        sys.argv[0]), file=sys.stderr)
+    sys.exit(1)
+
+beginning_of_time = time.clock_gettime(time.CLOCK_MONOTONIC)
+
+print("Connecting to {}".format(remote), file=sys.stderr)
+socket.connect(remote)
+to_send = [sys.argv[1].encode(), b'tagxyz123']
+to_send += (x.encode() for x in sys.argv[2:])
+print("Sending {}".format(to_send[0]), file=sys.stderr)
+socket.send_multipart(to_send)
+if socket.poll(timeout=5000):
+    m = socket.recv_multipart()
+    recv_time = time.clock_gettime(time.CLOCK_MONOTONIC)
+    if len(m) < 3 or m[0:2] != [b'REPLY', b'tagxyz123']:
+        print("Received unexpected {}-part reply:".format(len(m)), file=sys.stderr)
+        for x in m:
+            print("- {}".format(x))
+    else: # m[2] is numeric value, m[3] is data part, and will become m[2] <- changed
+        print("Received reply in {:.6f}s:".format(recv_time - beginning_of_time), file=sys.stderr)
+        if len(m) < 3:
+            print("(empty reply data)", file=sys.stderr)
+        else:
+            for x in m[2:]:
+                print("{} bytes data part:".format(len(x)), file=sys.stderr)
+                if any(x.startswith(y) for y in (b'd', b'l', b'i')) and x.endswith(b'e'):
+                    sys.stdout.buffer.write(x)
+                else:
+                    print(x.decode(), end="\n\n")
+
+else:
+    print("Request timed out", file=sys.stderr)
+    socket.close(linger=0)
+    sys.exit(1)
+
+# sample usage:
+# ./omq-rpc.py ipc://$HOME/.oxen/testnet/oxend.sock 'llarp.get_service_nodes' | jq

contrib/windows-configure.sh

@@ -32,7 +32,6 @@ cmake \
     -DBUILD_PACKAGE=ON \
     -DBUILD_SHARED_LIBS=OFF \
     -DBUILD_TESTING=OFF \
-    -DBUILD_LIBLOKINET=OFF \
     -DWITH_TESTS=OFF \
     -DWITH_BOOTSTRAP=OFF \
     -DNATIVE_BUILD=OFF \

crypto/CMakeLists.txt

@@ -20,7 +20,7 @@ add_library(lokinet-cryptography
   libntrup/src/ref/rq.c
 )
 
-target_include_directories(lokinet-cryptography PUBLIC libntrup/include)
+target_include_directories(lokinet-cryptography PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/libntrup/include)
 
 # The avx implementation uses runtime CPU feature detection to enable itself, so we *always* want to
 # compile it with avx2/fma support when supported by the compiler even if we aren't compiling with

daemon/lokinet-vpn.cpp

@@ -1,11 +1,16 @@
 #include <oxenmq/oxenmq.h>
 #include <nlohmann/json.hpp>
 #include <fmt/core.h>
-#include <cxxopts.hpp>
 #include <future>
 #include <vector>
 #include <array>
 #include <llarp/net/net.hpp>
+#include <string_view>
+
+#include <CLI/App.hpp>
+#include <CLI/Formatter.hpp>
+#include <CLI/Config.hpp>
+#include "oxenmq/address.h"
 
 #ifdef _WIN32
 // add the unholy windows headers for iphlpapi
@@ -53,25 +58,29 @@ OMQ_Request(
 
 namespace
 {
-  template <typename T>
-  constexpr bool is_optional = false;
-  template <typename T>
-  constexpr bool is_optional<std::optional<T>> = true;
-
-  // Extracts a value from a cxxopts result and assigns it into `value` if present.  The value can
-  // either be a plain value or a std::optional.  If not present, `value` is not touched.
-  template <typename T>
-  void
-  extract_option(const cxxopts::ParseResult& r, const std::string& name, T& value)
+  struct command_line_options
   {
-    if (r.count(name))
-    {
-      if constexpr (is_optional<T>)
-        value = r[name].as<typename T::value_type>();
-      else
-        value = r[name].as<T>();
-    }
-  }
+    // bool options
+    bool verbose = false;
+    bool help = false;
+    bool vpnUp = false;
+    bool vpnDown = false;
+    bool swap = false;
+    bool printStatus = false;
+    bool killDaemon = false;
+
+    // string options
+    std::string exitAddress;
+    std::string rpc;
+    std::string endpoint = "default";
+    std::string token;
+    std::optional<std::string> range;
+    std::vector<std::string> swapExits;
+
+    // oxenmq
+    oxenmq::address rpcURL{};
+    oxenmq::LogLevel logLevel = oxenmq::LogLevel::warn;
+  };
 
   // Takes a code, prints a message, and returns the code.  Intended use is:
   //     return exit_error(1, "blah: {}", 42);
@@ -98,127 +107,131 @@ namespace
 int
 main(int argc, char* argv[])
 {
-  cxxopts::Options opts("lokinet-vpn", "LokiNET vpn control utility");
-
-  // clang-format off
-  opts.add_options()
-    ("v,verbose", "Verbose", cxxopts::value<bool>())
-    ("h,help", "help", cxxopts::value<bool>())
-    ("kill", "kill the daemon", cxxopts::value<bool>())
-    ("up", "put vpn up", cxxopts::value<bool>())
-    ("down", "put vpn down", cxxopts::value<bool>())
-    ("exit", "specify exit node address", cxxopts::value<std::string>())
-    ("rpc", "rpc url for lokinet", cxxopts::value<std::string>())
-    ("endpoint", "endpoint to use", cxxopts::value<std::string>())
-    ("token", "exit auth token to use", cxxopts::value<std::string>())
-    ("auth", "exit auth token to use", cxxopts::value<std::string>())
-    ("status", "print status and exit", cxxopts::value<bool>())
-    ("range", "ip range to map", cxxopts::value<std::string>())
-    ;
-  // clang-format on
-  oxenmq::address rpcURL("tcp://127.0.0.1:1190");
-  std::string exitAddress;
-  std::string endpoint = "default";
-  std::string token;
-  std::optional<std::string> range;
-  oxenmq::LogLevel logLevel = oxenmq::LogLevel::warn;
-  bool goUp = false;
-  bool goDown = false;
-  bool printStatus = false;
-  bool killDaemon = false;
-  try
-  {
-    const auto result = opts.parse(argc, argv);
+  CLI::App cli{"lokiNET vpn control utility", "lokinet-vpn"};
+  command_line_options options{};
 
-    if (result.count("help") > 0)
-    {
-      std::cout << opts.help() << std::endl;
-      return 0;
-    }
+  // flags: boolean values in command_line_options struct
+  cli.add_flag("-v,--verbose", options.verbose, "Verbose");
+  cli.add_flag("--add,--up", options.vpnUp, "Map VPN connection to exit node [--up is deprecated]");
+  cli.add_flag(
+      "--remove,--down",
+      options.vpnDown,
+      "Unmap VPN connection to exit node [--down is deprecated]");
+  cli.add_flag("--status", options.printStatus, "Print VPN status and exit");
+  cli.add_flag("-k,--kill", options.killDaemon, "Kill lokinet daemon");
 
-    if (result.count("verbose") > 0)
-    {
-      logLevel = oxenmq::LogLevel::debug;
-    }
-    goUp = result.count("up") > 0;
-    goDown = result.count("down") > 0;
-    printStatus = result.count("status") > 0;
-    killDaemon = result.count("kill") > 0;
-
-    extract_option(result, "rpc", rpcURL);
-    extract_option(result, "exit", exitAddress);
-    extract_option(result, "endpoint", endpoint);
-    extract_option(result, "token", token);
-    extract_option(result, "auth", token);
-    extract_option(result, "range", range);
+  // options: string values in command_line_options struct
+  cli.add_option("--exit", options.exitAddress, "Specify exit node address")->capture_default_str();
+  cli.add_option("--endpoint", options.endpoint, "Endpoint to use")->capture_default_str();
+  cli.add_option("--token,--auth", options.token, "Exit auth token to use")->capture_default_str();
+  cli.add_option("--range", options.range, "IP range to map exit to")->capture_default_str();
+  cli.add_option(
+         "--swap", options.swapExits, "Exit addresses to swap mapped connection to [old] [new]")
+      ->expected(2)
+      ->capture_default_str();
+
+  // options: oxenmq values in command_line_options struct
+  cli.add_option("--rpc", options.rpc, "Specify RPC URL for lokinet")->capture_default_str();
+  cli.add_option(
+         "--log-level", options.logLevel, "Log verbosity level, see log levels for accepted values")
+      ->type_name("LEVEL")
+      ->capture_default_str();
+
+  try
+  {
+    cli.parse(argc, argv);
+  }
+  catch (const CLI::ParseError& e)
+  {
+    return cli.exit(e);
   }
-  catch (const cxxopts::option_not_exists_exception& ex)
+
+  try
   {
-    return exit_error(2, "{}\n{}", ex.what(), opts.help());
+    if (options.verbose)
+      options.logLevel = oxenmq::LogLevel::debug;
   }
-  catch (std::exception& ex)
+  catch (const CLI::OptionNotFound& e)
   {
-    return exit_error(2, "{}", ex.what());
+    cli.exit(e);
   }
+  catch (const CLI::Error& e)
+  {
+    cli.exit(e);
+  };
 
-  int num_commands = goUp + goDown + printStatus + killDaemon;
+  int numCommands = options.vpnUp + options.vpnDown + options.printStatus + options.killDaemon
+      + (not options.swapExits.empty());
 
-  if (num_commands == 0)
-    return exit_error(3, "One of --up/--down/--status/--kill must be specified");
-  if (num_commands != 1)
-    return exit_error(3, "Only one of --up/--down/--status/--kill may be specified");
+  switch (numCommands)
+  {
+    case 0:
+      return exit_error(3, "One of --add/--remove/--swap/--status/--kill must be specified");
+    case 1:
+      break;
+    default:
+      return exit_error(3, "Only one of --add/--remove/--swap/--status/--kill may be specified");
+  }
 
-  if (goUp and exitAddress.empty())
-    return exit_error("no exit address provided");
+  if (options.vpnUp and options.exitAddress.empty())
+    return exit_error("No exit address provided, must specify --exit <address>");
 
   oxenmq::OxenMQ omq{
       [](oxenmq::LogLevel lvl, const char* file, int line, std::string msg) {
         std::cout << lvl << " [" << file << ":" << line << "] " << msg << std::endl;
       },
-      logLevel};
+      options.logLevel};
+
+  options.rpcURL = oxenmq::address{(options.rpc.empty()) ? "tcp://127.0.0.1:1190" : options.rpc};
 
   omq.start();
 
   std::promise<bool> connectPromise;
 
-  const auto connID = omq.connect_remote(
-      rpcURL,
+  const auto connectionID = omq.connect_remote(
+      options.rpcURL,
       [&connectPromise](auto) { connectPromise.set_value(true); },
       [&connectPromise](auto, std::string_view msg) {
-        std::cout << "failed to connect to lokinet RPC: " << msg << std::endl;
+        std::cout << "Failed to connect to lokinet RPC: " << msg << std::endl;
         connectPromise.set_value(false);
       });
 
   auto ftr = connectPromise.get_future();
   if (not ftr.get())
-  {
     return 1;
-  }
 
-  if (killDaemon)
+  if (options.killDaemon)
   {
-    if (not OMQ_Request(omq, connID, "llarp.halt"))
-      return exit_error("call to llarp.halt failed");
-    return 0;
+    auto maybe_halt = OMQ_Request(omq, connectionID, "llarp.halt");
+
+    if (not maybe_halt)
+      return exit_error("Call to llarp.halt failed");
+
+    if (auto err_it = maybe_halt->find("error");
+        err_it != maybe_halt->end() and not err_it.value().is_null())
+    {
+      return exit_error("{}", err_it.value().dump());
+    }
   }
 
-  if (printStatus)
+  if (options.printStatus)
   {
-    const auto maybe_status = OMQ_Request(omq, connID, "llarp.status");
+    const auto maybe_status = OMQ_Request(omq, connectionID, "llarp.status");
+
     if (not maybe_status)
-      return exit_error("call to llarp.status failed");
+      return exit_error("Call to llarp.status failed");
 
     try
     {
-      const auto& ep = maybe_status->at("result").at("services").at(endpoint);
-      const auto exitMap = ep.at("exitMap");
-      if (exitMap.empty())
+      const auto& ep = maybe_status->at("result").at("services").at(options.endpoint).at("exitMap");
+
+      if (ep.empty())
       {
-        std::cout << "no exits" << std::endl;
+        std::cout << "No exits found" << std::endl;
       }
       else
       {
-        for (const auto& [range, exit] : exitMap.items())
+        for (const auto& [range, exit] : ep.items())
         {
           std::cout << range << " via " << exit.get<std::string>() << std::endl;
         }
@@ -226,34 +239,60 @@ main(int argc, char* argv[])
     }
     catch (std::exception& ex)
     {
-      return exit_error("failed to parse result: {}", ex.what());
+      return exit_error("Failed to parse result: {}", ex.what());
     }
     return 0;
   }
-  if (goUp)
+
+  if (not options.swapExits.empty())
   {
-    nlohmann::json opts{{"exit", exitAddress}, {"token", token}};
-    if (range)
-      opts["range"] = *range;
+    nlohmann::json opts{{"exit_addresses", std::move(options.swapExits)}};
+
+    auto maybe_swap = OMQ_Request(omq, connectionID, "llarp.swap_exits", std::move(opts));
 
-    auto maybe_result = OMQ_Request(omq, connID, "llarp.exit", std::move(opts));
+    if (not maybe_swap)
+      return exit_error("Failed to swap exit node connections");
+
+    if (auto err_it = maybe_swap->find("error");
+        err_it != maybe_swap->end() and not err_it.value().is_null())
+    {
+      return exit_error("{}", err_it.value().dump());
+    }
+  }
+
+  if (options.vpnUp)
+  {
+    nlohmann::json opts{{"address", options.exitAddress}, {"token", options.token}};
+    if (options.range)
+      opts["ip_range"] = *options.range;
+
+    auto maybe_result = OMQ_Request(omq, connectionID, "llarp.map_exit", std::move(opts));
 
     if (not maybe_result)
-      return exit_error("could not add exit");
+      return exit_error("Could not add exit");
 
     if (auto err_it = maybe_result->find("error");
         err_it != maybe_result->end() and not err_it.value().is_null())
     {
-      return exit_error("{}", err_it.value());
+      return exit_error("{}", err_it.value().dump());
     }
   }
-  if (goDown)
+  if (options.vpnDown)
   {
-    nlohmann::json opts{{"unmap", true}};
-    if (range)
-      opts["range"] = *range;
-    if (not OMQ_Request(omq, connID, "llarp.exit", std::move(opts)))
-      return exit_error("failed to unmap exit");
+    nlohmann::json opts{{"unmap_exit", true}};
+    if (options.range)
+      opts["ip_range"] = *options.range;
+
+    auto maybe_down = OMQ_Request(omq, connectionID, "llarp.unmap_exit", std::move(opts));
+
+    if (not maybe_down)
+      return exit_error("Failed to unmap exit node connection");
+
+    if (auto err_it = maybe_down->find("error");
+        err_it != maybe_down->end() and not err_it.value().is_null())
+    {
+      return exit_error("{}", err_it.value().dump());
+    }
   }
 
   return 0;

docs/refactor_notes.md

@@ -0,0 +1,97 @@
+# High Level Iterative Approach
+
+the desired outcome of this refactor will be splitting the existing code up into a stack of new components.
+a layer hides all functionality of the layer below it to reduce the complexity like the OSI stack intends to.
+the refactor starts at the top layer, wiring up the old implementation piecewise to the top layer.
+once the top layer is wired up to the old implementation we will move down to the next layer.
+this will repeat until we reach the bottom layer.
+once the old implementation is wired up into these new clearly defined layers, we can fixup or replace different parts of each layer one at a time as needed.
+
+working down from each layer will let us pick apart the old implementation (if needed) that we would wire up to the new base classes for that layer we are defining now without worrying about what is below it (yet).
+
+this refactor is very able to be split up into small work units that (ideally) do not confict with each other.
+
+
+PDU: https://en.wikipedia.org/wiki/Protocol_data_unit
+
+# The New Layers
+
+from top to bottom the new layers are:
+
+* Platform Layer
+* Flow Layer
+* Routing Layer
+* Onion Layer
+* Link Layer
+* Wire Layer
+
+
+## Platform Layer
+
+this is the top layer, it is responsibile ONLY to act as a handler of reading data from the "user" (via tun interface or whatever) to forward to the flow layer as desired, and to take data from the flow layer and send it to the "user".
+any kind of IP/dns mapping or traffic isolation details are done here. embedded lokinet would be implemented in this layer as well, as it is without a full tun interface.
+
+Platform layer PDU are what the OS gives us and we internally convert them into flow layer PDU and hand them off to the flow layer.
+
+
+## Flow Layer
+
+this layer is tl;dr mean to multiplex data from the platform layer across the routing layer and propagating PDU from the routing to the platform layer if needed.
+
+the flow layer is responsible for sending platform layer PDU across path we have already established.
+this layer is informed by the routing layer below it of state changes in what paths are available for use.
+the flow layer requests from the layer below to make new paths if it wishes to get new ones on demand.
+this layer will recieve routing layer PDU from the routing layer and apply any congestion control needed to buffer things to the os if it is needed at all.
+
+flow layer PDU are (data, ethertype, src-pubkey, dst-pubkey, isolation-metric) tuples.
+data is the datum we are tunneling over lokinet. ethertype tells us what kind of datum this is, e.g. plainquic/ipv4/ipv6/auth/etc.
+src-pubkey and dst-pubkey are public the ed25519 public keys of each end of the flow in use.
+the isolation metric is a piece of metadata we use to distinguish unique flows (convotag). in this new seperation convotags explicitly do not hand over across paths.
+
+
+## Routing Layer
+
+this layer is tl;dr meant for path management but not path building.
+
+the routing layer is responsible for sending/recieving flow layer PDU, DHT requests/responses, latency testing PDU and any other kind of PDU we send/recieve over the onion layer.
+this layer will be responsible for managing paths we have already built across lokinet.
+the routing layer will periodically measure path status/latency, and do any other kinds of perioidic path related tasks post build.
+this layer when asked for a new path from the flow layer will use one that has been prebuilt already and if the number of prebuilt paths is below a threshold we will tell the onion layer to build more paths.
+the routing layer will recieve path build results be their success/fail/timeout from the onion layer that were requested and apply any congestion control needed at the pivot router.
+
+routing layer PDU are (data, src-path, dst-path) tuples.
+data is the datum we are transferring between paths.
+src-path and dst-path are (pathid, router id) tuples, the source being which path this routing layer PDU originated from, destination being which path it is going to.
+in the old model, router id is always the router that recieves it as the pivot router, this remains the same unless we explicitly provide router-id.
+this lets us propagate hints to DHT related PDU held inside the datum.
+
+
+## Onion Layer
+
+the onion layer is repsonsible for path builds, path selection logic and low level details of encrypted/decrypting PDU that are onion routed over paths.
+this layer is requested by the routing layer to build a path to a pivot router with an optional additional constraints (e.g. unique cidr/operator/geoip/etc, latency constaints, hop length, path lifetime).
+the onion layer will encrypt PDU and send them to link layer as (frame/edge router id) tuples, and recieve link layer frames from edge routers, decrypt them and propagate them as needed to the routing layer.
+this layer also handles transit onion traffic and transit path build responsibilities as a snode and apply congestion control as needed per transit path.
+
+the onion layer PDU are (data, src-path, dst-path) tuples.
+src-path and dst-path are (router-id, path-id) tuples which contain the ed25519 pubkey of the node and the 128 bit path-id it was associated with.
+data is some datum we are onion routing that we would apply symettric encryption as needed before propagating to upper or lower layers.
+
+
+## Link Layer
+
+the link layer is responsbile for transmission of frames between nodes.
+this layer will handle queuing and congestion control between wire proto sessions between nodes.
+the link layer is will initate and recieve wire session to/from remote nodes.
+
+the link layer PDU is (data, src-router-id, dst-router-id) tuples.
+data is a datum of a link layer frame.
+src-router-id and dst-router-id are (ed25519-pubkey, net-addr, wire-proto-info) tuples.
+the ed25519 pubkey is a .snode address, (clients have these too but they are ephemeral).
+net-addr is an (ip, port) tuple the node is reachable via the wire protocol.
+wire-proto-info is dialect specific wire protocol specific info.
+
+## Wire Layer
+
+the wire layer is responsible for transmitting link layer frames between nodes.
+all details here are specific to each wire proto dialect.

external/CLI11

@@ -0,0 +1 @@
+Subproject commit 4c7c8ddc45d2ef74584e5cd945f7a4d27c987748

external/CMakeLists.txt

@@ -1,4 +1,3 @@
-
 option(SUBMODULE_CHECK "Enables checking that vendored library submodules are up to date" ON)
 if(SUBMODULE_CHECK)
   find_package(Git)
@@ -26,7 +25,6 @@ if(SUBMODULE_CHECK)
 
     message(STATUS "Checking submodules")
     check_submodule(nlohmann)
-    check_submodule(cxxopts)
     check_submodule(ghc-filesystem)
     check_submodule(oxen-logging fmt spdlog)
     check_submodule(pybind11)
@@ -36,6 +34,7 @@ if(SUBMODULE_CHECK)
     check_submodule(uvw)
     check_submodule(cpr)
     check_submodule(ngtcp2)
+    check_submodule(CLI11)
   endif()
 endif()
 
@@ -78,7 +77,7 @@ if(WITH_HIVE)
   add_subdirectory(pybind11 EXCLUDE_FROM_ALL)
 endif()
 
-add_subdirectory(cxxopts EXCLUDE_FROM_ALL)
+system_or_submodule(CLI11 CLI11 CLI11>=2.2.0 CLI11)
 
 if(WITH_PEERSTATS)
   add_library(sqlite_orm INTERFACE)
@@ -140,3 +139,27 @@ if(WITH_BOOTSTRAP)
   endif()
 
 endif()
+
+
+# libcrypt defaults, only on with macos and non static linux
+set(default_libcrypt OFF)
+
+if(LINUX AND NOT STATIC_LINK)
+  set(default_libcrypt ON)
+endif()
+if(MACOS)
+  set(default_libcrypt ON)
+endif()
+
+option(WITH_LIBCRYPT "enable fast password hash with libcrypt" ${default_libcrypt})
+
+add_library(lokinet-libcrypt INTERFACE)
+if(WITH_LIBCRYPT)
+  pkg_check_modules(LIBCRYPT libcrypt IMPORTED_TARGET REQUIRED)
+  add_definitions(-DHAVE_CRYPT)
+  target_link_libraries(lokinet-libcrypt INTERFACE PkgConfig::LIBCRYPT)
+  message(STATUS "using libcrypt ${LIBCRYPT_VERSION}")
+else()
+  # TODO static build lib crypt?
+  message(STATUS "not building with libcrypt")
+endif()

external/cxxopts

@@ -1 +0,0 @@
-Subproject commit c74846a891b3cc3bfa992d588b1295f528d43039

external/oxen-logging

@@ -1 +1 @@
-Subproject commit 9f2323a2db5fc54fe8394892769eff859967f735
+Subproject commit b6e70ad3aa3b2d718f8607599864cd50a951166a

external/oxen-mq

@@ -1 +1 @@
-Subproject commit ac6ef82ff6fd20437b7d073466dbef82a95a2173
+Subproject commit 4f6dc35ea13722a5f9dcd0c3d65b6b7ac3d0f0c5

external/readme.md

@@ -1,7 +1,7 @@
 directory for git submodules
 
+* CLI11: cli argument parser
 * cpr: curl for people, used by lokinet-bootstrap toolchain (to be removed)
-* cxxopts: cli argument parser (to be removed)
 * ghc-filesystem: `std::filesystem` shim lib for older platforms (like macos)
 * ngtcp2: quic implementation
 * nlohmann: json parser
@@ -10,4 +10,4 @@ directory for git submodules
 * oxen-mq: zmq wrapper library for threadpool and rpc
 * pybind11: for pybind modules
 * sqlite_orm: for peer stats db
-* uvw: libuv header only library for main event loop
+* uvw: libuv header only library for main event loop

include/llarp.hpp

@@ -43,7 +43,6 @@ namespace llarp
     std::shared_ptr<AbstractRouter> router = nullptr;
     std::shared_ptr<EventLoop> loop = nullptr;
     std::shared_ptr<NodeDB> nodedb = nullptr;
-    std::string nodedb_dir;
 
     Context();
     virtual ~Context() = default;

llarp/CMakeLists.txt

@@ -1,5 +1,12 @@
 include(Version)
 
+target_sources(lokinet-cryptography PRIVATE
+  crypto/crypto_libsodium.cpp
+  crypto/crypto.cpp
+  crypto/encrypted_frame.cpp
+  crypto/types.cpp
+)
+
 add_library(lokinet-util
   STATIC
   ${CMAKE_CURRENT_BINARY_DIR}/constants/version.cpp
@@ -15,19 +22,9 @@ add_library(lokinet-util
   util/thread/threading.cpp
   util/time.cpp)
 
-
 add_dependencies(lokinet-util genversion)
 
-target_include_directories(lokinet-util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR})
-
-target_link_libraries(lokinet-util PUBLIC
-  lokinet-cryptography
-  nlohmann_json::nlohmann_json
-  filesystem
-  oxenc::oxenc
-  oxen::logging
-)
-
+# lokinet-platform holds all platform specific code
 add_library(lokinet-platform
   STATIC
   # for networking
@@ -45,9 +42,6 @@ add_library(lokinet-platform
   vpn/platform.cpp
 )
 
-target_link_libraries(lokinet-platform PUBLIC lokinet-cryptography lokinet-util Threads::Threads base_libs uvw)
-target_link_libraries(lokinet-platform PRIVATE oxenmq::oxenmq)
-
 if (ANDROID)
   target_sources(lokinet-platform PRIVATE android/ifaddrs.c util/nop_service_manager.cpp)
 endif()
@@ -66,36 +60,26 @@ if (WIN32)
     net/win32.cpp
     vpn/win32.cpp
     win32/service_manager.cpp
-    win32/exec.cpp)
-  add_library(lokinet-win32 STATIC
+    win32/exec.cpp
     win32/dll.cpp
-    win32/exception.cpp)
-  add_library(lokinet-wintun STATIC
-    win32/wintun.cpp)
-  add_library(lokinet-windivert STATIC
+    win32/exception.cpp
+    win32/wintun.cpp
     win32/windivert.cpp)
-    
-  # wintun and windivert are privated linked by lokinet-platform
-  # this is so their details do not leak out to deps of lokinet-platform
-  # wintun and windivert still need things from lokinet-platform
-  target_compile_options(lokinet-wintun PUBLIC -I${CMAKE_BINARY_DIR}/wintun/include/)
-  target_compile_options(lokinet-windivert PUBLIC -I${CMAKE_BINARY_DIR}/WinDivert-${WINDIVERT_VERSION}/include/)
-  target_include_directories(lokinet-windivert PUBLIC ${PROJECT_SOURCE_DIR})
-  target_link_libraries(lokinet-wintun PUBLIC lokinet-platform lokinet-util lokinet-config)
-  target_link_libraries(lokinet-win32 PUBLIC lokinet-util)
-  target_link_libraries(lokinet-windivert PUBLIC oxen-logging)
-  target_link_libraries(lokinet-windivert PRIVATE lokinet-win32)
-  target_link_libraries(lokinet-platform PRIVATE lokinet-win32 lokinet-wintun lokinet-windivert)
+  target_include_directories(lokinet-platform PRIVATE ${CMAKE_BINARY_DIR}/wintun/include/ ${CMAKE_BINARY_DIR}/WinDivert-${WINDIVERT_VERSION}/include/)
 else()
   target_sources(lokinet-platform PRIVATE
     net/posix.cpp)
 endif()
 
-
-if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
-  target_include_directories(lokinet-platform SYSTEM PUBLIC /usr/local/include)
+if(APPLE)
+  add_subdirectory(apple)
+  target_sources(lokinet-platform PRIVATE util/nop_service_manager.cpp)
 endif()
 
+# lokinet-dns is the dns parsing and hooking library that we use to
+# parse modify and reconstitute dns wire proto, dns queries and RR
+# should have no concept of dns caching, this is left as an implementation
+# detail of dns resolvers (LATER: make separate lib for dns resolvers)
 add_library(lokinet-dns
   STATIC
   dns/message.cpp
@@ -107,13 +91,50 @@ add_library(lokinet-dns
   dns/server.cpp
   dns/srv_data.cpp)
 
+# platform specific bits and bobs for setting dns
+add_library(lokinet-dns-platform INTERFACE)
 if(WITH_SYSTEMD)
-  target_sources(lokinet-dns PRIVATE dns/nm_platform.cpp dns/sd_platform.cpp)
+  add_library(lokinet-dns-systemd STATIC dns/nm_platform.cpp dns/sd_platform.cpp)
+  target_link_libraries(lokinet-dns-platform INTERFACE lokinet-dns-systemd)
 endif()
 
-target_link_libraries(lokinet-dns PUBLIC lokinet-platform uvw)
-target_link_libraries(lokinet-dns PRIVATE libunbound lokinet-config)
+# lokinet-nodedb holds all types and logic for storing parsing and constructing
+# nodedb data published to the network and versions of it stored locally
+add_library(lokinet-nodedb
+  STATIC
+  bootstrap.cpp
+  net/address_info.cpp
+  net/exit_info.cpp
+  net/traffic_policy.cpp
+  nodedb.cpp
+  pow.cpp
+  profiling.cpp
+  router_contact.cpp
+  router_id.cpp
+  router_version.cpp
+)
+
+set(BOOTSTRAP_FALLBACKS)
+foreach(bs IN ITEMS MAINNET TESTNET)
+  if(BOOTSTRAP_FALLBACK_${bs})
+    message(STATUS "Building with ${bs} fallback boostrap path \"${BOOTSTRAP_FALLBACK_${bs}}\"")
+    file(READ "${BOOTSTRAP_FALLBACK_${bs}}" bs_data HEX)
+    if(bs STREQUAL TESTNET)
+      set(network "gamma")
+    elseif(bs STREQUAL MAINNET)
+      set(network "lokinet")
+    else()
+      string(TOLOWER "${bs}" network)
+    endif()
+    string(REGEX REPLACE "([0-9a-f][0-9a-f])" "\\\\x\\1" bs_data "${bs_data}")
+    set(BOOTSTRAP_FALLBACKS "${BOOTSTRAP_FALLBACKS}{\"${network}\"s, \"${bs_data}\"sv},\n")
+  endif()
+endforeach()
+configure_file("bootstrap-fallbacks.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/bootstrap-fallbacks.cpp" @ONLY)
+target_sources(lokinet-nodedb PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/bootstrap-fallbacks.cpp")
+
 
+# lokinet-config is for all configuration types and parsers
 add_library(lokinet-config
   STATIC
   config/config.cpp
@@ -121,18 +142,15 @@ add_library(lokinet-config
   config/ini.cpp
   config/key_manager.cpp)
 
-target_link_libraries(lokinet-config PUBLIC lokinet-dns lokinet-platform oxenmq::oxenmq)
-
-add_library(lokinet-amalgum
+# lokinet-consensus is for deriving and tracking network consensus state for both service nodes and clients
+add_library(lokinet-consensus
   STATIC
   consensus/reachability_testing.cpp
+)
 
-  bootstrap.cpp
-  context.cpp
-  crypto/crypto_libsodium.cpp
-  crypto/crypto.cpp
-  crypto/encrypted_frame.cpp
-  crypto/types.cpp
+# lokinet-dht holds all logic related to interacting with and participating in the DHT hashring
+add_library(lokinet-dht
+  STATIC
   dht/context.cpp
   dht/dht.cpp
   dht/explorenetworkjob.cpp
@@ -151,44 +169,56 @@ add_library(lokinet-amalgum
   dht/recursiverouterlookup.cpp
   dht/serviceaddresslookup.cpp
   dht/taglookup.cpp
+)
 
-  endpoint_base.cpp
+# lokinet-layer-flow is the flow layer which sits atop the routing layer which manages
+# flows between lokinet snapp endpoints be they .loki or .snode
+add_library(lokinet-layer-flow
+  STATIC
+  layers/flow/stub.cpp # todo: remove me
+)
 
-  exit/context.cpp
-  exit/endpoint.cpp
-  exit/exit_messages.cpp
-  exit/policy.cpp
-  exit/session.cpp
-  handlers/exit.cpp
-  handlers/tun.cpp
+
+# lokinet-layer-onion is the "dumb" onion routing layer with builds manages and does i/o
+# with onion paths. onion paths anonymize routing layer pdu.
+add_library(lokinet-layer-onion
+  STATIC
+  path/ihophandler.cpp
+  path/path_context.cpp
+  path/path.cpp
+  path/pathbuilder.cpp
+  path/pathset.cpp
+  path/transit_hop.cpp
+  messages/relay.cpp
+  messages/relay_commit.cpp
+  messages/relay_status.cpp
+)
+
+# lokinet-layer-wire is a layer 1 analog which splits up
+# layer 2 frames into layer 1 symbols which in the case of iwp are encrypted udp/ip packets
+add_library(lokinet-layer-wire
+  STATIC
   iwp/iwp.cpp
   iwp/linklayer.cpp
   iwp/message_buffer.cpp
   iwp/session.cpp
+)
+
+# lokinet-layer-link is for our layer 2 analog which splits up layer 2 frames into
+# a series of layer 1 symbols which are then transmitted between lokinet instances
+add_library(lokinet-layer-link
+  STATIC
   link/link_manager.cpp
   link/session.cpp
   link/server.cpp
   messages/dht_immediate.cpp
   messages/link_intro.cpp
   messages/link_message_parser.cpp
-  messages/relay.cpp
-  messages/relay_commit.cpp
-  messages/relay_status.cpp
-  net/address_info.cpp
-  net/exit_info.cpp
-  net/traffic_policy.cpp
-  nodedb.cpp
-  path/ihophandler.cpp
-  path/path_context.cpp
-  path/path.cpp
-  path/pathbuilder.cpp
-  path/pathset.cpp
-  path/transit_hop.cpp
-  peerstats/peer_db.cpp
-  peerstats/types.cpp
-  pow.cpp
-  profiling.cpp
-  
+)
+
+# lokinet-plainquic is for holding the tunneled plainquic code, not quic wire protocol code
+add_library(lokinet-plainquic
+  STATIC
   quic/address.cpp
   quic/client.cpp
   quic/connection.cpp
@@ -197,27 +227,64 @@ add_library(lokinet-amalgum
   quic/server.cpp
   quic/stream.cpp
   quic/tunnel.cpp
+)
 
-  router_contact.cpp
-  router_id.cpp
-  router_version.cpp
-  service/name.cpp
+# lokinet-context holds the contextualized god objects for a lokinet instance
+# it is what any main function would link to in practice but it is hidden behind an interface library (lokinet-amalgum)
+add_library(lokinet-context
+  STATIC
+  context.cpp
+  link/link_manager.cpp
   router/outbound_message_handler.cpp
   router/outbound_session_maker.cpp
   router/rc_lookup_handler.cpp
   router/rc_gossiper.cpp
   router/router.cpp
   router/route_poker.cpp
+)
+
+# lokinet-rpc holds all rpc related compilation units
+add_library(lokinet-rpc
+  STATIC
+  rpc/json_binary_proxy.cpp
+  rpc/json_conversions.cpp
+  rpc/lokid_rpc_client.cpp
+  rpc/rpc_request_parser.cpp
+  rpc/rpc_server.cpp
+  rpc/endpoint_rpc.cpp
+)
+
+# optional peer stats library
+add_library(lokinet-peerstats
+  STATIC
+  peerstats/peer_db.cpp
+  peerstats/types.cpp
+)
 
+# lokinet-layer-routing holds logic related to the routing layer
+# routing layer is anonymized over the onion layer
+add_library(lokinet-layer-routing
+  STATIC
   routing/dht_message.cpp
   routing/message_parser.cpp
   routing/path_confirm_message.cpp
   routing/path_latency_message.cpp
   routing/path_transfer_message.cpp
   routing/transfer_traffic_message.cpp
-  rpc/lokid_rpc_client.cpp
-  rpc/rpc_server.cpp
-  rpc/endpoint_rpc.cpp
+)
+
+# kitchen sink to be removed after refactor
+add_library(lokinet-service-deprecated-kitchensink
+  STATIC
+  endpoint_base.cpp
+  exit/context.cpp
+  exit/endpoint.cpp
+  exit/exit_messages.cpp
+  exit/policy.cpp
+  exit/session.cpp
+  handlers/exit.cpp
+  handlers/tun.cpp
+  service/name.cpp
   service/address.cpp
   service/async_key_exchange.cpp
   service/auth.cpp
@@ -242,65 +309,216 @@ add_library(lokinet-amalgum
   service/tag.cpp
 )
 
-set(BOOTSTRAP_FALLBACKS)
-foreach(bs IN ITEMS MAINNET TESTNET)
-  if(BOOTSTRAP_FALLBACK_${bs})
-    message(STATUS "Building with ${bs} fallback boostrap path \"${BOOTSTRAP_FALLBACK_${bs}}\"")
-    file(READ "${BOOTSTRAP_FALLBACK_${bs}}" bs_data HEX)
-    if(bs STREQUAL TESTNET)
-      set(network "gamma")
-    elseif(bs STREQUAL MAINNET)
-      set(network "lokinet")
-    else()
-      string(TOLOWER "${bs}" network)
-    endif()
-    string(REGEX REPLACE "([0-9a-f][0-9a-f])" "\\\\x\\1" bs_data "${bs_data}")
-    set(BOOTSTRAP_FALLBACKS "${BOOTSTRAP_FALLBACKS}{\"${network}\"s, \"${bs_data}\"sv},\n")
-  endif()
-endforeach()
-configure_file("bootstrap-fallbacks.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/bootstrap-fallbacks.cpp" @ONLY)
-target_sources(lokinet-amalgum PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/bootstrap-fallbacks.cpp")
+add_library(lokinet-layer-platform
+  STATIC
+  layers/platform/stub.cpp # todo: remove me
+)
 
-if(WITH_PEERSTATS_BACKEND)
-  target_compile_definitions(lokinet-amalgum PRIVATE -DLOKINET_PEERSTATS_BACKEND)
-  target_link_libraries(lokinet-amalgum PUBLIC sqlite_orm)
-endif()
 
+# interal tooling for pybind
+add_library(lokinet-tooling INTERFACE)
 if(WITH_HIVE)
-  target_sources(lokinet-amalgum PRIVATE
+  add_library(lokinet-hive-tooling
+    STATIC
     tooling/router_hive.cpp
     tooling/hive_router.cpp
     tooling/hive_context.cpp
   )
+  target_link_libraries(lokinet-tooling INTERFACE lokinet-hive-tooling)
 endif()
 
-# TODO: make libunbound hidden behind a feature flag like sqlite for embedded lokinet
-target_link_libraries(lokinet-amalgum PRIVATE libunbound)
 
-target_link_libraries(lokinet-amalgum PUBLIC
-    cxxopts
-    oxenc::oxenc
+# interface library for setting commone includes, linkage and flags.
+add_library(lokinet-base INTERFACE)
+target_include_directories(lokinet-base
+  INTERFACE ${PROJECT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/include
+)
+target_link_libraries(lokinet-base INTERFACE oxen::logging lokinet-cryptography)
+
+if(WITH_PEERSTATS)
+  target_compile_definitions(lokinet-base INTERFACE -DLOKINET_PEERSTATS_BACKEND)
+  target_link_libraries(lokinet-base INTERFACE sqlite_orm)
+endif()
+
+# interface libraries for internal linkage
+add_library(lokinet-layers INTERFACE)
+add_library(lokinet-amalgum INTERFACE)
+
+
+# helper function to link a library to lokinet-base, enable lto, add to lokinet-amalgum and then link to other libs
+function(lokinet_link_lib libname)
+  message(DEBUG "created target: ${libname}")
+  enable_lto(${libname})
+  target_link_libraries(${libname} PUBLIC lokinet-base ${ARGN})
+  target_link_libraries(lokinet-amalgum INTERFACE ${libname})
+endfunction()
+
+# internal public linkages of components
+lokinet_link_lib(lokinet-util)
+lokinet_link_lib(lokinet-cryptography lokinet-libcrypt lokinet-util)
+lokinet_link_lib(lokinet-peerstats lokinet-context)
+lokinet_link_lib(lokinet-consensus lokinet-context)
+lokinet_link_lib(lokinet-layer-link lokinet-peerstats)
+
+if(TARGET lokinet-hive-tooling)
+  lokinet_link_lib(lokinet-hive-tooling lokinet-context)
+endif()
+
+if(TARGET lokinet-dns-systemd)
+  lokinet_link_lib(lokinet-dns-systemd
+    lokinet-dns
     lokinet-platform
-    lokinet-config
+  )
+endif()
+
+lokinet_link_lib(lokinet-platform lokinet-util)
+
+lokinet_link_lib(lokinet-config
+  lokinet-util
+  lokinet-nodedb
+  lokinet-dns
+  lokinet-platform
+)
+
+lokinet_link_lib(lokinet-context
+  lokinet-config
+  lokinet-platform
+  lokinet-peerstats
+  lokinet-layers
+  lokinet-consensus
+  lokinet-rpc
+)
+
+lokinet_link_lib(lokinet-dht
+  lokinet-util
+  lokinet-nodedb
+)
+
+lokinet_link_lib(lokinet-plainquic
+  lokinet-platform
+  lokinet-config
+)
+
+lokinet_link_lib(lokinet-dns
+  lokinet-platform
+  lokinet-dns-platform
+  lokinet-config
+)
+
+lokinet_link_lib(lokinet-nodedb
+  lokinet-util
+  lokinet-platform
+)
+
+lokinet_link_lib(lokinet-util
+  lokinet-nodedb
+  lokinet-platform
+)
+
+lokinet_link_lib(lokinet-rpc
+  lokinet-context
+  lokinet-peerstats
+  lokinet-util
+)
+
+# inter lokinet-layer public/private linkage.
+# when linking each layer, we consider the layer directly below private linkage and the layer above public linkage.
+# this lets us hide functionality of layers below us when depended on by another component.
+#
+# from highest to lowest layer, the above layers are stacked as follows:
+#
+# platform (what lokinet snapps interact with, be it l3 os interaction or embedded lokinet)
+# flow     (how we want to route and stripe over our onion routing)
+# routing  (what we are onion routing)
+# onion    (how the onion routing happens)
+# link     (what we want to send over the wire and to where)
+# wire     (what is actually sent over the wire)
+#
+function(link_lokinet_layers)
+  set(lib ${ARGV0})
+  if(${ARGC} GREATER 1)
+    lokinet_link_lib(${ARGV1} ${lib})
+    list(REMOVE_AT ARGV 1)
+    target_link_libraries(${lib} PRIVATE ${ARGV1})
+    # recursion :D 
+    link_lokinet_layers(${ARGV})
+  else()
+    lokinet_link_lib(${lib})
+  endif()
+endfunction()
+
+link_lokinet_layers(
+  lokinet-layer-platform
+  lokinet-layer-flow
+  lokinet-layer-routing
+  lokinet-layer-onion
+  lokinet-layer-link
+  lokinet-layer-wire
+)
+
+# set me to OFF to disable old codepath
+set(use_old_impl ON)
+if(use_old_impl)
+  # flow layer deprecated-kitchensink (remove me after refactor)
+  lokinet_link_lib(lokinet-service-deprecated-kitchensink
     lokinet-dns
-    lokinet-util
-    lokinet-cryptography
-    ngtcp2_static
-    oxenmq::oxenmq)
-
-enable_lto(lokinet-util lokinet-platform lokinet-dns lokinet-config lokinet-amalgum)
-
-pkg_check_modules(CRYPT libcrypt IMPORTED_TARGET)
-if(CRYPT_FOUND AND NOT CMAKE_CROSSCOMPILING)
-  add_definitions(-DHAVE_CRYPT)
-  add_library(libcrypt INTERFACE)
-  target_link_libraries(libcrypt INTERFACE PkgConfig::CRYPT)
-  target_link_libraries(lokinet-amalgum PRIVATE libcrypt)
-  message(STATUS "using libcrypt ${CRYPT_VERSION}")
+    lokinet-nodedb
+    lokinet-context
+    lokinet-plainquic
+    lokinet-layer-routing
+    lokinet-layer-onion
+    lokinet-dht
+    lokinet-platform
+    lokinet-rpc
+  )
+  target_link_libraries(lokinet-layers INTERFACE lokinet-service-deprecated-kitchensink)
 endif()
 
+target_link_libraries(lokinet-layers INTERFACE
+  lokinet-layer-platform
+  lokinet-layer-flow
+  lokinet-layer-routing
+  lokinet-layer-onion
+  lokinet-layer-link
+  lokinet-layer-wire
+)
+
+
+# per component external deps
+
+target_link_libraries(lokinet-config PUBLIC oxenmq::oxenmq)
+target_link_libraries(lokinet-platform PUBLIC oxenmq::oxenmq)
+target_link_libraries(lokinet-dns PUBLIC libunbound)
+
+target_link_libraries(lokinet-cryptography PUBLIC
+  oxenc::oxenc
+  sodium
+)
+
+target_link_libraries(lokinet-context PUBLIC
+  CLI11
+  oxenmq::oxenmq
+  uvw
+)
+
+target_link_libraries(lokinet-platform PUBLIC
+  Threads::Threads
+  base_libs
+  uvw
+)
+
+target_link_libraries(lokinet-util PUBLIC
+  nlohmann_json::nlohmann_json
+  filesystem
+  oxenc::oxenc
+)
+
+target_link_libraries(lokinet-plainquic PUBLIC
+  ngtcp2_static
+  uvw
+)
 
-if(BUILD_LIBLOKINET)
+if(WITH_EMBEDDED_LOKINET)
   include(GNUInstallDirs)
   add_library(lokinet-shared SHARED lokinet_shared.cpp)
   target_link_libraries(lokinet-shared PUBLIC lokinet-amalgum)
@@ -316,11 +534,5 @@ if(BUILD_LIBLOKINET)
   endif()
 endif()
 
-if(APPLE)
-  add_subdirectory(apple)
-  target_sources(lokinet-platform PRIVATE util/nop_service_manager.cpp)
-endif()
-
 file(GLOB_RECURSE docs_SRC */*.hpp *.hpp)
-
 set(DOCS_SRC ${docs_SRC} PARENT_SCOPE)

llarp/config/config.cpp

@@ -1,8 +1,7 @@
-#include <chrono>
 #include "config.hpp"
-
-#include "config/definition.hpp"
+#include "definition.hpp"
 #include "ini.hpp"
+
 #include <llarp/constants/files.hpp>
 #include <llarp/constants/platform.hpp>
 #include <llarp/constants/version.hpp>
@@ -18,6 +17,7 @@
 
 #include <llarp/service/name.hpp>
 
+#include <chrono>
 #include <cstdlib>
 #include <ios>
 #include <iostream>
@@ -609,7 +609,7 @@ namespace llarp
         "ifname",
         Comment{
             "Interface name for lokinet traffic. If unset lokinet will look for a free name",
-            "matching 'lokinetN', starting at N=0 (e.g. lokinet0, lokinet1, ...).",
+            "matching 'lokitunN', starting at N=0 (e.g. lokitun0, lokitun1, ...).",
         },
         AssignmentAcceptor(m_ifname));
 
@@ -1114,7 +1114,12 @@ namespace llarp
   void
   ApiConfig::defineConfigOptions(ConfigDefinition& conf, const ConfigGenParameters& params)
   {
-    constexpr Default DefaultRPCBindAddr{"tcp://127.0.0.1:1190"};
+    constexpr std::array DefaultRPCBind{
+        Default{"tcp://127.0.0.1:1190"},
+#ifndef _WIN32
+        Default{"ipc://rpc.sock"},
+#endif
+    };
 
     conf.defineOption<bool>(
         "api",
@@ -1128,20 +1133,22 @@ namespace llarp
     conf.defineOption<std::string>(
         "api",
         "bind",
-        DefaultRPCBindAddr,
-        [this](std::string arg) {
-          if (arg.empty())
+        DefaultRPCBind,
+        MultiValue,
+        [this, first = true](std::string arg) mutable {
+          if (first)
           {
-            arg = DefaultRPCBindAddr.val;
+            m_rpcBindAddresses.clear();
+            first = false;
           }
           if (arg.find("://") == std::string::npos)
           {
             arg = "tcp://" + arg;
           }
-          m_rpcBindAddr = std::move(arg);
+          m_rpcBindAddresses.emplace_back(arg);
         },
         Comment{
-            "IP address and port to bind to.",
+            "IP addresses and ports to bind to.",
             "Recommend localhost-only for security purposes.",
         });
 

llarp/config/config.hpp

@@ -1,14 +1,15 @@
 #pragma once
+#include "ini.hpp"
+#include "definition.hpp"
 
 #include <chrono>
+
 #include <llarp/bootstrap.hpp>
 #include <llarp/crypto/types.hpp>
 #include <llarp/router_contact.hpp>
 #include <llarp/util/fs.hpp>
 #include <llarp/util/str.hpp>
 #include <llarp/util/logging.hpp>
-#include "ini.hpp"
-#include "definition.hpp"
 #include <llarp/constants/files.hpp>
 #include <llarp/net/ip_address.hpp>
 #include <llarp/net/net_int.hpp>
@@ -16,7 +17,6 @@
 #include <llarp/service/address.hpp>
 #include <llarp/service/auth.hpp>
 #include <llarp/dns/srv_data.hpp>
-
 #include <llarp/router_contact.hpp>
 
 #include <cstdlib>
@@ -189,7 +189,7 @@ namespace llarp
   struct ApiConfig
   {
     bool m_enableRPCServer = false;
-    std::string m_rpcBindAddr;
+    std::vector<oxenmq::address> m_rpcBindAddresses;
 
     void
     defineConfigOptions(ConfigDefinition& conf, const ConfigGenParameters& params);

llarp/config/definition.hpp

@@ -105,11 +105,9 @@ namespace llarp
     constexpr bool is_default_array<U&> = is_default_array<remove_cvref_t<U>>;
 
     template <typename T, typename Option>
-    constexpr bool is_option =
-        std::is_base_of_v<
-            option_flag,
-            remove_cvref_t<
-                Option>> or std::is_same_v<Comment, Option> or is_default<Option> or is_default_array<Option> or std::is_invocable_v<remove_cvref_t<Option>, T>;
+    constexpr bool is_option = std::is_base_of_v<option_flag, remove_cvref_t<Option>>
+        or std::is_same_v<Comment, Option> or is_default<Option> or is_default_array<Option>
+        or std::is_invocable_v<remove_cvref_t<Option>, T>;
   }  // namespace config
 
   /// A base class for specifying config options and their constraints. The basic to/from string
@@ -298,7 +296,12 @@ namespace llarp
         std::vector<std::string> def_strs;
         def_strs.reserve(defaultValues.size());
         for (const auto& v : defaultValues)
-          def_strs.push_back(fmt::format("{}", v));
+        {
+          if constexpr (std::is_same_v<bool, T>)
+            def_strs.push_back(fmt::format("{}", (bool)v));
+          else
+            def_strs.push_back(fmt::format("{}", v));
+        }
         return def_strs;
       }
     }
@@ -308,8 +311,7 @@ namespace llarp
     {
       if (not multiValued and parsedValues.size() > 0)
       {
-        throw std::invalid_argument{
-            fmt::format("duplicate value for {}, previous value: {}", name, parsedValues[0])};
+        throw std::invalid_argument{fmt::format("duplicate value for {}", name)};
       }
 
       parsedValues.emplace_back(fromString(input));
@@ -329,8 +331,7 @@ namespace llarp
         iss >> t;
         if (iss.fail())
           throw std::invalid_argument{fmt::format("{} is not a valid {}", input, typeid(T).name())};
-        else
-          return t;
+        return t;
       }
     }
 
@@ -342,7 +343,12 @@ namespace llarp
       std::vector<std::string> result;
       result.reserve(parsedValues.size());
       for (const auto& v : parsedValues)
-        result.push_back(fmt::format("{}", v));
+      {
+        if constexpr (std::is_same_v<bool, T>)
+          result.push_back(fmt::format("{}", (bool)v));
+        else
+          result.push_back(fmt::format("{}", v));
+      }
       return result;
     }
 

llarp/config/ini.cpp

@@ -9,6 +9,7 @@
 #include <list>
 #include <iostream>
 #include <cassert>
+#include <stdexcept>
 
 namespace llarp
 {
@@ -30,6 +31,14 @@ namespace llarp
     return Parse();
   }
 
+  bool
+  ConfigParser::LoadNewFromStr(std::string_view str)
+  {
+    m_Data.resize(str.size());
+    std::copy(str.begin(), str.end(), m_Data.begin());
+    return ParseAll();
+  }
+
   bool
   ConfigParser::LoadFromStr(std::string_view str)
   {
@@ -52,6 +61,78 @@ namespace llarp
     return std::isspace(static_cast<unsigned char>(ch)) != 0;
   }
 
+  /// Differs from Parse() as ParseAll() does NOT skip comments
+  /// ParseAll() is only used by RPC endpoint 'config' for
+  /// reading new .ini files from string and writing them
+  bool
+  ConfigParser::ParseAll()
+  {
+    std::list<std::string_view> lines;
+    {
+      auto itr = m_Data.begin();
+      // split into lines
+      while (itr != m_Data.end())
+      {
+        auto beg = itr;
+        while (itr != m_Data.end() && *itr != '\n' && *itr != '\r')
+          ++itr;
+        lines.emplace_back(std::addressof(*beg), std::distance(beg, itr));
+        if (itr == m_Data.end())
+          break;
+        ++itr;
+      }
+    }
+
+    std::string_view sectName;
+    size_t lineno = 0;
+    for (auto line : lines)
+    {
+      lineno++;
+      // Trim whitespace
+      while (!line.empty() && whitespace(line.front()))
+        line.remove_prefix(1);
+      while (!line.empty() && whitespace(line.back()))
+        line.remove_suffix(1);
+
+      // Skip blank lines but NOT comments
+      if (line.empty())
+        continue;
+
+      if (line.front() == '[' && line.back() == ']')
+      {
+        // section header
+        line.remove_prefix(1);
+        line.remove_suffix(1);
+        sectName = line;
+      }
+      else if (auto kvDelim = line.find('='); kvDelim != std::string_view::npos)
+      {
+        // key value pair
+        std::string_view k = line.substr(0, kvDelim);
+        std::string_view v = line.substr(kvDelim + 1);
+        // Trim inner whitespace
+        while (!k.empty() && whitespace(k.back()))
+          k.remove_suffix(1);
+        while (!v.empty() && whitespace(v.front()))
+          v.remove_prefix(1);
+
+        if (k.empty())
+        {
+          throw std::runtime_error(
+              fmt::format("{} invalid line ({}): '{}'", m_FileName, lineno, line));
+        }
+        LogDebug(m_FileName, ": [", sectName, "]:", k, "=", v);
+        m_Config[std::string{sectName}].emplace(k, v);
+      }
+      else  // malformed?
+      {
+        throw std::runtime_error(
+            fmt::format("{} invalid line ({}): '{}'", m_FileName, lineno, line));
+      }
+    }
+    return true;
+  }
+
   bool
   ConfigParser::Parse()
   {
@@ -82,7 +163,7 @@ namespace llarp
       while (!line.empty() && whitespace(line.back()))
         line.remove_suffix(1);
 
-      // Skip blank lines and comments
+      // Skip blank lines
       if (line.empty() or line.front() == ';' or line.front() == '#')
         continue;
 
@@ -106,16 +187,16 @@ namespace llarp
 
         if (k.empty())
         {
-          LogError(m_FileName, " invalid line (", lineno, "): '", line, "'");
-          return false;
+          throw std::runtime_error(
+              fmt::format("{} invalid line ({}): '{}'", m_FileName, lineno, line));
         }
         LogDebug(m_FileName, ": [", sectName, "]:", k, "=", v);
         m_Config[std::string{sectName}].emplace(k, v);
       }
       else  // malformed?
       {
-        LogError(m_FileName, " invalid line (", lineno, "): '", line, "'");
-        return false;
+        throw std::runtime_error(
+            fmt::format("{} invalid line ({}): '{}'", m_FileName, lineno, line));
       }
     }
     return true;
@@ -168,4 +249,31 @@ namespace llarp
     m_Overrides.clear();
   }
 
+  void
+  ConfigParser::SaveNew() const
+  {
+    if (not m_Overrides.empty())
+    {
+      throw std::invalid_argument("Override specified when attempting new .ini save");
+    }
+    if (m_Config.empty())
+    {
+      throw std::invalid_argument("New config not loaded when attempting new .ini save");
+    }
+    if (m_FileName.empty())
+    {
+      throw std::invalid_argument("New config cannot be saved with filepath specified");
+    }
+
+    std::ofstream ofs(m_FileName);
+    for (const auto& [section, values] : m_Config)
+    {
+      ofs << std::endl << "[" << section << "]" << std::endl;
+      for (const auto& [key, value] : values)
+      {
+        ofs << key << "=" << value << std::endl;
+      }
+    }
+  }
+
 }  // namespace llarp

llarp/config/ini.hpp

@@ -24,6 +24,12 @@ namespace llarp
     bool
     LoadFile(const fs::path& fname);
 
+    /// load new .ini file from string (calls ParseAll() rather than Parse())
+    /// return true on success
+    /// return false on error
+    bool
+    LoadNewFromStr(std::string_view str);
+
     /// load from string
     /// return true on success
     /// return false on error
@@ -47,6 +53,10 @@ namespace llarp
     void
     Save();
 
+    /// save new .ini config file to path
+    void
+    SaveNew() const;
+
     inline void
     Filename(fs::path f)
     {
@@ -54,6 +64,9 @@ namespace llarp
     };
 
    private:
+    bool
+    ParseAll();
+
     bool
     Parse();
 

llarp/constants/version.cpp.in

@@ -1,5 +1,5 @@
-#include <constants/version.hpp>
-#include <constants/proto.hpp>
+#include <llarp/constants/version.hpp>
+#include <llarp/constants/proto.hpp>
 
 namespace llarp
 {

llarp/context.cpp

@@ -14,7 +14,10 @@
 
 #include <llarp/util/service_manager.hpp>
 
-#include <cxxopts.hpp>
+#include <CLI/App.hpp>
+#include <CLI/Formatter.hpp>
+#include <CLI/Config.hpp>
+
 #include <csignal>
 #include <stdexcept>
 
@@ -22,10 +25,10 @@
 #include <pthread_np.h>
 #endif
 
-static auto logcat = llarp::log::Cat("llarp-context");
-
 namespace llarp
 {
+  static auto logcat = llarp::log::Cat("llarp-context");
+
   bool
   Context::CallSafe(std::function<void(void)> f)
   {
@@ -42,8 +45,6 @@ namespace llarp
       throw std::runtime_error("Config already exists");
 
     config = std::move(conf);
-
-    nodedb_dir = fs::path{config->router.m_dataDir / nodedb_dirname}.string();
   }
 
   bool
@@ -89,7 +90,7 @@ namespace llarp
   Context::makeNodeDB()
   {
     return std::make_shared<NodeDB>(
-        nodedb_dir, [r = router.get()](auto call) { r->QueueDiskIO(std::move(call)); });
+        nodedb_dirname, [r = router.get()](auto call) { r->QueueDiskIO(std::move(call)); });
   }
 
   std::shared_ptr<AbstractRouter>

llarp/dns/rr.cpp

@@ -1,6 +1,6 @@
 #include "rr.hpp"
 #include "dns.hpp"
-#include "util/formattable.hpp"
+#include <llarp/util/formattable.hpp>
 #include <llarp/util/mem.hpp>
 #include <llarp/util/logging.hpp>
 

llarp/dns/server.cpp

@@ -177,10 +177,7 @@ namespace llarp::dns
       void
       AddUpstreamResolver(const SockAddr& dns)
       {
-        std::string str = dns.hostString();
-
-        if (const auto port = dns.getPort(); port != 53)
-          fmt::format_to(std::back_inserter(str), "@{}", port);
+        std::string str = fmt::format("{}@{}", dns.hostString(false), dns.getPort());
 
         if (auto err = ub_ctx_set_fwd(m_ctx, str.c_str()))
         {

llarp/ev/libuv.cpp

@@ -75,8 +75,11 @@ namespace llarp::uv
     std::optional<SockAddr>
     LocalAddr() const override
     {
-      auto addr = handle->sock<uvw::IPv4>();
-      return SockAddr{addr.ip, huint16_t{static_cast<uint16_t>(addr.port)}};
+      if (auto addr = handle->sock<uvw::IPv4>(); not addr.ip.empty())
+        return SockAddr{addr.ip, huint16_t{static_cast<uint16_t>(addr.port)}};
+      if (auto addr = handle->sock<uvw::IPv6>(); not addr.ip.empty())
+        return SockAddr{addr.ip, huint16_t{static_cast<uint16_t>(addr.port)}};
+      return std::nullopt;
     }
 
     std::optional<int>

llarp/exit/session.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "exit_messages.hpp"
-#include "service/protocol_type.hpp"
+#include <llarp/service/protocol_type.hpp>
 #include <llarp/net/ip_packet.hpp>
 #include <llarp/path/pathbuilder.hpp>
 #include <llarp/routing/transfer_traffic_message.hpp>

llarp/handlers/exit.cpp

@@ -11,7 +11,7 @@
 #include <llarp/router/i_rc_lookup_handler.hpp>
 
 #include <cassert>
-#include "service/protocol_type.hpp"
+#include <llarp/service/protocol_type.hpp>
 
 namespace llarp
 {

llarp/messages/relay_commit.hpp

@@ -2,7 +2,7 @@
 
 #include <llarp/crypto/encrypted_frame.hpp>
 #include <llarp/crypto/types.hpp>
-#include "link_message.hpp"
+#include <llarp/messages/link_message.hpp>
 #include <llarp/path/path_types.hpp>
 #include <llarp/pow.hpp>
 

llarp/net/ip_range.hpp

@@ -8,6 +8,7 @@
 
 #include <list>
 #include <optional>
+#include <stdexcept>
 #include <string>
 
 namespace llarp
@@ -24,6 +25,12 @@ namespace llarp
         : addr{std::move(address)}, netmask_bits{std::move(netmask)}
     {}
 
+    explicit IPRange(std::string _range)
+    {
+      if (not FromString(_range))
+        throw std::invalid_argument{"IP string '{}' cannot be parsed as IP range"_format(_range)};
+    }
+
     static constexpr IPRange
     V4MappedRange()
     {
@@ -40,7 +47,8 @@ namespace llarp
     FromIPv4(net::ipv4addr_t addr, net::ipv4addr_t netmask)
     {
       return IPRange{
-          net::ExpandV4(ToHost(addr)), netmask_ipv6_bits(bits::count_bits(netmask) + 96)};
+          net::ExpandV4(llarp::net::ToHost(addr)),
+          netmask_ipv6_bits(bits::count_bits(netmask) + 96)};
     }
 
     /// return true if this iprange is in the IPv4 mapping range for containing ipv4 addresses
@@ -103,7 +111,7 @@ namespace llarp
     inline bool
     Contains(const net::ipaddr_t& ip) const
     {
-      return var::visit([this](auto&& ip) { return Contains(ToHost(ip)); }, ip);
+      return var::visit([this](auto&& ip) { return Contains(llarp::net::ToHost(ip)); }, ip);
     }
 
     /// get the highest address on this range

llarp/net/ip_range_map.hpp

@@ -2,6 +2,7 @@
 
 #include "ip_range.hpp"
 #include <llarp/util/status.hpp>
+#include <set>
 #include <vector>
 
 namespace llarp

llarp/net/sock_addr.cpp

@@ -290,25 +290,21 @@ namespace llarp
   }
 
   std::string
-  SockAddr::hostString() const
+  SockAddr::hostString(bool ipv6_brackets) const
   {
-    std::string str;
-    char buf[INET6_ADDRSTRLEN] = {0x0};
+    std::array<char, 128> buf{};
     if (isIPv4())
     {
-      // handle IPv4 mapped addrs
-      inet_ntop(AF_INET, &m_addr4.sin_addr.s_addr, buf, sizeof(buf));
-      str = buf;
+      // IPv4 mapped addrs
+      inet_ntop(AF_INET, &m_addr4.sin_addr.s_addr, buf.data(), buf.size());
+      return buf.data();
     }
-    else
-    {
-      inet_ntop(AF_INET6, &m_addr.sin6_addr.s6_addr, buf, sizeof(buf));
-      str.reserve(std::strlen(buf) + 2);
-      str.append("[");
-      str.append(buf);
-      str.append("]");
-    }
-    return str;
+
+    inet_ntop(AF_INET6, &m_addr.sin6_addr.s6_addr, buf.data(), buf.size());
+    if (not ipv6_brackets)
+      return buf.data();
+
+    return fmt::format("[{}]", buf.data());
   }
 
   bool

llarp/net/sock_addr.hpp

@@ -86,8 +86,10 @@ namespace llarp
     std::string
     ToString() const;
 
+    /// convert ip address to string; ipv6_brackets - if true or omitted we add [...] around the
+    /// IPv6 address, otherwise we return it bare.
     std::string
-    hostString() const;
+    hostString(bool ipv6_brackets = true) const;
 
     inline int
     Family() const

llarp/path/pathbuilder.cpp

@@ -1,10 +1,10 @@
 #include "pathbuilder.hpp"
+#include "path_context.hpp"
 
 #include <llarp/crypto/crypto.hpp>
 #include <llarp/messages/relay_commit.hpp>
 #include <llarp/nodedb.hpp>
-#include "path_context.hpp"
-#include "util/logging.hpp"
+#include <llarp/util/logging.hpp>
 #include <llarp/profiling.hpp>
 #include <llarp/router/abstractrouter.hpp>
 #include <llarp/router/i_rc_lookup_handler.hpp>

llarp/path/pathset.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "path_types.hpp"
-#include "service/protocol_type.hpp"
+#include <llarp/service/protocol_type.hpp>
 #include <llarp/router_id.hpp>
 #include <llarp/routing/message.hpp>
 #include <llarp/service/intro_set.hpp>

llarp/path/transit_hop.cpp

@@ -33,7 +33,9 @@ namespace llarp
     }
 
     TransitHop::TransitHop()
-        : m_UpstreamGather(transit_hop_queue_size), m_DownstreamGather(transit_hop_queue_size)
+        : IHopHandler{}
+        , m_UpstreamGather{transit_hop_queue_size}
+        , m_DownstreamGather{transit_hop_queue_size}
     {
       m_UpstreamGather.enable();
       m_DownstreamGather.enable();

llarp/path/transit_hop.hpp

@@ -1,8 +1,8 @@
 #pragma once
 
 #include <llarp/constants/path.hpp>
-#include "ihophandler.hpp"
-#include "path_types.hpp"
+#include <llarp/path/ihophandler.hpp>
+#include <llarp/path/path_types.hpp>
 #include <llarp/routing/handler.hpp>
 #include <llarp/router_id.hpp>
 #include <llarp/util/compare_ptr.hpp>

llarp/quic/connection.cpp

@@ -90,7 +90,7 @@ namespace llarp::quic
         void* user_data)
     {
       std::basic_string_view data{rawdata, rawdatalen};
-      LogTrace("Receiving crypto data @ level ", crypto_level, " ", buffer_printer{data});
+      log::trace(log_cat, "Receiving crypto data: {}", buffer_printer{data});
 
       auto& conn = *static_cast<Connection*>(user_data);
       switch (crypto_level)
@@ -101,6 +101,7 @@ namespace llarp::quic
           return FAIL;
 
         case NGTCP2_CRYPTO_LEVEL_INITIAL:
+          log::trace(log_cat, "Receiving initial crypto...");
           // "Initial" level means we are still handshaking; if we are server then we receive
           // the client's transport params (sent in client_initial, above) and blast ours
           // back.  If we are a client then getting here means we received a response from the
@@ -120,6 +121,7 @@ namespace llarp::quic
           break;
 
         case NGTCP2_CRYPTO_LEVEL_HANDSHAKE:
+          log::trace(log_cat, "Receiving handshake crypto...");
           if (!ngtcp2_conn_is_server(conn))
           {
             if (auto rv = conn.recv_transport_params(data); rv != 0)
@@ -144,13 +146,8 @@ namespace llarp::quic
           conn.complete_handshake();
           break;
 
-        case NGTCP2_CRYPTO_LEVEL_APPLICATION:
-          // if (!conn.init_tx_key())
-          //    return FAIL;
-          break;
-
         default:
-          LogWarn("Unhandled crypto_level ", crypto_level);
+          LogWarn("Unhandled crypto_level");
           return FAIL;
       }
       conn.io_ready();

llarp/quic/connection.hpp

@@ -21,6 +21,11 @@ extern "C"
 #include <uvw/poll.h>
 #include <uvw/timer.h>
 
+namespace
+{
+  static auto log_cat = llarp::log::Cat("lokinet.quic");
+}  // namespace
+
 namespace llarp::quic
 {
   // We send and verify this in the initial connection and handshake; this is designed to allow

llarp/quic/tunnel.cpp

@@ -1,7 +1,7 @@
 #include "tunnel.hpp"
-#include "service/convotag.hpp"
-#include "service/endpoint.hpp"
-#include "service/name.hpp"
+#include <llarp/service/convotag.hpp>
+#include <llarp/service/endpoint.hpp>
+#include <llarp/service/name.hpp>
 #include "stream.hpp"
 #include <limits>
 #include <llarp/util/logging.hpp>
@@ -196,10 +196,11 @@ namespace llarp::quic
       auto peer = client.peer();
       LogDebug("Closing connection to ", peer.ip, ":", peer.port);
       client.clear();
-      if (error_code)
-        client.close();
-      else
-        client.close();
+      // TOFIX: this logic
+      //   if (error_code)
+      // client.close();
+      //   else
+      client.close();
     }
 
   }  // namespace
@@ -265,7 +266,7 @@ namespace llarp::quic
       LogInfo("quic stream from ", lokinet_addr, " to ", port, " tunnelling to ", *tunnel_to);
 
       auto tcp = get_loop()->resource<uvw::TCPHandle>();
-      auto error_handler = tcp->once<uvw::ErrorEvent>(
+      [[maybe_unused]] auto error_handler = tcp->once<uvw::ErrorEvent>(
           [&stream, to = *tunnel_to](const uvw::ErrorEvent&, uvw::TCPHandle&) {
             LogWarn("Failed to connect to ", to, ", shutting down quic stream");
             stream.close(tunnel::ERROR_CONNECT);
@@ -275,8 +276,7 @@ namespace llarp::quic
       // tunnel to let the other end know the connection was successful, then set up regular
       // stream handling to handle any other to/from data.
       tcp->once<uvw::ConnectEvent>(
-          [streamw = stream.weak_from_this(), error_handler = std::move(error_handler)](
-              const uvw::ConnectEvent&, uvw::TCPHandle& tcp) {
+          [streamw = stream.weak_from_this()](const uvw::ConnectEvent&, uvw::TCPHandle& tcp) {
             auto peer = tcp.peer();
             auto stream = streamw.lock();
             if (!stream)

llarp/router/route_poker.cpp

@@ -1,6 +1,6 @@
 #include "route_poker.hpp"
-#include "abstractrouter.hpp"
-#include "net/sock_addr.hpp"
+#include <llarp/router/abstractrouter.hpp>
+#include <llarp/net/sock_addr.hpp>
 #include <llarp/service/context.hpp>
 #include <llarp/dns/platform.hpp>
 #include <unordered_set>

llarp/router/router.cpp

@@ -361,6 +361,7 @@ namespace llarp
           _identity = RpcClient()->ObtainIdentityKey();
           const RouterID pk{pubkey()};
           LogWarn("Obtained lokid identity key: ", pk);
+          RpcClient()->StartPings();
           break;
         }
         catch (const std::exception& e)
@@ -411,12 +412,10 @@ namespace llarp
     if (log::get_level_default() != log::Level::off)
       log::reset_level(conf.logging.m_logLevel);
     log::clear_sinks();
-    log::add_sink(log_type, conf.logging.m_logFile);
-
-    enableRPCServer = conf.api.m_enableRPCServer;
+    log::add_sink(log_type, log_type == log::Type::System ? "lokinet" : conf.logging.m_logFile);
 
     // re-add rpc log sink if rpc enabled, else free it
-    if (enableRPCServer and llarp::logRingBuffer)
+    if (m_Config->api.m_enableRPCServer and llarp::logRingBuffer)
       log::add_sink(llarp::logRingBuffer, llarp::log::DEFAULT_PATTERN_MONO);
     else
       llarp::logRingBuffer = nullptr;
@@ -430,9 +429,6 @@ namespace llarp
       m_lokidRpcClient = std::make_shared<rpc::LokidRpcClient>(m_lmq, weak_from_this());
     }
 
-    if (enableRPCServer)
-      rpcBindAddr = oxenmq::address(conf.api.m_rpcBindAddr);
-
     log::debug(logcat, "Starting RPC server");
     if (not StartRpcServer())
       throw std::runtime_error("Failed to start rpc server");
@@ -628,8 +624,7 @@ namespace llarp
     }
     if (IsServiceNode())
       return SaveRC();
-    else
-      return true;
+    return true;
   }
 
   bool
@@ -1260,12 +1255,8 @@ namespace llarp
   bool
   Router::StartRpcServer()
   {
-    if (enableRPCServer)
-    {
-      m_RPCServer.reset(new rpc::RpcServer{m_lmq, this});
-      m_RPCServer->AsyncServeRPC(rpcBindAddr);
-      LogInfo("Bound RPC server to ", rpcBindAddr.full_address());
-    }
+    if (m_Config->api.m_enableRPCServer)
+      m_RPCServer = std::make_unique<rpc::RPCServer>(m_lmq, *this);
 
     return true;
   }

llarp/router/router.hpp

@@ -298,10 +298,7 @@ namespace llarp
     void
     PumpLL();
 
-    const oxenmq::address DefaultRPCBindAddr = oxenmq::address::tcp("127.0.0.1", 1190);
-    bool enableRPCServer = false;
-    oxenmq::address rpcBindAddr = DefaultRPCBindAddr;
-    std::unique_ptr<rpc::RpcServer> m_RPCServer;
+    std::unique_ptr<rpc::RPCServer> m_RPCServer;
 
     const llarp_time_t _randomStartDelay;
 

llarp/rpc/json_binary_proxy.cpp

@@ -0,0 +1,63 @@
+#include "json_binary_proxy.hpp"
+#include <oxenc/hex.h>
+#include <oxenc/base64.h>
+
+namespace llarp::rpc
+{
+
+  void
+  load_binary_parameter_impl(
+      std::string_view bytes, size_t raw_size, bool allow_raw, uint8_t* val_data)
+  {
+    if (allow_raw && bytes.size() == raw_size)
+    {
+      std::memcpy(val_data, bytes.data(), bytes.size());
+      return;
+    }
+    else if (bytes.size() == raw_size * 2)
+    {
+      if (oxenc::is_hex(bytes))
+      {
+        oxenc::from_hex(bytes.begin(), bytes.end(), val_data);
+        return;
+      }
+    }
+    else
+    {
+      const size_t b64_padded = (raw_size + 2) / 3 * 4;
+      const size_t b64_padding = raw_size % 3 == 1 ? 2 : raw_size % 3 == 2 ? 1 : 0;
+      const size_t b64_unpadded = b64_padded - b64_padding;
+      const std::string_view b64_padding_string = b64_padding == 2 ? "=="sv
+          : b64_padding == 1                                       ? "="sv
+                                                                   : ""sv;
+      if (bytes.size() == b64_unpadded
+          || (b64_padding > 0 && bytes.size() == b64_padded
+              && bytes.substr(b64_unpadded) == b64_padding_string))
+      {
+        if (oxenc::is_base64(bytes))
+        {
+          oxenc::from_base64(bytes.begin(), bytes.end(), val_data);
+          return;
+        }
+      }
+    }
+
+    throw std::runtime_error{"Invalid binary value: unexpected size and/or encoding"};
+  }
+
+  nlohmann::json&
+  json_binary_proxy::operator=(std::string_view binary_data)
+  {
+    switch (format)
+    {
+      case fmt::bt:
+        return e = binary_data;
+      case fmt::hex:
+        return e = oxenc::to_hex(binary_data);
+      case fmt::base64:
+        return e = oxenc::to_base64(binary_data);
+    }
+    throw std::runtime_error{"Internal error: invalid binary encoding"};
+  }
+
+}  // namespace llarp::rpc

llarp/rpc/json_binary_proxy.hpp

@@ -0,0 +1,158 @@
+#pragma once
+
+#include <string_view>
+#include <nlohmann/json.hpp>
+#include <unordered_set>
+
+using namespace std::literals;
+
+namespace llarp::rpc
+{
+
+  // Binary types that we support for rpc input/output.  For json, these must be specified as hex or
+  // base64; for bt-encoded requests these can be accepted as binary, hex, or base64.
+  template <typename T>
+  inline constexpr bool json_is_binary = false;
+
+  template <typename T>
+  inline constexpr bool json_is_binary_container = false;
+  template <typename T>
+  inline constexpr bool json_is_binary_container<std::vector<T>> = json_is_binary<T>;
+  template <typename T>
+  inline constexpr bool json_is_binary_container<std::unordered_set<T>> = json_is_binary<T>;
+
+  // De-referencing wrappers around the above:
+  template <typename T>
+  inline constexpr bool json_is_binary<const T&> = json_is_binary<T>;
+  template <typename T>
+  inline constexpr bool json_is_binary<T&&> = json_is_binary<T>;
+  template <typename T>
+  inline constexpr bool json_is_binary_container<const T&> = json_is_binary_container<T>;
+  template <typename T>
+  inline constexpr bool json_is_binary_container<T&&> = json_is_binary_container<T>;
+
+  void
+  load_binary_parameter_impl(
+      std::string_view bytes, size_t raw_size, bool allow_raw, uint8_t* val_data);
+
+  // Loads a binary value from a string_view which may contain hex, base64, and (optionally) raw
+  // bytes.
+  template <typename T, typename = std::enable_if_t<json_is_binary<T>>>
+  void
+  load_binary_parameter(std::string_view bytes, bool allow_raw, T& val)
+  {
+    load_binary_parameter_impl(bytes, sizeof(T), allow_raw, reinterpret_cast<uint8_t*>(&val));
+  }
+
+  // Wrapper around a nlohmann::json that assigns a binary value either as binary (for bt-encoding);
+  // or as hex or base64 (for json-encoding).
+  class json_binary_proxy
+  {
+   public:
+    nlohmann::json& e;
+    enum class fmt
+    {
+      bt,
+      hex,
+      base64
+    } format;
+    explicit json_binary_proxy(nlohmann::json& elem, fmt format) : e{elem}, format{format}
+    {}
+    json_binary_proxy() = delete;
+
+    json_binary_proxy(const json_binary_proxy&) = default;
+    json_binary_proxy(json_binary_proxy&&) = default;
+
+    /// Dereferencing a proxy element accesses the underlying nlohmann::json
+    nlohmann::json&
+    operator*()
+    {
+      return e;
+    }
+    nlohmann::json*
+    operator->()
+    {
+      return &e;
+    }
+
+    /// Descends into the json object, returning a new binary value proxy around the child element.
+    template <typename T>
+    json_binary_proxy
+    operator[](T&& key)
+    {
+      return json_binary_proxy{e[std::forward<T>(key)], format};
+    }
+
+    /// Returns a binary value proxy around the first/last element (requires an underlying list)
+    json_binary_proxy
+    front()
+    {
+      return json_binary_proxy{e.front(), format};
+    }
+    json_binary_proxy
+    back()
+    {
+      return json_binary_proxy{e.back(), format};
+    }
+
+    /// Assigns binary data from a string_view/string/etc.
+    nlohmann::json&
+    operator=(std::string_view binary_data);
+
+    /// Assigns binary data from a string_view over a 1-byte, non-char type (e.g. unsigned char or
+    /// uint8_t).
+    template <
+        typename Char,
+        std::enable_if_t<sizeof(Char) == 1 && !std::is_same_v<Char, char>, int> = 0>
+    nlohmann::json&
+    operator=(std::basic_string_view<Char> binary_data)
+    {
+      return *this = std::string_view{
+                 reinterpret_cast<const char*>(binary_data.data()), binary_data.size()};
+    }
+
+    /// Takes a trivial, no-padding data structure (e.g. a crypto::hash) as the value and dumps its
+    /// contents as the binary value.
+    template <typename T, std::enable_if_t<json_is_binary<T>, int> = 0>
+    nlohmann::json&
+    operator=(const T& val)
+    {
+      return *this = std::string_view{reinterpret_cast<const char*>(&val), sizeof(val)};
+    }
+
+    /// Takes a vector of some json_binary_proxy-assignable type and builds an array by assigning
+    /// each one into a new array of binary values.
+    template <typename T, std::enable_if_t<json_is_binary_container<T>, int> = 0>
+    nlohmann::json&
+    operator=(const T& vals)
+    {
+      auto a = nlohmann::json::array();
+      for (auto& val : vals)
+        json_binary_proxy{a.emplace_back(), format} = val;
+      return e = std::move(a);
+    }
+    /// Emplaces a new nlohman::json to the end of an underlying list and returns a
+    /// json_binary_proxy wrapping it.
+    ///
+    /// Example:
+    ///
+    ///     auto child = wrappedelem.emplace_back({"key1": 1}, {"key2": 2});
+    ///     child["binary-key"] = some_binary_thing;
+    template <typename... Args>
+    json_binary_proxy
+    emplace_back(Args&&... args)
+    {
+      return json_binary_proxy{e.emplace_back(std::forward<Args>(args)...), format};
+    }
+
+    /// Adds an element to an underlying list, then copies or moves the given argument onto it via
+    /// json_binary_proxy assignment.
+    template <typename T>
+    void
+    push_back(T&& val)
+    {
+      emplace_back() = std::forward<T>(val);
+    }
+  };
+
+}  // namespace llarp::rpc

llarp/rpc/json_bt.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+#include <oxenc/bt_value.h>
+
+using nlohmann::json;
+
+namespace llarp::rpc
+{
+
+  inline oxenc::bt_value
+  json_to_bt(json&& j)
+  {
+    if (j.is_object())
+    {
+      oxenc::bt_dict res;
+      for (auto& [k, v] : j.items())
+      {
+        if (v.is_null())
+          continue;  // skip k-v pairs with a null v (for other nulls we fail).
+        res[k] = json_to_bt(std::move(v));
+      }
+      return res;
+    }
+    if (j.is_array())
+    {
+      oxenc::bt_list res;
+      for (auto& v : j)
+        res.push_back(json_to_bt(std::move(v)));
+      return res;
+    }
+    if (j.is_string())
+    {
+      return std::move(j.get_ref<std::string&>());
+    }
+    if (j.is_boolean())
+      return j.get<bool>() ? 1 : 0;
+    if (j.is_number_unsigned())
+      return j.get<uint64_t>();
+    if (j.is_number_integer())
+      return j.get<int64_t>();
+    throw std::domain_error{
+        "internal error: encountered some unhandled/invalid type in json-to-bt translation"};
+  }
+
+}  // namespace llarp::rpc

llarp/rpc/json_conversions.cpp

@@ -0,0 +1,18 @@
+#include "json_conversions.hpp"
+#include <nlohmann/json.hpp>
+
+namespace llarp
+{
+  void
+  to_json(nlohmann::json& j, const IPRange& ipr)
+  {
+    j = ipr.ToString();
+  }
+
+  void
+  from_json(const nlohmann::json& j, IPRange& ipr)
+  {
+    ipr = IPRange{j.get<std::string>()};
+  }
+
+}  // namespace llarp

llarp/rpc/json_conversions.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <llarp/net/ip_range.hpp>
+#include <nlohmann/json_fwd.hpp>
+#include "json_binary_proxy.hpp"
+
+namespace llarp
+{
+  void
+  to_json(nlohmann::json& j, const IPRange& ipr);
+  void
+  from_json(const nlohmann::json& j, IPRange& ipr);
+}  // namespace llarp
+
+namespace nlohmann
+{
+  // Specializations of binary types for deserialization; when receiving these from json we expect
+  // them encoded in hex or base64.  These may *not* be used for serialization, and will throw if so
+  // invoked; for serialization you need to use RPC_COMMAND::response_hex (or _b64) instead.
+  template <typename T>
+  struct adl_serializer<T, std::enable_if_t<llarp::rpc::json_is_binary<T>>>
+  {
+    static_assert(std::is_trivially_copyable_v<T> && std::has_unique_object_representations_v<T>);
+
+    static void
+    to_json(json&, const T&)
+    {
+      throw std::logic_error{"Internal error: binary types are not directly serializable"};
+    }
+    static void
+    from_json(const json& j, T& val)
+    {
+      llarp::rpc::load_binary_parameter(j.get<std::string_view>(), false /*no raw*/, val);
+    }
+  };
+
+}  // namespace nlohmann

llarp/rpc/lokid_rpc_client.cpp

@@ -64,7 +64,7 @@ namespace llarp
         LogInfo("connecting to lokid via LMQ at ", url.full_address());
         m_Connection = m_lokiMQ->connect_remote(
             url,
-            [self = shared_from_this()](oxenmq::ConnectionID) { self->Connected(); },
+            [](oxenmq::ConnectionID) {},
             [self = shared_from_this(), url](oxenmq::ConnectionID, std::string_view f) {
               llarp::LogWarn("Failed to connect to lokid: ", f);
               if (auto router = self->m_Router.lock())
@@ -126,7 +126,7 @@ namespace llarp
            }},
       };
       if (!m_LastUpdateHash.empty())
-        request["fields"]["poll_block_hash"] = m_LastUpdateHash;
+        request["poll_block_hash"] = m_LastUpdateHash;
 
       Request(
           "rpc.get_service_nodes",
@@ -168,10 +168,15 @@ namespace llarp
     }
 
     void
-    LokidRpcClient::Connected()
+    LokidRpcClient::StartPings()
     {
       constexpr auto PingInterval = 30s;
-      auto makePingRequest = [self = shared_from_this()]() {
+
+      auto router = m_Router.lock();
+      if (not router)
+        return;
+
+      auto makePingRequest = router->loop()->make_caller([self = shared_from_this()]() {
         // send a ping
         PubKey pk{};
         auto r = self->m_Router.lock();
@@ -208,10 +213,11 @@ namespace llarp
         // reason (e.g. oxend restarts and loses the subscription); we poll using the last known
         // hash so that the poll is very cheap (basically empty) if the block hasn't advanced.
         self->UpdateServiceNodeList();
-      };
+      });
+
       // Fire one ping off right away to get things going.
       makePingRequest();
-      m_lokiMQ->add_timer(makePingRequest, PingInterval);
+      m_lokiMQ->add_timer(std::move(makePingRequest), PingInterval);
     }
 
     void

llarp/rpc/lokid_rpc_client.hpp

@@ -46,11 +46,10 @@ namespace llarp
       void
       InformConnection(RouterID router, bool success);
 
-     private:
-      /// called when we have connected to lokid via lokimq
       void
-      Connected();
+      StartPings();
 
+     private:
       /// do a lmq command on the current connection
       void
       Command(std::string_view cmd);

llarp/rpc/param_parser.hpp

@@ -0,0 +1,367 @@
+#pragma once
+
+#include "json_binary_proxy.hpp"
+#include "json_bt.hpp"
+#include "json_conversions.hpp"
+#include <oxenc/bt_serialize.h>
+#include <nlohmann/json.hpp>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <optional>
+
+namespace llarp::rpc
+{
+  using json_range = std::pair<nlohmann::json::const_iterator, nlohmann::json::const_iterator>;
+  using rpc_input = std::variant<std::monostate, nlohmann::json, oxenc::bt_dict_consumer>;
+
+  // Checks that key names are given in ascending order
+  template <typename... Ignore>
+  void
+  check_ascending_names(std::string_view name1, std::string_view name2, const Ignore&...)
+  {
+    if (!(name2 > name1))
+      throw std::runtime_error{
+          "Internal error: request values must be retrieved in ascending order"};
+  }
+
+  // Wrapper around a reference for get_values that is used to indicate that the value is
+  // required, in which case an exception will be raised if the value is not found.  Usage:
+  //
+  //     int a_optional = 0, b_required;
+  //     get_values(input,
+  //         "a", a_optional,
+  //         "b", required{b_required},
+  //         // ...
+  //     );
+  template <typename T>
+  struct required
+  {
+    T& value;
+    required(T& ref) : value{ref}
+    {}
+  };
+  template <typename T>
+  constexpr bool is_required_wrapper = false;
+  template <typename T>
+  constexpr bool is_required_wrapper<required<T>> = true;
+
+  template <typename T>
+  constexpr bool is_std_optional = false;
+  template <typename T>
+  constexpr bool is_std_optional<std::optional<T>> = true;
+
+  // Wrapper around a reference for get_values that adds special handling to act as if the value was
+  // not given at all if the value is given as an empty string.  This sucks, but is necessary for
+  // backwards compatibility (especially with wallet2 clients).
+  //
+  // Usage:
+  //
+  //     std::string x;
+  //     get_values(input,
+  //         "x", ignore_empty_string{x},
+  //         // ...
+  //     );
+  template <typename T>
+  struct ignore_empty_string
+  {
+    T& value;
+    ignore_empty_string(T& ref) : value{ref}
+    {}
+
+    bool
+    should_ignore(oxenc::bt_dict_consumer& d)
+    {
+      if (d.is_string())
+      {
+        auto d2{d};  // Copy because we want to leave d intact
+        if (d2.consume_string_view().empty())
+          return true;
+      }
+      return false;
+    }
+
+    bool
+    should_ignore(json_range& it_range)
+    {
+      auto& e = *it_range.first;
+      return (e.is_string() && e.get<std::string_view>().empty());
+    }
+  };
+
+  template <typename T>
+  constexpr bool is_ignore_empty_string_wrapper = false;
+  template <typename T>
+  constexpr bool is_ignore_empty_string_wrapper<ignore_empty_string<T>> = true;
+
+  // Advances the dict consumer to the first element >= the given name.  Returns true if found,
+  // false if it advanced beyond the requested name.  This is exactly the same as
+  // `d.skip_until(name)`, but is here so we can also overload an equivalent function for json
+  // iteration.
+  inline bool
+  skip_until(oxenc::bt_dict_consumer& d, std::string_view name)
+  {
+    return d.skip_until(name);
+  }
+  // Equivalent to the above but for a json object iterator.
+  inline bool
+  skip_until(json_range& it_range, std::string_view name)
+  {
+    auto& [it, end] = it_range;
+    while (it != end && it.key() < name)
+      ++it;
+    return it != end && it.key() == name;
+  }
+
+  // List types that are expandable; for these we emplace_back for each element of the input
+  template <typename T>
+  constexpr bool is_expandable_list = false;
+  template <typename T>
+  constexpr bool is_expandable_list<std::vector<T>> = true;
+
+  // Types that are constructible from string
+  template <typename T>
+  constexpr bool is_string_constructible = false;
+  template <>
+  inline constexpr bool is_string_constructible<IPRange> = true;
+
+  // Fixed size elements: tuples, pairs, and std::array's; we accept list input as long as the
+  // list length matches exactly.
+  template <typename T>
+  constexpr bool is_tuple_like = false;
+  template <typename T, size_t N>
+  constexpr bool is_tuple_like<std::array<T, N>> = true;
+  template <typename S, typename T>
+  constexpr bool is_tuple_like<std::pair<S, T>> = true;
+  template <typename... T>
+  constexpr bool is_tuple_like<std::tuple<T...>> = true;
+
+  // True if T is a `std::unordered_map<std::string, ANYTHING...>`
+  template <typename T>
+  constexpr bool is_unordered_string_map = false;
+  template <typename... ValueEtc>
+  constexpr bool is_unordered_string_map<std::unordered_map<std::string, ValueEtc...>> = true;
+
+  template <typename TupleLike, size_t... Is>
+  void
+  load_tuple_values(oxenc::bt_list_consumer&, TupleLike&, std::index_sequence<Is...>);
+
+  // Consumes the next value from the dict consumer into `val`
+  template <
+      typename BTConsumer,
+      typename T,
+      std::enable_if_t<
+          std::is_same_v<BTConsumer, oxenc::bt_dict_consumer>
+              || std::is_same_v<BTConsumer, oxenc::bt_list_consumer>,
+          int> = 0>
+  void
+  load_value(BTConsumer& c, T& target)
+  {
+    if constexpr (std::is_integral_v<T>)
+      target = c.template consume_integer<T>();
+    else if constexpr (std::is_same_v<T, std::string> || std::is_same_v<T, std::string_view>)
+      target = c.consume_string_view();
+    else if constexpr (is_string_constructible<T>)
+      target = T{c.consume_string()};
+    else if constexpr (llarp::rpc::json_is_binary<T>)
+      llarp::rpc::load_binary_parameter(c.consume_string_view(), true /*allow raw*/, target);
+    else if constexpr (is_expandable_list<T>)
+    {
+      auto lc = c.consume_list_consumer();
+      target.clear();
+      while (!lc.is_finished())
+        load_value(lc, target.emplace_back());
+    }
+    else if constexpr (is_tuple_like<T>)
+    {
+      auto lc = c.consume_list_consumer();
+      load_tuple_values(lc, target, std::make_index_sequence<std::tuple_size_v<T>>{});
+    }
+    else if constexpr (is_unordered_string_map<T>)
+    {
+      auto dc = c.consume_dict_consumer();
+      target.clear();
+      while (!dc.is_finished())
+        load_value(dc, target[std::string{dc.key()}]);
+    }
+    else
+      static_assert(std::is_same_v<T, void>, "Unsupported load_value type");
+  }
+
+  // Copies the next value from the json range into `val`, and advances the iterator.  Throws
+  // on unconvertible values.
+  template <typename T>
+  void
+  load_value(json_range& range_itr, T& target)
+  {
+    auto& key = range_itr.first.key();
+    auto& current = *range_itr.first;  // value currently pointed to by range_itr.first
+    if constexpr (std::is_same_v<T, bool>)
+    {
+      if (current.is_boolean())
+        target = current.get<bool>();
+      else if (current.is_number_unsigned())
+      {
+        // Also accept 0 or 1 for bools (mainly to be compatible with bt-encoding which doesn't
+        // have a distinct bool type).
+        auto b = current.get<uint64_t>();
+        if (b <= 1)
+          target = b;
+        else
+          throw std::domain_error{"Invalid value for '" + key + "': expected boolean"};
+      }
+      else
+      {
+        throw std::domain_error{"Invalid value for '" + key + "': expected boolean"};
+      }
+    }
+    else if constexpr (std::is_unsigned_v<T>)
+    {
+      if (!current.is_number_unsigned())
+        throw std::domain_error{"Invalid value for '" + key + "': non-negative value required"};
+      auto i = current.get<uint64_t>();
+      if (sizeof(T) < sizeof(uint64_t) && i > std::numeric_limits<T>::max())
+        throw std::domain_error{"Invalid value for '" + key + "': value too large"};
+      target = i;
+    }
+    else if constexpr (std::is_integral_v<T>)
+    {
+      if (!current.is_number_integer())
+        throw std::domain_error{"Invalid value for '" + key + "': value is not an integer"};
+      auto i = current.get<int64_t>();
+      if (sizeof(T) < sizeof(int64_t))
+      {
+        if (i < std::numeric_limits<T>::lowest())
+          throw std::domain_error{
+              "Invalid value for '" + key + "': negative value magnitude is too large"};
+        if (i > std::numeric_limits<T>::max())
+          throw std::domain_error{"Invalid value for '" + key + "': value is too large"};
+      }
+      target = i;
+    }
+    else if constexpr (std::is_same_v<T, std::string> || std::is_same_v<T, std::string_view>)
+    {
+      target = current.get<std::string_view>();
+    }
+    else if constexpr (
+        llarp::rpc::json_is_binary<T> || is_expandable_list<T> || is_tuple_like<T>
+        || is_unordered_string_map<T>)
+    {
+      try
+      {
+        current.get_to(target);
+      }
+      catch (const std::exception& e)
+      {
+        throw std::domain_error{"Invalid values in '" + key + "'"};
+      }
+    }
+    else
+    {
+      static_assert(std::is_same_v<T, void>, "Unsupported load type");
+    }
+    ++range_itr.first;
+  }
+
+  template <typename TupleLike, size_t... Is>
+  void
+  load_tuple_values(oxenc::bt_list_consumer& c, TupleLike& val, std::index_sequence<Is...>)
+  {
+    (load_value(c, std::get<Is>(val)), ...);
+  }
+
+  // Takes a json object iterator or bt_dict_consumer and loads the current value at the iterator.
+  // This calls itself recursively, if needed, to unwrap optional/required/ignore_empty_string
+  // wrappers.
+  template <typename In, typename T>
+  void
+  load_curr_value(In& in, T& val)
+  {
+    if constexpr (is_required_wrapper<T>)
+    {
+      load_curr_value(in, val.value);
+    }
+    else if constexpr (is_ignore_empty_string_wrapper<T>)
+    {
+      if (!val.should_ignore(in))
+        load_curr_value(in, val.value);
+    }
+    else if constexpr (is_std_optional<T>)
+    {
+      load_curr_value(in, val.emplace());
+    }
+    else
+    {
+      load_value(in, val);
+    }
+  }
+
+  // Gets the next value from a json object iterator or bt_dict_consumer.  Leaves the iterator at
+  // the next value, i.e.  found + 1 if found, or the next greater value if not found.  (NB:
+  // nlohmann::json objects are backed by an *ordered* map and so both nlohmann iterators and
+  // bt_dict_consumer behave analogously here).
+  template <typename In, typename T>
+  void
+  get_next_value(In& in, [[maybe_unused]] std::string_view name, T& val)
+  {
+    if constexpr (std::is_same_v<std::monostate, In>)
+      ;
+    else if (skip_until(in, name))
+      load_curr_value(in, val);
+    else if constexpr (is_required_wrapper<T>)
+      throw std::runtime_error{"Required key '" + std::string{name} + "' not found"};
+  }
+
+  // Accessor for simple, flat value retrieval from a json or bt_dict_consumer.  In the later
+  // case note that the given bt_dict_consumer will be advanced, so you *must* take care to
+  // process keys in order, both for the keys passed in here *and* for use before and after this
+  // call.
+  template <typename Input, typename T, typename... More>
+  void
+  get_values(Input& in, std::string_view name, T&& val, More&&... more)
+  {
+    if constexpr (std::is_same_v<rpc_input, Input>)
+    {
+      if (auto* json_in = std::get_if<nlohmann::json>(&in))
+      {
+        json_range r{json_in->cbegin(), json_in->cend()};
+        get_values(r, name, val, std::forward<More>(more)...);
+      }
+      else if (auto* dict = std::get_if<oxenc::bt_dict_consumer>(&in))
+      {
+        get_values(*dict, name, val, std::forward<More>(more)...);
+      }
+      else
+      {
+        // A monostate indicates that no parameters field was provided at all
+        get_values(var::get<std::monostate>(in), name, val, std::forward<More>(more)...);
+      }
+    }
+    else if constexpr (std::is_same_v<std::string_view, Input>)
+    {
+      if (in.front() == 'd')
+      {
+        oxenc::bt_dict_consumer d{in};
+        get_values(d, name, val, std::forward<More>(more)...);
+      }
+      else
+      {
+        auto json_in = nlohmann::json::parse(in);
+        json_range r{json_in.cbegin(), json_in.cend()};
+        get_values(r, name, val, std::forward<More>(more)...);
+      }
+    }
+    else
+    {
+      static_assert(
+          std::is_same_v<json_range, Input> || std::is_same_v<oxenc::bt_dict_consumer, Input>
+          || std::is_same_v<std::monostate, Input>);
+      get_next_value(in, name, val);
+      if constexpr (sizeof...(More) > 0)
+      {
+        check_ascending_names(name, more...);
+        get_values(in, std::forward<More>(more)...);
+      }
+    }
+  }
+}  // namespace llarp::rpc

llarp/rpc/rpc_request.hpp

@@ -0,0 +1,71 @@
+#pragma once
+
+#include "rpc_server.hpp"
+#include "rpc_request_parser.hpp"
+#include "rpc_request_decorators.hpp"
+#include "rpc_request_definitions.hpp"
+#include "json_bt.hpp"
+#include <string_view>
+#include <llarp/config/config.hpp>
+#include <llarp/router/abstractrouter.hpp>
+#include <oxenmq/oxenmq.h>
+#include <oxenmq/address.h>
+#include <oxen/log/omq_logger.hpp>
+
+namespace llarp::rpc
+{
+  using nlohmann::json;
+
+  template <typename RPC>
+  auto
+  make_invoke()
+  {
+    return [](oxenmq::Message& m, RPCServer& server) {
+      EndpointHandler<RPC> handler{server, m.send_later()};
+      auto& rpc = handler.rpc;
+
+      if (m.data.size() > 1)
+      {
+        m.send_reply(nlohmann::json{
+            {"error",
+             "Bad Request: RPC requests must have at most one data part (received {})"_format(
+                 m.data.size())}}
+                         .dump());
+        return;
+      }
+      // parsing input as bt or json
+      //    hand off to parse_request (overloaded versions)
+      try
+      {
+        if (m.data.empty() or m.data[0].empty())
+        {
+          parse_request(rpc, nlohmann::json::object());
+        }
+        else if (m.data[0].front() == 'd')
+        {
+          rpc.set_bt();
+          parse_request(rpc, oxenc::bt_dict_consumer{m.data[0]});
+        }
+        else
+        {
+          parse_request(rpc, nlohmann::json::parse(m.data[0]));
+        }
+      }
+      catch (const std::exception& e)
+      {
+        m.send_reply(nlohmann::json{{"Failed to parse request parameters: "s + e.what()}}.dump());
+        return;
+      }
+
+      if (not std::is_base_of_v<Immediate, RPC>)
+      {
+        server.m_Router.loop()->call_soon(std::move(handler));
+      }
+      else
+      {
+        handler();
+      }
+    };
+  }
+
+}  // namespace llarp::rpc

llarp/rpc/rpc_request_decorators.hpp

@@ -0,0 +1,128 @@
+#pragma once
+
+#include "json_binary_proxy.hpp"
+#include "json_bt.hpp"
+#include <nlohmann/json_fwd.hpp>
+#include <string_view>
+#include <llarp/config/config.hpp>
+#include <oxenmq/oxenmq.h>
+#include <oxenmq/address.h>
+#include <oxen/log/omq_logger.hpp>
+
+namespace tools
+{
+  //  Type wrapper that contains an arbitrary list of types.
+  template <typename...>
+  struct type_list
+  {};
+}  // namespace tools
+
+namespace llarp::rpc
+{
+  //  Base class that all RPC requests will expand for each endpoint type
+  struct RPCRequest
+  {
+   private:
+    bool bt = false;
+
+   public:
+    //  Returns true if response is bt-encoded, and false for json
+    //  Note: do not set value
+    bool
+    is_bt() const
+    {
+      return bt;
+    }
+
+    //  Callable method to indicate request is bt-encoded
+    void
+    set_bt()
+    {
+      bt = true;
+      response_b64.format = llarp::rpc::json_binary_proxy::fmt::bt;
+      response_hex.format = llarp::rpc::json_binary_proxy::fmt::bt;
+    }
+
+    //  Invoked if this.replier is still present. If it is "stolen" by endpoint (moved from
+    //  RPC struct), then endpoint handles sending reply
+    void
+    send_response()
+    {
+      replier->reply(
+          is_bt() ? oxenc::bt_serialize(json_to_bt(std::move(response))) : response.dump());
+    }
+
+    void
+    send_response(nlohmann::json _response)
+    {
+      response = std::move(_response);
+      send_response();
+    }
+
+    //  Response Data:
+    //  bt-encoded are converted in real-time
+    //    - bool becomes 0 or 1
+    //    - key:value where value == null are omitted
+    //    - other nulls will raise an exception if found in json
+    //    - no doubles
+    //      - to store doubles: encode bt in endpoint-specific way
+    //    - binary strings will fail json serialization; caller must
+    //
+    //        std::string binary = some_binary_data();
+    //        request.response["binary_value"] = is_bt ? binary : oxenmq::to_hex(binary)
+    //
+    nlohmann::json response;
+
+    //  Proxy Object:
+    //  Sets binary data in "response"
+    //    - if return type is json, encodes as hex
+    //    - if return type is bt, then binary is untouched
+    //
+    //  Usage:
+    //    std::string data = "abc";
+    //    request.response_hex["foo"]["bar"] = data; // json: "616263", bt: "abc"
+    //
+    llarp::rpc::json_binary_proxy response_hex{response, llarp::rpc::json_binary_proxy::fmt::hex};
+
+    //  Proxy Object:
+    //  Encodes binary data as base_64 for json-encoded responses, leaves as binary for bt-encoded
+    //  responses
+    //
+    //    Usage:
+    //      std::string data = "abc"
+    //      request.response_b64["foo"]["bar"] = data; json: "YWJj", bt: "abc"
+    //
+    llarp::rpc::json_binary_proxy response_b64{
+        response, llarp::rpc::json_binary_proxy::fmt::base64};
+
+    //  The oxenmq deferred send object into which the response will be sent when the `invoke`
+    //  method returns.  If the response needs to happen later (i.e. not immediately after `invoke`
+    //  returns) then you should call `defer()` to extract and clear this and then send the response
+    //  via the returned DeferredSend object yourself.
+    std::optional<oxenmq::Message::DeferredSend> replier;
+
+    // Called to clear the current replier and return it.  After this call the automatic reply will
+    // not be generated; the caller is responsible for calling `->reply` on the returned optional
+    // itself.  This is typically used where a call has to be deferred, for example because it
+    // depends on some network response to build the reply.
+    oxenmq::Message::DeferredSend
+    move()
+    {
+      auto r{std::move(*replier)};
+      replier.reset();
+      return r;
+    }
+  };
+
+  //  Tag types that are inherited to set RPC endpoint properties
+
+  //  RPC call wil take no input arguments
+  //    Parameter dict can be passed, but will be ignored
+  struct NoArgs : virtual RPCRequest
+  {};
+
+  // RPC call will be executed immediately
+  struct Immediate : virtual RPCRequest
+  {};
+
+}  // namespace llarp::rpc

llarp/rpc/rpc_request_definitions.hpp

@@ -0,0 +1,311 @@
+#pragma once
+
+#include "rpc_request_decorators.hpp"
+#include "llarp/net/ip_range.hpp"
+#include "llarp/router/abstractrouter.hpp"
+#include "llarp/router/route_poker.hpp"
+#include "llarp/service/address.hpp"
+#include "llarp/service/endpoint.hpp"
+#include "llarp/service/outbound_context.hpp"
+#include <string_view>
+#include <llarp/config/config.hpp>
+#include <oxenmq/oxenmq.h>
+#include <oxenmq/address.h>
+#include <oxen/log/omq_logger.hpp>
+#include <unordered_map>
+#include <vector>
+
+namespace llarp::rpc
+{
+  //  RPC: halt
+  //    Stops lokinet router
+  //
+  //  Inputs: none
+  //
+  struct Halt : NoArgs, Immediate
+  {
+    static constexpr auto name = "halt"sv;
+  };
+
+  //  RPC: version
+  //    Returns version and uptime information
+  //
+  //  Inputs: none
+  //
+  //  Returns: "OK"
+  //    "uptime"
+  //    "version"
+  //
+  struct Version : NoArgs, Immediate
+  {
+    static constexpr auto name = "version"sv;
+  };
+
+  //  RPC: status
+  //    Returns that current activity status of lokinet router
+  //    Calls router::extractstatus
+  //
+  //  Inputs: none
+  //
+  //  Returns: massive dump of status info including
+  //    "running"
+  //    "numNodesKnown"
+  //    "dht"
+  //    "services"
+  //    "exit"
+  //    "links"
+  //    "outboundMessages"
+  //    etc
+  //
+  struct Status : NoArgs
+  {
+    static constexpr auto name = "status"sv;
+  };
+
+  //  RPC: get_status
+  //    Returns current summary status
+  //
+  //  Inputs: none
+  //
+  //  Returns: slightly smaller dump of status info including
+  //    "authcodes"
+  //    "exitMap"
+  //    "lokiAddress"
+  //    "networkReady"
+  //    "numPathsBuilt"
+  //    "numPeersConnected"
+  //    etc
+  //
+  struct GetStatus : NoArgs
+  {
+    static constexpr auto name = "get_status"sv;
+  };
+
+  //  RPC: quic_connect
+  //    Initializes QUIC connection tunnel
+  //    Passes request parameters in nlohmann::json format
+  //
+  //  Inputs:
+  //    "endpoint" : endpoint id (string)
+  //    "bindAddr" : bind address (string, ex: "127.0.0.1:1142")
+  //    "host" : remote host ID (string)
+  //    "port" : port to bind to (int)
+  //    "close" : close connection to port or host ID
+  //
+  //  Returns:
+  //    "id" : connection ID
+  //    "addr" : connection local address
+  //
+  struct QuicConnect : RPCRequest
+  {
+    static constexpr auto name = "quic_connect"sv;
+
+    struct request_parameters
+    {
+      std::string bindAddr;
+      int closeID;
+      std::string endpoint;
+      uint16_t port;
+      std::string remoteHost;
+    } request;
+  };
+
+  //  RPC: quic_listener
+  //    Connects to QUIC interface on local endpoint
+  //    Passes request parameters in nlohmann::json format
+  //
+  //  Inputs:
+  //    "endpoint" : endpoint id (string)
+  //    "host" : remote host ID (string)
+  //    "port" : port to bind to (int)
+  //    "close" : close connection to port or host ID
+  //    "srv-proto" :
+  //
+  //  Returns:
+  //    "id" : connection ID
+  //    "addr" : connection local address
+  //
+  struct QuicListener : RPCRequest
+  {
+    static constexpr auto name = "quic_listener"sv;
+
+    struct request_parameters
+    {
+      int closeID;
+      std::string endpoint;
+      uint16_t port;
+      std::string remoteHost;
+      std::string srvProto;
+    } request;
+  };
+
+  //  RPC: lookup_snode
+  //    Look up service node
+  //    Passes request parameters in nlohmann::json format
+  //
+  //  Inputs:
+  //    "routerID" : router ID to query (string)
+  //
+  //  Returns:
+  //    "ip" : snode IP address
+  //
+  struct LookupSnode : RPCRequest
+  {
+    static constexpr auto name = "lookup_snode"sv;
+
+    struct request_parameters
+    {
+      std::string routerID;
+    } request;
+  };
+
+  //  RPC: map_exit
+  //    Map a new connection to an exit node
+  //
+  //  Inputs:
+  //    "address" : ID of endpoint to map
+  //    "range" : IP range to map to exit node
+  //    "token" : auth token
+  //
+  //  Returns:
+  //
+  struct MapExit : RPCRequest
+  {
+    MapExit()
+    {
+      if constexpr (platform::supports_ipv6)
+        request.ip_range.emplace_back("::/0");
+      else
+        request.ip_range.emplace_back("0.0.0.0/0");
+    }
+
+    static constexpr auto name = "map_exit"sv;
+
+    struct request_parameters
+    {
+      std::string address;
+      std::vector<IPRange> ip_range;
+      std::string token;
+    } request;
+  };
+
+  //  RPC: list_exits
+  //    List all currently mapped exit node connections
+  //
+  //  Inputs: none
+  //
+  //  Returns:
+  //
+  struct ListExits : NoArgs
+  {
+    static constexpr auto name = "list_exits"sv;
+  };
+
+  //  RPC: unmap_exit
+  //    Unmap a connection to an exit node
+  //
+  //  Inputs:
+  //    "endpoint" : ID of endpoint to map
+  //    "range" : IP range to map to exit node
+  //    "token" : auth token
+  //
+  //  Returns:
+  //
+  struct UnmapExit : RPCRequest
+  {
+    UnmapExit()
+    {
+      if constexpr (platform::supports_ipv6)
+        request.ip_range.emplace_back("::/0");
+      else
+        request.ip_range.emplace_back("0.0.0.0/0");
+    }
+
+    static constexpr auto name = "unmap_exit"sv;
+
+    struct request_parameters
+    {
+      std::vector<IPRange> ip_range;
+    } request;
+  };
+
+  //  RPC: swap_exit
+  //    Swap a connection from one exit to another
+  //
+  //  Inputs:
+  //    "exits" : exit nodes to swap mappings from (index 0 = old exit, index 1 = new exit)
+  //
+  //  Returns:
+  //
+  struct SwapExits : RPCRequest
+  {
+    static constexpr auto name = "swap_exits"sv;
+
+    struct request_parameters
+    {
+      std::vector<std::string> exit_addresses;
+      std::string token;
+    } request;
+  };
+
+  //  RPC: dns_query
+  //    Attempts to query endpoint by domain name
+  //
+  //  Inputs:
+  //    "endpoint" : endpoint ID to query (string)
+  //    "qname" : query name (string)
+  //    "qtype" : query type (int)
+  //
+  //  Returns:
+  //
+  struct DNSQuery : Immediate
+  {
+    static constexpr auto name = "dns_query"sv;
+
+    struct request_parameters
+    {
+      std::string endpoint;
+      uint16_t qtype;
+      std::string qname;
+    } request;
+  };
+
+  //  RPC: config
+  //    Runs lokinet router using .ini config file passed as path
+  //
+  //  Inputs:
+  //    "filename" : name of .ini file to either save or delete
+  //    "ini" : .ini chunk to save in new file
+  //    "del" : boolean specifying whether to delete file "filename" or save it
+  //
+  //  Returns:
+  //
+  struct Config : Immediate
+  {
+    static constexpr auto name = "config"sv;
+
+    struct request_parameters
+    {
+      bool del;
+      std::string filename;
+      std::string ini;
+    } request;
+  };
+
+  // List of all RPC request structs to allow compile-time enumeration of all supported types
+  using rpc_request_types = tools::type_list<
+      Halt,
+      Version,
+      Status,
+      GetStatus,
+      QuicConnect,
+      QuicListener,
+      LookupSnode,
+      MapExit,
+      ListExits,
+      SwapExits,
+      UnmapExit,
+      DNSQuery,
+      Config>;
+
+}  // namespace llarp::rpc

llarp/rpc/rpc_request_parser.cpp

@@ -0,0 +1,110 @@
+#include "rpc_request_parser.hpp"
+#include "llarp/rpc/rpc_request_definitions.hpp"
+#include "param_parser.hpp"
+#include <string_view>
+#include <llarp/config/config.hpp>
+#include <oxenmq/oxenmq.h>
+#include <oxenmq/address.h>
+#include <oxen/log/omq_logger.hpp>
+
+namespace llarp::rpc
+{
+  using nlohmann::json;
+
+  void
+  parse_request(QuicConnect& quicconnect, rpc_input input)
+  {
+    get_values(
+        input,
+        "bindAddr",
+        quicconnect.request.bindAddr,
+        "closeID",
+        quicconnect.request.closeID,
+        "endpoint",
+        quicconnect.request.endpoint,
+        "port",
+        quicconnect.request.port,
+        "remoteHost",
+        quicconnect.request.remoteHost);
+  }
+
+  void
+  parse_request(QuicListener& quiclistener, rpc_input input)
+  {
+    get_values(
+        input,
+        "closeID",
+        quiclistener.request.closeID,
+        "endpoint",
+        quiclistener.request.endpoint,
+        "port",
+        quiclistener.request.port,
+        "remoteHost",
+        quiclistener.request.remoteHost,
+        "srvProto",
+        quiclistener.request.srvProto);
+  }
+
+  void
+  parse_request(LookupSnode& lookupsnode, rpc_input input)
+  {
+    get_values(input, "routerID", lookupsnode.request.routerID);
+  }
+
+  void
+  parse_request(MapExit& mapexit, rpc_input input)
+  {
+    get_values(
+        input,
+        "address",
+        mapexit.request.address,
+        "ip_range",
+        mapexit.request.ip_range,
+        "token",
+        mapexit.request.token);
+  }
+
+  void
+  parse_request(UnmapExit& unmapexit, rpc_input input)
+  {
+    get_values(input, "ip_range", unmapexit.request.ip_range);
+  }
+
+  void
+  parse_request(SwapExits& swapexits, rpc_input input)
+  {
+    get_values(
+        input,
+        "exit_addresses",
+        swapexits.request.exit_addresses,
+        "token",
+        swapexits.request.token);
+  }
+
+  void
+  parse_request(DNSQuery& dnsquery, rpc_input input)
+  {
+    get_values(
+        input,
+        "endpoint",
+        dnsquery.request.endpoint,
+        "qname",
+        dnsquery.request.qname,
+        "qtype",
+        dnsquery.request.qtype);
+  }
+
+  void
+  parse_request(Config& config, rpc_input input)
+  {
+    get_values(
+        input,
+        "delete",
+        config.request.del,
+        "filename",
+        config.request.filename,
+        "ini",
+        config.request.ini);
+  }
+
+}  // namespace llarp::rpc

llarp/rpc/rpc_request_parser.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "rpc_request_definitions.hpp"
+#include <string_view>
+#include <llarp/config/config.hpp>
+#include <oxenmq/oxenmq.h>
+#include <oxenmq/address.h>
+#include <oxen/log/omq_logger.hpp>
+
+namespace llarp::rpc
+{
+  using rpc_input = std::variant<std::monostate, nlohmann::json, oxenc::bt_dict_consumer>;
+
+  inline void
+  parse_request(NoArgs&, rpc_input)
+  {}
+
+  void
+  parse_request(QuicConnect& quicconnect, rpc_input input);
+  void
+  parse_request(QuicListener& quiclistener, rpc_input input);
+  void
+  parse_request(LookupSnode& lookupsnode, rpc_input input);
+  void
+  parse_request(MapExit& mapexit, rpc_input input);
+  void
+  parse_request(UnmapExit& unmapexit, rpc_input input);
+  void
+  parse_request(SwapExits& swapexits, rpc_input input);
+  void
+  parse_request(DNSQuery& dnsquery, rpc_input input);
+  void
+  parse_request(Config& config, rpc_input input);
+
+}  // namespace llarp::rpc

llarp/rpc/rpc_server.hpp

@@ -1,33 +1,165 @@
 #pragma once
 
+#include "rpc_request_definitions.hpp"
+#include "json_bt.hpp"
+#include <nlohmann/json_fwd.hpp>
+#include <stdexcept>
 #include <string_view>
+#include <llarp/config/config.hpp>
 #include <oxenmq/oxenmq.h>
+#include <oxenmq/message.h>
 #include <oxenmq/address.h>
 #include <oxen/log/omq_logger.hpp>
 
 namespace llarp
 {
   struct AbstractRouter;
-}
+}  // namespace llarp
+
+namespace
+{
+  static auto logcat = llarp::log::Cat("lokinet.rpc");
+}  // namespace
 
 namespace llarp::rpc
 {
   using LMQ_ptr = std::shared_ptr<oxenmq::OxenMQ>;
+  using DeferredSend = oxenmq::Message::DeferredSend;
+
+  class RPCServer;
 
-  struct RpcServer
+  //  Stores RPC request callback
+  struct rpc_callback
   {
-    explicit RpcServer(LMQ_ptr, AbstractRouter*);
-    ~RpcServer() = default;
-    void
-    AsyncServeRPC(oxenmq::address addr);
+    using result_type = std::variant<oxenc::bt_value, nlohmann::json, std::string>;
+    //  calls with incoming request data; returns response body or throws exception
+    void (*invoke)(oxenmq::Message&, RPCServer&);
+  };
+
+  //  RPC request registration
+  //    Stores references to RPC requests in a unordered map for ease of reference
+  //    when adding to server. To add endpoints, define in rpc_request_definitions.hpp
+  //    and register in rpc_server.cpp
+  extern const std::unordered_map<std::string, rpc_callback> rpc_request_map;
+
+  //  Exception used to signal various types of errors with a request back to the caller.  This
+  //  exception indicates that the caller did something wrong: bad data, invalid value, etc., but
+  //  don't indicate a local problem (and so we'll log them only at debug).  For more serious,
+  //  internal errors a command should throw some other stl error (e.g. std::runtime_error or
+  //  perhaps std::logic_error), which will result in a local daemon warning (and a generic internal
+  //  error response to the user).
+  //
+  //  For JSON RPC these become an error response with the code as the error.code value and the
+  //  string as the error.message.
+  //  For HTTP JSON these become a 500 Internal Server Error response with the message as the body.
+  //  For OxenMQ the code becomes the first part of the response and the message becomes the
+  //  second part of the response.
+  struct rpc_error : std::runtime_error
+  {
+    /// \param message - a message to send along with the error code (see general description
+    /// above).
+    rpc_error(std::string message)
+        : std::runtime_error{"RPC error: " + message}, message{std::move(message)}
+    {}
+
+    std::string message;
+  };
+
+  template <typename Result_t>
+  void
+  SetJSONResponse(Result_t result, json& j)
+  {
+    j["result"] = result;
+  }
+
+  inline void
+  SetJSONError(std::string_view msg, json& j)
+  {
+    j["error"] = msg;
+  }
+
+  class RPCServer
+  {
+   public:
+    explicit RPCServer(LMQ_ptr, AbstractRouter&);
+    ~RPCServer() = default;
 
-   private:
     void
     HandleLogsSubRequest(oxenmq::Message& m);
 
-    LMQ_ptr m_LMQ;
-    AbstractRouter* const m_Router;
+    void
+    AddCategories();
 
+    void
+    invoke(Halt& halt);
+    void
+    invoke(Version& version);
+    void
+    invoke(Status& status);
+    void
+    invoke(GetStatus& getstatus);
+    void
+    invoke(QuicConnect& quicconnect);
+    void
+    invoke(QuicListener& quiclistener);
+    void
+    invoke(LookupSnode& lookupsnode);
+    void
+    invoke(MapExit& mapexit);
+    void
+    invoke(ListExits& listexits);
+    void
+    invoke(UnmapExit& unmapexit);
+    void
+    invoke(SwapExits& swapexits);
+    void
+    invoke(DNSQuery& dnsquery);
+    void
+    invoke(Config& config);
+
+    LMQ_ptr m_LMQ;
+    AbstractRouter& m_Router;
     oxen::log::PubsubLogger log_subs;
   };
+
+  template <typename RPC>
+  class EndpointHandler
+  {
+   public:
+    RPCServer& server;
+    RPC rpc{};
+
+    EndpointHandler(RPCServer& _server, DeferredSend _replier) : server{_server}
+    {
+      rpc.replier.emplace(std::move(_replier));
+    }
+
+    void
+    operator()()
+    {
+      try
+      {
+        server.invoke(rpc);
+      }
+      catch (const rpc_error& e)
+      {
+        log::info(logcat, "RPC request 'rpc.{}' failed with: {}", rpc.name, e.what());
+        SetJSONError(
+            fmt::format("RPC request 'rpc.{}' failed with: {}", rpc.name, e.what()), rpc.response);
+      }
+      catch (const std::exception& e)
+      {
+        log::info(logcat, "RPC request 'rpc.{}' raised an exception: {}", rpc.name, e.what());
+        SetJSONError(
+            fmt::format("RPC request 'rpc.{}' raised an exception: {}", rpc.name, e.what()),
+            rpc.response);
+      }
+
+      if (rpc.replier.has_value())
+      {
+        rpc.send_response();
+      }
+    }
+  };
+
 }  // namespace llarp::rpc

llarp/service/convotag.cpp

@@ -1,5 +1,5 @@
 #include "convotag.hpp"
-#include "net/ip.hpp"
+#include <llarp/net/ip.hpp>
 
 namespace llarp::service
 {

llarp/service/endpoint.cpp

@@ -1,7 +1,16 @@
-#include <chrono>
-#include <memory>
 #include "endpoint.hpp"
+#include "endpoint_state.hpp"
+#include "endpoint_util.hpp"
+#include "hidden_service_address_lookup.hpp"
+#include "auth.hpp"
+#include "llarp/util/logging.hpp"
+#include "outbound_context.hpp"
+#include "protocol.hpp"
+#include "info.hpp"
+#include "protocol_type.hpp"
 
+#include <llarp/net/ip.hpp>
+#include <llarp/net/ip_range.hpp>
 #include <llarp/dht/context.hpp>
 #include <llarp/dht/key.hpp>
 #include <llarp/dht/messages/findintro.hpp>
@@ -14,29 +23,22 @@
 #include <llarp/nodedb.hpp>
 #include <llarp/profiling.hpp>
 #include <llarp/router/abstractrouter.hpp>
+#include <llarp/router/route_poker.hpp>
 #include <llarp/routing/dht_message.hpp>
 #include <llarp/routing/path_transfer_message.hpp>
-#include "endpoint_state.hpp"
-#include "endpoint_util.hpp"
-#include "hidden_service_address_lookup.hpp"
-#include "net/ip.hpp"
-#include "outbound_context.hpp"
-#include "protocol.hpp"
-#include "service/info.hpp"
-#include "service/protocol_type.hpp"
+
 #include <llarp/util/str.hpp>
 #include <llarp/util/buffer.hpp>
 #include <llarp/util/meta/memfn.hpp>
 #include <llarp/link/link_manager.hpp>
 #include <llarp/tooling/dht_event.hpp>
+#include <llarp/quic/server.hpp>
 #include <llarp/quic/tunnel.hpp>
 #include <llarp/util/priority_queue.hpp>
 
 #include <optional>
+#include <type_traits>
 #include <utility>
-
-#include <llarp/quic/server.hpp>
-#include <llarp/quic/tunnel.hpp>
 #include <uvw.hpp>
 #include <variant>
 
@@ -216,6 +218,75 @@ namespace llarp
       return std::nullopt;
     }
 
+    void
+    Endpoint::map_exit(
+        std::string name,
+        std::string token,
+        std::vector<IPRange> ranges,
+        std::function<void(bool, std::string)> result_handler)
+    {
+      if (ranges.empty())
+      {
+        result_handler(false, "no ranges provided");
+        return;
+      }
+
+      LookupNameAsync(
+          name,
+          [ptr = std::static_pointer_cast<Endpoint>(GetSelf()),
+           name,
+           auth = AuthInfo{token},
+           ranges,
+           result_handler,
+           poker = m_router->routePoker()](auto maybe_addr) {
+            if (not maybe_addr)
+            {
+              result_handler(false, "exit not found: {}"_format(name));
+              return;
+            }
+            if (auto* addr_ptr = std::get_if<Address>(&*maybe_addr))
+            {
+              Address addr{*addr_ptr};
+
+              ptr->SetAuthInfoForEndpoint(addr, auth);
+              ptr->MarkAddressOutbound(addr);
+              auto result = ptr->EnsurePathToService(
+                  addr,
+                  [ptr, name, ranges, result_handler, poker](auto addr, auto* ctx) {
+                    if (ctx == nullptr)
+                    {
+                      result_handler(false, "could not establish flow to {}"_format(name));
+                      return;
+                    }
+
+                    // make a lambda that sends the reply after doing auth
+                    auto apply_result =
+                        [ptr, poker, addr, result_handler, ranges](AuthResult result) {
+                          if (result.code != AuthResultCode::eAuthAccepted)
+                          {
+                            result_handler(false, result.reason);
+                            return;
+                          }
+                          for (const auto& range : ranges)
+                            ptr->MapExitRange(range, addr);
+
+                          if (poker)
+                            poker->Up();
+                          result_handler(true, result.reason);
+                        };
+
+                    ctx->AsyncSendAuth(apply_result);
+                  },
+                  ptr->PathAlignmentTimeout());
+
+              if (not result)
+                result_handler(false, "did not build path to {}"_format(name));
+            }
+            else
+              result_handler(false, "exit via snode not supported");
+          });
+    }
+
     void
     Endpoint::LookupServiceAsync(
         std::string name,
@@ -2087,6 +2158,11 @@ namespace llarp
     void
     Endpoint::SetAuthInfoForEndpoint(Address addr, AuthInfo info)
     {
+      if (info.token.empty())
+      {
+        m_RemoteAuthInfos.erase(addr);
+        return;
+      }
       m_RemoteAuthInfos[addr] = std::move(info);
     }
 
@@ -2113,6 +2189,26 @@ namespace llarp
         LogInfo(Name(), " unmap ", item.first, " exit range mapping");
         return true;
       });
+
+      if (m_ExitMap.Empty())
+        m_router->routePoker()->Down();
+    }
+
+    void
+    Endpoint::UnmapRangeByExit(IPRange range, std::string exit)
+    {
+      // unmap all ranges that match the given exit when hot swapping
+      m_ExitMap.RemoveIf([&](const auto& item) -> bool {
+        if ((range.Contains(item.first)) and (item.second.ToString() == exit))
+        {
+          log::info(logcat, "{} unmap {} range mapping to exit node {}", Name(), item.first, exit);
+          return true;
+        }
+        return false;
+      });
+
+      if (m_ExitMap.Empty())
+        m_router->routePoker()->Down();
     }
 
     std::optional<AuthInfo>

llarp/service/endpoint.hpp

@@ -6,24 +6,27 @@
 #include <llarp/net/net.hpp>
 #include <llarp/path/path.hpp>
 #include <llarp/path/pathbuilder.hpp>
-#include "address.hpp"
-#include "handler.hpp"
-#include "identity.hpp"
-#include "pendingbuffer.hpp"
-#include "protocol.hpp"
-#include "sendcontext.hpp"
-#include "service/protocol_type.hpp"
-#include "session.hpp"
-#include "lookup.hpp"
 #include <llarp/util/compare_ptr.hpp>
+
+// --- begin kitchen sink headers ----
+#include <llarp/service/address.hpp>
+#include <llarp/service/handler.hpp>
+#include <llarp/service/identity.hpp>
+#include <llarp/service/pendingbuffer.hpp>
+#include <llarp/service/protocol.hpp>
+#include <llarp/service/sendcontext.hpp>
+#include <llarp/service/protocol_type.hpp>
+#include <llarp/service/session.hpp>
+#include <llarp/service/lookup.hpp>
+#include <llarp/service/endpoint_types.hpp>
+#include <llarp/endpoint_base.hpp>
+#include <llarp/service/auth.hpp>
+// ----- end kitchen sink headers -----
+
 #include <optional>
 #include <unordered_map>
 #include <variant>
 #include <oxenc/variant.h>
-#include "endpoint_types.hpp"
-#include "llarp/endpoint_base.hpp"
-
-#include "auth.hpp"
 
 #include <llarp/vpn/egres_packet_router.hpp>
 #include <llarp/dns/server.hpp>
@@ -281,6 +284,16 @@ namespace llarp
       void
       UnmapExitRange(IPRange range);
 
+      void
+      UnmapRangeByExit(IPRange range, std::string exit);
+
+      void
+      map_exit(
+          std::string name,
+          std::string token,
+          std::vector<IPRange> ranges,
+          std::function<void(bool, std::string)> result);
+
       void
       PutLookup(IServiceLookup* lookup, uint64_t txid) override;
 

llarp/service/intro.cpp

@@ -1,5 +1,5 @@
 #include "intro.hpp"
-#include "util/time.hpp"
+#include <llarp/util/time.hpp>
 
 namespace llarp
 {

llarp/service/outbound_context.cpp

@@ -1,16 +1,15 @@
 #include "outbound_context.hpp"
-
-#include <llarp/router/abstractrouter.hpp>
 #include "async_key_exchange.hpp"
 #include "hidden_service_address_lookup.hpp"
 #include "endpoint.hpp"
+#include "endpoint_util.hpp"
+#include "protocol_type.hpp"
+
+#include <llarp/router/abstractrouter.hpp>
 #include <llarp/nodedb.hpp>
 #include <llarp/profiling.hpp>
 #include <llarp/util/meta/memfn.hpp>
 
-#include "endpoint_util.hpp"
-#include "service/protocol_type.hpp"
-
 #include <random>
 #include <algorithm>
 

llarp/service/sendcontext.cpp

@@ -125,8 +125,7 @@ namespace llarp
     void
     SendContext::AsyncSendAuth(std::function<void(AuthResult)> resultHandler)
     {
-      const auto maybe = m_Endpoint->MaybeGetAuthInfoForEndpoint(remoteIdent.Addr());
-      if (maybe.has_value())
+      if (const auto maybe = m_Endpoint->MaybeGetAuthInfoForEndpoint(remoteIdent.Addr()))
       {
         // send auth message
         const llarp_buffer_t authdata{maybe->token};
@@ -134,7 +133,7 @@ namespace llarp
         authResultListener = resultHandler;
       }
       else
-        resultHandler({AuthResultCode::eAuthFailed, "no auth for given endpoint"});
+        resultHandler({AuthResultCode::eAuthAccepted, "no auth needed"});
     }
 
     void

llarp/tooling/router_hive.hpp

@@ -3,8 +3,8 @@
 #include "router_event.hpp"
 
 #include <llarp.hpp>
-#include <config/config.hpp>
-#include <tooling/hive_context.hpp>
+#include <llarp/config/config.hpp>
+#include <llarp/tooling/hive_context.hpp>
 
 #include <vector>
 #include <deque>

llarp/util/thread/queue_manager.hpp

@@ -1,5 +1,5 @@
 #pragma once
-
+#include <cstdint>
 #include <algorithm>
 #include <atomic>
 #include <cassert>

llarp/vpn/win32.cpp

@@ -1,4 +1,4 @@
-#include "vpn/win32.hpp"
+#include "win32.hpp"
 #include <llarp/win32/windivert.hpp>
 #include <llarp/win32/wintun.hpp>
 #include <fmt/core.h>

pybind/llarp/config.cpp

@@ -80,7 +80,7 @@ namespace llarp
     py::class_<ApiConfig>(mod, "ApiConfig")
         .def(py::init<>())
         .def_readwrite("enableRPCServer", &ApiConfig::m_enableRPCServer)
-        .def_readwrite("rpcBindAddr", &ApiConfig::m_rpcBindAddr);
+        .def_readwrite("rpcBindAddresses", &ApiConfig::m_rpcBindAddresses);
 
     py::class_<LokidConfig>(mod, "LokidConfig")
         .def(py::init<>())

pybind/llarp/context.cpp

@@ -3,7 +3,8 @@
 #include <llarp/tooling/hive_context.hpp>
 #include <llarp/router/router.hpp>
 #include <llarp/handlers/pyhandler.hpp>
-#include "service/protocol_type.hpp"
+#include <llarp/service/protocol_type.hpp>
+
 namespace llarp
 {
   void

test/CMakeLists.txt

@@ -60,8 +60,4 @@ if(WIN32)
     target_link_libraries(testAll PUBLIC ws2_32 iphlpapi shlwapi)
 endif()
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
-    target_link_directories(testAll PRIVATE /usr/local/lib)
-endif()
-
 add_custom_target(check COMMAND testAll)

test/check_main.cpp

@@ -1,8 +1,8 @@
 #define CATCH_CONFIG_RUNNER
 #include <catch2/catch.hpp>
 
-#include <util/logging.hpp>
-#include <util/service_manager.hpp>
+#include <llarp/util/logging.hpp>
+#include <llarp/util/service_manager.hpp>
 
 #ifdef _WIN32
 #include <winsock2.h>

test/config/test_llarp_config_definition.cpp

@@ -1,4 +1,4 @@
-#include <config/definition.hpp>
+#include <llarp/config/definition.hpp>
 
 #include <catch2/catch.hpp>
 

test/config/test_llarp_config_ini.cpp

@@ -1,4 +1,4 @@
-#include <config/ini.hpp>
+#include <llarp/config/ini.hpp>
 
 #include <catch2/catch.hpp>
 
@@ -42,12 +42,12 @@ TEST_CASE("ConfigParser", "[config]")
 
   SECTION("No key")
   {
-    REQUIRE_FALSE(parser.LoadFromStr("[test]\n=1090\n"));
+    REQUIRE_THROWS(parser.LoadFromStr("[test]\n=1090\n"));
   }
 
   SECTION("Parse invalid")
   {
-    REQUIRE_FALSE(
+    REQUIRE_THROWS(
         parser.LoadFromStr("srged5ghe5\nf34wtge5\nw34tgfs4ygsd5yg=4;\n#"
                            "g4syhgd5\n"));
   }

test/config/test_llarp_config_output.cpp

@@ -1,4 +1,4 @@
-#include <config/definition.hpp>
+#include <llarp/config/definition.hpp>
 
 #include <catch2/catch.hpp>
 

test/crypto/test_llarp_crypto.cpp

@@ -1,4 +1,4 @@
-#include <crypto/crypto_libsodium.hpp>
+#include <llarp/crypto/crypto_libsodium.hpp>
 
 #include <iostream>
 

test/crypto/test_llarp_crypto_types.cpp

@@ -1,9 +1,9 @@
-#include <crypto/types.hpp>
+#include <llarp/crypto/types.hpp>
 
 #include <fstream>
 #include <string>
 
-#include <test_util.hpp>
+#include "test_util.hpp"
 #include <catch2/catch.hpp>
 
 extern "C" {

test/crypto/test_llarp_key_manager.cpp

@@ -1,14 +1,15 @@
-#include <config/key_manager.hpp>
+#include "llarp_test.hpp"
+#include "test_util.hpp"
 
-#include <crypto/crypto.hpp>
-#include <crypto/crypto_libsodium.hpp>
-#include <llarp_test.hpp>
+#include <llarp/config/key_manager.hpp>
+
+#include <llarp/crypto/crypto.hpp>
+#include <llarp/crypto/crypto_libsodium.hpp>
 
 #include <functional>
 #include <random>
 
 #include <string>
-#include <test_util.hpp>
 #include <catch2/catch.hpp>
 
 using namespace ::llarp;

test/dns/test_llarp_dns_dns.cpp

@@ -1,11 +1,11 @@
 #include <catch2/catch.hpp>
-#include <dns/dns.hpp>
-#include <dns/message.hpp>
-#include <dns/name.hpp>
-#include <dns/rr.hpp>
-#include <net/net.hpp>
-#include <net/ip.hpp>
-#include <util/buffer.hpp>
+#include <llarp/dns/dns.hpp>
+#include <llarp/dns/message.hpp>
+#include <llarp/dns/name.hpp>
+#include <llarp/dns/rr.hpp>
+#include <llarp/net/net.hpp>
+#include <llarp/net/ip.hpp>
+#include <llarp/util/buffer.hpp>
 
 #include <algorithm>
 

test/llarp_test.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <crypto/crypto_libsodium.hpp>
+#include <llarp/crypto/crypto_libsodium.hpp>
 #include <catch2/catch.hpp>
 
 namespace llarp::test

test/net/test_ip_address.cpp

@@ -1,4 +1,4 @@
-#include <net/ip_address.hpp>
+#include <llarp/net/ip_address.hpp>
 
 #include <catch2/catch.hpp>
 

test/net/test_llarp_net.cpp

@@ -1,7 +1,7 @@
-#include <net/net_int.hpp>
-#include <net/ip.hpp>
-#include <net/ip_range.hpp>
-#include <net/net.hpp>
+#include <llarp/net/net_int.hpp>
+#include <llarp/net/ip.hpp>
+#include <llarp/net/ip_range.hpp>
+#include <llarp/net/net.hpp>
 #include <oxenc/hex.h>
 
 #include <catch2/catch.hpp>

test/net/test_sock_addr.cpp

@@ -1,6 +1,6 @@
-#include <util/mem.hpp>
-#include <net/sock_addr.hpp>
-#include <net/net_if.hpp>
+#include <llarp/util/mem.hpp>
+#include <llarp/net/sock_addr.hpp>
+#include <llarp/net/net_if.hpp>
 #include <llarp/util/logging.hpp>
 
 #include <catch2/catch.hpp>

test/nodedb/test_nodedb.cpp

@@ -1,8 +1,8 @@
 #include <catch2/catch.hpp>
-#include "config/config.hpp"
 
-#include <router_contact.hpp>
-#include <nodedb.hpp>
+#include <llarp/config/config.hpp>
+#include <llarp/router_contact.hpp>
+#include <llarp/nodedb.hpp>
 
 using llarp_nodedb = llarp::NodeDB;
 

test/path/test_path.cpp

@@ -1,4 +1,4 @@
-#include <path/path.hpp>
+#include <llarp/path/path.hpp>
 #include <catch2/catch.hpp>
 
 using Path_t   = llarp::path::Path;

test/peerstats/test_peer_db.cpp

@@ -1,12 +1,12 @@
-#include <peerstats/peer_db.hpp>
+#include <llarp/peerstats/peer_db.hpp>
 #include <test_util.hpp>
 
 #include <numeric>
 #include <catch2/catch.hpp>
-#include "peerstats/types.hpp"
-#include "router_contact.hpp"
-#include "util/logging.hpp"
-#include "util/time.hpp"
+#include <llarp/peerstats/types.hpp>
+#include <llarp/router_contact.hpp>
+#include <llarp/util/logging.hpp>
+#include <llarp/util/time.hpp>
 
 TEST_CASE("Test PeerDb PeerStats memory storage", "[PeerDb]")
 {

test/peerstats/test_peer_types.cpp

@@ -1,5 +1,5 @@
 #include <numeric>
-#include <peerstats/types.hpp>
+#include <llarp/peerstats/types.hpp>
 #include <test_util.hpp>
 
 #include <catch2/catch.hpp>