Merge pull request #2224 from tewinget/rc-gossip

RC gossip

llarp/CMakeLists.txt

@@ -12,7 +12,6 @@ endfunction()
 
 lokinet_add_library(lokinet-cryptography
   crypto/crypto.cpp
-  crypto/encrypted_frame.cpp
   crypto/types.cpp
 )
 
@@ -25,7 +24,6 @@ lokinet_add_library(lokinet-core-utils
   exit/policy.cpp   # net/
   handlers/exit.cpp # link/ exit/
   handlers/tun.cpp
-  router/rc_gossiper.cpp
   service/auth.cpp  # config/
   service/context.cpp
   service/identity.cpp
@@ -110,7 +108,6 @@ lokinet_add_library(lokinet-time-place
 # lokinet-platform holds all platform specific code
 lokinet_add_library(lokinet-platform
   net/interface_info.cpp
-  router/rc_lookup_handler.cpp
   vpn/packet_router.cpp
   vpn/platform.cpp
 )
@@ -207,7 +204,6 @@ lokinet_add_library(lokinet-config
 
 # All path objects; link directly to lokinet-core
 lokinet_add_library(lokinet-path
-  messages/relay.cpp
   path/abstracthophandler.cpp
   path/path.cpp
   path/path_context.cpp

llarp/consensus/reachability_testing.cpp

@@ -104,7 +104,7 @@ namespace llarp::consensus
     // We exhausted the queue so repopulate it and try again
 
     testing_queue.clear();
-    const auto all = router->router_whitelist();
+    const auto all = router->get_whitelist();
     testing_queue.insert(testing_queue.begin(), all.begin(), all.end());
 
     std::shuffle(testing_queue.begin(), testing_queue.end(), llarp::csrng);

llarp/crypto/crypto.cpp

@@ -65,7 +65,7 @@ namespace llarp
 
   static bool
   dh_client_priv(
-      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const TunnelNonce& n)
+      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const SymmNonce& n)
   {
     llarp::SharedSecret dh_result;
 
@@ -81,13 +81,13 @@ namespace llarp
 
   static bool
   dh_server_priv(
-      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const TunnelNonce& n)
+      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const SymmNonce& n)
   {
     llarp::SharedSecret dh_result;
 
     if (dh(dh_result, pk, sk.toPublic(), pk.data(), sk))
     {
-      return crypto_generichash_blake2b(shared.data(), 32, n.data(), 32, dh_result.data(), 32)
+      return crypto_generichash_blake2b(shared.data(), 32, n.data(), n.size(), dh_result.data(), 32)
           != -1;
     }
 
@@ -102,7 +102,7 @@ namespace llarp
 
     if (dh(dh_result.data(), pk, sk, pk, sk))
     {
-      return crypto_generichash_blake2b(shared, 32, nonce, 32, dh_result.data(), 32) != -1;
+      return crypto_generichash_blake2b(shared, 32, nonce, 24, dh_result.data(), 32) != -1;
     }
 
     llarp::LogWarn("crypto::dh_server - dh failed");
@@ -143,7 +143,7 @@ namespace llarp
   }
 
   bool
-  crypto::xchacha20(uint8_t* buf, size_t size, const SharedSecret& k, const TunnelNonce& n)
+  crypto::xchacha20(uint8_t* buf, size_t size, const SharedSecret& k, const SymmNonce& n)
   {
     return xchacha20(buf, size, n.data(), k.data());
   }
@@ -154,16 +154,31 @@ namespace llarp
     return crypto_stream_xchacha20_xor(buf, buf, size, nonce, secret) == 0;
   }
 
+  // do a round of chacha for and return the nonce xor the given xor_factor
+  SymmNonce
+  crypto::onion(
+      unsigned char* buf,
+      size_t size,
+      const SharedSecret& k,
+      const SymmNonce& nonce,
+      const SymmNonce& xor_factor)
+  {
+    if (!crypto::xchacha20(buf, size, k, nonce))
+      throw std::runtime_error{"chacha failed during onion step"};
+
+    return nonce ^ xor_factor;
+  }
+
   bool
   crypto::dh_client(
-      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const TunnelNonce& n)
+      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const SymmNonce& n)
   {
     return dh_client_priv(shared, pk, sk, n);
   }
   /// path dh relay side
   bool
   crypto::dh_server(
-      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const TunnelNonce& n)
+      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const SymmNonce& n)
   {
     return dh_server_priv(shared, pk, sk, n);
   }
@@ -177,20 +192,6 @@ namespace llarp
   {
     return dh_server_priv(shared_secret, other_pk, local_pk, nonce);
   }
-  /// transport dh client side
-  bool
-  crypto::transport_dh_client(
-      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const TunnelNonce& n)
-  {
-    return dh_client_priv(shared, pk, sk, n);
-  }
-  /// transport dh server side
-  bool
-  crypto::transport_dh_server(
-      llarp::SharedSecret& shared, const PubKey& pk, const SecretKey& sk, const TunnelNonce& n)
-  {
-    return dh_server_priv(shared, pk, sk, n);
-  }
 
   bool
   crypto::shorthash(ShortHash& result, uint8_t* buf, size_t size)

llarp/crypto/crypto.hpp

@@ -23,28 +23,30 @@ namespace llarp
 
     /// xchacha symmetric cipher
     bool
-    xchacha20(uint8_t*, size_t size, const SharedSecret&, const TunnelNonce&);
+    xchacha20(uint8_t*, size_t size, const SharedSecret&, const SymmNonce&);
     bool
     xchacha20(uint8_t*, size_t size, const uint8_t*, const uint8_t*);
 
+    SymmNonce
+    onion(
+        unsigned char* buf,
+        size_t size,
+        const SharedSecret& k,
+        const SymmNonce& nonce,
+        const SymmNonce& xor_factor);
+
     /// path dh creator's side
     bool
-    dh_client(SharedSecret&, const PubKey&, const SecretKey&, const TunnelNonce&);
+    dh_client(SharedSecret&, const PubKey&, const SecretKey&, const SymmNonce&);
     /// path dh relay side
     bool
-    dh_server(SharedSecret&, const PubKey&, const SecretKey&, const TunnelNonce&);
+    dh_server(SharedSecret&, const PubKey&, const SecretKey&, const SymmNonce&);
     bool
     dh_server(
         uint8_t* shared_secret,
         const uint8_t* other_pk,
         const uint8_t* local_pk,
         const uint8_t* nonce);
-    /// transport dh client side
-    bool
-    transport_dh_client(SharedSecret&, const PubKey&, const SecretKey&, const TunnelNonce&);
-    /// transport dh server side
-    bool
-    transport_dh_server(SharedSecret&, const PubKey&, const SecretKey&, const TunnelNonce&);
     /// blake2b 256 bit
     bool
     shorthash(ShortHash&, uint8_t*, size_t size);

llarp/crypto/encrypted_frame.cpp

@@ -1,127 +0,0 @@
-#include "encrypted_frame.hpp"
-
-#include "crypto.hpp"
-
-#include <llarp/util/logging.hpp>
-
-namespace llarp
-{
-  bool
-  EncryptedFrame::DoEncrypt(const SharedSecret& shared, bool noDH)
-  {
-    uint8_t* hash_ptr = data();
-    uint8_t* nonce_ptr = hash_ptr + SHORTHASHSIZE;
-    uint8_t* pubkey_ptr = nonce_ptr + TUNNONCESIZE;
-    uint8_t* body_ptr = pubkey_ptr + PUBKEYSIZE;
-
-    if (noDH)
-    {
-      crypto::randbytes(nonce_ptr, TUNNONCESIZE);
-      crypto::randbytes(pubkey_ptr, PUBKEYSIZE);
-    }
-
-    TunnelNonce nonce(nonce_ptr);
-
-    // encrypt body
-    if (!crypto::xchacha20(body_ptr, size() - EncryptedFrameOverheadSize, shared, nonce))
-    {
-      llarp::LogError("encrypt failed");
-      return false;
-    }
-
-    if (!crypto::hmac(hash_ptr, nonce_ptr, size() - SHORTHASHSIZE, shared))
-    {
-      llarp::LogError("Failed to generate message auth");
-      return false;
-    }
-
-    return true;
-  }
-
-  bool
-  EncryptedFrame::EncryptInPlace(const SecretKey& ourSecretKey, const PubKey& otherPubkey)
-  {
-    // format of frame is
-    // <32 bytes keyed hash of following data>
-    // <32 bytes nonce>
-    // <32 bytes pubkey>
-    // <N bytes encrypted payload>
-    //
-    byte_t* hash = data();
-    byte_t* noncePtr = hash + SHORTHASHSIZE;
-    byte_t* pubkey = noncePtr + TUNNONCESIZE;
-
-    SharedSecret shared;
-
-    // set our pubkey
-    memcpy(pubkey, ourSecretKey.toPublic().data(), PUBKEYSIZE);
-    // randomize nonce
-    crypto::randbytes(noncePtr, TUNNONCESIZE);
-    TunnelNonce nonce(noncePtr);
-
-    // derive shared key
-    if (!crypto::dh_client(shared, otherPubkey, ourSecretKey, nonce))
-    {
-      llarp::LogError("DH failed");
-      return false;
-    }
-
-    return DoEncrypt(shared, false);
-  }
-
-  bool
-  EncryptedFrame::DoDecrypt(const SharedSecret& shared)
-  {
-    uint8_t* hash_ptr = data();
-    uint8_t* nonce_ptr = hash_ptr + SHORTHASHSIZE;
-    uint8_t* body_ptr = hash_ptr + EncryptedFrameOverheadSize;
-
-    TunnelNonce nonce(nonce_ptr);
-
-    ShortHash digest;
-    if (!crypto::hmac(digest.data(), nonce_ptr, size() - SHORTHASHSIZE, shared))
-    {
-      llarp::LogError("Digest failed");
-      return false;
-    }
-
-    if (!std::equal(digest.begin(), digest.end(), hash_ptr))
-    {
-      llarp::LogError("message authentication failed");
-      return false;
-    }
-
-    if (!crypto::xchacha20(body_ptr, size() - EncryptedFrameOverheadSize, shared, nonce))
-    {
-      llarp::LogError("decrypt failed");
-      return false;
-    }
-
-    return true;
-  }
-
-  bool
-  EncryptedFrame::DecryptInPlace(const SecretKey& ourSecretKey)
-  {
-    // format of frame is
-    // <32 bytes keyed hash of following data>
-    // <32 bytes nonce>
-    // <32 bytes pubkey>
-    // <N bytes encrypted payload>
-    //
-    byte_t* noncePtr = data() + SHORTHASHSIZE;
-    TunnelNonce nonce(noncePtr);
-    PubKey otherPubkey(noncePtr + TUNNONCESIZE);
-
-    SharedSecret shared;
-
-    // use dh_server because we are not the creator of this message
-    if (!crypto::dh_server(shared, otherPubkey, ourSecretKey, nonce))
-    {
-      llarp::LogError("DH failed");
-      return false;
-    }
-
-    return DoDecrypt(shared);
-  }
-}  // namespace llarp

llarp/crypto/encrypted_frame.hpp

@@ -1,87 +0,0 @@
-#pragma once
-
-#include "encrypted.hpp"
-#include "types.hpp"
-
-#include <llarp/util/buffer.hpp>
-#include <llarp/util/mem.h>
-
-#include <utility>
-
-namespace llarp
-{
-  static constexpr size_t EncryptedFrameOverheadSize = PUBKEYSIZE + TUNNONCESIZE + SHORTHASHSIZE;
-  static constexpr size_t EncryptedFrameBodySize = 128 * 6;
-  static constexpr size_t EncryptedFrameSize = EncryptedFrameOverheadSize + EncryptedFrameBodySize;
-
-  struct EncryptedFrame : public Encrypted<EncryptedFrameSize>
-  {
-    EncryptedFrame() : EncryptedFrame(EncryptedFrameBodySize)
-    {}
-
-    EncryptedFrame(size_t sz)
-        : Encrypted<EncryptedFrameSize>(
-            std::min(sz, EncryptedFrameBodySize) + EncryptedFrameOverheadSize)
-    {}
-
-    void
-    Resize(size_t sz)
-    {
-      if (sz <= EncryptedFrameSize)
-      {
-        _sz = sz;
-        UpdateBuffer();
-      }
-    }
-
-    bool
-    DoEncrypt(const SharedSecret& shared, bool noDH = false);
-
-    bool
-    DecryptInPlace(const SecretKey& seckey);
-
-    bool
-    DoDecrypt(const SharedSecret& shared);
-
-    bool
-    EncryptInPlace(const SecretKey& seckey, const PubKey& other);
-  };
-
-  template <typename User>
-  struct AsyncFrameDecrypter
-  {
-    using User_ptr = std::shared_ptr<User>;
-    using DecryptHandler = std::function<void(llarp_buffer_t*, User_ptr)>;
-
-    void
-    Decrypt(User_ptr user)
-    {
-      if (target.DecryptInPlace(seckey))
-      {
-        auto buf = target.Buffer();
-        buf->cur = buf->base + EncryptedFrameOverheadSize;
-        result(buf, user);
-      }
-      else
-        result(nullptr, user);
-    }
-
-    AsyncFrameDecrypter(const SecretKey& secretkey, DecryptHandler h)
-        : result(std::move(h)), seckey(secretkey)
-    {}
-
-    DecryptHandler result;
-    const SecretKey& seckey;
-    EncryptedFrame target;
-
-    using WorkFunc_t = std::function<void(void)>;
-    using WorkerFunction_t = std::function<void(WorkFunc_t)>;
-
-    void
-    AsyncDecrypt(const EncryptedFrame& frame, User_ptr u, WorkerFunction_t worker)
-    {
-      target = frame;
-      worker([this, u = std::move(u)]() mutable { Decrypt(std::move(u)); });
-    }
-  };
-}  // namespace llarp

llarp/crypto/types.hpp

@@ -164,10 +164,6 @@ namespace llarp
   /// PKE(result, publickey, secretkey, nonce)
   using path_dh_func = bool (*)(SharedSecret&, const PubKey&, const SecretKey&, const TunnelNonce&);
 
-  /// TKE(result, publickey, secretkey, nonce)
-  using transport_dh_func =
-      bool (*)(SharedSecret&, const PubKey&, const SecretKey&, const TunnelNonce&);
-
   /// SH(result, body)
   using shorthash_func = bool (*)(ShortHash&, const llarp_buffer_t&);
 }  // namespace llarp

llarp/endpoint_base.hpp

@@ -131,7 +131,7 @@ namespace llarp
         llarp_time_t timeout) = 0;
 
     virtual void
-    lookup_name(std::string name, std::function<void(oxen::quic::message)> func) = 0;
+    lookup_name(std::string name, std::function<void(std::string, bool)> func) = 0;
 
     virtual const EventLoop_ptr&
     Loop() = 0;

llarp/ev/libuv.hpp

@@ -2,7 +2,6 @@
 #include "ev.hpp"
 #include "udp_handle.hpp"
 
-#include <llarp/util/meta/memfn.hpp>
 #include <llarp/util/thread/queue.hpp>
 
 // #include <uvw.hpp>

llarp/exit/endpoint.cpp

@@ -57,7 +57,8 @@ namespace llarp::exit
     if (!parent->UpdateEndpointPath(remote_signkey, nextPath))
       return false;
     const RouterID us{parent->GetRouter()->pubkey()};
-    current_path = parent->GetRouter()->path_context().GetByUpstream(us, nextPath);
+    // TODO: is this getting a Path or a TransitHop?
+    // current_path = parent->GetRouter()->path_context().GetByUpstream(us, nextPath);
     return true;
   }
 

llarp/exit/session.cpp

@@ -272,35 +272,8 @@ namespace llarp::exit
 
       if (numHops == 1)
       {
-        auto r = router;
+        if (const auto maybe = router->node_db()->get_rc(exit_router); maybe.has_value())
-        if (const auto maybe = r->node_db()->get_rc(exit_router); maybe.has_value())
+          router->connect_to(*maybe);
-          r->connect_to(*maybe);
-        else
-          r->lookup_router(exit_router, [r](oxen::quic::message m) mutable {
-            if (m)
-            {
-              std::string payload;
-
-              try
-              {
-                oxenc::bt_dict_consumer btdc{m.body()};
-                payload = btdc.require<std::string>("RC");
-              }
-              catch (...)
-              {
-                log::warning(link_cat, "Failed to parse Find Router response!");
-                throw;
-              }
-
-              RemoteRC result{std::move(payload)};
-              r->node_db()->put_rc_if_newer(result);
-              r->connect_to(result);
-            }
-            else
-            {
-              r->link_manager().handle_find_router_error(std::move(m));
-            }
-          });
       }
       else if (UrgentBuild(now))
         BuildOneAlignedTo(exit_router);

llarp/handlers/exit.cpp

@@ -2,8 +2,8 @@
 
 #include <llarp/dns/dns.hpp>
 #include <llarp/net/net.hpp>
+#include <llarp/nodedb.hpp>
 #include <llarp/path/path_context.hpp>
-#include <llarp/router/rc_lookup_handler.hpp>
 #include <llarp/router/router.hpp>
 #include <llarp/service/protocol_type.hpp>
 
@@ -20,9 +20,9 @@ namespace llarp::handlers
   ExitEndpoint::~ExitEndpoint() = default;
 
   void
-  ExitEndpoint::lookup_name(std::string, std::function<void(oxen::quic::message)>)
+  ExitEndpoint::lookup_name(std::string, std::function<void(std::string, bool)>)
   {
-    // TODO: implement me
+    // TODO: implement me (or does EndpointBase having this method as virtual even make sense?)
   }
 
   void
@@ -243,9 +243,8 @@ namespace llarp::handlers
     {
       if (msg.questions[0].IsName("random.snode"))
       {
-        RouterID random;
+        if (auto random = GetRouter()->GetRandomGoodRouter())
-        if (GetRouter()->GetRandomGoodRouter(random))
+          msg.AddCNAMEReply(random->ToString(), 1);
-          msg.AddCNAMEReply(random.ToString(), 1);
         else
           msg.AddNXReply();
       }
@@ -263,11 +262,10 @@ namespace llarp::handlers
       const bool isV4 = msg.questions[0].qtype == dns::qTypeA;
       if (msg.questions[0].IsName("random.snode"))
       {
-        RouterID random;
+        if (auto random = GetRouter()->GetRandomGoodRouter())
-        if (GetRouter()->GetRandomGoodRouter(random))
         {
-          msg.AddCNAMEReply(random.ToString(), 1);
+          msg.AddCNAMEReply(random->ToString(), 1);
-          auto ip = ObtainServiceNodeIP(random);
+          auto ip = ObtainServiceNodeIP(*random);
           msg.AddINReply(ip, false);
         }
         else
@@ -333,7 +331,7 @@ namespace llarp::handlers
   void
   ExitEndpoint::ObtainSNodeSession(const RouterID& rid, exit::SessionReadyFunc obtain_cb)
   {
-    if (not router->rc_lookup_handler().is_session_allowed(rid))
+    if (not router->node_db()->is_connection_allowed(rid))
     {
       obtain_cb(nullptr);
       return;
@@ -766,7 +764,9 @@ namespace llarp::handlers
   {
     if (wantInternet && !permit_exit)
       return false;
-    path::HopHandler_ptr handler = router->path_context().GetByUpstream(router->pubkey(), path);
+    // TODO: is this getting a path or a transit hop or...somehow possibly either?
+    // path::HopHandler_ptr handler = router->path_context().GetByUpstream(router->pubkey(), path);
+    path::HopHandler_ptr handler{};
     if (handler == nullptr)
       return false;
     auto ip = GetIPForIdent(pk);

llarp/handlers/exit.hpp

@@ -49,7 +49,7 @@ namespace llarp
           llarp_time_t timeout) override;
 
       void
-      lookup_name(std::string name, std::function<void(oxen::quic::message)> func) override;
+      lookup_name(std::string name, std::function<void(std::string, bool)> func) override;
 
       const EventLoop_ptr&
       Loop() override;

llarp/handlers/tun.cpp

@@ -606,34 +606,11 @@ namespace llarp::handlers
       RouterID snode;
       if (snode.FromString(qname))
       {
-        router()->lookup_router(
+        if (auto rc = router()->node_db()->get_rc(snode))
-            snode, [r = router(), msg = std::move(msg), reply](oxen::quic::message m) mutable {
+          msg.AddTXTReply(std::string{rc->view()});
-              if (m)
+        else
-              {
+          msg.AddNXReply();
-                std::string payload;
+        reply(msg);
-
-                try
-                {
-                  oxenc::bt_dict_consumer btdc{m.body()};
-                  payload = btdc.require<std::string>("RC");
-                }
-                catch (...)
-                {
-                  log::warning(link_cat, "Failed to parse Find Router response!");
-                  throw;
-                }
-
-                r->node_db()->put_rc_if_newer(RemoteRC{payload});
-                msg.AddTXTReply(payload);
-              }
-              else
-              {
-                msg.AddNXReply();
-                r->link_manager().handle_find_router_error(std::move(m));
-              }
-
-              reply(msg);
-            });
 
         return true;
       }
@@ -683,28 +660,17 @@ namespace llarp::handlers
       }
       else if (service::is_valid_name(ons_name))
       {
-        lookup_name(ons_name, [msg, ons_name, reply](oxen::quic::message m) mutable {
+        lookup_name(
-          if (m)
+            ons_name, [msg, ons_name, reply](std::string name_result, bool success) mutable {
-          {
+              if (success)
-            std::string result;
+              {
-            try
+                msg.AddMXReply(name_result, 1);
-            {
+              }
-              oxenc::bt_dict_consumer btdc{m.body()};
+              else
-              result = btdc.require<std::string>("NAME");
+                msg.AddNXReply();
-            }
-            catch (...)
-            {
-              log::warning(logcat, "Failed to parse find name response!");
-              throw;
-            }
-
-            msg.AddMXReply(result, 1);
-          }
-          else
-            msg.AddNXReply();
 
-          reply(msg);
+              reply(msg);
-        });
+            });
 
         return true;
       }
@@ -716,10 +682,9 @@ namespace llarp::handlers
     {
       if (is_random_snode(msg))
       {
-        RouterID random;
+        if (auto random = router()->GetRandomGoodRouter())
-        if (router()->GetRandomGoodRouter(random))
         {
-          msg.AddCNAMEReply(random.ToString(), 1);
+          msg.AddCNAMEReply(random->ToString(), 1);
         }
         else
           msg.AddNXReply();
@@ -766,11 +731,10 @@ namespace llarp::handlers
       // on MacOS this is a typeA query
       else if (is_random_snode(msg))
       {
-        RouterID random;
+        if (auto random = router()->GetRandomGoodRouter())
-        if (router()->GetRandomGoodRouter(random))
         {
-          msg.AddCNAMEReply(random.ToString(), 1);
+          msg.AddCNAMEReply(random->ToString(), 1);
-          return ReplyToSNodeDNSWhenReady(random, std::make_shared<dns::Message>(msg), isV6);
+          return ReplyToSNodeDNSWhenReady(*random, std::make_shared<dns::Message>(msg), isV6);
         }
 
         msg.AddNXReply();
@@ -837,29 +801,15 @@ namespace llarp::handlers
              ons_name,
              isV6,
              reply,
-             ReplyToDNSWhenReady](oxen::quic::message m) mutable {
+             ReplyToDNSWhenReady](std::string name_result, bool success) mutable {
-              if (m)
+              if (not success)
-              {
-                std::string name;
-                try
-                {
-                  oxenc::bt_dict_consumer btdc{m.body()};
-                  name = btdc.require<std::string>("NAME");
-                }
-                catch (...)
-                {
-                  log::warning(logcat, "Failed to parse find name response!");
-                  throw;
-                }
-
-                ReplyToDNSWhenReady(name, msg, isV6);
-              }
-              else
               {
                 log::warning(logcat, "{} (ONS name: {}) not resolved", name, ons_name);
                 msg->AddNXReply();
                 reply(*msg);
               }
+
+              ReplyToDNSWhenReady(name_result, msg, isV6);
             });
         return true;
       }

llarp/link/contacts.cpp

@@ -71,13 +71,6 @@ namespace llarp
     return obj;
   }
 
-  bool
-  Contacts::lookup_router(const RouterID& rid, std::function<void(oxen::quic::message)> func)
-  {
-    return _router.send_control_message(
-        rid, "find_router", FindRouterMessage::serialize(rid, false, false), std::move(func));
-  }
-
   void
   Contacts::put_rc_node_async(const dht::RCNode& val)
   {

llarp/link/contacts.hpp

@@ -45,9 +45,6 @@ namespace llarp
     util::StatusObject
     ExtractStatus() const;
 
-    bool
-    lookup_router(const RouterID&, std::function<void(oxen::quic::message)> = nullptr);
-
     void
     put_rc_node_async(const dht::RCNode& val);
 

llarp/link/link_manager.hpp

@@ -4,7 +4,8 @@
 
 #include <llarp/constants/path.hpp>
 #include <llarp/crypto/crypto.hpp>
-#include <llarp/router/rc_lookup_handler.hpp>
+#include <llarp/messages/common.hpp>
+#include <llarp/path/transit_hop.hpp>
 #include <llarp/router_contact.hpp>
 #include <llarp/util/compare_ptr.hpp>
 #include <llarp/util/decaying_hashset.hpp>
@@ -25,12 +26,7 @@ namespace
 namespace llarp
 {
   struct LinkManager;
-
+  class NodeDB;
-  inline std::string
-  serialize_response(oxenc::bt_dict supplement = {})
-  {
-    return oxenc::bt_serialize(supplement);
-  }
 
   namespace link
   {
@@ -171,8 +167,6 @@ namespace llarp
     friend struct link::Endpoint;
 
     std::atomic<bool> is_stopping;
-    // DISCUSS: is this necessary? can we reduce the amount of locking and nuke this
-    mutable util::Mutex m;  // protects persisting_conns
 
     // sessions to persist -> timestamp to end persist at
     std::unordered_map<RouterID, llarp_time_t> persisting_conns;
@@ -182,7 +176,6 @@ namespace llarp
 
     util::DecayingHashSet<RouterID> clients{path::DEFAULT_LIFETIME};
 
-    RCLookupHandler* rc_lookup;
     std::shared_ptr<NodeDB> node_db;
 
     oxen::quic::Address addr;
@@ -227,6 +220,12 @@ namespace llarp
       return addr;
     }
 
+    void
+    gossip_rc(const RouterID& rc_rid, std::string serialized_rc);
+
+    void
+    handle_gossip_rc(oxen::quic::message m);
+
     bool
     have_connection_to(const RouterID& remote, bool client_only = false) const;
 
@@ -267,7 +266,7 @@ namespace llarp
     extract_status() const;
 
     void
-    init(RCLookupHandler* rcLookup);
+    init();
 
     void
     for_each_connection(std::function<void(link::Connection&)> func);
@@ -288,18 +287,18 @@ namespace llarp
 
    private:
     // DHT messages
-    void handle_find_name(oxen::quic::message);      // relay
+    void
-    void handle_find_intro(oxen::quic::message);     // relay
+    handle_find_name(std::string_view body, std::function<void(std::string)> respond);  // relay
-    void handle_publish_intro(oxen::quic::message);  // relay
+    void
-    void handle_find_router(oxen::quic::message);    // relay + path
+    handle_find_intro(std::string_view body, std::function<void(std::string)> respond);  // relay
+    void
+    handle_publish_intro(std::string_view body, std::function<void(std::string)> respond);  // relay
 
     // Path messages
-    void handle_path_build(oxen::quic::message);     // relay
+    void
-    void handle_path_confirm(oxen::quic::message);   // relay
+    handle_path_build(oxen::quic::message, const RouterID& from);  // relay
-    void handle_path_latency(oxen::quic::message);   // relay
+    void handle_path_latency(oxen::quic::message);                 // relay
-    void handle_path_transfer(oxen::quic::message);  // relay
+    void handle_path_transfer(oxen::quic::message);                // relay
-    void handle_relay_commit(oxen::quic::message);   // relay
-    void handle_relay_status(oxen::quic::message);   // relay
 
     // Exit messages
     void handle_obtain_exit(oxen::quic::message);  // relay
@@ -309,33 +308,49 @@ namespace llarp
     // Misc
     void handle_convo_intro(oxen::quic::message);
 
-    std::unordered_map<std::string, void (LinkManager::*)(oxen::quic::message)> rpc_commands = {
+    // These requests come over a path (as a "path_control" request),
-        {"find_name", &LinkManager::handle_find_name},
+    // may or may not need to make a request to another relay,
-        {"find_router", &LinkManager::handle_find_router},
+    // then respond (onioned) back along the path.
-        {"publish_intro", &LinkManager::handle_publish_intro},
+    std::unordered_map<
-        {"find_intro", &LinkManager::handle_find_intro},
+        std::string_view,
-        {"path_build", &LinkManager::handle_path_build},
+        void (LinkManager::*)(std::string_view body, std::function<void(std::string)> respond)>
-        {"path_confirm", &LinkManager::handle_path_confirm},
+        path_requests = {
-        {"path_latency", &LinkManager::handle_path_latency},
+            {"find_name"sv, &LinkManager::handle_find_name},
-        {"update_exit", &LinkManager::handle_update_exit},
+            {"publish_intro"sv, &LinkManager::handle_publish_intro},
-        {"obtain_exit", &LinkManager::handle_obtain_exit},
+            {"find_intro"sv, &LinkManager::handle_find_intro}};
-        {"close_exit", &LinkManager::handle_close_exit},
+    /*
-        {"convo_intro", &LinkManager::handle_convo_intro}};
+    {"path_confirm", &LinkManager::handle_path_confirm},
+    {"path_latency", &LinkManager::handle_path_latency},
+    {"update_exit", &LinkManager::handle_update_exit},
+    {"obtain_exit", &LinkManager::handle_obtain_exit},
+    {"close_exit", &LinkManager::handle_close_exit},
+    {"convo_intro", &LinkManager::handle_convo_intro}};
+    */
+
+    // these requests are direct, i.e. not over a path;
+    // the rest are relay->relay
+    // TODO: new RC fetch endpoint (which will be both client->relay and relay->relay)
+    std::unordered_map<
+        std::string_view,
+        void (LinkManager::*)(std::string_view body, std::function<void(std::string)> respond)>
+        direct_requests = {
+            {"publish_intro"sv, &LinkManager::handle_publish_intro},
+            {"find_intro"sv, &LinkManager::handle_find_intro}};
 
     // Path relaying
-    void handle_path_control(oxen::quic::message);
+    void
+    handle_path_control(oxen::quic::message, const RouterID& from);
+
+    void
+    handle_inner_request(
+        oxen::quic::message m, std::string payload, std::shared_ptr<path::TransitHop> hop);
 
     // DHT responses
     void handle_find_name_response(oxen::quic::message);
     void handle_find_intro_response(oxen::quic::message);
     void handle_publish_intro_response(oxen::quic::message);
-    void handle_find_router_response(oxen::quic::message);
 
     // Path responses
-    void handle_path_build_response(oxen::quic::message);
-    void handle_relay_commit_response(oxen::quic::message);
-    void handle_relay_status_response(oxen::quic::message);
-    void handle_path_confirm_response(oxen::quic::message);
     void handle_path_latency_response(oxen::quic::message);
     void handle_path_transfer_response(oxen::quic::message);
 
@@ -346,18 +361,11 @@ namespace llarp
 
     std::unordered_map<std::string, void (LinkManager::*)(oxen::quic::message)> rpc_responses = {
         {"find_name", &LinkManager::handle_find_name_response},
-        {"find_router", &LinkManager::handle_find_router_response},
         {"publish_intro", &LinkManager::handle_publish_intro_response},
         {"find_intro", &LinkManager::handle_find_intro_response},
         {"update_exit", &LinkManager::handle_update_exit_response},
         {"obtain_exit", &LinkManager::handle_obtain_exit_response},
         {"close_exit", &LinkManager::handle_close_exit_response}};
-
-   public:
-    // Public response functions and error handling functions invoked elsehwere. These take
-    // r-value references s.t. that message is taken out of calling scope
-    void
-    handle_find_router_error(oxen::quic::message&& m);
   };
 
   namespace link
@@ -421,7 +429,6 @@ namespace llarp
 std::unordered_map<std::string, void (llarp::link::LinkManager::*)(oxen::quic::message)>
 rpc_commands = {
     {"find_name", &handle_find_name},
-    {"find_router", &handle_find_router},
     // ...
 };
 

llarp/messages/common.hpp

@@ -18,6 +18,22 @@ namespace
 
 namespace llarp
 {
+  namespace messages
+  {
+
+    inline std::string
+    serialize_response(oxenc::bt_dict supplement = {})
+    {
+      return oxenc::bt_serialize(supplement);
+    }
+
+    // ideally STATUS is the first key in a bt-dict, so use a single, early ascii char
+    inline const auto STATUS_KEY = "!"s;
+    inline const auto TIMEOUT_RESPONSE = serialize_response({{STATUS_KEY, "TIMEOUT"}});
+    inline const auto ERROR_RESPONSE = serialize_response({{STATUS_KEY, "ERROR"}});
+    inline const auto OK_RESPONSE = serialize_response({{STATUS_KEY, "OK"}});
+  }  // namespace messages
+
   /// abstract base class for serialized messages
   struct AbstractSerializable
   {

llarp/messages/dht.hpp

@@ -6,7 +6,6 @@ namespace llarp
 {
   namespace FindRouterMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto RETRY_EXP = "RETRY AS EXPLORATORY"sv;
     inline auto RETRY_ITER = "RETRY AS ITERATIVE"sv;
     inline auto RETRY_NEW = "RETRY WITH NEW RECIPIENT"sv;
@@ -52,9 +51,7 @@ namespace llarp
 
   namespace FindIntroMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto NOT_FOUND = "NOT FOUND"sv;
-    inline auto TIMED_OUT = "TIMED OUT"sv;
     inline auto INVALID_ORDER = "INVALID ORDER"sv;
     inline auto INSUFFICIENT_NODES = "INSUFFICIENT NODES"sv;
 
@@ -80,7 +77,6 @@ namespace llarp
 
   namespace FindNameMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto NOT_FOUND = "NOT FOUND"sv;
 
     inline static std::string
@@ -120,7 +116,6 @@ namespace llarp
 
   namespace PublishIntroMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto INVALID_INTROSET = "INVALID INTROSET"sv;
     inline auto EXPIRED = "EXPIRED INTROSET"sv;
     inline auto INSUFFICIENT = "INSUFFICIENT NODES"sv;

llarp/messages/exit.hpp

@@ -12,7 +12,6 @@ namespace llarp
 
   namespace ObtainExitMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
 
     // flag: 0 = Exit, 1 = Snode
     inline std::string
@@ -63,7 +62,6 @@ namespace llarp
 
   namespace UpdateExitMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto UPDATE_FAILED = "EXIT UPDATE FAILED"sv;
 
     inline std::string
@@ -113,7 +111,6 @@ namespace llarp
 
   namespace CloseExitMessage
   {
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto UPDATE_FAILED = "CLOSE EXIT FAILED"sv;
 
     inline std::string

llarp/messages/path.hpp

@@ -6,8 +6,6 @@ namespace llarp
 {
   namespace PathBuildMessage
   {
-    inline auto OK = "OK"sv;
-    inline auto EXCEPTION = "EXCEPTION"sv;
     inline auto BAD_FRAMES = "BAD_FRAMES"sv;
     inline auto BAD_CRYPTO = "BAD_CRYPTO"sv;
     inline auto NO_TRANSIT = "NOT ALLOWING TRANSIT"sv;
@@ -29,7 +27,9 @@ namespace llarp
         throw std::runtime_error{std::move(err)};
       }
       // generate nonceXOR value self->hop->pathKey
-      crypto::shorthash(hop.nonceXOR, hop.shared.data(), hop.shared.size());
+      ShortHash hash;
+      crypto::shorthash(hash, hop.shared.data(), hop.shared.size());
+      hop.nonceXOR = hash.data();  // nonceXOR is 24 bytes, ShortHash is 32; this will truncate
 
       hop.upstream = nextHop;
     }
@@ -56,7 +56,7 @@ namespace llarp
       crypto::encryption_keygen(framekey);
 
       SharedSecret shared;
-      TunnelNonce outer_nonce;
+      SymmNonce outer_nonce;
       outer_nonce.Randomize();
 
       // derive (outer) shared key

llarp/messages/relay.cpp

@@ -1,121 +0,0 @@
-#include "relay.hpp"
-
-#include <llarp/path/path_context.hpp>
-#include <llarp/router/router.hpp>
-#include <llarp/util/bencode.hpp>
-
-namespace llarp
-{
-  void
-  RelayUpstreamMessage::clear()
-  {
-    pathid.Zero();
-    enc.Clear();
-    nonce.Zero();
-    version = 0;
-  }
-
-  std::string
-  RelayUpstreamMessage::bt_encode() const
-  {
-    oxenc::bt_dict_producer btdp;
-
-    try
-    {
-      btdp.append("a", "u");
-      btdp.append("p", pathid.ToView());
-      btdp.append("v", llarp::constants::proto_version);
-      btdp.append("x", std::string_view{reinterpret_cast<const char*>(enc.data()), enc.size()});
-      btdp.append("y", std::string_view{reinterpret_cast<const char*>(nonce.data()), nonce.size()});
-    }
-    catch (...)
-    {
-      log::critical(link_cat, "Error: RelayUpstreamMessage failed to bt encode contents!");
-    }
-
-    return std::move(btdp).str();
-  }
-
-  bool
-  RelayUpstreamMessage::decode_key(const llarp_buffer_t& key, llarp_buffer_t* buf)
-  {
-    bool read = false;
-    if (!BEncodeMaybeReadDictEntry("p", pathid, read, key, buf))
-      return false;
-    if (!BEncodeMaybeVerifyVersion("v", version, llarp::constants::proto_version, read, key, buf))
-      return false;
-    if (!BEncodeMaybeReadDictEntry("x", enc, read, key, buf))
-      return false;
-    if (!BEncodeMaybeReadDictEntry("y", nonce, read, key, buf))
-      return false;
-    return read;
-  }
-
-  bool
-  RelayUpstreamMessage::handle_message(Router* r) const
-  {
-    auto path = r->path_context().GetByDownstream(conn->remote_rc.router_id(), pathid);
-    if (path)
-    {
-      return path->HandleUpstream(llarp_buffer_t(enc), nonce, r);
-    }
-    return false;
-  }
-
-  void
-  RelayDownstreamMessage::clear()
-  {
-    pathid.Zero();
-    enc.Clear();
-    nonce.Zero();
-    version = 0;
-  }
-
-  std::string
-  RelayDownstreamMessage::bt_encode() const
-  {
-    oxenc::bt_dict_producer btdp;
-
-    try
-    {
-      btdp.append("a", "d");
-      btdp.append("p", pathid.ToView());
-      btdp.append("v", llarp::constants::proto_version);
-      btdp.append("x", std::string_view{reinterpret_cast<const char*>(enc.data()), enc.size()});
-      btdp.append("y", std::string_view{reinterpret_cast<const char*>(nonce.data()), nonce.size()});
-    }
-    catch (...)
-    {
-      log::critical(link_cat, "Error: RelayDownstreamMessage failed to bt encode contents!");
-    }
-
-    return std::move(btdp).str();
-  }
-
-  bool
-  RelayDownstreamMessage::decode_key(const llarp_buffer_t& key, llarp_buffer_t* buf)
-  {
-    bool read = false;
-    if (!BEncodeMaybeReadDictEntry("p", pathid, read, key, buf))
-      return false;
-    if (!BEncodeMaybeVerifyVersion("v", version, llarp::constants::proto_version, read, key, buf))
-      return false;
-    if (!BEncodeMaybeReadDictEntry("x", enc, read, key, buf))
-      return false;
-    if (!BEncodeMaybeReadDictEntry("y", nonce, read, key, buf))
-      return false;
-    return read;
-  }
-
-  bool
-  RelayDownstreamMessage::handle_message(Router* r) const
-  {
-    auto path = r->path_context().GetByUpstream(conn->remote_rc.router_id(), pathid);
-    if (path)
-    {
-      return path->HandleDownstream(llarp_buffer_t(enc), nonce, r);
-    }
-    llarp::LogWarn("no path for downstream message id=", pathid);
-    return false;
-  }
-}  // namespace llarp

llarp/messages/relay.hpp

@@ -1,75 +0,0 @@
-#pragma once
-
-#include "link_message.hpp"
-
-#include <llarp/crypto/encrypted.hpp>
-#include <llarp/crypto/types.hpp>
-#include <llarp/path/path_types.hpp>
-
-#include <vector>
-
-namespace llarp
-{
-  /*
-      Data messages to be sent via quic datagrams
-  */
-
-  struct RelayUpstreamMessage final : public AbstractLinkMessage
-  {
-    Encrypted<MAX_LINK_MSG_SIZE - 128> enc;
-    TunnelNonce nonce;
-
-    bool
-    decode_key(const llarp_buffer_t& key, llarp_buffer_t* buf) override;
-
-    std::string
-    bt_encode() const override;
-
-    bool
-    handle_message(Router* router) const override;
-
-    void
-    clear() override;
-
-    const char*
-    name() const override
-    {
-      return "RelayUpstream";
-    }
-    uint16_t
-    priority() const override
-    {
-      return 0;
-    }
-  };
-
-  struct RelayDownstreamMessage final : public AbstractLinkMessage
-  {
-    Encrypted<MAX_LINK_MSG_SIZE - 128> enc;
-    TunnelNonce nonce;
-
-    bool
-    decode_key(const llarp_buffer_t& key, llarp_buffer_t* buf) override;
-
-    std::string
-    bt_encode() const override;
-
-    bool
-    handle_message(Router* router) const override;
-
-    void
-    clear() override;
-
-    const char*
-    name() const override
-    {
-      return "RelayDownstream";
-    }
-
-    uint16_t
-    priority() const override
-    {
-      return 0;
-    }
-  };
-}  // namespace llarp

llarp/nodedb.cpp

@@ -14,9 +14,6 @@ static const std::string RC_FILE_EXT = ".signed";
 
 namespace llarp
 {
-  NodeDB::Entry::Entry(RemoteRC value) : rc(std::move(value)), insertedAt(llarp::time_now_ms())
-  {}
-
   static void
   EnsureSkiplist(fs::path nodedbDir)
   {
@@ -72,8 +69,8 @@ namespace llarp
         // make copy of all rcs
         std::vector<RemoteRC> copy;
 
-        for (const auto& item : entries)
+        for (const auto& item : known_rcs)
-          copy.push_back(item.second.rc);
+          copy.push_back(item.second);
 
         // flush them to disk in one big job
         // TODO: split this up? idk maybe some day...
@@ -99,6 +96,81 @@ namespace llarp
     return m_Root / skiplistDir / fname;
   }
 
+  bool
+  NodeDB::want_rc(const RouterID& rid) const
+  {
+    if (not router.is_service_node())
+      return true;
+    return registered_routers.count(rid);
+  }
+
+  void
+  NodeDB::set_bootstrap_routers(const std::set<RemoteRC>& rcs)
+  {
+    bootstraps.clear();  // this function really shouldn't be called more than once, but...
+    for (const auto& rc : rcs)
+      bootstraps.emplace(rc.router_id(), rc);
+  }
+
+  void
+  NodeDB::set_router_whitelist(
+      const std::vector<RouterID>& whitelist,
+      const std::vector<RouterID>& greylist,
+      const std::vector<RouterID>& greenlist)
+  {
+    if (whitelist.empty())
+      return;
+
+    registered_routers.clear();
+    registered_routers.insert(whitelist.begin(), whitelist.end());
+    registered_routers.insert(greylist.begin(), greylist.end());
+    registered_routers.insert(greenlist.begin(), greenlist.end());
+
+    router_whitelist.clear();
+    router_whitelist.insert(whitelist.begin(), whitelist.end());
+    router_greylist.clear();
+    router_greylist.insert(greylist.begin(), greylist.end());
+    router_greenlist.clear();
+    router_greenlist.insert(greenlist.begin(), greenlist.end());
+
+    log::info(
+        logcat, "lokinet service node list now has ", router_whitelist.size(), " active routers");
+  }
+
+  std::optional<RouterID>
+  NodeDB::get_random_whitelist_router() const
+  {
+    const auto sz = router_whitelist.size();
+    if (sz == 0)
+      return std::nullopt;
+    auto itr = router_whitelist.begin();
+    if (sz > 1)
+      std::advance(itr, randint() % sz);
+    return *itr;
+  }
+
+  bool
+  NodeDB::is_connection_allowed(const RouterID& remote) const
+  {
+    if (pinned_edges.size() && pinned_edges.count(remote) == 0 && bootstraps.count(remote) == 0)
+    {
+      return false;
+    }
+
+    if (not router.is_service_node())
+      return true;
+
+    return router_whitelist.count(remote) or router_greylist.count(remote);
+  }
+
+  bool
+  NodeDB::is_first_hop_allowed(const RouterID& remote) const
+  {
+    if (pinned_edges.size() && pinned_edges.count(remote) == 0)
+      return false;
+    return true;
+  }
+
   void
   NodeDB::load_from_disk()
   {
@@ -137,10 +209,10 @@ namespace llarp
             return true;
           }
 
-          // validate signature and purge entries with invalid signatures
+          // validate signature and purge known_rcs with invalid signatures
           // load ones with valid signatures
           if (rc.verify())
-            entries.emplace(rc.router_id(), rc);
+            known_rcs.emplace(rc.router_id(), rc);
           else
             purge.emplace(f);
 
@@ -165,36 +237,33 @@ namespace llarp
       return;
 
     router.loop()->call([this]() {
-      for (const auto& item : entries)
+      for (const auto& item : known_rcs)
-        item.second.rc.write(get_path_by_pubkey(item.first));
+        item.second.write(get_path_by_pubkey(item.first));
     });
   }
 
   bool
-  NodeDB::has_router(RouterID pk) const
+  NodeDB::has_rc(RouterID pk) const
   {
-    return router.loop()->call_get(
+    return known_rcs.count(pk);
-        [this, pk]() -> bool { return entries.find(pk) != entries.end(); });
   }
 
   std::optional<RemoteRC>
   NodeDB::get_rc(RouterID pk) const
   {
-    return router.loop()->call_get([this, pk]() -> std::optional<RemoteRC> {
+    const auto itr = known_rcs.find(pk);
-      const auto itr = entries.find(pk);
 
-      if (itr == entries.end())
+    if (itr == known_rcs.end())
-        return std::nullopt;
+      return std::nullopt;
 
-      return itr->second.rc;
+    return itr->second;
-    });
   }
 
   void
   NodeDB::remove_router(RouterID pk)
   {
     router.loop()->call([this, pk]() {
-      entries.erase(pk);
+      known_rcs.erase(pk);
       remove_many_from_disk_async({pk});
     });
   }
@@ -202,54 +271,37 @@ namespace llarp
   void
   NodeDB::remove_stale_rcs(std::unordered_set<RouterID> keep, llarp_time_t cutoff)
   {
-    router.loop()->call([this, keep, cutoff]() {
+    (void)keep;
-      std::unordered_set<RouterID> removed;
+    (void)cutoff;
-      auto itr = entries.begin();
+    // TODO: handling of "stale" is pending change, removing here for now.
-      while (itr != entries.end())
-      {
-        if (itr->second.insertedAt < cutoff and keep.count(itr->second.rc.router_id()) == 0)
-        {
-          removed.insert(itr->second.rc.router_id());
-          itr = entries.erase(itr);
-        }
-        else
-          ++itr;
-      }
-      if (not removed.empty())
-        remove_many_from_disk_async(std::move(removed));
-    });
   }
 
-  void
+  bool
   NodeDB::put_rc(RemoteRC rc)
   {
-    router.loop()->call([this, rc]() {
+    const auto& rid = rc.router_id();
-      const auto& rid = rc.router_id();
+    if (not want_rc(rid))
-      entries.erase(rid);
+      return false;
-      entries.emplace(rid, rc);
+    known_rcs.erase(rid);
-    });
+    known_rcs.emplace(rid, std::move(rc));
+    return true;
   }
 
   size_t
   NodeDB::num_loaded() const
   {
-    return router.loop()->call_get([this]() { return entries.size(); });
+    return router.loop()->call_get([this]() { return known_rcs.size(); });
   }
 
-  void
+  bool
   NodeDB::put_rc_if_newer(RemoteRC rc)
   {
-    router.loop()->call([this, rc]() {
+    auto itr = known_rcs.find(rc.router_id());
-      auto itr = entries.find(rc.router_id());
+    if (itr == known_rcs.end() or itr->second.other_is_newer(rc))
-      if (itr == entries.end() or itr->second.rc.other_is_newer(rc))
+    {
-      {
+      return put_rc(std::move(rc));
-        // delete if existing
+    }
-        if (itr != entries.end())
+    return false;
-          entries.erase(itr);
-        // add new entry
-        entries.emplace(rc.router_id(), rc);
-      }
-    });
   }
 
   void
@@ -296,10 +348,10 @@ namespace llarp
     return router.loop()->call_get([this, location, numRouters]() -> std::vector<RemoteRC> {
       std::vector<const RemoteRC*> all;
 
-      all.reserve(entries.size());
+      all.reserve(known_rcs.size());
-      for (auto& entry : entries)
+      for (auto& entry : known_rcs)
       {
-        all.push_back(&entry.second.rc);
+        all.push_back(&entry.second);
       }
 
       auto it_mid = numRouters < all.size() ? all.begin() + numRouters : all.end();

llarp/nodedb.hpp

@@ -24,16 +24,7 @@ namespace llarp
 
   class NodeDB
   {
-    struct Entry
+    std::unordered_map<RouterID, RemoteRC> known_rcs;
-    {
-      const RemoteRC rc;
-      llarp_time_t insertedAt;
-      explicit Entry(RemoteRC rc);
-    };
-
-    using NodeMap = std::unordered_map<RouterID, Entry>;
-
-    NodeMap entries;
 
     const Router& router;
     const fs::path m_Root;
@@ -49,14 +40,95 @@ namespace llarp
     fs::path
     get_path_by_pubkey(RouterID pk) const;
 
+    std::unordered_map<RouterID, RemoteRC> bootstraps;
+
+    // whitelist = active routers
+    std::unordered_set<RouterID> router_whitelist;
+    // greylist = fully funded, but decommissioned routers
+    std::unordered_set<RouterID> router_greylist;
+    // greenlist = registered but not fully-staked routers
+    std::unordered_set<RouterID> router_greenlist;
+
+    // all registered relays (snodes)
+    std::unordered_set<RouterID> registered_routers;
+
+    // only ever use to specific edges as path first-hops
+    std::unordered_set<RouterID> pinned_edges;
+
+    bool
+    want_rc(const RouterID& rid) const;
+
    public:
+    void
+    set_bootstrap_routers(const std::set<RemoteRC>& rcs);
+
+    const std::unordered_set<RouterID>&
+    whitelist() const
+    {
+      return router_whitelist;
+    }
+
+    const std::unordered_set<RouterID>&
+    greylist() const
+    {
+      return router_greylist;
+    }
+
+    const std::unordered_set<RouterID>&
+    get_registered_routers() const
+    {
+      return registered_routers;
+    }
+
+    void
+    set_router_whitelist(
+        const std::vector<RouterID>& whitelist,
+        const std::vector<RouterID>& greylist,
+        const std::vector<RouterID>& greenlist);
+
+    std::optional<RouterID>
+    get_random_whitelist_router() const;
+
+    // client:
+    //   if pinned edges were specified, connections are allowed only to those and
+    //   to the configured bootstrap nodes.  otherwise, always allow.
+    //
+    // relay:
+    //   outgoing connections are allowed only to other registered, funded relays
+    //   (whitelist and greylist, respectively).
+    bool
+    is_connection_allowed(const RouterID& remote) const;
+
+    // client:
+    //   same as is_connection_allowed
+    //
+    // server:
+    //   we only build new paths through registered, not decommissioned relays
+    //   (i.e. whitelist)
+    bool
+    is_path_allowed(const RouterID& remote) const
+    {
+      return router_whitelist.count(remote);
+    }
+
+    // if pinned edges were specified, the remote must be in that set, else any remote
+    // is allowed as first hop.
+    bool
+    is_first_hop_allowed(const RouterID& remote) const;
+
+    const std::unordered_set<RouterID>&
+    get_pinned_edges() const
+    {
+      return pinned_edges;
+    }
+
     explicit NodeDB(
         fs::path rootdir, std::function<void(std::function<void()>)> diskCaller, Router* r);
 
     /// in memory nodedb
     NodeDB();
 
-    /// load all entries from disk syncrhonously
+    /// load all known_rcs from disk syncrhonously
     void
     load_from_disk();
 
@@ -82,7 +154,7 @@ namespace llarp
 
     /// return true if we have an rc by its ident pubkey
     bool
-    has_router(RouterID pk) const;
+    has_rc(RouterID pk) const;
 
     /// maybe get an rc by its ident pubkey
     std::optional<RemoteRC>
@@ -93,44 +165,30 @@ namespace llarp
     GetRandom(Filter visit) const
     {
       return router.loop()->call_get([visit]() -> std::optional<RemoteRC> {
-        std::vector<const decltype(entries)::value_type*> entries;
+        std::vector<const decltype(known_rcs)::value_type*> known_rcs;
-        for (const auto& entry : entries)
+        for (const auto& entry : known_rcs)
-          entries.push_back(entry);
+          known_rcs.push_back(entry);
 
-        std::shuffle(entries.begin(), entries.end(), llarp::csrng);
+        std::shuffle(known_rcs.begin(), known_rcs.end(), llarp::csrng);
 
-        for (const auto entry : entries)
+        for (const auto entry : known_rcs)
         {
-          if (visit(entry->second.rc))
+          if (visit(entry->second))
-            return entry->second.rc;
+            return entry->second;
         }
 
         return std::nullopt;
       });
     }
 
-    /// visit all entries
+    /// visit all known_rcs
     template <typename Visit>
     void
     VisitAll(Visit visit) const
     {
       router.loop()->call([this, visit]() {
-        for (const auto& item : entries)
+        for (const auto& item : known_rcs)
-          visit(item.second.rc);
+          visit(item.second);
-      });
-    }
-
-    /// visit all entries inserted before a timestamp
-    template <typename Visit>
-    void
-    VisitInsertedBefore(Visit visit, llarp_time_t insertedBefore)
-    {
-      router.loop()->call([this, visit, insertedBefore]() {
-        for (const auto& item : entries)
-        {
-          if (item.second.insertedAt < insertedBefore)
-            visit(item.second.rc);
-        }
       });
     }
 
@@ -145,13 +203,13 @@ namespace llarp
     {
       router.loop()->call([this, visit]() {
         std::unordered_set<RouterID> removed;
-        auto itr = entries.begin();
+        auto itr = known_rcs.begin();
-        while (itr != entries.end())
+        while (itr != known_rcs.end())
         {
-          if (visit(itr->second.rc))
+          if (visit(itr->second))
           {
-            removed.insert(itr->second.rc.router_id());
+            removed.insert(itr->second.router_id());
-            itr = entries.erase(itr);
+            itr = known_rcs.erase(itr);
           }
           else
             ++itr;
@@ -165,12 +223,14 @@ namespace llarp
     void
     remove_stale_rcs(std::unordered_set<RouterID> keep, llarp_time_t cutoff);
 
-    /// put this rc into the cache if it is not there or newer than the one there already
+    /// put (or replace) the RC if we consider it valid (want_rc).  returns true if put.
-    void
+    bool
-    put_rc_if_newer(RemoteRC rc);
-
-    /// unconditional put of rc into cache
-    void
     put_rc(RemoteRC rc);
+
+    /// if we consider it valid (want_rc),
+    /// put this rc into the cache if it is not there or is newer than the one there already
+    /// returns true if the rc was inserted
+    bool
+    put_rc_if_newer(RemoteRC rc);
   };
 }  // namespace llarp

llarp/path/abstracthophandler.cpp

@@ -4,34 +4,26 @@
 
 namespace llarp::path
 {
-  // handle data in upstream direction
+  std::string
-  bool
+  make_onion_payload(
-  AbstractHopHandler::HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router* r)
+      const SymmNonce& nonce, const PathID_t& path_id, const std::string_view& inner_payload)
   {
-    auto& pkt = m_UpstreamQueue.emplace_back();
+    return make_onion_payload(
-    pkt.first.resize(X.sz);
+        nonce,
-    std::copy_n(X.base, X.sz, pkt.first.begin());
+        path_id,
-    pkt.second = Y;
+        ustring_view{
-    r->TriggerPump();
+            reinterpret_cast<const unsigned char*>(inner_payload.data()), inner_payload.size()});
-    return true;
   }
 
-  // handle data in downstream direction
+  std::string
-  bool
+  make_onion_payload(
-  AbstractHopHandler::HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router* r)
+      const SymmNonce& nonce, const PathID_t& path_id, const ustring_view& inner_payload)
   {
-    auto& pkt = m_DownstreamQueue.emplace_back();
+    oxenc::bt_dict_producer next_dict;
-    pkt.first.resize(X.sz);
+    next_dict.append("NONCE", nonce.ToView());
-    std::copy_n(X.base, X.sz, pkt.first.begin());
+    next_dict.append("PATHID", path_id.ToView());
-    pkt.second = Y;
+    next_dict.append("PAYLOAD", inner_payload);
-    r->TriggerPump();
-    return true;
-  }
 
-  void
+    return std::move(next_dict).str();
-  AbstractHopHandler::DecayFilters(llarp_time_t now)
-  {
-    m_UpstreamReplayFilter.Decay(now);
-    m_DownstreamReplayFilter.Decay(now);
   }
 }  // namespace llarp::path

llarp/path/abstracthophandler.hpp

@@ -1,8 +1,8 @@
 #pragma once
 
-#include <llarp/crypto/encrypted_frame.hpp>
+#include "path_types.hpp"
+
 #include <llarp/crypto/types.hpp>
-#include <llarp/messages/relay.hpp>
 #include <llarp/util/decaying_hashset.hpp>
 #include <llarp/util/types.hpp>
 
@@ -22,41 +22,38 @@ namespace llarp
 
   namespace path
   {
+
+    std::string
+    make_onion_payload(
+        const SymmNonce& nonce, const PathID_t& path_id, const std::string_view& inner_payload);
+    std::string
+    make_onion_payload(
+        const SymmNonce& nonce, const PathID_t& path_id, const ustring_view& inner_payload);
+
     struct AbstractHopHandler
     {
-      using TrafficEvent_t = std::pair<std::vector<byte_t>, TunnelNonce>;
-      using TrafficQueue_t = std::list<TrafficEvent_t>;
-
       virtual ~AbstractHopHandler() = default;
 
       virtual PathID_t
       RXID() const = 0;
 
-      void
-      DecayFilters(llarp_time_t now);
-
       virtual bool
       Expired(llarp_time_t now) const = 0;
 
       virtual bool
       ExpiresSoon(llarp_time_t now, llarp_time_t dlt) const = 0;
 
+      /// sends a control request along a path
+      ///
+      /// performs the necessary onion encryption before sending.
+      /// func will be called when a timeout occurs or a response is received.
+      /// if a response is received, onion decryption is performed before func is called.
+      ///
+      /// func is called with a bt-encoded response string (if applicable), and
+      /// a timeout flag (if set, response string will be empty)
       virtual bool
       send_path_control_message(
-          std::string method,
+          std::string method, std::string body, std::function<void(std::string)> func) = 0;
-          std::string body,
-          std::function<void(oxen::quic::message m)> func) = 0;
-
-      /// send routing message and increment sequence number
-      virtual bool
-      SendRoutingMessage(std::string payload, Router* r) = 0;
-
-      // handle data in upstream direction
-      virtual bool
-      HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router*);
-      // handle data in downstream direction
-      virtual bool
-      HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router*);
 
       /// return timestamp last remote activity happened at
       virtual llarp_time_t
@@ -68,29 +65,8 @@ namespace llarp
         return m_SequenceNum++;
       }
 
-      virtual void
-      FlushUpstream(Router* r) = 0;
-
-      virtual void
-      FlushDownstream(Router* r) = 0;
-
      protected:
       uint64_t m_SequenceNum = 0;
-      TrafficQueue_t m_UpstreamQueue;
-      TrafficQueue_t m_DownstreamQueue;
-      util::DecayingHashSet<TunnelNonce> m_UpstreamReplayFilter;
-      util::DecayingHashSet<TunnelNonce> m_DownstreamReplayFilter;
-
-      virtual void
-      UpstreamWork(TrafficQueue_t queue, Router* r) = 0;
-
-      virtual void
-      DownstreamWork(TrafficQueue_t queue, Router* r) = 0;
-
-      virtual void
-      HandleAllUpstream(std::vector<RelayUpstreamMessage> msgs, Router* r) = 0;
-      virtual void
-      HandleAllDownstream(std::vector<RelayDownstreamMessage> msgs, Router* r) = 0;
     };
 
     using HopHandler_ptr = std::shared_ptr<AbstractHopHandler>;

llarp/path/path.cpp

@@ -8,6 +8,7 @@
 
 namespace llarp::path
 {
+
   Path::Path(
       Router* rtr,
       const std::vector<RemoteRC>& h,
@@ -48,10 +49,7 @@ namespace llarp::path
 
   bool
   Path::obtain_exit(
-      SecretKey sk,
+      SecretKey sk, uint64_t flag, std::string tx_id, std::function<void(std::string)> func)
-      uint64_t flag,
-      std::string tx_id,
-      std::function<void(oxen::quic::message m)> func)
   {
     return send_path_control_message(
         "obtain_exit",
@@ -60,7 +58,7 @@ namespace llarp::path
   }
 
   bool
-  Path::close_exit(SecretKey sk, std::string tx_id, std::function<void(oxen::quic::message m)> func)
+  Path::close_exit(SecretKey sk, std::string tx_id, std::function<void(std::string)> func)
   {
     return send_path_control_message(
         "close_exit", CloseExitMessage::sign_and_serialize(sk, std::move(tx_id)), std::move(func));
@@ -71,82 +69,92 @@ namespace llarp::path
       const dht::Key_t& location,
       bool is_relayed,
       uint64_t order,
-      std::function<void(oxen::quic::message m)> func)
+      std::function<void(std::string)> func)
   {
     return send_path_control_message(
         "find_intro", FindIntroMessage::serialize(location, is_relayed, order), std::move(func));
   }
 
   bool
-  Path::find_name(std::string name, std::function<void(oxen::quic::message m)> func)
+  Path::find_name(std::string name, std::function<void(std::string)> func)
   {
     return send_path_control_message(
         "find_name", FindNameMessage::serialize(std::move(name)), std::move(func));
   }
 
-  bool
-  Path::find_router(std::string rid, std::function<void(oxen::quic::message m)> func)
-  {
-    return send_path_control_message(
-        "find_router", FindRouterMessage::serialize(std::move(rid), false, false), std::move(func));
-  }
-
   bool
   Path::send_path_control_message(
-      std::string method, std::string body, std::function<void(oxen::quic::message m)> func)
+      std::string method, std::string body, std::function<void(std::string)> func)
   {
-    std::string payload;
+    oxenc::bt_dict_producer btdp;
+    btdp.append("BODY", body);
+    btdp.append("METHOD", method);
+    auto payload = std::move(btdp).str();
 
-    {
+    // TODO: old impl padded messages if smaller than a certain size; do we still want to?
-      oxenc::bt_dict_producer btdp;
+    SymmNonce nonce;
-      btdp.append("BODY", body);
-      btdp.append("METHOD", method);
-      payload = std::move(btdp).str();
-    }
-
-    TunnelNonce nonce;
     nonce.Randomize();
 
+    // chacha and mutate nonce for each hop
     for (const auto& hop : hops)
     {
-      // do a round of chacha for each hop and mutate the nonce with that hop's nonce
+      nonce = crypto::onion(
-      crypto::xchacha20(
+          reinterpret_cast<unsigned char*>(payload.data()),
-          reinterpret_cast<unsigned char*>(payload.data()), payload.size(), hop.shared, nonce);
+          payload.size(),
-
+          hop.shared,
-      nonce ^= hop.nonceXOR;
+          nonce,
+          hop.nonceXOR);
     }
 
-    oxenc::bt_dict_producer outer_dict;
+    auto outer_payload = make_onion_payload(nonce, TXID(), payload);
-    outer_dict.append("NONCE", nonce.ToView());
-    outer_dict.append("PATHID", TXID().ToView());
-    outer_dict.append("PAYLOAD", payload);
 
     return router.send_control_message(
         upstream(),
         "path_control",
-        std::move(outer_dict).str(),
+        std::move(outer_payload),
-        [response_cb = std::move(func)](oxen::quic::message m) {
+        [response_cb = std::move(func), weak = weak_from_this()](oxen::quic::message m) {
-          if (m)
+          auto self = weak.lock();
+          // TODO: do we want to allow empty callback here?
+          if ((not self) or (not response_cb))
+            return;
+
+          if (m.timed_out)
           {
-            // do path hop logic here
+            response_cb(messages::TIMEOUT_RESPONSE);
+            return;
           }
-        });
-  }
 
-  bool
+          SymmNonce nonce{};
-  Path::HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router* r)
+          std::string payload;
-  {
+          try
-    if (not m_UpstreamReplayFilter.Insert(Y))
+          {
-      return false;
+            oxenc::bt_dict_consumer btdc{m.body()};
-    return AbstractHopHandler::HandleUpstream(X, Y, r);
-  }
 
-  bool
+            auto nonce = SymmNonce{btdc.require<ustring_view>("NONCE").data()};
-  Path::HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router* r)
+            auto payload = btdc.require<std::string>("PAYLOAD");
-  {
+          }
-    if (not m_DownstreamReplayFilter.Insert(Y))
+          catch (const std::exception& e)
-      return false;
+          {
-    return AbstractHopHandler::HandleDownstream(X, Y, r);
+            log::warning(path_cat, "Error parsing path control message response: {}", e.what());
+            response_cb(messages::ERROR_RESPONSE);
+            return;
+          }
+
+          for (const auto& hop : self->hops)
+          {
+            nonce = crypto::onion(
+                reinterpret_cast<unsigned char*>(payload.data()),
+                payload.size(),
+                hop.shared,
+                nonce,
+                hop.nonceXOR);
+          }
+
+          // TODO: should we do anything (even really simple) here to check if the decrypted
+          //       response is sensible (e.g. is a bt dict)?  Parsing and handling of the
+          //       contents (errors or otherwise) is the currently responsibility of the callback.
+          response_cb(payload);
+        });
   }
 
   RouterID
@@ -216,6 +224,9 @@ namespace llarp::path
   void
   Path::EnterState(PathStatus st, llarp_time_t now)
   {
+    if (now == 0s)
+      now = router.now();
+
     if (st == ePathFailed)
     {
       _status = st;
@@ -286,8 +297,6 @@ namespace llarp::path
         {"ready", IsReady()},
         {"txRateCurrent", m_LastTXRate},
         {"rxRateCurrent", m_LastRXRate},
-        {"replayTX", m_UpstreamReplayFilter.Size()},
-        {"replayRX", m_DownstreamReplayFilter.Size()},
         {"hasExit", SupportsAnyRoles(ePathRoleExit)}};
 
     std::vector<util::StatusObject> hopsObj;
@@ -421,73 +430,6 @@ namespace llarp::path
     }
   }
 
-  void
-  Path::HandleAllUpstream(std::vector<RelayUpstreamMessage> msgs, Router* r)
-  {
-    for (const auto& msg : msgs)
-    {
-      if (r->send_data_message(upstream(), msg.bt_encode()))
-      {
-        m_TXRate += msg.enc.size();
-      }
-      else
-      {
-        LogDebug("failed to send upstream to ", upstream());
-      }
-    }
-    r->TriggerPump();
-  }
-
-  void
-  Path::UpstreamWork(TrafficQueue_t msgs, Router* r)
-  {
-    std::vector<RelayUpstreamMessage> sendmsgs(msgs.size());
-    size_t idx = 0;
-    for (auto& ev : msgs)
-    {
-      TunnelNonce n = ev.second;
-
-      uint8_t* buf = ev.first.data();
-      size_t sz = ev.first.size();
-
-      for (const auto& hop : hops)
-      {
-        crypto::xchacha20(buf, sz, hop.shared, n);
-        n ^= hop.nonceXOR;
-      }
-      auto& msg = sendmsgs[idx];
-      std::memcpy(msg.enc.data(), buf, sz);
-      msg.nonce = ev.second;
-      msg.pathid = TXID();
-      ++idx;
-    }
-    r->loop()->call([self = shared_from_this(), data = std::move(sendmsgs), r]() mutable {
-      self->HandleAllUpstream(std::move(data), r);
-    });
-  }
-
-  void
-  Path::FlushUpstream(Router* r)
-  {
-    if (not m_UpstreamQueue.empty())
-    {
-      r->queue_work([self = shared_from_this(),
-                     data = std::exchange(m_UpstreamQueue, {}),
-                     r]() mutable { self->UpstreamWork(std::move(data), r); });
-    }
-  }
-
-  void
-  Path::FlushDownstream(Router* r)
-  {
-    if (not m_DownstreamQueue.empty())
-    {
-      r->queue_work([self = shared_from_this(),
-                     data = std::exchange(m_DownstreamQueue, {}),
-                     r]() mutable { self->DownstreamWork(std::move(data), r); });
-    }
-  }
-
   /// how long we wait for a path to become active again after it times out
   constexpr auto PathReanimationTimeout = 45s;
 
@@ -515,47 +457,6 @@ namespace llarp::path
     return fmt::format("TX={} RX={}", TXID(), RXID());
   }
 
-  void
-  Path::DownstreamWork(TrafficQueue_t msgs, Router* r)
-  {
-    std::vector<RelayDownstreamMessage> sendMsgs(msgs.size());
-    size_t idx = 0;
-    for (auto& ev : msgs)
-    {
-      sendMsgs[idx].nonce = ev.second;
-
-      uint8_t* buf = ev.first.data();
-      size_t sz = ev.first.size();
-
-      for (const auto& hop : hops)
-      {
-        sendMsgs[idx].nonce ^= hop.nonceXOR;
-        crypto::xchacha20(buf, sz, hop.shared, sendMsgs[idx].nonce);
-      }
-
-      std::memcpy(sendMsgs[idx].enc.data(), buf, sz);
-      ++idx;
-    }
-    r->loop()->call([self = shared_from_this(), msgs = std::move(sendMsgs), r]() mutable {
-      self->HandleAllDownstream(std::move(msgs), r);
-    });
-  }
-
-  void
-  Path::HandleAllDownstream(std::vector<RelayDownstreamMessage> msgs, Router* /* r */)
-  {
-    for (const auto& msg : msgs)
-    {
-      const llarp_buffer_t buf{msg.enc};
-      m_RXRate += buf.sz;
-      // if (HandleRoutingMessage(buf, r))
-      // {
-      //   r->TriggerPump();
-      //   m_LastRecvMessage = r->now();
-      // }
-    }
-  }
-
   /** Note: this is one of two places where AbstractRoutingMessage::bt_encode() is called, the
       other of which is llarp/path/transit_hop.cpp in TransitHop::SendRoutingMessage(). For now,
       we will default to the override of ::bt_encode() that returns an std::string. The role that
@@ -571,6 +472,7 @@ namespace llarp::path
       functions it calls and so on) will need to be modified to take an std::string that we can
       std::move around.
   */
+  /* TODO: replace this with sending an onion-ed data message
   bool
   Path::SendRoutingMessage(std::string payload, Router*)
   {
@@ -594,6 +496,7 @@ namespace llarp::path
 
     return true;
   }
+  */
 
   template <typename Samples_t>
   static llarp_time_t

llarp/path/path.hpp

@@ -5,9 +5,7 @@
 #include "pathset.hpp"
 
 #include <llarp/constants/path.hpp>
-#include <llarp/crypto/encrypted_frame.hpp>
 #include <llarp/crypto/types.hpp>
-#include <llarp/messages/relay.hpp>
 #include <llarp/router_id.hpp>
 #include <llarp/service/intro.hpp>
 #include <llarp/util/aligned.hpp>
@@ -33,8 +31,6 @@ namespace llarp
     struct TransitHopInfo;
     struct PathHopConfig;
 
-    using TransitHop_ptr = std::shared_ptr<TransitHop>;
-
     struct Ptr_hash;
 
     struct Endpoint_Hash;
@@ -122,14 +118,6 @@ namespace llarp
         return _status;
       }
 
-      // handle data in upstream direction
-      bool
-      HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router*) override;
-      // handle data in downstream direction
-
-      bool
-      HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y, Router*) override;
-
       const std::string&
       ShortName() const;
 
@@ -143,7 +131,7 @@ namespace llarp
       }
 
       void
-      EnterState(PathStatus st, llarp_time_t now);
+      EnterState(PathStatus st, llarp_time_t now = 0s);
 
       llarp_time_t
       ExpireTime() const
@@ -186,39 +174,38 @@ namespace llarp
       Tick(llarp_time_t now, Router* r);
 
       bool
-      find_name(std::string name, std::function<void(oxen::quic::message m)> func = nullptr);
+      find_name(std::string name, std::function<void(std::string)> func = nullptr);
-
-      bool
-      find_router(std::string rid, std::function<void(oxen::quic::message m)> func = nullptr);
 
       bool
       find_intro(
           const dht::Key_t& location,
           bool is_relayed = false,
           uint64_t order = 0,
-          std::function<void(oxen::quic::message m)> func = nullptr);
+          std::function<void(std::string)> func = nullptr);
 
       bool
-      close_exit(
+      close_exit(SecretKey sk, std::string tx_id, std::function<void(std::string)> func = nullptr);
-          SecretKey sk,
-          std::string tx_id,
-          std::function<void(oxen::quic::message m)> func = nullptr);
 
       bool
       obtain_exit(
           SecretKey sk,
           uint64_t flag,
           std::string tx_id,
-          std::function<void(oxen::quic::message m)> func = nullptr);
+          std::function<void(std::string)> func = nullptr);
-
+
+      /// sends a control request along a path
+      ///
+      /// performs the necessary onion encryption before sending.
+      /// func will be called when a timeout occurs or a response is received.
+      /// if a response is received, onion decryption is performed before func is called.
+      ///
+      /// func is called with a bt-encoded response string (if applicable), and
+      /// a timeout flag (if set, response string will be empty)
       bool
       send_path_control_message(
           std::string method,
           std::string body,
-          std::function<void(oxen::quic::message m)> func = nullptr) override;
+          std::function<void(std::string)> func = nullptr) override;
-
-      bool
-      SendRoutingMessage(std::string payload, Router* r) override;
 
       bool
       IsReady() const;
@@ -246,25 +233,6 @@ namespace llarp
       std::string
       name() const;
 
-      void
-      FlushUpstream(Router* r) override;
-
-      void
-      FlushDownstream(Router* r) override;
-
-     protected:
-      void
-      UpstreamWork(TrafficQueue_t queue, Router* r) override;
-
-      void
-      DownstreamWork(TrafficQueue_t queue, Router* r) override;
-
-      void
-      HandleAllUpstream(std::vector<RelayUpstreamMessage> msgs, Router* r) override;
-
-      void
-      HandleAllDownstream(std::vector<RelayDownstreamMessage> msgs, Router* r) override;
-
      private:
       bool
       SendLatencyMessage(Router* r);

llarp/path/path_context.cpp

@@ -69,195 +69,80 @@ namespace llarp::path
   bool
   PathContext::HopIsUs(const RouterID& k) const
   {
-    return std::equal(_router->pubkey(), _router->pubkey() + PUBKEYSIZE, k.begin());
+    return _router->pubkey() == k;
   }
 
-  PathContext::EndpointPathPtrSet
+  std::vector<std::shared_ptr<Path>>
   PathContext::FindOwnedPathsWithEndpoint(const RouterID& r)
   {
-    EndpointPathPtrSet found;
+    std::vector<std::shared_ptr<Path>> found;
-    m_OurPaths.ForEach([&](const Path_ptr& p) {
+    for (const auto& [pathid, path] : own_paths)
-      if (p->Endpoint() == r && p->IsReady())
-        found.insert(p);
-    });
-    return found;
-  }
-
-  template <
-      typename Lock_t,
-      typename Map_t,
-      typename Key_t,
-      typename CheckValue_t,
-      typename GetFunc_t,
-      typename Return_ptr = HopHandler_ptr>
-  Return_ptr
-  MapGet(Map_t& map, const Key_t& k, CheckValue_t check, GetFunc_t get)
-  {
-    Lock_t lock(map.first);
-    auto range = map.second.equal_range(k);
-    for (auto i = range.first; i != range.second; ++i)
-    {
-      if (check(i->second))
-        return get(i->second);
-    }
-    return nullptr;
-  }
-
-  template <typename Lock_t, typename Map_t, typename Key_t, typename CheckValue_t>
-  bool
-  MapHas(Map_t& map, const Key_t& k, CheckValue_t check)
-  {
-    Lock_t lock(map.first);
-    auto range = map.second.equal_range(k);
-    for (auto i = range.first; i != range.second; ++i)
     {
-      if (check(i->second))
+      // each path is stored in this map twice, once for each pathid at the first hop
-        return true;
+      // This will make the output deduplicated without needing a std::set
-    }
+      // TODO: we should only need to map one pathid; as the path owner we only send/receive
-    return false;
+      //       packets with the first hop's RXID; its TXID is for packets between it and hop 2.
-  }
+      // TODO: Also, perhaps we want a bit of data duplication here, e.g. a map from
-
+      //       RouterID (terminal hop) to shared_ptr<Path>.
-  template <typename Lock_t, typename Map_t, typename Key_t, typename Value_t>
+      if (path->TXID() == pathid)
-  void
+        continue;
-  MapPut(Map_t& map, const Key_t& k, const Value_t& v)
+
-  {
+      if (path->Endpoint() == r && path->IsReady())
-    Lock_t lock(map.first);
+        found.push_back(path);
-    map.second.emplace(k, v);
-  }
-
-  template <typename Lock_t, typename Map_t, typename Visit_t>
-  void
-  MapIter(Map_t& map, Visit_t v)
-  {
-    Lock_t lock(map.first);
-    for (const auto& item : map.second)
-      v(item);
-  }
-
-  template <typename Lock_t, typename Map_t, typename Key_t, typename Check_t>
-  void
-  MapDel(Map_t& map, const Key_t& k, Check_t check)
-  {
-    Lock_t lock(map.first);
-    auto range = map.second.equal_range(k);
-    for (auto i = range.first; i != range.second;)
-    {
-      if (check(i->second))
-        i = map.second.erase(i);
-      else
-        ++i;
     }
+    return found;
   }
 
   void
   PathContext::AddOwnPath(PathSet_ptr set, Path_ptr path)
   {
     set->AddPath(path);
-    MapPut<util::Lock>(m_OurPaths, path->TXID(), path);
+    own_paths[path->TXID()] = path;
-    MapPut<util::Lock>(m_OurPaths, path->RXID(), path);
+    own_paths[path->RXID()] = path;
   }
 
   bool
   PathContext::HasTransitHop(const TransitHopInfo& info)
   {
-    return MapHas<SyncTransitMap_t::Lock_t>(
+    TransitHopID downstream{info.downstream, info.rxID};
-        m_TransitPaths, info.txID, [info](const std::shared_ptr<TransitHop>& hop) -> bool {
+    if (transit_hops.count(downstream))
-          return info == hop->info;
+      return true;
-        });
-  }
 
-  std::optional<std::weak_ptr<TransitHop>>
+    TransitHopID upstream{info.upstream, info.txID};
-  PathContext::TransitHopByInfo(const TransitHopInfo& info)
+    if (transit_hops.count(upstream))
-  {
+      return true;
-    // this is ugly as sin
-    auto own = MapGet<
-        SyncTransitMap_t::Lock_t,
-        decltype(m_TransitPaths),
-        PathID_t,
-        std::function<bool(const std::shared_ptr<TransitHop>&)>,
-        std::function<TransitHop*(const std::shared_ptr<TransitHop>&)>,
-        TransitHop*>(
-        m_TransitPaths,
-        info.txID,
-        [info](const auto& hop) -> bool { return hop->info == info; },
-        [](const auto& hop) -> TransitHop* { return hop.get(); });
-    if (own)
-      return own->weak_from_this();
-    return std::nullopt;
-  }
 
-  std::optional<std::weak_ptr<TransitHop>>
+    return false;
-  PathContext::TransitHopByUpstream(const RouterID& upstream, const PathID_t& id)
-  {
-    // this is ugly as sin as well
-    auto own = MapGet<
-        SyncTransitMap_t::Lock_t,
-        decltype(m_TransitPaths),
-        PathID_t,
-        std::function<bool(const std::shared_ptr<TransitHop>&)>,
-        std::function<TransitHop*(const std::shared_ptr<TransitHop>&)>,
-        TransitHop*>(
-        m_TransitPaths,
-        id,
-        [upstream](const auto& hop) -> bool { return hop->info.upstream == upstream; },
-        [](const auto& hop) -> TransitHop* { return hop.get(); });
-    if (own)
-      return own->weak_from_this();
-    return std::nullopt;
   }
 
-  HopHandler_ptr
+  std::shared_ptr<TransitHop>
-  PathContext::GetByUpstream(const RouterID& remote, const PathID_t& id)
+  PathContext::GetTransitHop(const RouterID& rid, const PathID_t& path_id)
   {
-    auto own = MapGet<util::Lock>(
+    if (auto itr = transit_hops.find({rid, path_id}); itr != transit_hops.end())
-        m_OurPaths,
+      return itr->second;
-        id,
+
-        [](const Path_ptr) -> bool {
+    return nullptr;
-          // TODO: is this right?
-          return true;
-        },
-        [](Path_ptr p) -> HopHandler_ptr { return p; });
-    if (own)
-      return own;
-
-    return MapGet<SyncTransitMap_t::Lock_t>(
-        m_TransitPaths,
-        id,
-        [remote](const std::shared_ptr<TransitHop>& hop) -> bool {
-          return hop->info.upstream == remote;
-        },
-        [](const std::shared_ptr<TransitHop>& h) -> HopHandler_ptr { return h; });
   }
 
-  HopHandler_ptr
+  Path_ptr
-  PathContext::GetByDownstream(const RouterID& remote, const PathID_t& id)
+  PathContext::GetPath(const PathID_t& path_id)
   {
-    return MapGet<SyncTransitMap_t::Lock_t>(
+    if (auto itr = own_paths.find(path_id); itr != own_paths.end())
-        m_TransitPaths,
+      return itr->second;
-        id,
+
-        [remote](const std::shared_ptr<TransitHop>& hop) -> bool {
+    return nullptr;
-          return hop->info.downstream == remote;
-        },
-        [](const std::shared_ptr<TransitHop>& h) -> HopHandler_ptr { return h; });
   }
 
   bool
-  PathContext::TransitHopPreviousIsRouter(const PathID_t& path, const RouterID& otherRouter)
+  PathContext::TransitHopPreviousIsRouter(const PathID_t& path_id, const RouterID& otherRouter)
   {
-    SyncTransitMap_t::Lock_t lock(m_TransitPaths.first);
+    return transit_hops.count({otherRouter, path_id});
-    auto itr = m_TransitPaths.second.find(path);
-    if (itr == m_TransitPaths.second.end())
-      return false;
-    return itr->second->info.downstream == otherRouter;
   }
 
   PathSet_ptr
   PathContext::GetLocalPathSet(const PathID_t& id)
   {
-    auto& map = m_OurPaths;
+    if (auto itr = own_paths.find(id); itr != own_paths.end())
-    util::Lock lock(map.first);
-    auto itr = map.second.find(id);
-    if (itr != map.second.end())
     {
       if (auto parent = itr->second->m_PathSet.lock())
         return parent;
@@ -271,54 +156,30 @@ namespace llarp::path
     return _router->pubkey();
   }
 
-  TransitHop_ptr
+  std::shared_ptr<TransitHop>
   PathContext::GetPathForTransfer(const PathID_t& id)
   {
-    const RouterID us(OurRouterID());
+    if (auto itr = transit_hops.find({OurRouterID(), id}); itr != transit_hops.end())
-    auto& map = m_TransitPaths;
     {
-      SyncTransitMap_t::Lock_t lock(map.first);
+      return itr->second;
-      auto range = map.second.equal_range(id);
-      for (auto i = range.first; i != range.second; ++i)
-      {
-        if (i->second->info.upstream == us)
-          return i->second;
-      }
     }
-    return nullptr;
-  }
 
-  void
+    return nullptr;
-  PathContext::PumpUpstream()
-  {
-    m_TransitPaths.ForEach([&](auto& ptr) { ptr->FlushUpstream(_router); });
-    m_OurPaths.ForEach([&](auto& ptr) { ptr->FlushUpstream(_router); });
-  }
-
-  void
-  PathContext::PumpDownstream()
-  {
-    m_TransitPaths.ForEach([&](auto& ptr) { ptr->FlushDownstream(_router); });
-    m_OurPaths.ForEach([&](auto& ptr) { ptr->FlushDownstream(_router); });
   }
 
   uint64_t
   PathContext::CurrentTransitPaths()
   {
-    SyncTransitMap_t::Lock_t lock(m_TransitPaths.first);
+    return transit_hops.size() / 2;
-    const auto& map = m_TransitPaths.second;
-    return map.size() / 2;
   }
 
   uint64_t
   PathContext::CurrentOwnedPaths(path::PathStatus st)
   {
     uint64_t num{};
-    util::Lock lock{m_OurPaths.first};
+    for (auto& own_path : own_paths)
-    auto& map = m_OurPaths.second;
-    for (auto itr = map.begin(); itr != map.end(); ++itr)
     {
-      if (itr->second->Status() == st)
+      if (own_path.second->Status() == st)
         num++;
     }
     return num / 2;
@@ -327,8 +188,10 @@ namespace llarp::path
   void
   PathContext::PutTransitHop(std::shared_ptr<TransitHop> hop)
   {
-    MapPut<SyncTransitMap_t::Lock_t>(m_TransitPaths, hop->info.txID, hop);
+    TransitHopID downstream{hop->info.downstream, hop->info.rxID};
-    MapPut<SyncTransitMap_t::Lock_t>(m_TransitPaths, hop->info.rxID, hop);
+    TransitHopID upstream{hop->info.upstream, hop->info.txID};
+    transit_hops.emplace(std::move(downstream), hop);
+    transit_hops.emplace(std::move(upstream), hop);
   }
 
   void
@@ -338,37 +201,30 @@ namespace llarp::path
     path_limits.Decay(now);
 
     {
-      SyncTransitMap_t::Lock_t lock(m_TransitPaths.first);
+      auto itr = transit_hops.begin();
-      auto& map = m_TransitPaths.second;
+      while (itr != transit_hops.end())
-      auto itr = map.begin();
-      while (itr != map.end())
       {
         if (itr->second->Expired(now))
         {
           // TODO: this
           // _router->outboundMessageHandler().RemovePath(itr->first);
-          itr = map.erase(itr);
+          itr = transit_hops.erase(itr);
         }
         else
         {
-          itr->second->DecayFilters(now);
           ++itr;
         }
       }
     }
     {
-      util::Lock lock(m_OurPaths.first);
+      for (auto itr = own_paths.begin(); itr != own_paths.end();)
-      auto& map = m_OurPaths.second;
-      auto itr = map.begin();
-      while (itr != map.end())
       {
         if (itr->second->Expired(now))
         {
-          itr = map.erase(itr);
+          itr = own_paths.erase(itr);
         }
         else
         {
-          itr->second->DecayFilters(now);
           ++itr;
         }
       }

llarp/path/path_context.hpp

@@ -5,7 +5,6 @@
 #include "pathset.hpp"
 #include "transit_hop.hpp"
 
-#include <llarp/crypto/encrypted_frame.hpp>
 #include <llarp/ev/ev.hpp>
 #include <llarp/net/ip_address.hpp>
 #include <llarp/util/compare_ptr.hpp>
@@ -25,149 +24,121 @@ namespace llarp
     struct TransitHop;
     struct TransitHopInfo;
 
-    using TransitHop_ptr = std::shared_ptr<TransitHop>;
+    struct TransitHopID
+    {
+      RouterID rid;
+      PathID_t path_id;
 
-    struct PathContext
+      bool
+      operator==(const TransitHopID& other) const
+      {
+        return rid == other.rid && path_id == other.path_id;
+      }
+    };
+  }  // namespace path
+}  // namespace llarp
+
+namespace std
+{
+  template <>
+  struct hash<llarp::path::TransitHopID>
+  {
+    size_t
+    operator()(const llarp::path::TransitHopID& obj) const noexcept
     {
-      explicit PathContext(Router* router);
+      return std::hash<llarp::PathID_t>{}(obj.path_id);
+    }
+  };
+}  // namespace std
 
-      /// called from router tick function
+namespace llarp::path
-      void
+{
-      ExpirePaths(llarp_time_t now);
+  struct PathContext
+  {
+    explicit PathContext(Router* router);
 
-      void
+    /// called from router tick function
-      PumpUpstream();
+    void
+    ExpirePaths(llarp_time_t now);
 
-      void
+    void
-      PumpDownstream();
+    AllowTransit();
 
-      void
+    void
-      AllowTransit();
+    RejectTransit();
 
-      void
+    bool
-      RejectTransit();
+    CheckPathLimitHitByIP(const IpAddress& ip);
 
-      bool
+    bool
-      CheckPathLimitHitByIP(const IpAddress& ip);
+    CheckPathLimitHitByIP(const std::string& ip);
 
-      bool
+    bool
-      CheckPathLimitHitByIP(const std::string& ip);
+    AllowingTransit() const;
 
-      bool
+    bool
-      AllowingTransit() const;
+    HasTransitHop(const TransitHopInfo& info);
 
-      bool
+    void
-      HasTransitHop(const TransitHopInfo& info);
+    PutTransitHop(std::shared_ptr<TransitHop> hop);
 
-      void
+    Path_ptr
-      PutTransitHop(std::shared_ptr<TransitHop> hop);
+    GetPath(const PathID_t& path_id);
 
-      HopHandler_ptr
+    bool
-      GetByUpstream(const RouterID& id, const PathID_t& path);
+    TransitHopPreviousIsRouter(const PathID_t& path, const RouterID& r);
 
-      bool
+    std::shared_ptr<TransitHop>
-      TransitHopPreviousIsRouter(const PathID_t& path, const RouterID& r);
+    GetPathForTransfer(const PathID_t& topath);
 
-      TransitHop_ptr
+    std::shared_ptr<TransitHop>
-      GetPathForTransfer(const PathID_t& topath);
+    GetTransitHop(const RouterID&, const PathID_t&);
 
-      HopHandler_ptr
+    PathSet_ptr
-      GetByDownstream(const RouterID& id, const PathID_t& path);
+    GetLocalPathSet(const PathID_t& id);
 
-      std::optional<std::weak_ptr<TransitHop>>
+    /// get a set of all paths that we own who's endpoint is r
-      TransitHopByInfo(const TransitHopInfo&);
+    std::vector<std::shared_ptr<Path>>
+    FindOwnedPathsWithEndpoint(const RouterID& r);
 
-      std::optional<std::weak_ptr<TransitHop>>
+    bool
-      TransitHopByUpstream(const RouterID&, const PathID_t&);
+    HopIsUs(const RouterID& k) const;
 
-      PathSet_ptr
+    void
-      GetLocalPathSet(const PathID_t& id);
+    AddOwnPath(PathSet_ptr set, Path_ptr p);
 
-      using EndpointPathPtrSet = std::set<Path_ptr, ComparePtr<Path_ptr>>;
+    void
-      /// get a set of all paths that we own who's endpoint is r
+    RemovePathSet(PathSet_ptr set);
-      EndpointPathPtrSet
-      FindOwnedPathsWithEndpoint(const RouterID& r);
 
-      bool
+    const EventLoop_ptr&
-      HopIsUs(const RouterID& k) const;
+    loop();
 
-      void
+    const SecretKey&
-      AddOwnPath(PathSet_ptr set, Path_ptr p);
+    EncryptionSecretKey();
 
-      void
+    const byte_t*
-      RemovePathSet(PathSet_ptr set);
+    OurRouterID() const;
 
-      using TransitHopsMap_t = std::unordered_multimap<PathID_t, TransitHop_ptr>;
+    /// current number of transit paths we have
+    uint64_t
+    CurrentTransitPaths();
 
-      struct SyncTransitMap_t
+    /// current number of paths we created in status
-      {
+    uint64_t
-        using Mutex_t = util::NullMutex;
+    CurrentOwnedPaths(path::PathStatus status = path::PathStatus::ePathEstablished);
-        using Lock_t = util::NullLock;
-
-        Mutex_t first;  // protects second
-        TransitHopsMap_t second;
-
-        /// Invokes a callback for each transit path; visit must be invokable with a `const
-        /// TransitHop_ptr&` argument.
-        template <typename TransitHopVisitor>
-        void
-        ForEach(TransitHopVisitor&& visit)
-        {
-          Lock_t lock(first);
-          for (const auto& item : second)
-            visit(item.second);
-        }
-      };
-
-      // maps path id -> pathset owner of path
-      using OwnedPathsMap_t = std::unordered_map<PathID_t, Path_ptr>;
-
-      struct SyncOwnedPathsMap_t
-      {
-        util::Mutex first;  // protects second
-        OwnedPathsMap_t second;
-
-        /// Invokes a callback for each owned path; visit must be invokable with a `const Path_ptr&`
-        /// argument.
-        template <typename OwnedHopVisitor>
-        void
-        ForEach(OwnedHopVisitor&& visit)
-        {
-          util::Lock lock(first);
-          for (const auto& item : second)
-            visit(item.second);
-        }
-      };
-
-      const EventLoop_ptr&
-      loop();
-
-      const SecretKey&
-      EncryptionSecretKey();
-
-      const byte_t*
-      OurRouterID() const;
-
-      /// current number of transit paths we have
-      uint64_t
-      CurrentTransitPaths();
-
-      /// current number of paths we created in status
-      uint64_t
-      CurrentOwnedPaths(path::PathStatus status = path::PathStatus::ePathEstablished);
-
-      Router*
-      router() const
-      {
-        return _router;
-      }
 
-     private:
+    Router*
-      Router* _router;
+    router() const
-      SyncTransitMap_t m_TransitPaths;
+    {
-      SyncOwnedPathsMap_t m_OurPaths;
+      return _router;
-      bool m_AllowTransit;
+    }
-      util::DecayingHashSet<IpAddress> path_limits;
+
-    };
+   private:
-  }  // namespace path
+    Router* _router;
-}  // namespace llarp
+
+    std::unordered_map<TransitHopID, std::shared_ptr<TransitHop>> transit_hops;
+    std::unordered_map<PathID_t, Path_ptr> own_paths;
+    bool m_AllowTransit;
+    util::DecayingHashSet<IpAddress> path_limits;
+  };
+}  // namespace llarp::path

llarp/path/path_types.hpp

@@ -32,11 +32,11 @@ namespace llarp
       /// shared secret at this hop
       SharedSecret shared;
       /// hash of shared secret used for nonce mutation
-      ShortHash nonceXOR;
+      SymmNonce nonceXOR;
       /// next hop's router id
       RouterID upstream;
       /// nonce for key exchange
-      TunnelNonce nonce;
+      SymmNonce nonce;
       // lifetime
       llarp_time_t lifetime = DEFAULT_LIFETIME;
 

llarp/path/pathbuilder.cpp

@@ -7,11 +7,13 @@
 #include <llarp/link/link_manager.hpp>
 #include <llarp/messages/path.hpp>
 #include <llarp/nodedb.hpp>
+#include <llarp/path/pathset.hpp>
 #include <llarp/profiling.hpp>
-#include <llarp/router/rc_lookup_handler.hpp>
 #include <llarp/router/router.hpp>
 #include <llarp/util/logging.hpp>
 
+#include <functional>
+
 namespace llarp
 {
   namespace
@@ -90,7 +92,9 @@ namespace llarp
         throw std::runtime_error{std::move(err)};
       }
       // generate nonceXOR value self->hop->pathKey
-      crypto::shorthash(hop.nonceXOR, hop.shared.data(), hop.shared.size());
+      ShortHash hash;
+      crypto::shorthash(hash, hop.shared.data(), hop.shared.size());
+      hop.nonceXOR = hash.data();  // nonceXOR is 24 bytes, ShortHash is 32; this will truncate
 
       hop.upstream = nextHop;
     }
@@ -117,7 +121,7 @@ namespace llarp
       crypto::encryption_keygen(framekey);
 
       SharedSecret shared;
-      TunnelNonce outer_nonce;
+      SymmNonce outer_nonce;
       outer_nonce.Randomize();
 
       // derive (outer) shared key
@@ -424,44 +428,75 @@ namespace llarp
           path_cat, "{} building path -> {} : {}", Name(), path->ShortName(), path->HopsString());
 
       oxenc::bt_list_producer frames;
-
+      std::vector<std::string> frame_str(path::MAX_LEN);
       auto& path_hops = path->hops;
       size_t n_hops = path_hops.size();
       size_t last_len{0};
 
-      for (size_t i = 0; i < n_hops; i++)
+      // each hop will be able to read the outer part of its frame and decrypt
+      // the inner part with that information.  It will then do an onion step on the
+      // remaining frames so the next hop can read the outer part of its frame,
+      // and so on.  As this de-onion happens from hop 1 to n, we create and onion
+      // the frames from hop n downto 1 (i.e. reverse order).  The first frame is
+      // not onioned.
+      //
+      // Onion-ing the frames in this way will prevent relays controlled by
+      // the same entity from knowing they are part of the same path
+      // (unless they're adjacent in the path; nothing we can do about that obviously).
+
+      // i from n_hops downto 0
+      size_t i = n_hops;
+      while (i > 0)
       {
+        i--;
         bool lastHop = (i == (n_hops - 1));
 
         const auto& nextHop =
             lastHop ? path_hops[i].rc.router_id() : path_hops[i + 1].rc.router_id();
 
         PathBuildMessage::setup_hop_keys(path_hops[i], nextHop);
-        auto frame_str = PathBuildMessage::serialize(path_hops[i]);
+        frame_str[i] = PathBuildMessage::serialize(path_hops[i]);
 
         // all frames should be the same length...not sure what that is yet
+        // it may vary if path lifetime is non-default, as that is encoded as an
+        // integer in decimal, but it should be constant for a given path
         if (last_len != 0)
-          assert(frame_str.size() == last_len);
+          assert(frame_str[i].size() == last_len);
+
+        last_len = frame_str[i].size();
 
-        last_len = frame_str.size();
+        // onion each previously-created frame using the established shared secret and
-        frames.append(std::move(frame_str));
+        // onion_nonce = path_hops[i].nonce ^ path_hops[i].nonceXOR, which the transit hop
+        // will have recovered after decrypting its frame.
+        // Note: final value passed to crypto::onion is xor factor, but that's for *after* the
+        // onion round to compute the return value, so we don't care about it.
+        for (size_t j = n_hops - 1; j > i; j--)
+        {
+          auto onion_nonce = path_hops[i].nonce ^ path_hops[i].nonceXOR;
+          crypto::onion(
+              reinterpret_cast<unsigned char*>(frame_str[j].data()),
+              frame_str[j].size(),
+              path_hops[i].shared,
+              onion_nonce,
+              onion_nonce);
+        }
       }
 
       std::string dummy;
       dummy.reserve(last_len);
-
       // append dummy frames; path build request must always have MAX_LEN frames
-      // TODO: with the data structured as it is now (bt-encoded dict as each frame)
+      for (i = n_hops; i < path::MAX_LEN; i++)
-      //       the dummy frames can't be completely random; they need to look like
-      //       normal frames
-      for (size_t i = 0; i < path::MAX_LEN - n_hops; i++)
       {
-        randombytes(reinterpret_cast<uint8_t*>(dummy.data()), dummy.size());
+        frame_str[i].resize(last_len);
-        frames.append(dummy);
+        randombytes(reinterpret_cast<uint8_t*>(frame_str[i].data()), frame_str[i].size());
       }
 
-      auto self = GetSelf();
+      for (auto& str : frame_str)  // NOLINT
-      router->path_context().AddOwnPath(self, path);
+      {
+        frames.append(std::move(str));
+      }
+
+      router->path_context().AddOwnPath(GetSelf(), path);
       PathBuildStarted(path);
 
       // TODO:
@@ -469,30 +504,33 @@ namespace llarp
       // handle these responses as well as how we store and use Paths as a whole might
       // be worth doing sooner rather than later.  Leaving some TODOs below where fail
       // and success live.
-      auto response_cb = [self](oxen::quic::message m) {
+      auto response_cb = [path](oxen::quic::message m) {
-        if (m)
+        try
         {
-          std::string status;
+          if (m)
-
-          try
           {
-            oxenc::bt_dict_consumer btdc{m.body()};
+            // TODO: inform success (what this means needs revisiting, badly)
-            status = btdc.require<std::string>("STATUS");
+            path->EnterState(path::ePathEstablished);
+            return;
           }
-          catch (...)
+          if (m.timed_out)
           {
-            log::warning(path_cat, "Error: Failed to parse path build response!", status);
+            log::warning(path_cat, "Path build timed out");
-            m.respond(serialize_response({{"STATUS", "EXCEPTION"}}), true);
+          }
-            throw;
+          else
+          {
+            oxenc::bt_dict_consumer d{m.body()};
+            auto status = d.require<std::string_view>(messages::STATUS_KEY);
+            log::warning(path_cat, "Path build returned failure status: {}", status);
           }
-
-          // TODO: success logic
         }
-        else
+        catch (const std::exception& e)
         {
-          log::warning(path_cat, "Path build request returned failure {}");
+          log::warning(path_cat, "Failed parsing path build response.");
-          // TODO: failure logic
         }
+
+        // TODO: inform failure (what this means needs revisiting, badly)
+        path->EnterState(path::ePathFailed);
       };
 
       if (not router->send_control_message(
@@ -536,30 +574,6 @@ namespace llarp
       router->router_profiling().MarkPathTimeout(p.get());
       PathSet::HandlePathBuildTimeout(p);
       DoPathBuildBackoff();
-      for (const auto& hop : p->hops)
-      {
-        const auto& target = hop.rc.router_id();
-        // look up router and see if it's still on the network
-        log::info(path_cat, "Looking up RouterID {} due to path build timeout", target);
-
-        router->rc_lookup_handler().get_rc(
-            target,
-            [this](auto rid, auto rc, auto success) {
-              if (success && rc)
-              {
-                log::info(path_cat, "Refreshed RouterContact for {}", rid);
-                router->node_db()->put_rc_if_newer(*rc);
-              }
-              else
-              {
-                // remove all connections to this router as it's probably not registered anymore
-                log::warning(path_cat, "Removing router {} due to path build timeout", rid);
-                router->link_manager().deregister_peer(rid);
-                router->node_db()->remove_router(rid);
-              }
-            },
-            true);
-      }
     }
 
     void

llarp/path/pathset.cpp

@@ -2,6 +2,8 @@
 
 #include "path.hpp"
 
+#include <llarp/crypto/crypto.hpp>
+
 namespace llarp::path
 {
   PathSet::PathSet(size_t num) : numDesiredPaths(num)
@@ -441,16 +443,4 @@ namespace llarp::path
     return chosen;
   }
 
-  void
-  PathSet::UpstreamFlush(Router* r)
-  {
-    ForEachPath([r](const Path_ptr& p) { p->FlushUpstream(r); });
-  }
-
-  void
-  PathSet::DownstreamFlush(Router* r)
-  {
-    ForEachPath([r](const Path_ptr& p) { p->FlushDownstream(r); });
-  }
-
 }  // namespace llarp::path

llarp/path/transit_hop.cpp

@@ -12,21 +12,43 @@ namespace llarp::path
         "[TransitHopInfo tx={} rx={} upstream={} downstream={}]", txID, rxID, upstream, downstream);
   }
 
-  TransitHop::TransitHop()
+  TransitHop::TransitHop() : AbstractHopHandler{}
-      : AbstractHopHandler{}
+  {}
-      , m_UpstreamGather{TRANSIT_HOP_QUEUE_SIZE}
+
-      , m_DownstreamGather{TRANSIT_HOP_QUEUE_SIZE}
+  void
+  TransitHop::onion(ustring& data, SymmNonce& nonce, bool randomize) const
+  {
+    if (randomize)
+      nonce.Randomize();
+    nonce = crypto::onion(data.data(), data.size(), pathKey, nonce, nonceXOR);
+  }
+
+  void
+  TransitHop::onion(std::string& data, SymmNonce& nonce, bool randomize) const
+  {
+    if (randomize)
+      nonce.Randomize();
+    nonce = crypto::onion(
+        reinterpret_cast<unsigned char*>(data.data()), data.size(), pathKey, nonce, nonceXOR);
+  }
+
+  std::string
+  TransitHop::onion_and_payload(
+      std::string& payload, PathID_t next_id, std::optional<SymmNonce> nonce) const
   {
-    m_UpstreamGather.enable();
+    SymmNonce n;
-    m_DownstreamGather.enable();
+    auto& nref = nonce ? *nonce : n;
-    m_UpstreamWorkCounter = 0;
+    onion(payload, nref, not nonce);
-    m_DownstreamWorkCounter = 0;
+
+    return path::make_onion_payload(nref, next_id, payload);
   }
 
   bool
-  TransitHop::send_path_control_message(
+  TransitHop::send_path_control_message(std::string, std::string, std::function<void(std::string)>)
-      std::string, std::string, std::function<void(oxen::quic::message m)>)
   {
+    // TODO: if we want terminal/pivot hops to be able to *initiate* a request rather than
+    //       simply responding/reacting to the client end's requests, this will need
+    //       an implementation.
     return true;
   }
 
@@ -60,6 +82,7 @@ namespace llarp::path
       functions it calls and so on) will need to be modified to take an std::string that we can
       std::move around.
   */
+  /* TODO: replace this with layer of onion + send data message
   bool
   TransitHop::SendRoutingMessage(std::string payload, Router* r)
   {
@@ -82,157 +105,7 @@ namespace llarp::path
 
     return true;
   }
-
+  */
-  void
-  TransitHop::DownstreamWork(TrafficQueue_t msgs, Router* r)
-  {
-    auto flushIt = [self = shared_from_this(), r]() {
-      std::vector<RelayDownstreamMessage> msgs;
-      while (auto maybe = self->m_DownstreamGather.tryPopFront())
-      {
-        msgs.push_back(*maybe);
-      }
-      self->HandleAllDownstream(std::move(msgs), r);
-    };
-    for (auto& ev : msgs)
-    {
-      RelayDownstreamMessage msg;
-
-      // const llarp_buffer_t buf(ev.first);
-      uint8_t* buf = ev.first.data();
-      size_t sz = ev.first.size();
-
-      msg.pathid = info.rxID;
-      msg.nonce = ev.second ^ nonceXOR;
-
-      crypto::xchacha20(buf, sz, pathKey, ev.second);
-      std::memcpy(msg.enc.data(), buf, sz);
-
-      llarp::LogDebug(
-          "relay ",
-          msg.enc.size(),
-          " bytes downstream from ",
-          info.upstream,
-          " to ",
-          info.downstream);
-      if (m_DownstreamGather.full())
-      {
-        r->loop()->call(flushIt);
-      }
-      if (m_DownstreamGather.enabled())
-        m_DownstreamGather.pushBack(msg);
-    }
-    r->loop()->call(flushIt);
-  }
-
-  void
-  TransitHop::UpstreamWork(TrafficQueue_t msgs, Router* r)
-  {
-    for (auto& ev : msgs)
-    {
-      RelayUpstreamMessage msg;
-
-      uint8_t* buf = ev.first.data();
-      size_t sz = ev.first.size();
-
-      crypto::xchacha20(buf, sz, pathKey, ev.second);
-
-      msg.pathid = info.txID;
-      msg.nonce = ev.second ^ nonceXOR;
-      std::memcpy(msg.enc.data(), buf, sz);
-
-      if (m_UpstreamGather.tryPushBack(msg) != thread::QueueReturn::Success)
-        break;
-    }
-
-    // Flush it:
-    r->loop()->call([self = shared_from_this(), r] {
-      std::vector<RelayUpstreamMessage> msgs;
-      while (auto maybe = self->m_UpstreamGather.tryPopFront())
-      {
-        msgs.push_back(*maybe);
-      }
-      self->HandleAllUpstream(std::move(msgs), r);
-    });
-  }
-
-  void
-  TransitHop::HandleAllUpstream(std::vector<RelayUpstreamMessage> msgs, Router* r)
-  {
-    if (IsEndpoint(r->pubkey()))
-    {
-      for (const auto& msg : msgs)
-      {
-        const llarp_buffer_t buf(msg.enc);
-        if (!r->ParseRoutingMessageBuffer(buf, *this, info.rxID))
-        {
-          LogWarn("invalid upstream data on endpoint ", info);
-        }
-        m_LastActivity = r->now();
-      }
-      FlushDownstream(r);
-      for (const auto& other : m_FlushOthers)
-      {
-        other->FlushDownstream(r);
-      }
-      m_FlushOthers.clear();
-    }
-    else
-    {
-      for (const auto& msg : msgs)
-      {
-        llarp::LogDebug(
-            "relay ",
-            msg.enc.size(),
-            " bytes upstream from ",
-            info.downstream,
-            " to ",
-            info.upstream);
-        r->send_data_message(info.upstream, msg.bt_encode());
-      }
-    }
-    r->TriggerPump();
-  }
-
-  void
-  TransitHop::HandleAllDownstream(std::vector<RelayDownstreamMessage> msgs, Router* r)
-  {
-    for (const auto& msg : msgs)
-    {
-      log::debug(
-          path_cat,
-          "Relaying {} bytes downstream from {} to {}",
-          msg.enc.size(),
-          info.upstream,
-          info.downstream);
-      // TODO: is this right?
-      r->send_data_message(info.downstream, msg.bt_encode());
-    }
-
-    r->TriggerPump();
-  }
-
-  void
-  TransitHop::FlushUpstream(Router* r)
-  {
-    if (not m_UpstreamQueue.empty())
-    {
-      r->queue_work([self = shared_from_this(),
-                     data = std::exchange(m_UpstreamQueue, {}),
-                     r]() mutable { self->UpstreamWork(std::move(data), r); });
-    }
-  }
-
-  void
-  TransitHop::FlushDownstream(Router* r)
-  {
-    if (not m_DownstreamQueue.empty())
-    {
-      r->queue_work([self = shared_from_this(),
-                     data = std::exchange(m_DownstreamQueue, {}),
-                     r]() mutable { self->DownstreamWork(std::move(data), r); });
-    }
-  }
 
   std::string
   TransitHop::ToString() const
@@ -244,8 +117,7 @@ namespace llarp::path
   void
   TransitHop::Stop()
   {
-    m_UpstreamGather.disable();
+    // TODO: still need this concept?
-    m_DownstreamGather.disable();
   }
 
   void

llarp/path/transit_hop.hpp

@@ -55,12 +55,29 @@ namespace llarp
 
       TransitHopInfo info;
       SharedSecret pathKey;
-      ShortHash nonceXOR;
+      SymmNonce nonceXOR;
       llarp_time_t started = 0s;
       // 10 minutes default
       llarp_time_t lifetime = DEFAULT_LIFETIME;
       llarp_proto_version_t version;
       llarp_time_t m_LastActivity = 0s;
+      bool terminal_hop{false};
+
+      // If randomize is given, first randomizes `nonce`
+      //
+      // Does xchacha20 on `data` in-place with `nonce` and `pathKey`, then
+      // mutates `nonce` = `nonce` ^ `nonceXOR` in-place.
+      void
+      onion(ustring& data, SymmNonce& nonce, bool randomize = false) const;
+
+      void
+      onion(std::string& data, SymmNonce& nonce, bool randomize = false) const;
+
+      std::string
+      onion_and_payload(
+          std::string& payload,
+          PathID_t next_id,
+          std::optional<SymmNonce> nonce = std::nullopt) const;
 
       PathID_t
       RXID() const override
@@ -106,47 +123,29 @@ namespace llarp
         return now >= ExpireTime() - dlt;
       }
 
+      // TODO: should this be a separate method indicating directionality?
+      //       Most control messages won't make sense to be sent to a client,
+      //       so perhaps control messages from a terminal relay to a client (rather than
+      //       the other way around) should be their own message type.
+      //
+      /// sends a control request along a path
+      ///
+      /// performs the necessary onion encryption before sending.
+      /// func will be called when a timeout occurs or a response is received.
+      /// if a response is received, onion decryption is performed before func is called.
+      ///
+      /// func is called with a bt-encoded response string (if applicable), and
+      /// a timeout flag (if set, response string will be empty)
       bool
       send_path_control_message(
-          std::string method,
+          std::string method, std::string body, std::function<void(std::string)> func) override;
-          std::string body,
-          std::function<void(oxen::quic::message m)> func) override;
-
-      // send routing message when end of path
-      bool
-      SendRoutingMessage(std::string payload, Router* r) override;
-
-      void
-      FlushUpstream(Router* r) override;
-
-      void
-      FlushDownstream(Router* r) override;
 
       void
       QueueDestroySelf(Router* r);
 
-     protected:
-      void
-      UpstreamWork(TrafficQueue_t queue, Router* r) override;
-
-      void
-      DownstreamWork(TrafficQueue_t queue, Router* r) override;
-
-      void
-      HandleAllUpstream(std::vector<RelayUpstreamMessage> msgs, Router* r) override;
-
-      void
-      HandleAllDownstream(std::vector<RelayDownstreamMessage> msgs, Router* r) override;
-
      private:
       void
       SetSelfDestruct();
-
-      std::set<std::shared_ptr<TransitHop>, ComparePtr<std::shared_ptr<TransitHop>>> m_FlushOthers;
-      thread::Queue<RelayUpstreamMessage> m_UpstreamGather;
-      thread::Queue<RelayDownstreamMessage> m_DownstreamGather;
-      std::atomic<uint32_t> m_UpstreamWorkCounter;
-      std::atomic<uint32_t> m_DownstreamWorkCounter;
     };
   }  // namespace path
 

llarp/router/rc_gossiper.cpp

@@ -1,151 +0,0 @@
-#include "rc_gossiper.hpp"
-
-#include <llarp/router_contact.hpp>
-#include <llarp/util/time.hpp>
-
-namespace llarp
-{
-  // 30 minutes
-  static constexpr auto RCGossipFilterDecayInterval = 30min;
-  // (30 minutes * 2) - 5 minutes
-  static constexpr auto GossipOurRCInterval = (RCGossipFilterDecayInterval * 2) - (5min);
-
-  RCGossiper::RCGossiper() : filter(std::chrono::duration_cast<Time_t>(RCGossipFilterDecayInterval))
-  {}
-
-  void
-  RCGossiper::Init(LinkManager* l, const RouterID& ourID, Router* r)
-  {
-    rid = ourID;
-    link_manager = l;
-    router = r;
-  }
-
-  bool
-  RCGossiper::ShouldGossipOurRC(Time_t now) const
-  {
-    return now >= (last_rc_gossip + GossipOurRCInterval);
-  }
-
-  bool
-  RCGossiper::IsOurRC(const LocalRC& rc) const
-  {
-    return rc.router_id() == rid;
-  }
-
-  void
-  RCGossiper::Decay(Time_t now)
-  {
-    filter.Decay(now);
-  }
-
-  void
-  RCGossiper::Forget(const RouterID& pk)
-  {
-    filter.Remove(pk);
-    if (rid == pk)
-      last_rc_gossip = 0s;
-  }
-
-  TimePoint_t
-  RCGossiper::NextGossipAt() const
-  {
-    if (auto maybe = LastGossipAt())
-      return *maybe + GossipOurRCInterval;
-    return DateClock_t::now();
-  }
-
-  std::optional<TimePoint_t>
-  RCGossiper::LastGossipAt() const
-  {
-    if (last_rc_gossip == 0s)
-      return std::nullopt;
-    return DateClock_t::time_point{last_rc_gossip};
-  }
-
-  bool
-  RCGossiper::GossipRC(const LocalRC& rc)
-  {
-    // only distribute public routers
-    if (not rc.is_public_router())
-      return false;
-    if (link_manager == nullptr)
-      return false;
-    const RouterID pubkey(rc.router_id());
-    // filter check
-    if (filter.Contains(pubkey))
-      return false;
-    filter.Insert(pubkey);
-
-    const auto now = time_now_ms();
-    // is this our rc?
-    if (IsOurRC(rc))
-    {
-      // should we gossip our rc?
-      if (not ShouldGossipOurRC(now))
-      {
-        // nah drop it
-        return false;
-      }
-      // ya pop it
-      last_rc_gossip = now;
-    }
-
-    // send a GRCM as gossip method
-    // DHTImmediateMessage gossip;
-    // gossip.msgs.emplace_back(new dht::GotRouterMessage(dht::Key_t{}, 0, {rc}, false));
-
-    // std::vector<RouterID> gossipTo;
-
-    /*
-     * TODO: gossip RC via libquic
-     *
-    // select peers to gossip to
-    m_LinkManager->ForEachPeer(
-        [&](const AbstractLinkSession* peerSession, bool) {
-          // ensure connected session
-          if (not(peerSession && peerSession->IsEstablished()))
-            return;
-          // check if public router
-          const auto other_rc = peerSession->GetRemoteRC();
-          if (not other_rc.IsPublicRouter())
-            return;
-          gossipTo.emplace_back(other_rc.pubkey);
-        },
-        true);
-
-    std::unordered_set<RouterID> keys;
-    // grab the keys we want to use
-    std::sample(
-        gossipTo.begin(), gossipTo.end(), std::inserter(keys, keys.end()), MaxGossipPeers,
-    llarp::csrng);
-
-    m_LinkManager->ForEachPeer([&](AbstractLinkSession* peerSession) {
-      if (not(peerSession && peerSession->IsEstablished()))
-        return;
-
-      // exclude from gossip as we have not selected to use it
-      if (keys.count(peerSession->GetPubKey()) == 0)
-        return;
-
-      // encode message
-      AbstractLinkSession::Message_t msg{};
-      msg.resize(MAX_LINK_MSG_SIZE / 2);
-      llarp_buffer_t buf(msg);
-      if (not gossip.BEncode(&buf))
-        return;
-      msg.resize(buf.cur - buf.base);
-
-      m_router->NotifyRouterEvent<tooling::RCGossipSentEvent>(m_router->pubkey(), rc);
-
-      // send message
-      peerSession->SendMessageBuffer(std::move(msg), nullptr, gossip.Priority());
-    });
-     *
-     *
-     */
-
-    return true;
-  }
-
-}  // namespace llarp

llarp/router/rc_gossiper.hpp

@@ -1,57 +0,0 @@
-#pragma once
-
-#include <llarp/router_id.hpp>
-#include <llarp/util/decaying_hashset.hpp>
-
-#include <optional>
-
-namespace llarp
-{
-  struct Router;
-
-  /// The maximum number of peers we will flood a gossiped RC to when propagating an RC
-  constexpr size_t MaxGossipPeers = 20;
-  struct LinkManager;
-  struct LocalRC;
-
-  struct RCGossiper
-  {
-    using Time_t = Duration_t;
-
-    RCGossiper();
-
-    ~RCGossiper() = default;
-
-    bool
-    GossipRC(const LocalRC& rc);
-
-    void
-    Decay(Time_t now);
-
-    bool
-    ShouldGossipOurRC(Time_t now) const;
-
-    bool
-    IsOurRC(const LocalRC& rc) const;
-
-    void
-    Init(LinkManager*, const RouterID&, Router*);
-
-    void
-    Forget(const RouterID& router);
-
-    TimePoint_t
-    NextGossipAt() const;
-
-    std::optional<TimePoint_t>
-    LastGossipAt() const;
-
-   private:
-    RouterID rid;
-    Time_t last_rc_gossip = 0s;
-    LinkManager* link_manager = nullptr;
-    util::DecayingHashSet<RouterID> filter;
-
-    Router* router;
-  };
-}  // namespace llarp

llarp/router/rc_lookup_handler.cpp

@@ -1,381 +0,0 @@
-#include "rc_lookup_handler.hpp"
-
-#include "router.hpp"
-
-#include <llarp/crypto/crypto.hpp>
-#include <llarp/link/contacts.hpp>
-#include <llarp/link/link_manager.hpp>
-#include <llarp/nodedb.hpp>
-#include <llarp/router_contact.hpp>
-#include <llarp/service/context.hpp>
-#include <llarp/util/types.hpp>
-
-#include <functional>
-#include <iterator>
-
-namespace llarp
-{
-  void
-  RCLookupHandler::add_valid_router(const RouterID& rid)
-  {
-    router->loop()->call([this, rid]() { router_whitelist.insert(rid); });
-  }
-
-  void
-  RCLookupHandler::remove_valid_router(const RouterID& rid)
-  {
-    router->loop()->call([this, rid]() { router_whitelist.erase(rid); });
-  }
-
-  static void
-  loadColourList(std::unordered_set<RouterID>& beigelist, const std::vector<RouterID>& new_beige)
-  {
-    beigelist.clear();
-    beigelist.insert(new_beige.begin(), new_beige.end());
-  }
-
-  void
-  RCLookupHandler::set_router_whitelist(
-      const std::vector<RouterID>& whitelist,
-      const std::vector<RouterID>& greylist,
-      const std::vector<RouterID>& greenlist)
-  {
-    if (whitelist.empty())
-      return;
-
-    router->loop()->call([this, whitelist, greylist, greenlist]() {
-      loadColourList(router_whitelist, whitelist);
-      loadColourList(router_greylist, greylist);
-      loadColourList(router_greenlist, greenlist);
-      LogInfo("lokinet service node list now has ", router_whitelist.size(), " active routers");
-    });
-  }
-
-  bool
-  RCLookupHandler::has_received_whitelist() const
-  {
-    return router->loop()->call_get([this]() { return not router_whitelist.empty(); });
-  }
-
-  std::unordered_set<RouterID>
-  RCLookupHandler::whitelist() const
-  {
-    return router->loop()->call_get([this]() { return router_whitelist; });
-  }
-
-  void
-  RCLookupHandler::get_rc(const RouterID& rid, RCRequestCallback callback, bool forceLookup)
-  {
-    RemoteRC remoteRC;
-
-    if (not forceLookup)
-    {
-      if (const auto maybe = node_db->get_rc(rid); maybe.has_value())
-      {
-        remoteRC = *maybe;
-
-        if (callback)
-        {
-          callback(rid, remoteRC, true);
-        }
-
-        return;
-      }
-    }
-
-    auto lookup_cb = [this, callback, rid](oxen::quic::message m) mutable {
-      auto& r = link_manager->router();
-
-      if (m)
-      {
-        std::string payload;
-
-        try
-        {
-          oxenc::bt_dict_consumer btdc{m.body()};
-          payload = btdc.require<std::string>("RC");
-        }
-        catch (...)
-        {
-          log::warning(link_cat, "Failed to parse Find Router response!");
-          throw;
-        }
-
-        RemoteRC result{std::move(payload)};
-
-        if (callback)
-          callback(result.router_id(), result, true);
-        else
-          r.node_db()->put_rc_if_newer(result);
-      }
-      else
-      {
-        if (callback)
-          callback(rid, std::nullopt, false);
-        else
-          link_manager->handle_find_router_error(std::move(m));
-      }
-    };
-
-    // if we are a client try using the hidden service endpoints
-    if (!isServiceNode)
-    {
-      bool sent = false;
-      LogInfo("Lookup ", rid, " anonymously");
-      hidden_service_context->ForEachService(
-          [&, cb = lookup_cb](
-              const std::string&, const std::shared_ptr<service::Endpoint>& ep) -> bool {
-            const bool success = ep->lookup_router(rid, cb);
-            sent = sent || success;
-            return !success;
-          });
-      if (sent)
-        return;
-      LogWarn("cannot lookup ", rid, " anonymously");
-    }
-
-    contacts->lookup_router(rid, lookup_cb);
-  }
-
-  bool
-  RCLookupHandler::is_grey_listed(const RouterID& remote) const
-  {
-    if (strict_connect_pubkeys.size() && strict_connect_pubkeys.count(remote) == 0
-        && !is_remote_in_bootstrap(remote))
-    {
-      return false;
-    }
-
-    if (not isServiceNode)
-      return false;
-
-    return router->loop()->call_get([this, remote]() { return router_greylist.count(remote); });
-  }
-
-  bool
-  RCLookupHandler::is_green_listed(const RouterID& remote) const
-  {
-    return router->loop()->call_get([this, remote]() { return router_greenlist.count(remote); });
-  }
-
-  bool
-  RCLookupHandler::is_registered(const RouterID& rid) const
-  {
-    return router->loop()->call_get([this, rid]() {
-      return router_whitelist.count(rid) || router_greylist.count(rid)
-          || router_greenlist.count(rid);
-    });
-  }
-
-  bool
-  RCLookupHandler::is_path_allowed(const RouterID& rid) const
-  {
-    return router->loop()->call_get([this, rid]() {
-      if (strict_connect_pubkeys.size() && strict_connect_pubkeys.count(rid) == 0
-          && !is_remote_in_bootstrap(rid))
-      {
-        return false;
-      }
-
-      if (not isServiceNode)
-        return true;
-
-      return router_whitelist.count(rid) != 0;
-    });
-  }
-
-  bool
-  RCLookupHandler::is_session_allowed(const RouterID& rid) const
-  {
-    return router->loop()->call_get([this, rid]() {
-      if (strict_connect_pubkeys.size() && strict_connect_pubkeys.count(rid) == 0
-          && !is_remote_in_bootstrap(rid))
-      {
-        return false;
-      }
-
-      if (not isServiceNode)
-        return true;
-
-      return router_whitelist.count(rid) or router_greylist.count(rid);
-    });
-  }
-
-  bool
-  RCLookupHandler::check_rc(const RemoteRC& rc) const
-  {
-    if (not is_session_allowed(rc.router_id()))
-    {
-      contacts->delete_rc_node_async(dht::Key_t{rc.router_id()});
-      return false;
-    }
-
-    if (not rc.verify())
-    {
-      log::info(link_cat, "Invalid RC (rid: {})", rc.router_id());
-      return false;
-    }
-
-    // update nodedb if required
-    if (rc.is_public_router())
-    {
-      log::info(link_cat, "Adding or updating RC (rid: {}) to nodeDB and DHT", rc.router_id());
-      node_db->put_rc_if_newer(rc);
-      contacts->put_rc_node_async(rc);
-    }
-
-    return true;
-  }
-
-  size_t
-  RCLookupHandler::num_strict_connect_routers() const
-  {
-    return strict_connect_pubkeys.size();
-  }
-
-  bool
-  RCLookupHandler::get_random_whitelist_router(RouterID& rid) const
-  {
-    return router->loop()->call_get([this, rid]() mutable {
-      const auto sz = router_whitelist.size();
-      auto itr = router_whitelist.begin();
-      if (sz == 0)
-        return false;
-      if (sz > 1)
-        std::advance(itr, randint() % sz);
-      rid = *itr;
-      return true;
-    });
-  }
-
-  void
-  RCLookupHandler::periodic_update(llarp_time_t now)
-  {
-    // try looking up stale routers
-    std::unordered_set<RouterID> routersToLookUp;
-
-    node_db->VisitInsertedBefore(
-        [&](const RouterContact& rc) { routersToLookUp.insert(rc.router_id()); },
-        now - RouterContact::REPUBLISH);
-
-    for (const auto& router : routersToLookUp)
-    {
-      get_rc(router, nullptr, true);
-    }
-
-    node_db->remove_stale_rcs(boostrap_rid_list, now - RouterContact::STALE);
-  }
-
-  void
-  RCLookupHandler::explore_network()
-  {
-    const size_t known = node_db->num_loaded();
-    if (bootstrap_rc_list.empty() && known == 0)
-    {
-      LogError("we have no bootstrap nodes specified");
-    }
-    else if (known <= bootstrap_rc_list.size())
-    {
-      for (const auto& rc : bootstrap_rc_list)
-      {
-        const auto& rid = rc.router_id();
-        log::info(link_cat, "Doing explore via bootstrap node: {}", rid);
-
-        // TODO: replace this concept
-        // dht->ExploreNetworkVia(dht::Key_t{rc.pubkey});
-      }
-    }
-
-    if (isServiceNode)
-    {
-      static constexpr size_t LookupPerTick = 5;
-
-      std::vector<RouterID> lookup_routers = router->loop()->call_get([this]() {
-        std::vector<RouterID> lookups;
-        lookups.reserve(LookupPerTick);
-
-        for (const auto& r : router_whitelist)
-        {
-          if (not node_db->has_router(r))
-            lookups.emplace_back(r);
-        }
-
-        return lookups;
-      });
-
-      if (lookup_routers.size() > LookupPerTick)
-      {
-        std::shuffle(lookup_routers.begin(), lookup_routers.end(), llarp::csrng);
-        lookup_routers.resize(LookupPerTick);
-      }
-
-      for (const auto& r : lookup_routers)
-        get_rc(r, nullptr, true);
-      return;
-    }
-    // service nodes gossip, not explore
-    if (contacts->router()->is_service_node())
-      return;
-
-    // explore via every connected peer
-    /*
-     * TODO: DHT explore via libquic
-     *
-    _linkManager->ForEachPeer([&](ILinkSession* s) {
-      if (!s->IsEstablished())
-        return;
-      const RouterContact rc = s->GetRemoteRC();
-      if (rc.IsPublicRouter() && (_bootstrapRCList.find(rc) == _bootstrapRCList.end()))
-      {
-        LogDebug("Doing explore via public node: ", RouterID(rc.pubkey));
-        _dht->impl->ExploreNetworkVia(dht::Key_t{rc.pubkey});
-      }
-    });
-     *
-     *
-     */
-  }
-
-  void
-  RCLookupHandler::init(
-      std::shared_ptr<Contacts> c,
-      std::shared_ptr<NodeDB> nodedb,
-      EventLoop_ptr l,
-      std::function<void(std::function<void()>)> dowork,
-      LinkManager* linkManager,
-      service::Context* hiddenServiceContext,
-      const std::unordered_set<RouterID>& strictConnectPubkeys,
-      const std::set<RemoteRC>& bootstrapRCList,
-      bool isServiceNode_arg)
-  {
-    contacts = c;
-    node_db = std::move(nodedb);
-    loop = std::move(l);
-    work_func = std::move(dowork);
-    hidden_service_context = hiddenServiceContext;
-    strict_connect_pubkeys = strictConnectPubkeys;
-    bootstrap_rc_list = bootstrapRCList;
-    link_manager = linkManager;
-    router = &link_manager->router();
-    isServiceNode = isServiceNode_arg;
-
-    for (const auto& rc : bootstrap_rc_list)
-    {
-      boostrap_rid_list.insert(rc.router_id());
-    }
-  }
-
-  bool
-  RCLookupHandler::is_remote_in_bootstrap(const RouterID& remote) const
-  {
-    for (const auto& rc : bootstrap_rc_list)
-    {
-      if (rc.router_id() == remote)
-      {
-        return true;
-      }
-    }
-    return false;
-  }
-
-}  // namespace llarp

llarp/router/rc_lookup_handler.hpp

@@ -1,143 +0,0 @@
-#pragma once
-
-#include <llarp/router_contact.hpp>
-#include <llarp/router_id.hpp>
-#include <llarp/util/thread/threading.hpp>
-
-#include <chrono>
-#include <list>
-#include <set>
-#include <unordered_map>
-#include <unordered_set>
-
-struct llarp_dht_context;
-
-namespace llarp
-{
-  class NodeDB;
-  struct Router;
-  class EventLoop;
-
-  namespace service
-  {
-    struct Context;
-  }  // namespace service
-
-  struct Contacts;
-  struct LinkManager;
-
-  enum class RCRequestResult
-  {
-    Success,
-    InvalidRouter,
-    RouterNotFound,
-    BadRC
-  };
-
-  using RCRequestCallback =
-      std::function<void(const RouterID&, std::optional<RemoteRC>, bool success)>;
-
-  struct RCLookupHandler
-  {
-   public:
-    ~RCLookupHandler() = default;
-
-    void
-    add_valid_router(const RouterID& router);
-
-    void
-    remove_valid_router(const RouterID& router);
-
-    void
-    set_router_whitelist(
-        const std::vector<RouterID>& whitelist,
-        const std::vector<RouterID>& greylist,
-        const std::vector<RouterID>& greenlist);
-
-    bool
-    has_received_whitelist() const;
-
-    void
-    get_rc(const RouterID& router, RCRequestCallback callback, bool forceLookup = false);
-
-    bool
-    is_path_allowed(const RouterID& remote) const;
-
-    bool
-    is_session_allowed(const RouterID& remote) const;
-
-    bool
-    is_grey_listed(const RouterID& remote) const;
-
-    // "greenlist" = new routers (i.e. "green") that aren't fully funded yet
-    bool
-    is_green_listed(const RouterID& remote) const;
-
-    // registered just means that there is at least an operator stake, but doesn't require the node
-    // be fully funded, active, or not decommed.  (In other words: it is any of the white, grey, or
-    // green list).
-    bool
-    is_registered(const RouterID& remote) const;
-
-    bool
-    check_rc(const RemoteRC& rc) const;
-
-    bool
-    get_random_whitelist_router(RouterID& router) const;
-
-    void
-    periodic_update(llarp_time_t now);
-
-    void
-    explore_network();
-
-    size_t
-    num_strict_connect_routers() const;
-
-    void
-    init(
-        std::shared_ptr<Contacts> contacts,
-        std::shared_ptr<NodeDB> nodedb,
-        std::shared_ptr<EventLoop> loop,
-        std::function<void(std::function<void()>)> dowork,
-        LinkManager* linkManager,
-        service::Context* hiddenServiceContext,
-        const std::unordered_set<RouterID>& strictConnectPubkeys,
-        const std::set<RemoteRC>& bootstrapRCList,
-        bool isServiceNode_arg);
-
-    std::unordered_set<RouterID>
-    whitelist() const;
-
-   private:
-    bool
-    is_remote_in_bootstrap(const RouterID& remote) const;
-
-    std::shared_ptr<Contacts> contacts = nullptr;
-    std::shared_ptr<NodeDB> node_db;
-    std::shared_ptr<EventLoop> loop;
-    std::function<void(std::function<void()>)> work_func = nullptr;
-    service::Context* hidden_service_context = nullptr;
-    LinkManager* link_manager = nullptr;
-    Router* router;
-
-    /// explicit whitelist of routers we will connect to directly (not for
-    /// service nodes)
-    std::unordered_set<RouterID> strict_connect_pubkeys;
-
-    std::set<RemoteRC> bootstrap_rc_list;
-    std::unordered_set<RouterID> boostrap_rid_list;
-
-    // Now that all calls are made through the event loop, any access to these
-    // booleans is not guarded by a mutex
-    std::atomic<bool> isServiceNode = false;
-
-    // whitelist = active routers
-    std::unordered_set<RouterID> router_whitelist;
-    // greylist = fully funded, but decommissioned routers
-    std::unordered_set<RouterID> router_greylist;
-    // greenlist = registered but not fully-staked routers
-    std::unordered_set<RouterID> router_greenlist;
-  };
-
-}  // namespace llarp

llarp/router/router.cpp

@@ -11,7 +11,6 @@
 #include <llarp/net/net.hpp>
 #include <llarp/nodedb.hpp>
 #include <llarp/util/logging.hpp>
-#include <llarp/util/meta/memfn.hpp>
 #include <llarp/util/status.hpp>
 
 #include <cstdlib>
@@ -75,8 +74,6 @@ namespace llarp
     llarp::LogTrace("Router::PumpLL() start");
     if (is_stopping.load())
       return;
-    paths.PumpDownstream();
-    paths.PumpUpstream();
     _hidden_service_context.Pump();
     llarp::LogTrace("Router::PumpLL() end");
   }
@@ -230,32 +227,19 @@ namespace llarp
     _link_manager.set_conn_persist(remote, until);
   }
 
-  void
+  std::optional<RouterID>
-  Router::GossipRCIfNeeded(const LocalRC rc)
+  Router::GetRandomGoodRouter()
-  {
-    /// if we are not a service node forget about gossip
-    if (not is_service_node())
-      return;
-    /// wait for random uptime
-    if (std::chrono::milliseconds{Uptime()} < _randomStartDelay)
-      return;
-    _rcGossiper.GossipRC(rc);
-  }
-
-  bool
-  Router::GetRandomGoodRouter(RouterID& router)
   {
     if (is_service_node())
     {
-      return _rc_lookup_handler.get_random_whitelist_router(router);
+      return node_db()->get_random_whitelist_router();
     }
 
     if (auto maybe = node_db()->GetRandom([](const auto&) -> bool { return true; }))
     {
-      router = maybe->router_id();
+      return maybe->router_id();
-      return true;
     }
-    return false;
+    return std::nullopt;
   }
 
   void
@@ -276,16 +260,6 @@ namespace llarp
     _link_manager.connect_to(rc);
   }
 
-  void
-  Router::lookup_router(RouterID rid, std::function<void(oxen::quic::message)> func)
-  {
-    _link_manager.send_control_message(
-        rid,
-        "find_router",
-        FindRouterMessage::serialize(std::move(rid), false, false),
-        std::move(func));
-  }
-
   bool
   Router::send_data_message(const RouterID& remote, std::string payload)
   {
@@ -482,25 +456,25 @@ namespace llarp
   bool
   Router::have_snode_whitelist() const
   {
-    return is_service_node() and _rc_lookup_handler.has_received_whitelist();
+    return whitelist_received;
   }
 
   bool
   Router::appears_decommed() const
   {
-    return have_snode_whitelist() and _rc_lookup_handler.is_grey_listed(pubkey());
+    return have_snode_whitelist() and node_db()->greylist().count(pubkey());
   }
 
   bool
   Router::appears_funded() const
   {
-    return have_snode_whitelist() and _rc_lookup_handler.is_session_allowed(pubkey());
+    return have_snode_whitelist() and node_db()->is_connection_allowed(pubkey());
   }
 
   bool
   Router::appears_registered() const
   {
-    return have_snode_whitelist() and _rc_lookup_handler.is_registered(pubkey());
+    return have_snode_whitelist() and node_db()->get_registered_routers().count(pubkey());
   }
 
   bool
@@ -512,7 +486,7 @@ namespace llarp
   bool
   Router::SessionToRouterAllowed(const RouterID& router) const
   {
-    return _rc_lookup_handler.is_session_allowed(router);
+    return node_db()->is_connection_allowed(router);
   }
 
   bool
@@ -523,7 +497,7 @@ namespace llarp
       // we are decom'd don't allow any paths outbound at all
       return false;
     }
-    return _rc_lookup_handler.is_path_allowed(router);
+    return node_db()->is_path_allowed(router);
   }
 
   size_t
@@ -545,16 +519,6 @@ namespace llarp
     queue_disk_io([&]() { router_contact.write(our_rc_file); });
   }
 
-  bool
-  Router::update_rc()
-  {
-    router_contact.resign();
-    if (is_service_node())
-      save_rc();
-
-    return true;
-  }
-
   bool
   Router::from_config(const Config& conf)
   {
@@ -682,23 +646,15 @@ namespace llarp
       it = bootstrap_rc_list.erase(it);
     }
 
+    node_db()->set_bootstrap_routers(bootstrap_rc_list);
+
     if (conf.bootstrap.seednode)
       LogInfo("we are a seed node");
     else
       LogInfo("Loaded ", bootstrap_rc_list.size(), " bootstrap routers");
 
     // Init components after relevant config settings loaded
-    _link_manager.init(&_rc_lookup_handler);
+    _link_manager.init();
-    _rc_lookup_handler.init(
-        _contacts,
-        _node_db,
-        _loop,
-        util::memFn(&Router::queue_work, this),
-        &_link_manager,
-        &_hidden_service_context,
-        strictConnectPubkeys,
-        bootstrap_rc_list,
-        _is_service_node);
 
     // FIXME: kludge for now, will be part of larger cleanup effort.
     if (_is_service_node)
@@ -799,12 +755,12 @@ namespace llarp
           " | {} active paths | block {} ",
           path_context().CurrentTransitPaths(),
           (_rpc_client ? _rpc_client->BlockHeight() : 0));
-      auto maybe_last = _rcGossiper.LastGossipAt();
+      bool have_gossiped = last_rc_gossip == std::chrono::system_clock::time_point::min();
       fmt::format_to(
           out,
           " | gossip: (next/last) {} / {}",
-          short_time_from_now(_rcGossiper.NextGossipAt()),
+          short_time_from_now(next_rc_gossip),
-          maybe_last ? short_time_from_now(*maybe_last) : "never");
+          have_gossiped ? short_time_from_now(last_rc_gossip) : "never");
     }
     else
     {
@@ -857,36 +813,28 @@ namespace llarp
       report_stats();
     }
 
-    _rcGossiper.Decay(now);
-
-    _rc_lookup_handler.periodic_update(now);
-
-    const bool has_whitelist = _rc_lookup_handler.has_received_whitelist();
     const bool is_snode = is_service_node();
     const bool is_decommed = appears_decommed();
-    bool should_gossip = appears_funded();
 
-    if (is_snode
+    // (relay-only) if we have fetched the relay list from oxend and
-        and (router_contact.expires_within_delta(now, std::chrono::milliseconds(randint() % 10000)) 
+    // we are registered and funded, we want to gossip our RC periodically
-            or (now - router_contact.timestamp().time_since_epoch()) > rc_regen_interval))
+    auto now_timepoint = std::chrono::system_clock::time_point(now);
-    {
+    if (is_snode and appears_funded() and (now_timepoint > next_rc_gossip))
-      LogInfo("regenerating RC");
-      if (update_rc())
-      {
-        // our rc changed so we should gossip it
-        should_gossip = true;
-        // remove our replay entry so it goes out
-        _rcGossiper.Forget(pubkey());
-      }
-      else
-        LogError("failed to update our RC");
-    }
-    if (should_gossip)
     {
-      // if we have the whitelist enabled, we have fetched the list and we are in either
+      log::info(logcat, "regenerating and gossiping RC");
-      // the white or grey list, we want to gossip our RC
+      router_contact.resign();
-      GossipRCIfNeeded(router_contact);
+      save_rc();
+      auto view = router_contact.view();
+      _link_manager.gossip_rc(
+          pubkey(), std::string{reinterpret_cast<const char*>(view.data()), view.size()});
+      last_rc_gossip = now_timepoint;
+
+      // 1min to 5min before "stale time" is next gossip time
+      auto random_delta =
+          std::chrono::seconds{std::uniform_int_distribution<size_t>{60, 300}(llarp::csrng)};
+      next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta;
     }
+
     // remove RCs for nodes that are no longer allowed by network policy
     node_db()->RemoveIf([&](const RemoteRC& rc) -> bool {
       // don't purge bootstrap nodes from nodedb
@@ -918,7 +866,7 @@ namespace llarp
       }
 
       // if we don't have the whitelist yet don't remove the entry
-      if (not has_whitelist)
+      if (not whitelist_received)
       {
         log::debug(logcat, "Skipping check on {}: don't have whitelist yet", rc.router_id());
         return false;
@@ -927,7 +875,7 @@ namespace llarp
       // the whitelist enabled and we got the whitelist
       // check against the whitelist and remove if it's not
       // in the whitelist OR if there is no whitelist don't remove
-      if (has_whitelist and not _rc_lookup_handler.is_session_allowed(rc.router_id()))
+      if (not node_db()->is_connection_allowed(rc.router_id()))
       {
         log::debug(logcat, "Removing {}: not a valid router", rc.router_id());
         return true;
@@ -935,7 +883,9 @@ namespace llarp
       return false;
     });
 
-    if (not is_snode or not has_whitelist)
+    /* TODO: this behavior seems incorrect, but fixing it will require discussion
+     *
+    if (not is_snode or not whitelist_received)
     {
       // find all deregistered relays
       std::unordered_set<RouterID> close_peers;
@@ -951,23 +901,18 @@ namespace llarp
       for (auto& peer : close_peers)
         _link_manager.deregister_peer(peer);
     }
+    */
 
     _link_manager.check_persisting_conns(now);
 
     size_t connected = NumberOfConnectedRouters();
 
-    const int interval = is_snode ? 5 : 2;
-    const auto timepoint_now = std::chrono::steady_clock::now();
-    if (timepoint_now >= _next_explore_at and not is_decommed)
-    {
-      _rc_lookup_handler.explore_network();
-      _next_explore_at = timepoint_now + std::chrono::seconds(interval);
-    }
     size_t connectToNum = _link_manager.min_connected_routers;
-    const auto strictConnect = _rc_lookup_handler.num_strict_connect_routers();
+    const auto& pinned_edges = _node_db->get_pinned_edges();
-    if (strictConnect > 0 && connectToNum > strictConnect)
+    const auto pinned_count = pinned_edges.size();
+    if (pinned_count > 0 && connectToNum > pinned_count)
     {
-      connectToNum = strictConnect;
+      connectToNum = pinned_count;
     }
 
     if (is_snode and now >= _next_decomm_warning)
@@ -1015,14 +960,6 @@ namespace llarp
 
     _node_db->Tick(now);
 
-    std::set<dht::Key_t> peer_keys;
-
-    for_each_connection(
-        [&peer_keys](link::Connection& conn) { peer_keys.emplace(conn.remote_rc.router_id()); });
-
-    _contacts->rc_nodes()->RemoveIf(
-        [&peer_keys](const dht::Key_t& k) -> bool { return peer_keys.count(k) == 0; });
-
     paths.ExpirePaths(now);
 
     // update tick timestamp
@@ -1035,13 +972,20 @@ namespace llarp
     return _link_manager.get_random_connected(result);
   }
 
+  const std::unordered_set<RouterID>&
+  Router::get_whitelist() const
+  {
+    return _node_db->whitelist();
+  }
+
   void
   Router::set_router_whitelist(
       const std::vector<RouterID>& whitelist,
       const std::vector<RouterID>& greylist,
       const std::vector<RouterID>& unfundedlist)
   {
-    _rc_lookup_handler.set_router_whitelist(whitelist, greylist, unfundedlist);
+    node_db()->set_router_whitelist(whitelist, greylist, unfundedlist);
+    whitelist_received = true;
   }
 
   bool
@@ -1079,7 +1023,6 @@ namespace llarp
 
       log::info(logcat, "Router initialized as service node!");
       const RouterID us = pubkey();
-      _rcGossiper.Init(&_link_manager, us, this);
       // relays do not use profiling
       router_profiling().Disable();
     }
@@ -1290,7 +1233,6 @@ namespace llarp
     _exit_context.Stop();
     llarp::sys::service_manager->stopping();
     log::debug(logcat, "final upstream pump");
-    paths.PumpUpstream();
     llarp::sys::service_manager->stopping();
     log::debug(logcat, "final links pump");
     _loop->call_later(200ms, [this] { AfterStopIssued(); });

llarp/router/router.hpp

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "rc_gossiper.hpp"
-#include "rc_lookup_handler.hpp"
 #include "route_poker.hpp"
 
 #include <llarp/bootstrap.hpp>
@@ -59,8 +57,6 @@ namespace llarp
   static constexpr size_t INTROSET_STORAGE_REDUNDANCY =
       (INTROSET_RELAY_REDUNDANCY * INTROSET_REQS_PER_RELAY);
 
-  static constexpr size_t RC_LOOKUP_STORAGE_REDUNDANCY{4};
-
   struct Contacts;
 
   struct Router : std::enable_shared_from_this<Router>
@@ -122,17 +118,16 @@ namespace llarp
     const llarp_time_t _randomStartDelay;
 
     std::shared_ptr<rpc::LokidRpcClient> _rpc_client;
+    bool whitelist_received{false};
 
     oxenmq::address rpc_addr;
     Profiling _router_profiling;
     fs::path _profile_file;
     LinkManager _link_manager{*this};
-    RCLookupHandler _rc_lookup_handler;
+    std::chrono::system_clock::time_point last_rc_gossip{
-    RCGossiper _rcGossiper;
+        std::chrono::system_clock::time_point::min()};
-
+    std::chrono::system_clock::time_point next_rc_gossip{
-    /// how often do we resign our RC? milliseconds.
+        std::chrono::system_clock::time_point::min()};
-    // TODO: make configurable
-    llarp_time_t rc_regen_interval = 1h;
 
     // should we be sending padded messages every interval?
     bool send_padding = false;
@@ -150,9 +145,6 @@ namespace llarp
     void
     save_rc();
 
-    bool
-    update_rc();
-
     bool
     from_config(const Config& conf);
 
@@ -163,9 +155,6 @@ namespace llarp
     void
     for_each_connection(std::function<void(link::Connection&)> func);
 
-    void
-    lookup_router(RouterID rid, std::function<void(oxen::quic::message)> = nullptr);
-
     void
     connect_to(const RouterID& rid);
 
@@ -211,12 +200,6 @@ namespace llarp
       return _link_manager;
     }
 
-    RCLookupHandler&
-    rc_lookup_handler()
-    {
-      return _rc_lookup_handler;
-    }
-
     inline int
     outbound_udp_socket() const
     {
@@ -292,11 +275,8 @@ namespace llarp
     util::StatusObject
     ExtractSummaryStatus() const;
 
-    std::unordered_set<RouterID>
+    const std::unordered_set<RouterID>&
-    router_whitelist() const
+    get_whitelist() const;
-    {
-      return _rc_lookup_handler.whitelist();
-    }
 
     void
     set_router_whitelist(
@@ -380,17 +360,14 @@ namespace llarp
     std::string
     status_line();
 
-    void
-    GossipRCIfNeeded(const LocalRC rc);
-
     void
     InitInboundLinks();
 
     void
     InitOutboundLinks();
 
-    bool
+    std::optional<RouterID>
-    GetRandomGoodRouter(RouterID& r);
+    GetRandomGoodRouter();
 
     /// initialize us as a service node
     /// return true on success

llarp/router_contact.hpp

@@ -56,19 +56,18 @@ namespace llarp
 
     /// Timespans for RCs:
 
-    /// How long (relative to its timestamp) before an RC becomes stale.  Stale records are used
+    /// How long (from its signing time) before an RC is considered "stale".  Relays republish
-    /// (e.g. for path building) only if there are no non-stale records available, such as might be
+    /// their RCs slightly more frequently than this so that ideally this won't happen.
+    static constexpr auto STALE_AGE = 6h;
+
+    /// How long (from its signing time) before an RC becomes "outdated".  Outdated records are used
+    /// (e.g. for path building) only if there are no newer records available, such as might be
     /// the case when a client has been turned off for a while.
-    static constexpr auto STALE = 12h;
+    static constexpr auto OUTDATED_AGE = 12h;
 
     /// How long before an RC becomes invalid (and thus deleted).
     static constexpr auto LIFETIME = 30 * 24h;
 
-    /// How long before a relay updates and re-publish its RC to the network.  (Relays can
-    /// re-publish more frequently than this if needed; this is meant to apply only if there are no
-    /// changes i.e. just to push out a new confirmation of the details).
-    static constexpr auto REPUBLISH = STALE / 2 - 5min;
-
     ustring_view
     view() const
     {

llarp/service/async_key_exchange.cpp

@@ -4,7 +4,6 @@
 
 #include <llarp/crypto/crypto.hpp>
 #include <llarp/crypto/types.hpp>
-#include <llarp/util/meta/memfn.hpp>
 
 #include <utility>
 
@@ -43,6 +42,9 @@ namespace llarp::service
   AsyncKeyExchange::Encrypt(
       std::shared_ptr<AsyncKeyExchange> self, std::shared_ptr<ProtocolFrameMessage> frame)
   {
+    (void)self;
+    (void)frame;
+    /* TODO: client<->client session ("conversation"/"convo") key exchange
     // derive ntru session key component
     SharedSecret secret;
     crypto::pqe_encrypt(frame->cipher, secret, self->introPubKey);
@@ -73,5 +75,6 @@ namespace llarp::service
     {
       LogError("failed to encrypt and sign");
     }
+    */
   }
 }  // namespace llarp::service

llarp/service/endpoint.cpp

@@ -139,63 +139,50 @@ namespace llarp::service
          auth = AuthInfo{token},
          ranges,
          result_handler,
-         poker = router()->route_poker()](oxen::quic::message m) mutable {
+         poker = router()->route_poker()](std::string name_result, bool success) mutable {
-          if (m)
+          if (not success)
           {
-            std::string name;
+            result_handler(false, "Exit {} not found!"_format(name));
-            try
+            return;
-            {
+          }
-              oxenc::bt_dict_consumer btdc{m.body()};
-              name = btdc.require<std::string>("NAME");
-            }
-            catch (...)
-            {
-              log::warning(link_cat, "Failed to parse find name response!");
-              throw;
-            }
-
-            if (auto saddr = service::Address(); saddr.FromString(name))
-            {
-              ptr->SetAuthInfoForEndpoint(saddr, auth);
-              ptr->MarkAddressOutbound(saddr);
 
-              auto result = ptr->EnsurePathToService(
+          if (auto saddr = service::Address(); saddr.FromString(name_result))
-                  saddr,
+          {
-                  [ptr, name, ranges, result_handler, poker](auto addr, OutboundContext* ctx) {
+            ptr->SetAuthInfoForEndpoint(saddr, auth);
-                    if (ctx == nullptr)
+            ptr->MarkAddressOutbound(saddr);
+
+            auto result = ptr->EnsurePathToService(
+                saddr,
+                [ptr, name, name_result, ranges, result_handler, poker](
+                    auto addr, OutboundContext* ctx) {
+                  if (ctx == nullptr)
+                  {
+                    result_handler(
+                        false, "could not establish flow to {} ({})"_format(name_result, name));
+                    return;
+                  }
+
+                  // make a lambda that sends the reply after doing auth
+                  auto apply_result = [ptr, poker, addr, result_handler, ranges](
+                                          std::string result, bool success) {
+                    if (success)
                     {
-                      result_handler(false, "could not establish flow to {}"_format(name));
+                      for (const auto& range : ranges)
-                      return;
+                        ptr->MapExitRange(range, addr);
-                    }
 
-                    // make a lambda that sends the reply after doing auth
+                      if (poker)
-                    auto apply_result = [ptr, poker, addr, result_handler, ranges](
+                        poker->put_up();
-                                            std::string result, bool success) {
+                    }
-                      if (success)
-                      {
-                        for (const auto& range : ranges)
-                          ptr->MapExitRange(range, addr);
-
-                        if (poker)
-                          poker->put_up();
-
-                        result_handler(true, result);
-                      }
 
-                      result_handler(false, result);
+                    result_handler(success, result);
-                    };
+                  };
 
-                    ctx->send_auth_async(apply_result);
+                  ctx->send_auth_async(apply_result);
-                  },
+                },
-                  ptr->PathAlignmentTimeout());
+                ptr->PathAlignmentTimeout());
 
-              if (not result)
+            if (not result)
-                result_handler(false, "Could not build path to {}"_format(name));
+              result_handler(false, "Could not build path to {} ({})"_format(name_result, name));
-            }
-          }
-          else
-          {
-            result_handler(false, "Exit {} not found!"_format(name));
           }
         });
   }
@@ -213,20 +200,22 @@ namespace llarp::service
     // If we fail along the way (e.g. it's a .snode, we can't build a path, or whatever else) then
     // we invoke the resultHandler with an empty vector.
     lookup_name(
-        name, [this, resultHandler, service = std::move(service)](oxen::quic::message m) mutable {
+        name,
-          if (!m)
+        [this, resultHandler, service = std::move(service)](
+            std::string name_result, bool success) mutable {
+          if (!success)
             return resultHandler({});
 
           std::string name;
           try
           {
-            oxenc::bt_dict_consumer btdc{m.body()};
+            oxenc::bt_dict_consumer btdc{name_result};
             name = btdc.require<std::string>("NAME");
           }
           catch (...)
           {
             log::warning(link_cat, "Failed to parse find name response!");
-            throw;
+            return resultHandler({});
           }
 
           auto saddr = service::Address();
@@ -316,35 +305,24 @@ namespace llarp::service
       {
         auto& name = item.first;
 
-        lookup_name(name, [this, name, info = item.second](oxen::quic::message m) mutable {
+        lookup_name(
-          if (m)
+            name, [this, name, info = item.second](std::string name_result, bool success) mutable {
-          {
+              if (not success)
-            std::string result;
+                return;
-            try
-            {
-              oxenc::bt_dict_consumer btdc{m.body()};
-              result = btdc.require<std::string>("NAME");
-            }
-            catch (...)
-            {
-              log::warning(link_cat, "Failed to parse find name response!");
-              throw;
-            }
 
-            const auto maybe_range = info.first;
+              const auto maybe_range = info.first;
-            const auto maybe_auth = info.second;
+              const auto maybe_auth = info.second;
 
-            _startup_ons_mappings.erase(name);
+              _startup_ons_mappings.erase(name);
 
-            if (auto saddr = service::Address(); saddr.FromString(result))
+              if (auto saddr = service::Address(); saddr.FromString(name_result))
-            {
+              {
-              if (maybe_range.has_value())
+                if (maybe_range.has_value())
-                _exit_map.Insert(*maybe_range, saddr);
+                  _exit_map.Insert(*maybe_range, saddr);
-              if (maybe_auth.has_value())
+                if (maybe_auth.has_value())
-                SetAuthInfoForEndpoint(saddr, *maybe_auth);
+                  SetAuthInfoForEndpoint(saddr, *maybe_auth);
-            }
+              }
-          }
+            });
-        });
       }
     }
   }
@@ -798,7 +776,7 @@ namespace llarp::service
   }
 
   void
-  Endpoint::lookup_name(std::string name, std::function<void(oxen::quic::message)> func)
+  Endpoint::lookup_name(std::string name, std::function<void(std::string, bool)> func)
   {
     // TODO: so fuck all this?
 
@@ -838,37 +816,34 @@ namespace llarp::service
     std::shuffle(chosenpaths.begin(), chosenpaths.end(), llarp::csrng);
     chosenpaths.resize(std::min(paths.size(), MAX_ONS_LOOKUP_ENDPOINTS));
 
-    for (const auto& path : chosenpaths)
+    // TODO: only want one successful response to call the callback, or failed if all fail
-    {
+    auto response_cb = [func = std::move(func)](std::string resp) {
-      log::info(link_cat, "{} lookup {} from {}", Name(), name, path->Endpoint());
+      std::string name{};
-      path->find_name(name, func);
+      try
-    }
+      {
-  }
+        oxenc::bt_dict_consumer btdc{resp};
-
+        auto status = btdc.require<std::string_view>(messages::STATUS_KEY);
-  void
+        if (status != "OK"sv)
-  Endpoint::EnsureRouterIsKnown(const RouterID& rid)
+        {
-  {
+          log::info(link_cat, "Error on ONS lookup: {}", status);
-    if (rid.IsZero())
+          func(std::string{status}, false);
-      return;
+        }
-    if (!router()->node_db()->has_router(rid))
+        name = btdc.require<std::string>("NAME");
-    {
+      }
-      lookup_router(rid);
+      catch (...)
-    }
+      {
-  }
+        log::warning(link_cat, "Failed to parse find name response!");
+        func("ERROR"s, false);
+      }
 
-  bool
+      func(std::move(name), true);
-  Endpoint::lookup_router(RouterID rid, std::function<void(oxen::quic::message)> func)
+    };
-  {
-    const auto& routers = _state->pending_routers;
 
-    if (routers.find(rid) == routers.end())
+    for (const auto& path : chosenpaths)
     {
-      auto path = GetEstablishedPathClosestTo(rid);
+      log::info(link_cat, "{} lookup {} from {}", Name(), name, path->Endpoint());
-      path->find_router("find_router", func);
+      path->find_name(name, response_cb);
-      return true;
     }
-
-    return false;
   }
 
   void
@@ -1254,7 +1229,8 @@ namespace llarp::service
           this);
       _state->snode_sessions[snode] = session;
     }
-    EnsureRouterIsKnown(snode);
+    if (not router()->node_db()->has_rc(snode))
+      return false;
     auto range = nodeSessions.equal_range(snode);
     auto itr = range.first;
     while (itr != range.second)
@@ -1327,29 +1303,43 @@ namespace llarp::service
     // address once.
     bool hookAdded = false;
 
+    auto got_it = std::make_shared<bool>(false);
+
+    // TODO: if all requests fail, call callback with failure?
     for (const auto& path : paths)
     {
-      path->find_intro(location, false, 0, [this, hook](oxen::quic::message m) mutable {
+      path->find_intro(location, false, 0, [this, hook, got_it](std::string resp) mutable {
-        if (m)
+        // asking many, use only first successful
-        {
+        if (*got_it)
-          std::string introset;
+          return;
 
-          try
+        std::string introset;
-          {
+
-            oxenc::bt_dict_consumer btdc{m.body()};
+        try
-            introset = btdc.require<std::string>("INTROSET");
+        {
-          }
+          oxenc::bt_dict_consumer btdc{resp};
-          catch (...)
+          auto status = btdc.require<std::string_view>(messages::STATUS_KEY);
+          if (status != "OK"sv)
           {
-            log::warning(link_cat, "Failed to parse find name response!");
+            log::info(link_cat, "Error in find intro set response: {}", status);
-            throw;
+            return;
           }
+          introset = btdc.require<std::string>("INTROSET");
+        }
+        catch (...)
+        {
+          log::warning(link_cat, "Failed to parse find name response!");
+          throw;
+        }
 
-          service::EncryptedIntroSet enc{introset};
+        service::EncryptedIntroSet enc{introset};
-          router()->contacts()->services()->PutNode(std::move(enc));
+        router()->contacts()->services()->PutNode(std::move(enc));
 
-          // TODO: finish this
+        // TODO: finish this
-        }
+        /*
+        if (good)
+          *got_it = true;
+        */
       });
     }
     return hookAdded;
@@ -1432,7 +1422,7 @@ namespace llarp::service
       queue.pop();
     }
 
-    auto r = router();
+    // auto r = router();
 
     // TODO: locking on this container
     // for (const auto& [addr, outctx] : _state->remote_sessions)
@@ -1452,8 +1442,6 @@ namespace llarp::service
     //   if (item.second->SendRoutingMessage(*item.first, r))
     //     ConvoTagTX(item.first->protocol_frame_msg.convo_tag);
     // }
-
-    UpstreamFlush(r);
   }
 
   std::optional<ConvoTag>

llarp/service/endpoint.hpp

@@ -229,18 +229,9 @@ namespace llarp
       bool
       ProcessDataMessage(std::shared_ptr<ProtocolMessage> msg);
 
-      /// ensure that we know a router, looks up if it doesn't
-      void
-      EnsureRouterIsKnown(const RouterID& router);
-
-      // "find router" via closest path
-      bool
-      lookup_router(RouterID router, std::function<void(oxen::quic::message)> func = nullptr);
-
       // "find name"
       void
-      lookup_name(
+      lookup_name(std::string name, std::function<void(std::string, bool)> func = nullptr) override;
-          std::string name, std::function<void(oxen::quic::message)> func = nullptr) override;
 
       // "find introset?"
       void

llarp/service/identity.cpp

@@ -167,7 +167,7 @@ namespace llarp::service
       return std::nullopt;
 
     IntroSet i{other_i};
-    encrypted.nounce.Randomize();
+    encrypted.nonce.Randomize();
     // set timestamp
     // TODO: round to nearest 1000 ms
     i.time_signed = now;
@@ -180,7 +180,7 @@ namespace llarp::service
     auto bte = i.bt_encode();
 
     const SharedSecret k{i.address_keys.Addr()};
-    crypto::xchacha20(reinterpret_cast<uint8_t*>(bte.data()), bte.size(), k, encrypted.nounce);
+    crypto::xchacha20(reinterpret_cast<uint8_t*>(bte.data()), bte.size(), k, encrypted.nonce);
 
     std::memcpy(encrypted.introsetPayload.data(), bte.data(), bte.size());
 

llarp/service/intro_set.cpp

@@ -14,7 +14,7 @@ namespace llarp::service
       std::string s)
       : signedAt{signed_at}
       , introsetPayload{reinterpret_cast<uint8_t*>(enc_payload.data()), enc_payload.size()}
-      , nounce{reinterpret_cast<uint8_t*>(nonce.data())}
+      , nonce{reinterpret_cast<uint8_t*>(nonce.data())}
   {
     derivedSigningKey = PubKey::from_string(signing_key);
     sig.from_string(std::move(s));
@@ -27,7 +27,7 @@ namespace llarp::service
       oxenc::bt_dict_consumer btdc{bt_payload};
 
       derivedSigningKey = PubKey::from_string(btdc.require<std::string>("d"));
-      nounce.from_string(btdc.require<std::string>("n"));
+      nonce.from_string(btdc.require<std::string>("n"));
       signedAt = std::chrono::milliseconds{btdc.require<uint64_t>("s")};
       introsetPayload = btdc.require<ustring>("x");
       sig.from_string(btdc.require<std::string>("z"));
@@ -54,7 +54,7 @@ namespace llarp::service
     try
     {
       btdp.append("d", derivedSigningKey.ToView());
-      btdp.append("n", nounce.ToView());
+      btdp.append("n", nonce.ToView());
       btdp.append("s", signedAt.count());
       btdp.append(
           "x",
@@ -88,7 +88,7 @@ namespace llarp::service
     if (not BEncodeMaybeReadDictEntry("d", derivedSigningKey, read, key, buf))
       return false;
 
-    if (not BEncodeMaybeReadDictEntry("n", nounce, read, key, buf))
+    if (not BEncodeMaybeReadDictEntry("n", nonce, read, key, buf))
       return false;
 
     if (not BEncodeMaybeReadDictInt("s", signedAt, read, key, buf))
@@ -111,7 +111,7 @@ namespace llarp::service
     return fmt::format(
         "[EncIntroSet d={} n={} s={} x=[{} bytes] z={}]",
         derivedSigningKey,
-        nounce,
+        nonce,
         signedAt.count(),
         introsetPayload.size(),
         sig);
@@ -124,7 +124,7 @@ namespace llarp::service
     std::string payload{
         reinterpret_cast<const char*>(introsetPayload.data()), introsetPayload.size()};
 
-    crypto::xchacha20(reinterpret_cast<uint8_t*>(payload.data()), payload.size(), k, nounce);
+    crypto::xchacha20(reinterpret_cast<uint8_t*>(payload.data()), payload.size(), k, nonce);
 
     return IntroSet{payload};
   }

llarp/service/intro_set.hpp

@@ -138,7 +138,7 @@ namespace llarp::service
     PubKey derivedSigningKey;
     llarp_time_t signedAt = 0s;
     ustring introsetPayload;
-    TunnelNonce nounce;
+    SymmNonce nonce;
     std::optional<Tag> topic;
     Signature sig;
 
@@ -203,8 +203,8 @@ namespace llarp::service
   inline bool
   operator==(const EncryptedIntroSet& lhs, const EncryptedIntroSet& rhs)
   {
-    return std::tie(lhs.signedAt, lhs.derivedSigningKey, lhs.nounce, lhs.sig)
+    return std::tie(lhs.signedAt, lhs.derivedSigningKey, lhs.nonce, lhs.sig)
-        == std::tie(rhs.signedAt, rhs.derivedSigningKey, rhs.nounce, rhs.sig);
+        == std::tie(rhs.signedAt, rhs.derivedSigningKey, rhs.nonce, rhs.sig);
   }
 
   inline bool

llarp/service/outbound_context.cpp

@@ -5,6 +5,7 @@
 #include "endpoint_util.hpp"
 #include "protocol_type.hpp"
 
+#include <llarp/nodedb.hpp>
 #include <llarp/router/router.hpp>
 
 #include <algorithm>
@@ -157,6 +158,7 @@ namespace llarp::service
     return "OBContext:" + current_intro.address_keys.Addr().ToString();
   }
 
+  // TODO: it seems a lot of this logic is duplicated in service/endpoint
   void
   OutboundContext::UpdateIntroSet()
   {
@@ -173,7 +175,7 @@ namespace llarp::service
 
     for (const auto& path : paths)
     {
-      path->find_intro(location, false, relayOrder, [this](oxen::quic::message m) mutable {
+      path->find_intro(location, false, relayOrder, [this](std::string resp) mutable {
         if (marked_bad)
         {
           log::info(link_cat, "Outbound context has been marked bad (whatever that means)");
@@ -182,43 +184,51 @@ namespace llarp::service
 
         updatingIntroSet = false;
 
-        if (m)
+        // TODO: this parsing is probably elsewhere, may need DRYed
+        std::string introset;
+        try
         {
-          std::string introset;
+          oxenc::bt_dict_consumer btdc{resp};
-
+          auto status = btdc.require<std::string_view>(messages::STATUS_KEY);
-          try
+          if (status != "OK"sv)
-          {
-            oxenc::bt_dict_consumer btdc{m.body()};
-            introset = btdc.require<std::string>("INTROSET");
-          }
-          catch (...)
           {
-            log::warning(link_cat, "Failed to parse find name response!");
+            log::info(link_cat, "Error in find intro set response: {}", status);
-            throw;
+            return;
           }
+          introset = btdc.require<std::string>("INTROSET");
+        }
+        catch (...)
+        {
+          log::warning(link_cat, "Failed to parse find name response!");
+          throw;
+        }
 
-          service::EncryptedIntroSet enc{introset};
+        service::EncryptedIntroSet enc{introset};
-          const auto intro = enc.decrypt(PubKey{addr.as_array()});
+        const auto intro = enc.decrypt(PubKey{addr.as_array()});
 
-          if (intro.time_signed == 0s)
+        if (intro.time_signed == 0s)
-          {
+        {
-            log::warning(link_cat, "{} recieved introset with zero timestamp");
+          log::warning(link_cat, "{} recieved introset with zero timestamp");
-            return;
+          return;
-          }
+        }
-          if (current_intro.time_signed > intro.time_signed)
+        if (current_intro.time_signed > intro.time_signed)
-          {
+        {
-            log::info(link_cat, "{} received outdated introset; dropping", Name());
+          log::info(link_cat, "{} received outdated introset; dropping", Name());
-            return;
+          return;
-          }
+        }
-          if (intro.IsExpired(llarp::time_now_ms()))
+
-          {
+        // don't "shift" to the same intro we're already using...
-            log::warning(link_cat, "{} received expired introset", Name());
+        if (current_intro == intro)
-            return;
+          return;
-          }
 
-          current_intro = intro;
+        if (intro.IsExpired(llarp::time_now_ms()))
-          ShiftIntroRouter();
+        {
+          log::warning(link_cat, "{} received expired introset", Name());
+          return;
         }
+
+        current_intro = intro;
+        ShiftIntroRouter();
       });
     }
   }
@@ -307,9 +317,6 @@ namespace llarp::service
         }
       }
     }
-    // lookup router in intro if set and unknown
-    if (not next_intro.router.IsZero())
-      ep.EnsureRouterIsKnown(next_intro.router);
 
     if (ReadyToSend())
     {
@@ -398,6 +405,18 @@ namespace llarp::service
 
     std::vector<Introduction> intros = current_intro.intros;
 
+    // don't consider intros for which we don't have the RC for the pivot
+    auto itr = intros.begin();
+    while (itr != intros.end())
+    {
+      if (not ep.router()->node_db()->has_rc(itr->router))
+      {
+        itr = intros.erase(itr);
+        continue;
+      }
+      itr++;
+    }
+
     if (intros.size() > 1)
     {
       std::shuffle(intros.begin(), intros.end(), llarp::csrng);
@@ -426,7 +445,6 @@ namespace llarp::service
       {
         if (ep.SnodeBlacklist().count(intro.router))
           continue;
-        ep.EnsureRouterIsKnown(intro.router);
         if (intro.ExpiresSoon(now))
           continue;
         if (next_intro != intro)
@@ -534,8 +552,9 @@ namespace llarp::service
       ex->msg.proto = ProtocolType::Auth;
 
     ex->hook = [this, path, cb = std::move(func)](auto frame) mutable {
-      auto hook = [&, frame, path](oxen::quic::message) {
+      auto hook = [&, frame, path](std::string resp) {
         // TODO: revisit this
+        (void)resp;
         ep.HandleHiddenServiceFrame(path, *frame.get());
       };
 

llarp/service/protocol.cpp

@@ -302,8 +302,11 @@ namespace llarp::service
 
       // PKE (A, B, N)
       SharedSecret shared_secret;
-      if (!self->m_LocalIdentity.KeyExchange(
+      if (!crypto::dh_server(
-              crypto::dh_server, shared_secret, self->msg->sender, self->frame.nonce))
+              shared_secret,
+              self->msg->sender.EncryptionPublicKey(),
+              self->m_LocalIdentity.enckey,
+              self->frame.nonce))
       {
         LogError("x25519 key exchange failed");
         Dump<MAX_PROTOCOL_MESSAGE_SIZE>(self->frame);

llarp/service/protocol.hpp

@@ -75,7 +75,7 @@ namespace llarp
       PQCipherBlock cipher;
       Encrypted<2048> enc;
       uint64_t flag;  // set to indicate in plaintext a nack, aka "dont try again"
-      KeyExchangeNonce nonce;
+      SymmNonce nonce;
       Signature sig;
       PathID_t path_id;
       service::ConvoTag convo_tag;

llarp/util/meta/memfn.hpp

@@ -1,72 +0,0 @@
-#ifndef LLARP_UTIL_MEMFN
-#define LLARP_UTIL_MEMFN
-
-#include <memory>
-#include <type_traits>
-#include <utility>
-
-namespace llarp::util
-{
-  // Wraps a member function and instance into a callable object that invokes
-  // the method (non-const overload).
-  template <
-      typename Return,
-      typename Class,
-      typename Derived,
-      typename... Arg,
-      typename = std::enable_if_t<std::is_base_of<Class, Derived>::value>>
-  auto
-  memFn(Return (Class::*f)(Arg...), Derived* self)
-  {
-    return [f, self](Arg... args) -> Return { return (self->*f)(std::forward<Arg>(args)...); };
-  }
-
-  // Wraps a member function and instance into a lambda that invokes the
-  // method (const overload).
-  template <
-      typename Return,
-      typename Class,
-      typename Derived,
-      typename... Arg,
-      typename = std::enable_if_t<std::is_base_of<Class, Derived>::value>>
-  auto
-  memFn(Return (Class::*f)(Arg...) const, const Derived* self)
-  {
-    return [f, self](Arg... args) -> Return { return (self->*f)(std::forward<Arg>(args)...); };
-  }
-
-  // Wraps a member function and shared pointer to an instance into a lambda
-  // that invokes the method.
-  template <
-      typename Return,
-      typename Class,
-      typename Derived,
-      typename... Arg,
-      typename = std::enable_if_t<std::is_base_of<Class, Derived>::value>>
-  auto
-  memFn(Return (Class::*f)(Arg...), std::shared_ptr<Derived> self)
-  {
-    return [f, self = std::move(self)](Arg... args) -> Return {
-      return (self.get()->*f)(std::forward<Arg>(args)...);
-    };
-  }
-
-  // Wraps a member function and shared pointer to an instance into a lambda
-  // that invokes the method (const method overload).
-  template <
-      typename Return,
-      typename Class,
-      typename Derived,
-      typename... Arg,
-      typename = std::enable_if_t<std::is_base_of<Class, Derived>::value>>
-  auto
-  memFn(Return (Class::*f)(Arg...) const, std::shared_ptr<Derived> self)
-  {
-    return [f, self = std::move(self)](Arg... args) -> Return {
-      return (self.get()->*f)(std::forward<Arg>(args)...);
-    };
-  }
-
-}  // namespace llarp::util
-
-#endif