Squashed misc testnet fixes

external/oxen-libquic

@@ -1 +1 @@
-Subproject commit a2d89aa79dd06cbd7ee864da285cf4d8ac1b09b9
+Subproject commit 92fa0e987e0513dfcd4efc683cb4dbff21de9622

llarp/bootstrap.cpp

@@ -85,5 +85,7 @@ namespace llarp
       }
       insert(rc);
     }
+
+    _curr = begin();
   }
 }  // namespace llarp

llarp/bootstrap.hpp

@@ -12,8 +12,13 @@ namespace llarp
 {
   struct BootstrapList final : public std::set<RemoteRC>
   {
-    size_t index;
+    std::set<RemoteRC>::iterator _curr;
-    std::set<RemoteRC>::iterator current;
+
+    const RemoteRC&
+    current()
+    {
+      return *_curr;
+    }
 
     bool
     bt_decode(std::string_view buf);
@@ -32,12 +37,12 @@ namespace llarp
     const RemoteRC&
     next()
     {
-      ++current;
+      ++_curr;
 
-      if (current == this->end())
+      if (_curr == this->end())
-        current = this->begin();
+        _curr = this->begin();
 
-      return *current;
+      return *_curr;
     }
 
     bool
@@ -46,7 +51,8 @@ namespace llarp
     void
     randomize()
     {
-      current = std::next(begin(), std::uniform_int_distribution<size_t>{0, size() - 1}(csrng));
+      if (size() > 1)
+        _curr = std::next(begin(), std::uniform_int_distribution<size_t>{0, size() - 1}(csrng));
     }
 
     void

llarp/link/link_manager.cpp

@@ -193,26 +193,43 @@ namespace llarp
           return on_conn_closed(ci, ec);
         },
         [this](oxen::quic::dgram_interface& di, bstring dgram) { recv_data_message(di, dgram); });
-    ep->listen(
+    tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) {
-        tls_creds,
+      bool result = false;
-        [&](oxen::quic::Connection& c,
+      RouterID other{key.data()};
-            oxen::quic::Endpoint& e,
+      if (auto itr = rids_pending_verification.find(other); itr != rids_pending_verification.end())
-            std::optional<int64_t> id) -> std::shared_ptr<oxen::quic::Stream> {
+      {
-          if (id && id == 0)
+        rids_pending_verification.erase(itr);
-          {
+        result = true;
-            auto s = std::make_shared<oxen::quic::BTRequestStream>(
+      }
-                c, e, [](oxen::quic::Stream& s, uint64_t error_code) {
+      if (_router.node_db()->has_rc(other))
-                  log::warning(
+        result = true;
-                      logcat,
+
-                      "BTRequestStream closed unexpectedly (ec:{}); closing connection...",
+      log::critical(logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Un", other);
-                      error_code);
+      return result;
-                  s.conn.close_connection(error_code);
+    });
-                });
+    if (_router.is_service_node())
-            register_commands(s);
+    {
-            return s;
+      ep->listen(
-          }
+          tls_creds,
-          return std::make_shared<oxen::quic::Stream>(c, e);
+          [&](oxen::quic::Connection& c,
-        });
+              oxen::quic::Endpoint& e,
+              std::optional<int64_t> id) -> std::shared_ptr<oxen::quic::Stream> {
+            if (id && id == 0)
+            {
+              auto s = std::make_shared<oxen::quic::BTRequestStream>(
+                  c, e, [](oxen::quic::Stream& s, uint64_t error_code) {
+                    log::warning(
+                        logcat,
+                        "BTRequestStream closed unexpectedly (ec:{}); closing connection...",
+                        error_code);
+                    s.conn.close_connection(error_code);
+                  });
+              register_commands(s);
+              return s;
+            }
+            return std::make_shared<oxen::quic::Stream>(c, e);
+          });
+    }
     return ep;
   }
 
@@ -346,6 +363,9 @@ namespace llarp
     }
 
     const auto& remote_addr = rc.addr();
+    const auto& rid = rc.router_id();
+
+    rids_pending_verification.insert(rid);
 
     // TODO: confirm remote end is using the expected pubkey (RouterID).
     // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation)
@@ -415,6 +435,10 @@ namespace llarp
       {
         const auto& rid = c_itr->second;
 
+        if (auto maybe = rids_pending_verification.find(rid);
+            maybe != rids_pending_verification.end())
+          rids_pending_verification.erase(maybe);
+        // in case this didn't clear earlier, do it now
         if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end())
           pending_conn_msg_queue.erase(p_itr);
 
@@ -582,9 +606,16 @@ namespace llarp
 
   void
   LinkManager::fetch_bootstrap_rcs(
-      const RouterID& source, std::string payload, std::function<void(oxen::quic::message m)> func)
+      const RemoteRC& source, std::string payload, std::function<void(oxen::quic::message m)> func)
   {
-    send_control_message(source, "bfetch_rcs", std::move(payload), std::move(func));
+    _router.loop()->call([this, source, payload, f = std::move(func)]() {
+      auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f);
+
+      auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue());
+      itr->second.push_back(std::move(pending));
+
+      connect_to(source);
+    });
   }
 
   void

llarp/link/link_manager.hpp

@@ -181,6 +181,9 @@ namespace llarp
 
     // holds any messages we attempt to send while connections are establishing
     std::unordered_map<RouterID, MessageQueue> pending_conn_msg_queue;
+    // when establishing a connection, the rid of the remote is placed here to be cross-
+    // checked by the tls verification callback
+    std::set<RouterID> rids_pending_verification;
 
     util::DecayingHashSet<RouterID> clients{path::DEFAULT_LIFETIME};
 
@@ -252,7 +255,7 @@ namespace llarp
 
     void
     fetch_bootstrap_rcs(
-        const RouterID& source,
+        const RemoteRC& source,
         std::string payload,
         std::function<void(oxen::quic::message m)> func);
 

llarp/nodedb.cpp

@@ -196,7 +196,8 @@ namespace llarp
   {
     // TODO: if this needs to be called more than once (ex: drastic failures), then
     // change this assert to a bootstraps.clear() call
-    assert(_bootstraps->empty());
+    if (_bootstraps)
+      assert(_bootstraps->empty());
 
     _bootstraps = std::move(from_router);
     _bootstraps->randomize();
@@ -343,8 +344,9 @@ namespace llarp
   void
   NodeDB::fetch_initial()
   {
-    if (known_rids.empty())
+    if (known_rcs.empty())
     {
+      log::critical(logcat, "No RC's held locally... BOOTSTRAP TIME");
       fallback_to_bootstrap();
     }
     else
@@ -605,6 +607,14 @@ namespace llarp
   {
     _router.last_rc_fetch = llarp::time_point_now();
 
+    if (_router.is_service_node())
+    {
+      _needs_rebootstrap = false;
+      fail_sources.clear();
+      fetch_failures = 0;
+      return;
+    }
+
     if (initial)
       fetch_rids(initial);
   }
@@ -619,7 +629,10 @@ namespace llarp
     _needs_rebootstrap = false;
 
     if (initial)
+    {
       _needs_initial_fetch = false;
+      _initial_completed = true;
+    }
   }
 
   void
@@ -656,7 +669,7 @@ namespace llarp
     _needs_rebootstrap = false;
 
     _router.link_manager().fetch_bootstrap_rcs(
-        fetch_source,
+        _bootstraps->current(),
         BootstrapFetchMessage::serialize(BOOTSTRAP_SOURCE_COUNT),
         [this](oxen::quic::message m) mutable {
           if (not m)
@@ -755,9 +768,6 @@ namespace llarp
     registered_routers.insert(greylist.begin(), greylist.end());
     registered_routers.insert(greenlist.begin(), greenlist.end());
 
-    for (const auto& rid : whitelist)
-      known_rids.insert(rid);
-
     router_whitelist.clear();
     router_whitelist.insert(whitelist.begin(), whitelist.end());
     router_greylist.clear();

llarp/nodedb.hpp

@@ -135,9 +135,9 @@ namespace llarp
         - gray: fully funded, but decommissioned routers
         - green: registered, but not fully-staked routers
     */
-    std::unordered_set<RouterID> router_whitelist;
+    std::set<RouterID> router_whitelist{};
-    std::unordered_set<RouterID> router_greylist;
+    std::set<RouterID> router_greylist{};
-    std::unordered_set<RouterID> router_greenlist;
+    std::set<RouterID> router_greenlist{};
 
     // All registered relays (service nodes)
     std::set<RouterID> registered_routers;
@@ -165,7 +165,7 @@ namespace llarp
     std::atomic<int> fetch_failures{0}, bootstrap_failures{0};
 
     std::atomic<bool> _using_bootstrap_fallback{false}, _needs_rebootstrap{false},
-        _needs_initial_fetch{true};
+        _needs_initial_fetch{true}, _initial_completed{false};
 
     bool
     want_rc(const RouterID& rid) const;
@@ -332,7 +332,7 @@ namespace llarp
       return known_rids;
     }
 
-    const std::unordered_set<RouterID>&
+    const std::set<RouterID>&
     greylist() const
     {
       return router_greylist;

llarp/router/router.cpp

@@ -670,6 +670,8 @@ namespace llarp
 
     if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode)
     {
+      log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node");
+
       auto fallbacks = llarp::load_bootstrap_fallbacks();
 
       if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end())
@@ -692,9 +694,12 @@ namespace llarp
       log::info(
           logcat, "Loaded {} default fallback bootstrap routers!", _bootstrap_rc_list->size());
       clear_bad_rcs();
-      node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list));
     }
 
+    log::critical(logcat, "We have {} bootstrap routers!", _bootstrap_rc_list->size());
+
+    node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list));
+
     if (conf.bootstrap.seednode)
       log::critical(logcat, "We are a bootstrap seed node!");
 
@@ -890,7 +895,8 @@ namespace llarp
 
     if (needs_initial_fetch())
     {
-      node_db()->fetch_initial();
+      if (not _config->bootstrap.seednode)
+        node_db()->fetch_initial();
     }
     else if (needs_rebootstrap() and next_bootstrap_attempt > now_timepoint)
     {