contrib/testnet: fix up testnet config generator to make super centralized topology

llarp/context.cpp, llarp/nodedb.{h,c}pp: load netdb AFTER whitelist
llarp/router/router.cpp: explore always
llarp/router/{i,}rc_lookup_handler.{h,c}pp explore with whitelist, update routers with lookup before stale

contrib/testnet/genconf.py

@@ -37,7 +37,6 @@ def main():
         exe = 'valgrind {}'.format(args.bin)
     else:
         exe = args.bin
-        
     basedir = os.path.abspath(args.dir)
 
     for nodeid in range(args.svc):
@@ -108,7 +107,7 @@ def main():
         fp = os.path.join(d, 'client.ini')
         with open(fp, 'w') as f:
             config.write(f)
-            for n in range(args.connect):
+            for n in [0]:
                 otherID = (n + nodeid) % args.svc
                 f.write("[bootstrap]\nadd-node={}\n".format(os.path.join(basedir,svcNodeName(otherID), 'rc.signed')))
         with open(hiddenservice, 'w') as f:

llarp/context.cpp

@@ -183,16 +183,6 @@ __        ___    ____  _   _ ___ _   _  ____
       llarp::LogError("nodedb_dir is incorrect");
       return 0;
     }
-    // llarp::LogInfo("nodedb_dir [", nodedb_dir, "] configured!");
-    ssize_t loaded = nodedb->load_dir(nodedb_dir.c_str());
-    llarp::LogInfo("nodedb_dir loaded ", loaded, " RCs from [", nodedb_dir,
-                   "]");
-    if(loaded < 0)
-    {
-      // shouldn't be possible
-      llarp::LogError("nodedb_dir directory doesn't exist");
-      return 0;
-    }
     return 1;
   }
 
@@ -211,7 +201,7 @@ __        ___    ____  _   _ ___ _   _  ____
 
     router = std::make_unique< Router >(worker, mainloop, logic);
 
-    nodedb = std::make_unique< llarp_nodedb >(router->diskworker());
+    nodedb = std::make_unique< llarp_nodedb >(router->diskworker(), nodedb_dir);
 
     if(!router->Configure(config.get(), nodedb.get()))
     {

llarp/dht/recursiverouterlookup.cpp

@@ -42,6 +42,7 @@ namespace llarp
     void
     RecursiveRouterLookup::DoNextRequest(const Key_t &peer)
     {
+      peersAsked.emplace(peer);
       parent->LookupRouterRecursive(target, whoasked.node, whoasked.txid, peer,
                                     resultHandler);
     }
@@ -49,6 +50,7 @@ namespace llarp
     void
     RecursiveRouterLookup::Start(const TXOwner &peer)
     {
+      peersAsked.emplace(peer.node);
       parent->DHTSendTo(peer.node.as_array(),
                         new FindRouterMessage(peer.txid, target));
     }

llarp/nodedb.cpp

@@ -440,22 +440,10 @@ llarp_nodedb::ensure_dir(const char *dir)
   return true;
 }
 
-void
-llarp_nodedb::set_dir(const char *dir)
-{
-  nodePath = dir;
-}
-
 ssize_t
-llarp_nodedb::load_dir(const char *dir)
+llarp_nodedb::LoadAll()
 {
-  std::error_code ec;
-  if(!fs::exists(dir, ec))
-  {
-    return -1;
-  }
-  set_dir(dir);
-  return Load(dir);
+  return Load(nodePath.c_str());
 }
 
 /// maybe rename to verify_and_set

llarp/nodedb.hpp

@@ -40,8 +40,10 @@ struct llarp_nodedb_iter
 
 struct llarp_nodedb
 {
-  explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker)
-      : disk(std::move(diskworker))
+  explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker,
+                        const std::string rootdir)
+      : disk(std::move(diskworker)), nodePath(rootdir)
+
   {
   }
 
@@ -127,7 +129,8 @@ struct llarp_nodedb
   set_dir(const char *dir);
 
   ssize_t
-  load_dir(const char *dir);
+  LoadAll();
+
   ssize_t
   store_dir(const char *dir);
 

llarp/router/i_rc_lookup_handler.hpp

@@ -37,7 +37,8 @@ namespace llarp
     SetRouterWhitelist(const std::vector< RouterID > &routers) = 0;
 
     virtual void
-    GetRC(const RouterID &router, RCRequestCallback callback) = 0;
+    GetRC(const RouterID &router, RCRequestCallback callback,
+          bool forceLookup = false) = 0;
 
     virtual bool
     RemoteIsAllowed(const RouterID &remote) const = 0;

llarp/router/rc_lookup_handler.cpp

@@ -10,6 +10,7 @@
 #include <util/thread/threading.hpp>
 #include <nodedb.hpp>
 #include <dht/context.hpp>
+#include <router/abstractrouter.hpp>
 
 #include <iterator>
 #include <functional>
@@ -48,20 +49,22 @@ namespace llarp
   }
 
   void
-  RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback)
+  RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback,
+                         bool forceLookup)
   {
     RouterContact remoteRC;
-
-    if(_nodedb->Get(router, remoteRC))
+    if(not forceLookup)
     {
-      if(callback)
+      if(_nodedb->Get(router, remoteRC))
       {
-        callback(router, &remoteRC, RCRequestResult::Success);
+        if(callback)
+        {
+          callback(router, &remoteRC, RCRequestResult::Success);
+        }
+        FinalizeRequest(router, &remoteRC, RCRequestResult::Success);
+        return;
       }
-      FinalizeRequest(router, &remoteRC, RCRequestResult::Success);
-      return;
     }
-
     bool shouldDoLookup = false;
 
     {
@@ -212,7 +215,7 @@ namespace llarp
 
     for(const auto &router : routersToLookUp)
     {
-      GetRC(router, nullptr);
+      GetRC(router, nullptr, true);
     }
 
     _nodedb->RemoveStaleRCs(_bootstrapRouterIDList,
@@ -235,6 +238,26 @@ namespace llarp
       LogError("we have no bootstrap nodes specified");
     }
 
+    if(useWhitelist)
+    {
+      std::set< RouterID > lookupRouters;
+      {
+        static constexpr size_t LookupPerTick = 25;
+        // if we are using a whitelist look up a few routers we don't have
+        util::Lock l(&_mutex);
+        for(const auto &r : whitelistRouters)
+        {
+          if(_nodedb->Has(r))
+            continue;
+          lookupRouters.emplace(r);
+          if(lookupRouters.size() >= LookupPerTick)
+            break;
+        }
+      }
+      for(const auto &r : lookupRouters)
+        GetRC(r, nullptr, true);
+      return;
+    }
     // TODO: only explore via random subset
     // explore via every connected peer
     _linkManager->ForEachPeer([&](ILinkSession *s) {

llarp/router/rc_lookup_handler.hpp

@@ -41,8 +41,8 @@ namespace llarp
         LOCKS_EXCLUDED(_mutex);
 
     void
-    GetRC(const RouterID &router, RCRequestCallback callback) override
-        LOCKS_EXCLUDED(_mutex);
+    GetRC(const RouterID &router, RCRequestCallback callback,
+          bool forceLookup = false) override LOCKS_EXCLUDED(_mutex);
 
     bool
     RemoteIsAllowed(const RouterID &remote) const override

llarp/router/router.cpp

@@ -696,18 +696,9 @@ namespace llarp
     {
       connected += _linkManager.NumberOfPendingConnections();
     }
-    const size_t N = nodedb()->num_loaded();
-    if(N < llarp::path::default_len)
-    {
-      LogInfo("We need at least ", llarp::path::default_len,
-              " service nodes to build paths but we have ", N, " in nodedb");
 
-      _rcLookupHandler.ExploreNetwork();
-    }
-    else if(isSvcNode)
-    {
-      _rcLookupHandler.ExploreNetwork();
-    }
+    _rcLookupHandler.ExploreNetwork();
+
     size_t connectToNum      = _outboundSessionMaker.minConnectedRouters;
     const auto strictConnect = _rcLookupHandler.NumberOfStrictConnectRouters();
     if(strictConnect > 0 && connectToNum > strictConnect)
@@ -1014,6 +1005,16 @@ namespace llarp
       return false;
     }
 
+    {
+      ssize_t loaded = _nodedb->LoadAll();
+      llarp::LogInfo("loaded ", loaded, " RCs");
+      if(loaded < 0)
+      {
+        // shouldn't be possible
+        return false;
+      }
+    }
+
     llarp_dht_context_start(dht(), pubkey());
 
     for(const auto &rc : bootstrapRCList)