Convert llarp_nodedb to be a true C++ class

pull/124/head
Michael 6 years ago
parent 6195d7e5ae
commit 6358b25db0
No known key found for this signature in database
GPG Key ID: 2D51757B47E2434C

@ -5,43 +5,19 @@
#include <llarp/router_contact.hpp>
#include <llarp/router_id.hpp>
#include <fs.hpp>
/**
* nodedb.hpp
*
* persistent storage API for router contacts
*/
struct llarp_nodedb;
namespace llarp
{
class Logic;
}
/// create an empty nodedb
struct llarp_nodedb *
llarp_nodedb_new(struct llarp_crypto *crypto);
/// free a nodedb and all loaded rc
void
llarp_nodedb_free(struct llarp_nodedb **n);
/// ensure a nodedb fs skiplist structure is at dir
/// create if not there.
bool
llarp_nodedb_ensure_dir(const char *dir);
void
llarp_nodedb_set_dir(struct llarp_nodedb *n, const char *dir);
/// load entire nodedb from fs skiplist at dir
ssize_t
llarp_nodedb_load_dir(struct llarp_nodedb *n, const char *dir);
/// store entire nodedb to fs skiplist at dir
ssize_t
llarp_nodedb_store_dir(struct llarp_nodedb *n, const char *dir);
struct llarp_nodedb_iter
{
void *user;
@ -50,40 +26,82 @@ struct llarp_nodedb_iter
bool (*visit)(struct llarp_nodedb_iter *);
};
/// iterate over all loaded rc with an iterator
int
llarp_nodedb_iterate_all(struct llarp_nodedb *n, struct llarp_nodedb_iter i);
struct llarp_nodedb
{
llarp_nodedb(llarp_crypto *c) : crypto(c)
{
}
/// visit all loaded rc
/// stop iteration if visit return false
void
llarp_nodedb_visit_loaded(
struct llarp_nodedb *n,
std::function< bool(const llarp::RouterContact &) > visit);
~llarp_nodedb()
{
Clear();
}
/// return number of RC loaded
size_t
llarp_nodedb_num_loaded(struct llarp_nodedb *n);
llarp_crypto *crypto;
llarp::util::Mutex access;
std::unordered_map< llarp::RouterID, llarp::RouterContact,
llarp::RouterID::Hash >
entries;
fs::path nodePath;
/**
put an rc into the node db
overwrites with new contents if already present
flushes the single entry to disk
returns true on success and false on error
*/
bool
llarp_nodedb_put_rc(struct llarp_nodedb *n, const llarp::RouterContact &rc);
bool
Remove(const byte_t *pk);
bool
llarp_nodedb_get_rc(struct llarp_nodedb *n, const llarp::RouterID &pk,
llarp::RouterContact &result);
void
Clear();
/**
remove rc by public key from nodedb
returns true if removed
*/
bool
llarp_nodedb_del_rc(struct llarp_nodedb *n, const llarp::RouterID &pk);
bool
Get(const byte_t *pk, llarp::RouterContact &result);
bool
Has(const byte_t *pk);
std::string
getRCFilePath(const byte_t *pubkey) const;
/// insert and write to disk
bool
Insert(const llarp::RouterContact &rc);
ssize_t
Load(const fs::path &path);
ssize_t
loadSubdir(const fs::path &dir);
bool
loadfile(const fs::path &fpath);
void
visit(std::function< bool(const llarp::RouterContact &) > visit);
bool
iterate(struct llarp_nodedb_iter &i);
void
set_dir(const char *dir);
ssize_t
load_dir(const char *dir);
ssize_t
store_dir(const char *dir);
int
iterate_all(llarp_nodedb_iter i);
size_t
num_loaded() const;
bool
select_random_exit(llarp::RouterContact &rc);
bool
select_random_hop(const llarp::RouterContact &prev,
llarp::RouterContact &result, size_t N);
static bool
ensure_dir(const char *dir);
};
/// struct for async rc verification
struct llarp_async_verify_rc;
@ -151,13 +169,4 @@ struct llarp_async_load_rc
void
llarp_nodedb_async_load_rc(struct llarp_async_load_rc *job);
bool
llarp_nodedb_select_random_exit(struct llarp_nodedb *n,
llarp::RouterContact &rc);
bool
llarp_nodedb_select_random_hop(struct llarp_nodedb *n,
const llarp::RouterContact &prev,
llarp::RouterContact &result, size_t N);
#endif

@ -82,15 +82,15 @@ namespace llarp
Context::LoadDatabase()
{
llarp_crypto_init(&crypto);
nodedb = llarp_nodedb_new(&crypto);
nodedb = new llarp_nodedb(&crypto);
if(!llarp_nodedb_ensure_dir(nodedb_dir.c_str()))
if(!llarp_nodedb::ensure_dir(nodedb_dir.c_str()))
{
llarp::LogError("nodedb_dir is incorrect");
return 0;
}
// llarp::LogInfo("nodedb_dir [", nodedb_dir, "] configured!");
ssize_t loaded = llarp_nodedb_load_dir(nodedb, nodedb_dir.c_str());
ssize_t loaded = nodedb->load_dir(nodedb_dir.c_str());
llarp::LogInfo("nodedb_dir loaded ", loaded, " RCs from [", nodedb_dir,
"]");
if(loaded < 0)
@ -105,7 +105,7 @@ namespace llarp
int
Context::IterateDatabase(struct llarp_nodedb_iter i)
{
return llarp_nodedb_iterate_all(nodedb, i);
return nodedb->iterate_all(i);
}
bool
@ -230,7 +230,7 @@ namespace llarp
llarp_free_threadpool(&worker);
llarp::LogDebug("free nodedb");
llarp_nodedb_free(&nodedb);
delete nodedb;
llarp::LogDebug("stopping event loop");
llarp_ev_loop_stop(mainloop);

@ -33,7 +33,7 @@ namespace llarp
Key_t k = K.data();
// check if we know this in our nodedb first
RouterContact found;
if(llarp_nodedb_get_rc(dht.router->nodedb, K, found))
if(dht.router->nodedb->Get(K, found))
{
replies.emplace_back(
new GotRouterMessage(K.data(), txid, {found}, false));
@ -159,7 +159,7 @@ namespace llarp
RouterContact found;
if(exploritory)
return dht.HandleExploritoryRouterLookup(From, txid, K, replies);
else if(llarp_nodedb_get_rc(dht.router->nodedb, K, found))
else if(dht.router->nodedb->Get(K, found))
{
replies.emplace_back(
new GotRouterMessage(K.data(), txid, {found}, false));

@ -39,7 +39,7 @@ namespace llarp
{
if(hop == numHops - 1)
{
return llarp_nodedb_get_rc(db, m_ExitRouter, cur);
return db->Get(m_ExitRouter, cur);
}
else
return path::Builder::SelectHop(db, prev, cur, hop, roles);

@ -14,208 +14,194 @@
static const char skiplist_subdirs[] = "0123456789abcdef";
static const std::string RC_FILE_EXT = ".signed";
struct llarp_nodedb
bool
llarp_nodedb::Remove(const byte_t *pk)
{
llarp_nodedb(llarp_crypto *c) : crypto(c)
{
}
llarp::util::Lock lock(access);
auto itr = entries.find(pk);
if(itr == entries.end())
return false;
entries.erase(itr);
fs::remove(fs::path(getRCFilePath(pk)));
return true;
}
llarp_crypto *crypto;
// std::map< llarp::pubkey, llarp_rc > entries;
llarp::util::Mutex access;
std::unordered_map< llarp::RouterID, llarp::RouterContact,
llarp::RouterID::Hash >
entries;
fs::path nodePath;
void
llarp_nodedb::Clear()
{
llarp::util::Lock lock(access);
entries.clear();
}
bool
Remove(const byte_t *pk)
{
llarp::util::Lock lock(access);
auto itr = entries.find(pk);
if(itr == entries.end())
return false;
entries.erase(itr);
fs::remove(fs::path(getRCFilePath(pk)));
return true;
}
bool
llarp_nodedb::Get(const byte_t *pk, llarp::RouterContact &result)
{
llarp::util::Lock lock(access);
auto itr = entries.find(pk);
if(itr == entries.end())
return false;
result = itr->second;
return true;
}
void
Clear()
{
llarp::util::Lock lock(access);
entries.clear();
}
bool
llarp_nodedb::Has(const byte_t *pk)
{
llarp::util::Lock lock(access);
return entries.find(pk) != entries.end();
}
bool
Get(const byte_t *pk, llarp::RouterContact &result)
std::string
llarp_nodedb::getRCFilePath(const byte_t *pubkey) const
{
char ftmp[68] = {0};
const char *hexname =
llarp::HexEncode< llarp::AlignedBuffer< 32 >, decltype(ftmp) >(pubkey,
ftmp);
std::string hexString(hexname);
std::string skiplistDir;
skiplistDir += hexString[hexString.length() - 1];
hexString += RC_FILE_EXT;
fs::path filepath = nodePath / skiplistDir / hexString;
return filepath.string();
}
/// insert and write to disk
bool
llarp_nodedb::Insert(const llarp::RouterContact &rc)
{
byte_t tmp[MAX_RC_SIZE];
auto buf = llarp::StackBuffer< decltype(tmp) >(tmp);
{
llarp::util::Lock lock(access);
auto itr = entries.find(pk);
if(itr == entries.end())
return false;
result = itr->second;
return true;
entries.insert(std::make_pair(rc.pubkey.data(), rc));
}
if(!rc.BEncode(&buf))
return false;
bool
Has(const byte_t *pk)
buf.sz = buf.cur - buf.base;
auto filepath = getRCFilePath(rc.pubkey);
llarp::LogDebug("saving RC.pubkey ", filepath);
std::ofstream ofs(
filepath,
std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
ofs.write((char *)buf.base, buf.sz);
ofs.close();
if(!ofs)
{
llarp::util::Lock lock(access);
return entries.find(pk) != entries.end();
llarp::LogError("Failed to write: ", filepath);
return false;
}
llarp::LogDebug("saved RC.pubkey: ", filepath);
return true;
}
std::string
getRCFilePath(const byte_t *pubkey) const
ssize_t
llarp_nodedb::Load(const fs::path &path)
{
std::error_code ec;
if(!fs::exists(path, ec))
{
char ftmp[68] = {0};
const char *hexname =
llarp::HexEncode< llarp::AlignedBuffer< 32 >, decltype(ftmp) >(pubkey,
ftmp);
std::string hexString(hexname);
std::string skiplistDir;
skiplistDir += hexString[hexString.length() - 1];
hexString += RC_FILE_EXT;
fs::path filepath = nodePath / skiplistDir / hexString;
return filepath.string();
return -1;
}
ssize_t loaded = 0;
/// insert and write to disk
bool
Insert(const llarp::RouterContact &rc)
for(const char &ch : skiplist_subdirs)
{
byte_t tmp[MAX_RC_SIZE];
auto buf = llarp::StackBuffer< decltype(tmp) >(tmp);
{
llarp::util::Lock lock(access);
entries.insert(std::make_pair(rc.pubkey.data(), rc));
}
if(!rc.BEncode(&buf))
return false;
if(!ch)
continue;
std::string p;
p += ch;
fs::path sub = path / p;
buf.sz = buf.cur - buf.base;
auto filepath = getRCFilePath(rc.pubkey);
llarp::LogDebug("saving RC.pubkey ", filepath);
std::ofstream ofs(
filepath,
std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
ofs.write((char *)buf.base, buf.sz);
ofs.close();
if(!ofs)
{
llarp::LogError("Failed to write: ", filepath);
return false;
}
llarp::LogDebug("saved RC.pubkey: ", filepath);
return true;
ssize_t l = loadSubdir(sub);
if(l > 0)
loaded += l;
}
return loaded;
}
ssize_t
Load(const fs::path &path)
{
std::error_code ec;
if(!fs::exists(path, ec))
{
return -1;
}
ssize_t loaded = 0;
ssize_t
llarp_nodedb::loadSubdir(const fs::path &dir)
{
ssize_t sz = 0;
llarp::util::IterDir(dir, [&](const fs::path &f) -> bool {
if(fs::is_regular_file(f) && loadfile(f))
sz++;
return true;
});
return sz;
}
for(const char &ch : skiplist_subdirs)
{
if(!ch)
continue;
std::string p;
p += ch;
fs::path sub = path / p;
ssize_t l = loadSubdir(sub);
if(l > 0)
loaded += l;
}
return loaded;
bool
llarp_nodedb::loadfile(const fs::path &fpath)
{
if(fpath.extension() != RC_FILE_EXT)
return false;
llarp::RouterContact rc;
if(!rc.Read(fpath.string().c_str()))
{
llarp::LogError("failed to read file ", fpath);
return false;
}
ssize_t
loadSubdir(const fs::path &dir)
if(!rc.Verify(crypto))
{
ssize_t sz = 0;
llarp::util::IterDir(dir, [&](const fs::path &f) -> bool {
if(fs::is_regular_file(f) && loadfile(f))
sz++;
return true;
});
return sz;
llarp::LogError(fpath, " contains invalid RC");
return false;
}
bool
loadfile(const fs::path &fpath)
{
if(fpath.extension() != RC_FILE_EXT)
return false;
llarp::RouterContact rc;
if(!rc.Read(fpath.string().c_str()))
{
llarp::LogError("failed to read file ", fpath);
return false;
}
if(!rc.Verify(crypto))
{
llarp::LogError(fpath, " contains invalid RC");
return false;
}
{
llarp::util::Lock lock(access);
entries.insert(std::make_pair(rc.pubkey.data(), rc));
}
return true;
llarp::util::Lock lock(access);
entries.insert(std::make_pair(rc.pubkey.data(), rc));
}
return true;
}
void
visit(std::function< bool(const llarp::RouterContact &) > visit)
void
llarp_nodedb::visit(std::function< bool(const llarp::RouterContact &) > visit)
{
llarp::util::Lock lock(access);
auto itr = entries.begin();
while(itr != entries.end())
{
llarp::util::Lock lock(access);
auto itr = entries.begin();
while(itr != entries.end())
{
if(!visit(itr->second))
return;
++itr;
}
if(!visit(itr->second))
return;
++itr;
}
}
bool
iterate(struct llarp_nodedb_iter &i)
bool
llarp_nodedb::iterate(struct llarp_nodedb_iter &i)
{
i.index = 0;
llarp::util::Lock lock(access);
auto itr = entries.begin();
while(itr != entries.end())
{
i.index = 0;
llarp::util::Lock lock(access);
auto itr = entries.begin();
while(itr != entries.end())
{
i.rc = &itr->second;
i.visit(&i);
i.rc = &itr->second;
i.visit(&i);
// advance
i.index++;
itr++;
}
return true;
// advance
i.index++;
itr++;
}
return true;
}
/*
bool Save()
/*
bool
llarp_nodedb::Save()
{
auto itr = entries.begin();
while(itr != entries.end())
{
auto itr = entries.begin();
while(itr != entries.end())
{
llarp::pubkey pk = itr->first;
llarp_rc *rc= itr->second;
llarp::pubkey pk = itr->first;
llarp_rc *rc= itr->second;
itr++; // advance
}
return true;
itr++; // advance
}
*/
};
return true;
}
*/
// call request hook
void
@ -284,32 +270,8 @@ nodedb_async_load_rc(void *user)
job->logic->queue_job({job, &nodedb_inform_load_rc});
}
struct llarp_nodedb *
llarp_nodedb_new(struct llarp_crypto *crypto)
{
return new llarp_nodedb(crypto);
}
void
llarp_nodedb_free(struct llarp_nodedb **n)
{
if(*n)
{
auto i = *n;
*n = nullptr;
i->Clear();
delete i;
}
}
bool
llarp_nodedb_put_rc(struct llarp_nodedb *n, const llarp::RouterContact &rc)
{
return n->Insert(rc);
}
bool
llarp_nodedb_ensure_dir(const char *dir)
llarp_nodedb::ensure_dir(const char *dir)
{
fs::path path(dir);
std::error_code ec;
@ -341,36 +303,28 @@ llarp_nodedb_ensure_dir(const char *dir)
}
void
llarp_nodedb_set_dir(struct llarp_nodedb *n, const char *dir)
llarp_nodedb::set_dir(const char *dir)
{
n->nodePath = dir;
nodePath = dir;
}
ssize_t
llarp_nodedb_load_dir(struct llarp_nodedb *n, const char *dir)
llarp_nodedb::load_dir(const char *dir)
{
std::error_code ec;
if(!fs::exists(dir, ec))
{
return -1;
}
llarp_nodedb_set_dir(n, dir);
return n->Load(dir);
set_dir(dir);
return Load(dir);
}
int
llarp_nodedb_iterate_all(struct llarp_nodedb *n, struct llarp_nodedb_iter i)
llarp_nodedb::iterate_all(struct llarp_nodedb_iter i)
{
n->iterate(i);
return n->entries.size();
}
void
llarp_nodedb_visit_loaded(
struct llarp_nodedb *n,
std::function< bool(const llarp::RouterContact &) > visit)
{
return n->visit(visit);
iterate(i);
return entries.size();
}
/// maybe rename to verify_and_set
@ -393,38 +347,23 @@ llarp_nodedb_async_load_rc(struct llarp_async_load_rc *job)
}
*/
bool
llarp_nodedb_get_rc(struct llarp_nodedb *n, const llarp::RouterID &pk,
llarp::RouterContact &result)
{
// llarp::LogInfo("llarp_nodedb_get_rc [", pk, "]");
return n->Get(pk, result);
}
size_t
llarp_nodedb_num_loaded(struct llarp_nodedb *n)
{
return n->entries.size();
}
bool
llarp_nodedb_del_rc(struct llarp_nodedb *n, const llarp::RouterID &pk)
llarp_nodedb::num_loaded() const
{
return n->Remove(pk);
return entries.size();
}
bool
llarp_nodedb_select_random_exit(struct llarp_nodedb *n,
llarp::RouterContact &result)
llarp_nodedb::select_random_exit(llarp::RouterContact &result)
{
const auto sz = n->entries.size();
auto itr = n->entries.begin();
const auto sz = entries.size();
auto itr = entries.begin();
if(sz < 3)
return false;
auto idx = llarp_randint() % sz;
if(idx)
std::advance(itr, idx - 1);
while(itr != n->entries.end())
while(itr != entries.end())
{
if(itr->second.IsExit())
{
@ -433,8 +372,8 @@ llarp_nodedb_select_random_exit(struct llarp_nodedb *n,
}
++itr;
}
// wrap arround
itr = n->entries.begin();
// wrap around
itr = entries.begin();
while(idx--)
{
if(itr->second.IsExit())
@ -448,13 +387,12 @@ llarp_nodedb_select_random_exit(struct llarp_nodedb *n,
}
bool
llarp_nodedb_select_random_hop(struct llarp_nodedb *n,
const llarp::RouterContact &prev,
llarp::RouterContact &result, size_t N)
llarp_nodedb::select_random_hop(const llarp::RouterContact &prev,
llarp::RouterContact &result, size_t N)
{
/// checking for "guard" status for N = 0 is done by caller inside of
/// pathbuilder's scope
auto sz = n->entries.size();
auto sz = entries.size();
if(sz < 3)
return false;
size_t tries = 5;
@ -462,7 +400,7 @@ llarp_nodedb_select_random_hop(struct llarp_nodedb *n,
{
do
{
auto itr = n->entries.begin();
auto itr = entries.begin();
if(sz > 1)
{
auto idx = llarp_randint() % sz;
@ -485,7 +423,7 @@ llarp_nodedb_select_random_hop(struct llarp_nodedb *n,
}
else
{
auto itr = n->entries.begin();
auto itr = entries.begin();
if(sz > 1)
{
auto idx = llarp_randint() % sz;

@ -20,7 +20,7 @@ namespace llarp
Handler result;
size_t idx = 0;
llarp::Router* router = nullptr;
llarp::Router* router = nullptr;
llarp_threadpool* worker = nullptr;
llarp::Logic* logic = nullptr;
llarp_crypto* crypto = nullptr;
@ -182,7 +182,7 @@ namespace llarp
do
{
--tries;
if(llarp_nodedb_select_random_hop(db, prev, cur, hop))
if(db->select_random_hop(prev, cur, hop))
return true;
} while(router->routerProfiling.IsBad(cur.pubkey) && tries > 0);
return false;

@ -68,7 +68,7 @@ struct TryConnectJob
if(!router->IsServiceNode())
{
if(router->routerProfiling.IsBad(rc.pubkey))
llarp_nodedb_del_rc(router->nodedb, rc.pubkey);
router->nodedb->Remove(rc.pubkey);
}
// delete this
router->pendingEstablishJobs.erase(rc.pubkey);
@ -283,7 +283,7 @@ namespace llarp
}
llarp::RouterContact remoteRC;
// we don't have an open session to that router right now
if(llarp_nodedb_get_rc(nodedb, remote, remoteRC))
if(nodedb->Get(remote, remoteRC))
{
// try connecting directly as the rc is loaded from disk
llarp_router_try_connect(this, remoteRC, 10);
@ -311,7 +311,7 @@ namespace llarp
}
if(results[0].Verify(&crypto))
{
llarp_nodedb_put_rc(nodedb, results[0]);
nodedb->Insert(results[0]);
llarp_router_try_connect(this, results[0], 10);
}
}
@ -347,7 +347,7 @@ namespace llarp
{
llarp::LogDebug("verified signature");
// store into filesystem
if(!llarp_nodedb_put_rc(nodedb, remote))
if(!nodedb->Insert(remote))
{
llarp::LogWarn("failed to store");
}
@ -534,7 +534,7 @@ namespace llarp
for(const auto &rc : results)
{
if(rc.Verify(&crypto))
llarp_nodedb_put_rc(nodedb, rc);
nodedb->Insert(rc);
else
return;
}
@ -555,7 +555,7 @@ namespace llarp
}
llarp::RouterContact rc;
if(llarp_nodedb_get_rc(nodedb, remote, rc))
if(nodedb->Get(remote, rc))
{
// try connecting async
llarp_router_try_connect(this, rc, 5);
@ -602,7 +602,7 @@ namespace llarp
if(whitelistRouters
&& lokinetRouters.find(result.pubkey) == lokinetRouters.end())
continue;
llarp_nodedb_put_rc(nodedb, result);
nodedb->Insert(result);
llarp_router_try_connect(this, result, 10);
}
}
@ -649,7 +649,7 @@ namespace llarp
if(inboundLinks.size() == 0)
{
size_t N = llarp_nodedb_num_loaded(nodedb);
size_t N = nodedb->num_loaded();
if(N < minRequiredRouters)
{
llarp::LogInfo("We need at least ", minRequiredRouters,
@ -948,7 +948,7 @@ namespace llarp
return false;
}
llarp::LogInfo("have ", llarp_nodedb_num_loaded(nodedb), " routers");
llarp::LogInfo("have ", nodedb->num_loaded(), " routers");
llarp::LogDebug("starting outbound link");
if(!outboundLink->Start(logic))
@ -1033,8 +1033,9 @@ namespace llarp
{
int wanted = want;
Router *self = this;
llarp_nodedb_visit_loaded(
self->nodedb, [self, &want](const llarp::RouterContact &other) -> bool {
self->nodedb->visit(
[self, &want](const llarp::RouterContact &other) -> bool {
// check if we really want to
if(!self->ConnectionToRouterAllowed(other.pubkey))
return want > 0;

@ -709,7 +709,7 @@ namespace llarp
if(router.IsZero())
return;
RouterContact rc;
if(!llarp_nodedb_get_rc(m_Router->nodedb, router, rc))
if(!m_Router->nodedb->Get(router, rc))
{
if(m_PendingRouters.find(router) == m_PendingRouters.end())
{
@ -1222,7 +1222,7 @@ namespace llarp
else if(hop == numHops - 1)
{
// last hop
if(!llarp_nodedb_get_rc(nodedb, remote, hops[hop]))
if(!nodedb->Get(remote, hops[hop]))
return false;
}
// middle hop
@ -1231,8 +1231,7 @@ namespace llarp
size_t tries = 5;
do
{
llarp_nodedb_select_random_hop(nodedb, hops[hop - 1], hops[hop],
hop);
nodedb->select_random_hop(hops[hop - 1], hops[hop], hop);
--tries;
} while(m_Endpoint->Router()->routerProfiling.IsBad(hops[hop].pubkey)
&& tries > 0);
@ -1594,7 +1593,7 @@ namespace llarp
return false;
if(hop == numHops - 1)
{
if(llarp_nodedb_get_rc(db, m_NextIntro.router, cur))
if(db->Get(m_NextIntro.router, cur))
{
return true;
}

Loading…
Cancel
Save