|
|
|
|
@ -12,9 +12,9 @@
|
|
|
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
|
#include <atomic>
|
|
|
|
|
#include <map>
|
|
|
|
|
#include <optional>
|
|
|
|
|
#include <set>
|
|
|
|
|
#include <unordered_map>
|
|
|
|
|
#include <unordered_set>
|
|
|
|
|
#include <utility>
|
|
|
|
|
|
|
|
|
|
@ -22,15 +22,164 @@ namespace llarp
|
|
|
|
|
{
|
|
|
|
|
struct Router;
|
|
|
|
|
|
|
|
|
|
class NodeDB
|
|
|
|
|
// TESTNET: the following constants have been shortened for testing purposes
|
|
|
|
|
|
|
|
|
|
/* RC Fetch Constants */
|
|
|
|
|
// fallback to bootstrap if we have less than this many RCs
|
|
|
|
|
inline constexpr size_t MIN_ACTIVE_RCS{6};
|
|
|
|
|
// max number of attempts we make in non-bootstrap fetch requests
|
|
|
|
|
inline constexpr int MAX_FETCH_ATTEMPTS{10};
|
|
|
|
|
// the total number of returned rcs that are held locally should be at least this
|
|
|
|
|
inline constexpr size_t MIN_GOOD_RC_FETCH_TOTAL{};
|
|
|
|
|
// the ratio of returned rcs found locally to to total returned should be above this ratio
|
|
|
|
|
inline constexpr double MIN_GOOD_RC_FETCH_THRESHOLD{};
|
|
|
|
|
|
|
|
|
|
/* RID Fetch Constants */
|
|
|
|
|
// the number of rid sources that we make rid fetch requests to
|
|
|
|
|
inline constexpr size_t RID_SOURCE_COUNT{8};
|
|
|
|
|
// upper limit on how many rid fetch requests to rid sources can fail
|
|
|
|
|
inline constexpr size_t MAX_RID_ERRORS{2};
|
|
|
|
|
// each returned rid must appear this number of times across all responses
|
|
|
|
|
inline constexpr int MIN_RID_FETCH_FREQ{RID_SOURCE_COUNT - MAX_RID_ERRORS - 1};
|
|
|
|
|
// the total number of accepted returned rids should be above this number
|
|
|
|
|
inline constexpr size_t MIN_GOOD_RID_FETCH_TOTAL{};
|
|
|
|
|
// the ratio of accepted:rejected rids must be above this ratio
|
|
|
|
|
inline constexpr double GOOD_RID_FETCH_THRESHOLD{};
|
|
|
|
|
|
|
|
|
|
/* Bootstrap Constants */
|
|
|
|
|
// the number of rc's we query the bootstrap for; service nodes pass 0, which means
|
|
|
|
|
// gimme all dat RCs
|
|
|
|
|
inline constexpr size_t SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT{0};
|
|
|
|
|
inline constexpr size_t CLIENT_BOOTSTRAP_SOURCE_COUNT{10};
|
|
|
|
|
// the maximum number of fetch requests we make across all bootstraps
|
|
|
|
|
inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{5};
|
|
|
|
|
// if all bootstraps fail, router will trigger re-bootstrapping after this cooldown
|
|
|
|
|
inline constexpr auto BOOTSTRAP_COOLDOWN{1min};
|
|
|
|
|
|
|
|
|
|
/* Other Constants */
|
|
|
|
|
// the maximum number of RC/RID fetches that can pass w/o an unconfirmed rc/rid appearing
|
|
|
|
|
inline constexpr int MAX_CONFIRMATION_ATTEMPTS{5};
|
|
|
|
|
// threshold amount of verifications to promote an unconfirmed rc/rid
|
|
|
|
|
inline constexpr int CONFIRMATION_THRESHOLD{3};
|
|
|
|
|
|
|
|
|
|
inline constexpr auto FLUSH_INTERVAL{5min};
|
|
|
|
|
|
|
|
|
|
template <
|
|
|
|
|
typename ID_t,
|
|
|
|
|
std::enable_if_t<std::is_same_v<ID_t, RouterID> || std::is_same_v<ID_t, RemoteRC>, int> = 0>
|
|
|
|
|
struct Unconfirmed
|
|
|
|
|
{
|
|
|
|
|
std::unordered_map<RouterID, RemoteRC> known_rcs;
|
|
|
|
|
const ID_t id;
|
|
|
|
|
int attempts = 0;
|
|
|
|
|
int verifications = 0;
|
|
|
|
|
|
|
|
|
|
Unconfirmed() = delete;
|
|
|
|
|
Unconfirmed(const ID_t& obj) : id{obj}
|
|
|
|
|
{}
|
|
|
|
|
Unconfirmed(ID_t&& obj) : id{std::move(obj)}
|
|
|
|
|
{}
|
|
|
|
|
|
|
|
|
|
int
|
|
|
|
|
strikes() const
|
|
|
|
|
{
|
|
|
|
|
return attempts;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Router& router;
|
|
|
|
|
const fs::path m_Root;
|
|
|
|
|
const std::function<void(std::function<void()>)> disk;
|
|
|
|
|
operator bool() const
|
|
|
|
|
{
|
|
|
|
|
return verifications == CONFIRMATION_THRESHOLD;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
llarp_time_t m_NextFlushAt;
|
|
|
|
|
bool
|
|
|
|
|
operator==(const Unconfirmed& other) const
|
|
|
|
|
{
|
|
|
|
|
return id == other.id;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
operator<(const Unconfirmed& other) const
|
|
|
|
|
{
|
|
|
|
|
return id < other.id;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class NodeDB
|
|
|
|
|
{
|
|
|
|
|
Router& _router;
|
|
|
|
|
const fs::path _root;
|
|
|
|
|
const std::function<void(std::function<void()>)> _disk;
|
|
|
|
|
|
|
|
|
|
llarp_time_t _next_flush_time;
|
|
|
|
|
|
|
|
|
|
/******** RouterID/RouterContacts ********/
|
|
|
|
|
|
|
|
|
|
/** RouterID mappings
|
|
|
|
|
Both the following are populated in NodeDB startup with RouterID's stored on disk.
|
|
|
|
|
- known_rids: meant to persist between lokinet sessions, and is only
|
|
|
|
|
populated during startup and RouterID fetching. This is meant to represent the
|
|
|
|
|
client instance's most recent perspective of the network, and record which RouterID's
|
|
|
|
|
were recently "active" and connected to
|
|
|
|
|
- unconfirmed_rids: holds new rids returned in fetch requests to be verified by subsequent
|
|
|
|
|
fetch requests
|
|
|
|
|
- known_rcs: populated during startup and when RC's are updated both during gossip
|
|
|
|
|
and periodic RC fetching
|
|
|
|
|
- unconfirmed_rcs: holds new rcs to be verified by subsequent fetch requests, similar to
|
|
|
|
|
the unknown_rids container
|
|
|
|
|
- rc_lookup: holds all the same rc's as known_rcs, but can be used to look them up by
|
|
|
|
|
their rid
|
|
|
|
|
- bootstrap_seeds: if we are the seed node, we insert the rc's of bootstrap fetch requests
|
|
|
|
|
senders into this container to "introduce" them to each other
|
|
|
|
|
- _bootstraps: the standard container for bootstrap RemoteRCs
|
|
|
|
|
*/
|
|
|
|
|
std::set<RouterID> known_rids;
|
|
|
|
|
std::set<Unconfirmed<RouterID>> unconfirmed_rids;
|
|
|
|
|
|
|
|
|
|
std::set<RemoteRC> known_rcs;
|
|
|
|
|
std::set<Unconfirmed<RemoteRC>> unconfirmed_rcs;
|
|
|
|
|
|
|
|
|
|
std::map<RouterID, const RemoteRC&> rc_lookup;
|
|
|
|
|
|
|
|
|
|
BootstrapList _bootstraps{};
|
|
|
|
|
|
|
|
|
|
/** RouterID lists // TODO: get rid of all these, replace with better decom/not staked sets
|
|
|
|
|
- white: active routers
|
|
|
|
|
- gray: fully funded, but decommissioned routers
|
|
|
|
|
- green: registered, but not fully-staked routers
|
|
|
|
|
*/
|
|
|
|
|
std::set<RouterID> _router_whitelist{};
|
|
|
|
|
std::set<RouterID> _router_greylist{};
|
|
|
|
|
std::set<RouterID> _router_greenlist{};
|
|
|
|
|
|
|
|
|
|
// All registered relays (service nodes)
|
|
|
|
|
std::set<RouterID> _registered_routers;
|
|
|
|
|
// timing (note: Router holds the variables for last rc and rid request times)
|
|
|
|
|
std::unordered_map<RouterID, rc_time> last_rc_update_times;
|
|
|
|
|
// if populated from a config file, lists specific exclusively used as path first-hops
|
|
|
|
|
std::set<RouterID> _pinned_edges;
|
|
|
|
|
// source of "truth" for RC updating. This relay will also mediate requests to the
|
|
|
|
|
// 12 selected active RID's for RID fetching
|
|
|
|
|
RouterID fetch_source;
|
|
|
|
|
// set of 12 randomly selected RID's from the client's set of routers
|
|
|
|
|
std::set<RouterID> rid_sources{};
|
|
|
|
|
// logs the RID's that resulted in an error during RID fetching
|
|
|
|
|
std::set<RouterID> fail_sources{};
|
|
|
|
|
// tracks the number of times each rid appears in the above responses
|
|
|
|
|
std::unordered_map<RouterID, int> fetch_counters{};
|
|
|
|
|
|
|
|
|
|
/** Failure counters:
|
|
|
|
|
- fetch_failures: tracks errors fetching RC's from the RC node and requesting RID's
|
|
|
|
|
from the 12 RID sources. Errors in the individual RID sets are NOT counted towards
|
|
|
|
|
this, their performance as a group is evaluated wholistically
|
|
|
|
|
- bootstrap_failures: tracks errors fetching both RC's from bootstrasps and RID requests
|
|
|
|
|
they mediate. This is a different counter as we only bootstrap in problematic cases
|
|
|
|
|
*/
|
|
|
|
|
std::atomic<int> fetch_failures{0}, bootstrap_attempts{0};
|
|
|
|
|
|
|
|
|
|
std::atomic<bool> _using_bootstrap_fallback{false}, _needs_rebootstrap{false},
|
|
|
|
|
_needs_initial_fetch{true}, _fetching_initial{false}, _initial_completed{false};
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
want_rc(const RouterID& rid) const;
|
|
|
|
|
|
|
|
|
|
/// asynchronously remove the files for a set of rcs on disk given their public ident key
|
|
|
|
|
void
|
|
|
|
|
@ -38,99 +187,101 @@ namespace llarp
|
|
|
|
|
|
|
|
|
|
/// get filename of an RC file given its public ident key
|
|
|
|
|
fs::path
|
|
|
|
|
get_path_by_pubkey(RouterID pk) const;
|
|
|
|
|
|
|
|
|
|
std::unordered_map<RouterID, RemoteRC> bootstraps;
|
|
|
|
|
|
|
|
|
|
// Router lists for snodes
|
|
|
|
|
// whitelist = active routers
|
|
|
|
|
std::unordered_set<RouterID> router_whitelist;
|
|
|
|
|
// greylist = fully funded, but decommissioned routers
|
|
|
|
|
std::unordered_set<RouterID> router_greylist;
|
|
|
|
|
// greenlist = registered but not fully-staked routers
|
|
|
|
|
std::unordered_set<RouterID> router_greenlist;
|
|
|
|
|
// all registered relays (snodes)
|
|
|
|
|
std::unordered_set<RouterID> registered_routers;
|
|
|
|
|
std::unordered_map<RouterID, rc_time> last_rc_update_times;
|
|
|
|
|
|
|
|
|
|
// Router list for clients
|
|
|
|
|
std::unordered_set<RouterID> client_known_routers;
|
|
|
|
|
get_path_by_pubkey(const RouterID& pk) const;
|
|
|
|
|
|
|
|
|
|
// only ever use to specific edges as path first-hops
|
|
|
|
|
std::unordered_set<RouterID> pinned_edges;
|
|
|
|
|
|
|
|
|
|
// rc update info
|
|
|
|
|
RouterID rc_fetch_source;
|
|
|
|
|
rc_time last_rc_update_relay_timestamp;
|
|
|
|
|
static constexpr auto ROUTER_ID_SOURCE_COUNT = 12;
|
|
|
|
|
std::unordered_set<RouterID> router_id_fetch_sources;
|
|
|
|
|
std::unordered_map<RouterID, std::vector<RouterID>> router_id_fetch_responses;
|
|
|
|
|
// process responses once all are received (or failed/timed out)
|
|
|
|
|
size_t router_id_response_count{0};
|
|
|
|
|
bool router_id_fetch_in_progress{false};
|
|
|
|
|
public:
|
|
|
|
|
explicit NodeDB(
|
|
|
|
|
fs::path rootdir, std::function<void(std::function<void()>)> diskCaller, Router* r);
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
want_rc(const RouterID& rid) const;
|
|
|
|
|
/// in memory nodedb
|
|
|
|
|
NodeDB();
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
void
|
|
|
|
|
set_bootstrap_routers(const std::set<RemoteRC>& rcs);
|
|
|
|
|
const std::set<RouterID>&
|
|
|
|
|
get_known_rids() const
|
|
|
|
|
{
|
|
|
|
|
return known_rids;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::unordered_set<RouterID>&
|
|
|
|
|
whitelist() const
|
|
|
|
|
const std::set<RemoteRC>&
|
|
|
|
|
get_known_rcs() const
|
|
|
|
|
{
|
|
|
|
|
return router_whitelist;
|
|
|
|
|
return known_rcs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::unordered_set<RouterID>&
|
|
|
|
|
greylist() const
|
|
|
|
|
std::optional<RemoteRC>
|
|
|
|
|
get_rc_by_rid(const RouterID& rid);
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
is_initial_fetching() const
|
|
|
|
|
{
|
|
|
|
|
return router_greylist;
|
|
|
|
|
return _fetching_initial;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::unordered_set<RouterID>&
|
|
|
|
|
get_registered_routers() const
|
|
|
|
|
bool
|
|
|
|
|
initial_fetch_completed() const
|
|
|
|
|
{
|
|
|
|
|
return registered_routers;
|
|
|
|
|
return _initial_completed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::unordered_map<RouterID, RemoteRC>&
|
|
|
|
|
get_rcs() const
|
|
|
|
|
bool
|
|
|
|
|
needs_initial_fetch() const
|
|
|
|
|
{
|
|
|
|
|
return known_rcs;
|
|
|
|
|
return _needs_initial_fetch;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::unordered_map<RouterID, rc_time>&
|
|
|
|
|
get_last_rc_update_times() const
|
|
|
|
|
bool
|
|
|
|
|
needs_rebootstrap() const
|
|
|
|
|
{
|
|
|
|
|
return last_rc_update_times;
|
|
|
|
|
return _needs_rebootstrap;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// If we receive a bad set of RCs from our current RC source relay, we consider
|
|
|
|
|
// that relay to be a bad source of RCs and we randomly choose a new one.
|
|
|
|
|
//
|
|
|
|
|
// When using a new RC fetch relay, we first re-fetch the full RC list and, if
|
|
|
|
|
// that aligns with our RouterID list, we go back to periodic updates from that relay.
|
|
|
|
|
//
|
|
|
|
|
// This will respect edge-pinning and attempt to use a relay we already have
|
|
|
|
|
// a connection with.
|
|
|
|
|
void
|
|
|
|
|
rotate_rc_source();
|
|
|
|
|
ingest_bootstrap_seed();
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
ingest_fetched_rcs(std::set<RemoteRC> rcs);
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
process_fetched_rcs(std::set<RemoteRC>& rcs);
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
ingest_rcs(RouterID source, std::vector<RemoteRC> rcs, rc_time timestamp);
|
|
|
|
|
ingest_rid_fetch_responses(const RouterID& source, std::set<RouterID> ids = {});
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
process_fetched_rids();
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
ingest_router_ids(RouterID source, std::vector<RouterID> ids);
|
|
|
|
|
fetch_initial(bool is_snode = false);
|
|
|
|
|
|
|
|
|
|
// RouterContact fetching
|
|
|
|
|
void
|
|
|
|
|
fetch_rcs();
|
|
|
|
|
fetch_rcs(bool initial = false);
|
|
|
|
|
void
|
|
|
|
|
post_fetch_rcs(bool initial = false);
|
|
|
|
|
void
|
|
|
|
|
fetch_rcs_result(bool initial = false, bool error = false);
|
|
|
|
|
|
|
|
|
|
// RouterID fetching
|
|
|
|
|
void
|
|
|
|
|
fetch_rids(bool initial = false);
|
|
|
|
|
void
|
|
|
|
|
fetch_router_ids();
|
|
|
|
|
post_fetch_rids(bool initial = false);
|
|
|
|
|
void
|
|
|
|
|
fetch_rids_result(bool initial = false);
|
|
|
|
|
|
|
|
|
|
// Bootstrap fallback fetching
|
|
|
|
|
void
|
|
|
|
|
fallback_to_bootstrap();
|
|
|
|
|
void
|
|
|
|
|
select_router_id_sources(std::unordered_set<RouterID> excluded = {});
|
|
|
|
|
post_snode_bootstrap();
|
|
|
|
|
void
|
|
|
|
|
bootstrap_cooldown();
|
|
|
|
|
|
|
|
|
|
// Populate rid_sources with random sample from known_rids. A set of rids is passed
|
|
|
|
|
// if only specific RID's need to be re-selected; to re-select all, pass the member
|
|
|
|
|
// variable ::known_rids
|
|
|
|
|
void
|
|
|
|
|
reselect_router_id_sources(std::set<RouterID> specific);
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
set_router_whitelist(
|
|
|
|
|
@ -160,7 +311,7 @@ namespace llarp
|
|
|
|
|
bool
|
|
|
|
|
is_path_allowed(const RouterID& remote) const
|
|
|
|
|
{
|
|
|
|
|
return router_whitelist.count(remote);
|
|
|
|
|
return known_rids.count(remote);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// if pinned edges were specified, the remote must be in that set, else any remote
|
|
|
|
|
@ -168,17 +319,83 @@ namespace llarp
|
|
|
|
|
bool
|
|
|
|
|
is_first_hop_allowed(const RouterID& remote) const;
|
|
|
|
|
|
|
|
|
|
const std::unordered_set<RouterID>&
|
|
|
|
|
get_pinned_edges() const
|
|
|
|
|
std::set<RouterID>&
|
|
|
|
|
pinned_edges()
|
|
|
|
|
{
|
|
|
|
|
return pinned_edges;
|
|
|
|
|
return _pinned_edges;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
explicit NodeDB(
|
|
|
|
|
fs::path rootdir, std::function<void(std::function<void()>)> diskCaller, Router* r);
|
|
|
|
|
void
|
|
|
|
|
store_bootstraps();
|
|
|
|
|
|
|
|
|
|
/// in memory nodedb
|
|
|
|
|
NodeDB();
|
|
|
|
|
size_t
|
|
|
|
|
num_bootstraps() const
|
|
|
|
|
{
|
|
|
|
|
return _bootstraps.size();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
has_bootstraps() const
|
|
|
|
|
{
|
|
|
|
|
return _bootstraps.empty();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const BootstrapList&
|
|
|
|
|
bootstrap_list() const
|
|
|
|
|
{
|
|
|
|
|
return _bootstraps;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
BootstrapList&
|
|
|
|
|
bootstrap_list()
|
|
|
|
|
{
|
|
|
|
|
return _bootstraps;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
set_bootstrap_routers(BootstrapList& from_router);
|
|
|
|
|
|
|
|
|
|
const std::set<RouterID>&
|
|
|
|
|
whitelist() const
|
|
|
|
|
{
|
|
|
|
|
return _router_whitelist;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::set<RouterID>&
|
|
|
|
|
greylist() const
|
|
|
|
|
{
|
|
|
|
|
return _router_greylist;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::set<RouterID>&
|
|
|
|
|
registered_routers()
|
|
|
|
|
{
|
|
|
|
|
return _registered_routers;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::set<RouterID>&
|
|
|
|
|
registered_routers() const
|
|
|
|
|
{
|
|
|
|
|
return _registered_routers;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::set<RemoteRC>&
|
|
|
|
|
get_rcs() const
|
|
|
|
|
{
|
|
|
|
|
return known_rcs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// const std::unordered_map<RouterID, RemoteRC>&
|
|
|
|
|
// get_rcs() const
|
|
|
|
|
// {
|
|
|
|
|
// return known_rcs;
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
|
|
const std::unordered_map<RouterID, rc_time>&
|
|
|
|
|
get_last_rc_update_times() const
|
|
|
|
|
{
|
|
|
|
|
return last_rc_update_times;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// load all known_rcs from disk syncrhonously
|
|
|
|
|
void
|
|
|
|
|
@ -190,7 +407,10 @@ namespace llarp
|
|
|
|
|
|
|
|
|
|
/// the number of RCs that are loaded from disk
|
|
|
|
|
size_t
|
|
|
|
|
num_loaded() const;
|
|
|
|
|
num_rcs() const;
|
|
|
|
|
|
|
|
|
|
size_t
|
|
|
|
|
num_rids() const;
|
|
|
|
|
|
|
|
|
|
/// do periodic tasks like flush to disk and expiration
|
|
|
|
|
void
|
|
|
|
|
@ -206,41 +426,78 @@ namespace llarp
|
|
|
|
|
|
|
|
|
|
/// return true if we have an rc by its ident pubkey
|
|
|
|
|
bool
|
|
|
|
|
has_rc(RouterID pk) const;
|
|
|
|
|
has_rc(const RouterID& pk) const;
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
has_rc(const RemoteRC& rc) const;
|
|
|
|
|
|
|
|
|
|
/// maybe get an rc by its ident pubkey
|
|
|
|
|
std::optional<RemoteRC>
|
|
|
|
|
get_rc(RouterID pk) const;
|
|
|
|
|
get_rc(const RouterID& pk) const;
|
|
|
|
|
|
|
|
|
|
template <typename Filter>
|
|
|
|
|
std::optional<RemoteRC>
|
|
|
|
|
GetRandom(Filter visit) const
|
|
|
|
|
{
|
|
|
|
|
return router.loop()->call_get([visit]() -> std::optional<RemoteRC> {
|
|
|
|
|
std::vector<const decltype(known_rcs)::value_type*> known_rcs;
|
|
|
|
|
for (const auto& entry : known_rcs)
|
|
|
|
|
known_rcs.push_back(entry);
|
|
|
|
|
get_random_rc() const;
|
|
|
|
|
|
|
|
|
|
std::shuffle(known_rcs.begin(), known_rcs.end(), llarp::csrng);
|
|
|
|
|
// Get `n` random RCs from all RCs we know about. If `exact` is true then we require n matches
|
|
|
|
|
// (and otherwise return nullopt); otherwise we return whatever we found, or nullopt if we find
|
|
|
|
|
// nothing at all.
|
|
|
|
|
std::optional<std::vector<RemoteRC>>
|
|
|
|
|
get_n_random_rcs(size_t n, bool exact = false) const;
|
|
|
|
|
|
|
|
|
|
for (const auto entry : known_rcs)
|
|
|
|
|
{
|
|
|
|
|
if (visit(entry->second))
|
|
|
|
|
return entry->second;
|
|
|
|
|
}
|
|
|
|
|
/** The following random conditional functions utilize a simple implementation of reservoir
|
|
|
|
|
sampling to return either 1 or n random RC's using only one pass through the set of RC's.
|
|
|
|
|
|
|
|
|
|
return std::nullopt;
|
|
|
|
|
});
|
|
|
|
|
Pseudocode:
|
|
|
|
|
- begin iterating through the set
|
|
|
|
|
- load the first n (or 1) that pass hook(n) into a list Selected[]
|
|
|
|
|
- for all that pass the hook, increment i, tracking the number seen thus far
|
|
|
|
|
- generate a random integer x from 0 to i
|
|
|
|
|
- x < n ? Selected[x] = current : continue;
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
std::optional<RemoteRC>
|
|
|
|
|
get_random_rc_conditional(std::function<bool(RemoteRC)> hook) const;
|
|
|
|
|
|
|
|
|
|
std::optional<std::vector<RemoteRC>>
|
|
|
|
|
get_n_random_rcs_conditional(
|
|
|
|
|
size_t n, std::function<bool(RemoteRC)> hook, bool exact = false) const;
|
|
|
|
|
|
|
|
|
|
// Updates `current` to not contain any of the elements of `replace` and resamples (up to
|
|
|
|
|
// `target_size`) from population to refill it.
|
|
|
|
|
template <typename T, typename RNG>
|
|
|
|
|
void
|
|
|
|
|
replace_subset(
|
|
|
|
|
std::set<T>& current,
|
|
|
|
|
const std::set<T>& replace,
|
|
|
|
|
std::set<T> population,
|
|
|
|
|
size_t target_size,
|
|
|
|
|
RNG&& rng)
|
|
|
|
|
{
|
|
|
|
|
// Remove the ones we are replacing from current:
|
|
|
|
|
current.erase(replace.begin(), replace.end());
|
|
|
|
|
|
|
|
|
|
// Remove ones we are replacing, and ones we already have, from the population so that we
|
|
|
|
|
// won't reselect them:
|
|
|
|
|
population.erase(replace.begin(), replace.end());
|
|
|
|
|
population.erase(current.begin(), current.end());
|
|
|
|
|
|
|
|
|
|
if (current.size() < target_size)
|
|
|
|
|
std::sample(
|
|
|
|
|
population.begin(),
|
|
|
|
|
population.end(),
|
|
|
|
|
std::inserter(current, current.end()),
|
|
|
|
|
target_size - current.size(),
|
|
|
|
|
rng);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// visit all known_rcs
|
|
|
|
|
template <typename Visit>
|
|
|
|
|
void
|
|
|
|
|
VisitAll(Visit visit) const
|
|
|
|
|
visit_all(Visit visit) const
|
|
|
|
|
{
|
|
|
|
|
router.loop()->call([this, visit]() {
|
|
|
|
|
_router.loop()->call([this, visit]() {
|
|
|
|
|
for (const auto& item : known_rcs)
|
|
|
|
|
visit(item.second);
|
|
|
|
|
visit(item);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -251,26 +508,74 @@ namespace llarp
|
|
|
|
|
/// remove an entry given a filter that inspects the rc
|
|
|
|
|
template <typename Filter>
|
|
|
|
|
void
|
|
|
|
|
RemoveIf(Filter visit)
|
|
|
|
|
remove_if(Filter visit)
|
|
|
|
|
{
|
|
|
|
|
router.loop()->call([this, visit]() {
|
|
|
|
|
_router.loop()->call([this, visit]() {
|
|
|
|
|
std::unordered_set<RouterID> removed;
|
|
|
|
|
auto itr = known_rcs.begin();
|
|
|
|
|
while (itr != known_rcs.end())
|
|
|
|
|
|
|
|
|
|
for (auto itr = rc_lookup.begin(); itr != rc_lookup.end();)
|
|
|
|
|
{
|
|
|
|
|
if (visit(itr->second))
|
|
|
|
|
{
|
|
|
|
|
removed.insert(itr->second.router_id());
|
|
|
|
|
itr = known_rcs.erase(itr);
|
|
|
|
|
removed.insert(itr->first);
|
|
|
|
|
known_rcs.erase(itr->second);
|
|
|
|
|
itr = rc_lookup.erase(itr);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
++itr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (not removed.empty())
|
|
|
|
|
remove_many_from_disk_async(std::move(removed));
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
template <
|
|
|
|
|
typename ID_t,
|
|
|
|
|
std::enable_if_t<std::is_same_v<ID_t, RouterID> || std::is_same_v<ID_t, RemoteRC>, int> = 0>
|
|
|
|
|
void
|
|
|
|
|
process_results(
|
|
|
|
|
std::set<ID_t> unconfirmed, std::set<Unconfirmed<ID_t>>& container, std::set<ID_t>& known)
|
|
|
|
|
{
|
|
|
|
|
// before we add the unconfirmed set, we check to see if our local set of unconfirmed
|
|
|
|
|
// rcs/rids appeared in the latest unconfirmed set; if so, we will increment their number
|
|
|
|
|
// of verifications and reset the attempts counter. Once appearing in 3 different requests,
|
|
|
|
|
// the rc/rid will be "verified" and promoted to the known_{rcs,rids} container
|
|
|
|
|
for (auto itr = container.begin(); itr != container.end();)
|
|
|
|
|
{
|
|
|
|
|
auto& id = itr->id;
|
|
|
|
|
auto& count = const_cast<int&>(itr->attempts);
|
|
|
|
|
auto& verifications = const_cast<int&>(itr->verifications);
|
|
|
|
|
|
|
|
|
|
if (auto found = unconfirmed.find(id); found != unconfirmed.end())
|
|
|
|
|
{
|
|
|
|
|
if (++verifications >= CONFIRMATION_THRESHOLD)
|
|
|
|
|
{
|
|
|
|
|
if constexpr (std::is_same_v<ID_t, RemoteRC>)
|
|
|
|
|
put_rc_if_newer(id);
|
|
|
|
|
else
|
|
|
|
|
known.emplace(id);
|
|
|
|
|
itr = container.erase(itr);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
// reset attempt counter and continue
|
|
|
|
|
count = 0;
|
|
|
|
|
++itr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unconfirmed.erase(found);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
itr = (++count >= MAX_CONFIRMATION_ATTEMPTS) ? container.erase(itr) : ++itr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (auto& id : unconfirmed)
|
|
|
|
|
{
|
|
|
|
|
container.emplace(std::move(id));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// remove rcs that are older than we want to keep. For relays, this is when
|
|
|
|
|
/// they become "outdated" (i.e. 12hrs). Clients will hang on to them until
|
|
|
|
|
/// they are fully "expired" (i.e. 30 days), as the client may go offline for
|
|
|
|
|
@ -280,18 +585,29 @@ namespace llarp
|
|
|
|
|
|
|
|
|
|
/// put (or replace) the RC if we consider it valid (want_rc). returns true if put.
|
|
|
|
|
bool
|
|
|
|
|
put_rc(
|
|
|
|
|
RemoteRC rc,
|
|
|
|
|
rc_time now =
|
|
|
|
|
std::chrono::time_point_cast<std::chrono::seconds>(std::chrono::system_clock::now()));
|
|
|
|
|
put_rc(RemoteRC rc, rc_time now = time_point_now());
|
|
|
|
|
|
|
|
|
|
/// if we consider it valid (want_rc),
|
|
|
|
|
/// put this rc into the cache if it is not there or is newer than the one there already
|
|
|
|
|
/// returns true if the rc was inserted
|
|
|
|
|
bool
|
|
|
|
|
put_rc_if_newer(
|
|
|
|
|
RemoteRC rc,
|
|
|
|
|
rc_time now =
|
|
|
|
|
std::chrono::time_point_cast<std::chrono::seconds>(std::chrono::system_clock::now()));
|
|
|
|
|
put_rc_if_newer(RemoteRC rc);
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
verify_gossip_bfetch_rc(const RemoteRC& rc);
|
|
|
|
|
|
|
|
|
|
bool
|
|
|
|
|
verify_store_gossip_rc(const RemoteRC& rc);
|
|
|
|
|
};
|
|
|
|
|
} // namespace llarp
|
|
|
|
|
|
|
|
|
|
namespace std
|
|
|
|
|
{
|
|
|
|
|
template <>
|
|
|
|
|
struct hash<llarp::Unconfirmed<llarp::RemoteRC>> : public hash<llarp::RemoteRC>
|
|
|
|
|
{};
|
|
|
|
|
|
|
|
|
|
template <>
|
|
|
|
|
struct hash<llarp::Unconfirmed<llarp::RouterID>> : hash<llarp::RouterID>
|
|
|
|
|
{};
|
|
|
|
|
} // namespace std
|
|
|
|
|
|
dr7ana
3 months ago
committed by
GitHub