From 5fa3c2be87ac4978a82087c291ee3d2cafd6668a Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 28 Nov 2023 04:55:01 -0800 Subject: [PATCH 01/93] housekeeping and groundwork, initial commit --- llarp/link/link_manager.cpp | 31 ++++--- llarp/link/link_manager.hpp | 2 +- llarp/messages/dht.hpp | 1 + llarp/messages/rc.hpp | 3 +- llarp/nodedb.cpp | 147 ++++++++++++++++++++++++-------- llarp/nodedb.hpp | 68 +++++++++------ llarp/router/router.cpp | 90 ++++++++++++------- llarp/router/router.hpp | 26 ++---- llarp/router_contact.cpp | 1 - llarp/router_contact.hpp | 6 +- llarp/router_contact_remote.cpp | 5 +- llarp/util/time.cpp | 6 ++ llarp/util/time.hpp | 5 ++ 13 files changed, 259 insertions(+), 132 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 77f699740..926c1ceac 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -461,10 +461,9 @@ namespace llarp auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; std::vector rcs; + while (not btlc.is_finished()) { - // TODO: maybe make RemoteRC constructor throw a bespoke exception type - // and catch it below so we know what about parsing failed? rcs.emplace_back(btlc.consume_dict_consumer()); } @@ -486,32 +485,35 @@ namespace llarp assert(_router.is_service_node()); const auto& rcs = node_db->get_rcs(); - const auto now = - std::chrono::time_point_cast(std::chrono::system_clock::now()); + const auto now = time_point_now(); + try { oxenc::bt_dict_consumer btdc{m.body()}; btdc.required("explicit_ids"); auto explicit_ids = btdc.consume_list>(); + auto since_time = rc_time{std::chrono::seconds{btdc.require("since")}}; if (explicit_ids.size() > (rcs.size() / 4)) { log::info( link_cat, "Remote requested too many relay IDs (greater than 1/4 of what we have)."); - m.respond(serialize_response({{messages::STATUS_KEY, RCFetchMessage::INVALID_REQUEST}})); + m.respond(RCFetchMessage::INVALID_REQUEST, true); return; } std::unordered_set explicit_relays; + for (auto& sv : explicit_ids) { if (sv.size() != RouterID::SIZE) { - m.respond(serialize_response({{messages::STATUS_KEY, RCFetchMessage::INVALID_REQUEST}})); + m.respond(RCFetchMessage::INVALID_REQUEST, true); return; } + explicit_relays.emplace(reinterpret_cast(sv.data())); } @@ -535,17 +537,17 @@ namespace llarp resp.append("time", now.time_since_epoch().count()); - m.respond(std::move(resp).str(), false); + m.respond(std::move(resp).str()); } catch (const std::exception& e) { log::info(link_cat, "Exception handling RC Fetch request: {}", e.what()); - m.respond(messages::ERROR_RESPONSE); + m.respond(messages::ERROR_RESPONSE, true); } } void - LinkManager::fetch_router_ids(const RouterID& source) + LinkManager::fetch_router_ids(const RouterID& source, std::function func) { if (ep.conns.empty()) { @@ -559,6 +561,7 @@ namespace llarp edge, "fetch_router_ids"s, RouterIDFetch::serialize(source), + (func) ? std::move(func) : [this, source = source, edge = std::move(edge)](oxen::quic::message m) { if (not m) { @@ -567,14 +570,17 @@ namespace llarp "Error fetching RouterIDs from source \"{}\" via edge \"{}\"", source, edge); - node_db->ingest_router_ids(edge, {}); // empty response == failure + node_db->ingest_router_ids(edge); // empty response == failure return; } + try { oxenc::bt_dict_consumer btdc{m.body()}; + btdc.required("routers"); auto router_id_strings = btdc.consume_list>(); + btdc.require_signature("signature", [&edge](ustring_view msg, ustring_view sig) { if (sig.size() != 64) throw std::runtime_error{"Invalid signature: not 64 bytes"}; @@ -582,7 +588,9 @@ namespace llarp throw std::runtime_error{ "Failed to verify signature for fetch RouterIDs response."}; }); + std::vector router_ids; + for (const auto& s : router_id_strings) { if (s.size() != RouterID::SIZE) @@ -592,6 +600,7 @@ namespace llarp } router_ids.emplace_back(s.data()); } + node_db->ingest_router_ids(edge, std::move(router_ids)); return; } @@ -599,7 +608,7 @@ namespace llarp { log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); } - node_db->ingest_router_ids(edge, {}); // empty response == failure + node_db->ingest_router_ids(edge); // empty response == failure }); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 899b25af4..cfa4b3c6d 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -233,7 +233,7 @@ namespace llarp handle_fetch_rcs(oxen::quic::message m); void - fetch_router_ids(const RouterID& source); + fetch_router_ids(const RouterID& source, std::function func = nullptr); void handle_fetch_router_ids(oxen::quic::message m); diff --git a/llarp/messages/dht.hpp b/llarp/messages/dht.hpp index 2a0e38805..2da262f3a 100644 --- a/llarp/messages/dht.hpp +++ b/llarp/messages/dht.hpp @@ -34,6 +34,7 @@ namespace llarp { inline auto NOT_FOUND = "NOT FOUND"sv; + // NOT USED inline static std::string serialize(dht::Key_t name_hash) { diff --git a/llarp/messages/rc.hpp b/llarp/messages/rc.hpp index d9eb2ab82..633b540d2 100644 --- a/llarp/messages/rc.hpp +++ b/llarp/messages/rc.hpp @@ -4,7 +4,8 @@ namespace llarp::RCFetchMessage { - inline constexpr auto INVALID_REQUEST = "Invalid relay ID requested."sv; + inline const auto INVALID_REQUEST = + messages::serialize_response({{messages::STATUS_KEY, "Invalid relay ID requested"}}); inline static std::string serialize(std::chrono::system_clock::time_point since, const std::vector& explicit_ids) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 7861b0041..15159f7d6 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -48,24 +48,24 @@ namespace llarp constexpr auto FlushInterval = 5min; NodeDB::NodeDB(fs::path root, std::function)> diskCaller, Router* r) - : router{*r} - , m_Root{std::move(root)} - , disk(std::move(diskCaller)) - , m_NextFlushAt{time_now_ms() + FlushInterval} + : _router{*r} + , _root{std::move(root)} + , _disk(std::move(diskCaller)) + , _next_flush_time{time_now_ms() + FlushInterval} { - EnsureSkiplist(m_Root); + EnsureSkiplist(_root); } void NodeDB::Tick(llarp_time_t now) { - if (m_NextFlushAt == 0s) + if (_next_flush_time == 0s) return; - if (now > m_NextFlushAt) + if (now > _next_flush_time) { - router.loop()->call([this]() { - m_NextFlushAt += FlushInterval; + _router.loop()->call([this]() { + _next_flush_time += FlushInterval; // make copy of all rcs std::vector copy; @@ -74,7 +74,7 @@ namespace llarp // flush them to disk in one big job // TODO: split this up? idk maybe some day... - disk([this, data = std::move(copy)]() { + _disk([this, data = std::move(copy)]() { for (const auto& rc : data) rc.write(get_path_by_pubkey(rc.router_id())); }); @@ -93,13 +93,13 @@ namespace llarp skiplistDir += hexString[0]; fname += RC_FILE_EXT; - return m_Root / skiplistDir / fname; + return _root / skiplistDir / fname; } bool NodeDB::want_rc(const RouterID& rid) const { - if (not router.is_service_node()) + if (not _router.is_service_node()) return true; return registered_routers.count(rid); } @@ -114,13 +114,31 @@ namespace llarp } } + bool + NodeDB::rotate_startup_rc_source() + { + if (client_known_routers.size() < 13) + { + // do something here + return false; + } + + RouterID temp = rc_fetch_source; + + while (temp == rc_fetch_source) + std::sample(client_known_routers.begin(), client_known_routers.end(), &temp, 1, csrng); + + rc_fetch_source = std::move(temp); + return true; + } + /// Called in normal operation when the relay we fetched RCs from gives either a "bad" /// response or a timeout. Attempts to switch to a new relay as our RC source, using /// existing connections if possible, and respecting pinned edges. void NodeDB::rotate_rc_source() { - auto conn_count = router.link_manager().get_num_connected(); + auto conn_count = _router.link_manager().get_num_connected(); // This function makes no sense to be called if we have no connections... if (conn_count == 0) @@ -131,7 +149,7 @@ namespace llarp throw std::runtime_error{"Cannot rotate RC source without RC source(s) to rotate to!"}; RemoteRC new_source{}; - router.link_manager().get_random_connected(new_source); + _router.link_manager().get_random_connected(new_source); if (conn_count == 1) { // if we only have one connection, it must be current rc fetch source @@ -149,8 +167,8 @@ namespace llarp } // only one connection, choose a new relay to connect to for rc fetching - RouterID r = rc_fetch_source; + while (r == rc_fetch_source) { std::sample(client_known_routers.begin(), client_known_routers.end(), &r, 1, csrng); @@ -162,7 +180,7 @@ namespace llarp // choose one of our other existing connections to use as the RC fetch source while (new_source.router_id() == rc_fetch_source) { - router.link_manager().get_random_connected(new_source); + _router.link_manager().get_random_connected(new_source); } rc_fetch_source = new_source.router_id(); } @@ -209,20 +227,72 @@ namespace llarp } } + void + NodeDB::fetch_initial() + { + int num_failures = 0; + + [[maybe_unused]] + bool use_bootstrap = false; + + RouterID fetch_src; + + // NodeDB::load_from_disk is called in Router::Run before any calls to Router::Tick are + // made, which trigger this function call. As a result, client_known_routers should be + // populated (if there was anything to populate it with) + auto num_known = client_known_routers.size(); + + if (num_known >= MIN_ACTIVE_RIDS) + { + std::sample(client_known_routers.begin(), client_known_routers.end(), &fetch_src, 1, csrng); + } + else + { + // DEFAULT TO BOOTSTRAP ROUTERS HERE + use_bootstrap = true; + log::debug( + logcat, + "Insufficient known active RID's to fetch ({}/{}); defaulting to bootstrap", + num_known, + MIN_ACTIVE_RIDS); + + assert(not bootstraps.empty()); + fetch_src = bootstraps.begin()->first; + } + + while (num_failures < MAX_FETCH_ATTEMPTS) + { + + + + } + + router_id_fetch_in_progress = true; + router_id_response_count = 0; + router_id_fetch_responses.clear(); + } + + bool + NodeDB::fetch_initial_rcs(const RouterID& src) + { + (void)src; + return true; + } + void NodeDB::fetch_rcs() { std::vector needed; - const auto now = - std::chrono::time_point_cast(std::chrono::system_clock::now()); + const auto now = time_point_now(); + for (const auto& [rid, rc] : known_rcs) { if (now - rc.timestamp() > RouterContact::OUTDATED_AGE) needed.push_back(rid); } - router.link_manager().fetch_rcs( + _router.link_manager().fetch_rcs( rc_fetch_source, last_rc_update_relay_timestamp, std::move(needed)); } @@ -231,6 +301,7 @@ namespace llarp { if (router_id_fetch_in_progress) return; + if (router_id_fetch_sources.empty()) select_router_id_sources(); @@ -244,8 +315,9 @@ namespace llarp router_id_fetch_in_progress = true; router_id_response_count = 0; router_id_fetch_responses.clear(); + for (const auto& rid : router_id_fetch_sources) - router.link_manager().fetch_router_ids(rid); + _router.link_manager().fetch_router_ids(rid); } void @@ -326,7 +398,7 @@ namespace llarp return false; } - if (not router.is_service_node()) + if (not _router.is_service_node()) return true; return router_whitelist.count(remote) or router_greylist.count(remote); @@ -343,7 +415,7 @@ namespace llarp void NodeDB::load_from_disk() { - if (m_Root.empty()) + if (_root.empty()) return; std::set purge; @@ -356,7 +428,7 @@ namespace llarp continue; std::string p; p += ch; - fs::path sub = m_Root / p; + fs::path sub = _root / p; llarp::util::IterDir(sub, [&](const fs::path& f) -> bool { // skip files that are not suffixed with .signed @@ -379,11 +451,13 @@ namespace llarp return true; } - known_rcs.emplace(rc.router_id(), rc); + const auto& rid = rc.router_id(); + + known_rcs.emplace(rid, rc); // TODO: the list of relays should be maintained and stored separately from // the RCs, as we keep older RCs around in case we go offline and need to // bootstrap, but they shouldn't be in the "good relays" list. - client_known_routers.insert(rc.router_id()); + client_known_routers.insert(rid); return true; }); @@ -401,10 +475,10 @@ namespace llarp void NodeDB::save_to_disk() const { - if (m_Root.empty()) + if (_root.empty()) return; - router.loop()->call([this]() { + _router.loop()->call([this]() { for (const auto& item : known_rcs) item.second.write(get_path_by_pubkey(item.first)); }); @@ -430,7 +504,7 @@ namespace llarp void NodeDB::remove_router(RouterID pk) { - router.loop()->call([this, pk]() { + _router.loop()->call([this, pk]() { known_rcs.erase(pk); remove_many_from_disk_async({pk}); }); @@ -439,9 +513,10 @@ namespace llarp void NodeDB::remove_stale_rcs() { - auto cutoff_time = - std::chrono::time_point_cast(std::chrono::system_clock::now()); - cutoff_time -= router.is_service_node() ? RouterContact::OUTDATED_AGE : RouterContact::LIFETIME; + auto cutoff_time = time_point_now(); + + cutoff_time -= + _router.is_service_node() ? RouterContact::OUTDATED_AGE : RouterContact::LIFETIME; for (auto itr = known_rcs.begin(); itr != known_rcs.end();) { if (cutoff_time > itr->second.timestamp()) @@ -469,7 +544,7 @@ namespace llarp size_t NodeDB::num_loaded() const { - return router.loop()->call_get([this]() { return known_rcs.size(); }); + return _router.loop()->call_get([this]() { return known_rcs.size(); }); } bool @@ -486,7 +561,7 @@ namespace llarp void NodeDB::remove_many_from_disk_async(std::unordered_set remove) const { - if (m_Root.empty()) + if (_root.empty()) return; // build file list std::set files; @@ -495,7 +570,7 @@ namespace llarp files.emplace(get_path_by_pubkey(std::move(id))); } // remove them from the disk via the diskio thread - disk([files]() { + _disk([files]() { for (auto fpath : files) fs::remove(fpath); }); @@ -504,7 +579,7 @@ namespace llarp RemoteRC NodeDB::find_closest_to(llarp::dht::Key_t location) const { - return router.loop()->call_get([this, location]() -> RemoteRC { + return _router.loop()->call_get([this, location]() -> RemoteRC { RemoteRC rc; const llarp::dht::XorMetric compare(location); @@ -524,7 +599,7 @@ namespace llarp std::vector NodeDB::find_many_closest_to(llarp::dht::Key_t location, uint32_t numRouters) const { - return router.loop()->call_get([this, location, numRouters]() -> std::vector { + return _router.loop()->call_get([this, location, numRouters]() -> std::vector { std::vector all; all.reserve(known_rcs.size()); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index f4c3b629d..fc396f4ed 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -22,15 +22,18 @@ namespace llarp { struct Router; + inline constexpr size_t MIN_ACTIVE_RIDS{24}; + inline constexpr int MAX_FETCH_ATTEMPTS{10}; + class NodeDB { std::unordered_map known_rcs; - Router& router; - const fs::path m_Root; - const std::function)> disk; + Router& _router; + const fs::path _root; + const std::function)> _disk; - llarp_time_t m_NextFlushAt; + llarp_time_t _next_flush_time; /// asynchronously remove the files for a set of rcs on disk given their public ident key void @@ -53,7 +56,7 @@ namespace llarp std::unordered_set registered_routers; std::unordered_map last_rc_update_times; - // Router list for clients + // Client list of active RouterID's std::unordered_set client_known_routers; // only ever use to specific edges as path first-hops @@ -61,9 +64,13 @@ namespace llarp // rc update info RouterID rc_fetch_source; + rc_time last_rc_update_relay_timestamp; + static constexpr auto ROUTER_ID_SOURCE_COUNT = 12; + std::unordered_set router_id_fetch_sources; + std::unordered_map> router_id_fetch_responses; // process responses once all are received (or failed/timed out) size_t router_id_response_count{0}; @@ -106,22 +113,39 @@ namespace llarp return last_rc_update_times; } - // If we receive a bad set of RCs from our current RC source relay, we consider - // that relay to be a bad source of RCs and we randomly choose a new one. - // - // When using a new RC fetch relay, we first re-fetch the full RC list and, if - // that aligns with our RouterID list, we go back to periodic updates from that relay. - // - // This will respect edge-pinning and attempt to use a relay we already have - // a connection with. + /// If we receive a bad set of RCs from our current RC source relay, we consider + /// that relay to be a bad source of RCs and we randomly choose a new one. + /// + /// When using a new RC fetch relay, we first re-fetch the full RC list and, if + /// that aligns with our RouterID list, we go back to periodic updates from that relay. + /// + /// This will respect edge-pinning and attempt to use a relay we already have + /// a connection with. void rotate_rc_source(); + /// This function is called during startup and initial fetching. When a lokinet client + /// instance performs its initial RC/RID fetching, it may need to randomly select a + /// node from its list of stale RC's to relay its requests. If there is a failure in + /// mediating these request, the client will randomly select another RC source + /// + /// Returns: + /// true - a new startup RC source was selected + /// false - a new startup RC source was NOT selected + bool + rotate_startup_rc_source(); + void ingest_rcs(RouterID source, std::vector rcs, rc_time timestamp); void - ingest_router_ids(RouterID source, std::vector ids); + ingest_router_ids(RouterID source, std::vector ids = {}); + + void + fetch_initial(); + + bool + fetch_initial_rcs(const RouterID& src); void fetch_rcs(); @@ -216,7 +240,7 @@ namespace llarp std::optional GetRandom(Filter visit) const { - return router.loop()->call_get([visit]() -> std::optional { + return _router.loop()->call_get([visit]() -> std::optional { std::vector known_rcs; for (const auto& entry : known_rcs) known_rcs.push_back(entry); @@ -238,7 +262,7 @@ namespace llarp void VisitAll(Visit visit) const { - router.loop()->call([this, visit]() { + _router.loop()->call([this, visit]() { for (const auto& item : known_rcs) visit(item.second); }); @@ -253,7 +277,7 @@ namespace llarp void RemoveIf(Filter visit) { - router.loop()->call([this, visit]() { + _router.loop()->call([this, visit]() { std::unordered_set removed; auto itr = known_rcs.begin(); while (itr != known_rcs.end()) @@ -280,18 +304,12 @@ namespace llarp /// put (or replace) the RC if we consider it valid (want_rc). returns true if put. bool - put_rc( - RemoteRC rc, - rc_time now = - std::chrono::time_point_cast(std::chrono::system_clock::now())); + put_rc(RemoteRC rc, rc_time now = time_point_now()); /// if we consider it valid (want_rc), /// put this rc into the cache if it is not there or is newer than the one there already /// returns true if the rc was inserted bool - put_rc_if_newer( - RemoteRC rc, - rc_time now = - std::chrono::time_point_cast(std::chrono::system_clock::now())); + put_rc_if_newer(RemoteRC rc, rc_time now = time_point_now()); }; } // namespace llarp diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 7a23a3748..da98cf31b 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -352,12 +352,14 @@ namespace llarp // Backwards compat: before 0.9.10 we used `type=file` with `file=|-|stdout` for print mode auto log_type = conf.logging.type; + if (log_type == log::Type::File && (conf.logging.file == "stdout" || conf.logging.file == "-" || conf.logging.file.empty())) log_type = log::Type::Print; if (log::get_level_default() != log::Level::off) log::reset_level(conf.logging.level); + log::clear_sinks(); log::add_sink(log_type, log_type == log::Type::System ? "lokinet" : conf.logging.file); @@ -524,6 +526,7 @@ namespace llarp { // Set netid before anything else log::debug(logcat, "Network ID set to {}", conf.router.net_id); + if (!conf.router.net_id.empty() && strcmp(conf.router.net_id.c_str(), llarp::LOKINET_DEFAULT_NETID) != 0) { @@ -534,8 +537,8 @@ namespace llarp "' which does not equal '", llarp::LOKINET_DEFAULT_NETID, "' you will run as a different network, good luck " - "and don't forget: something something MUH traffic " - "shape correlation !!!!"); + "and don't forget: something something traffic shape " + "correlation!!"); } // Router config @@ -575,16 +578,20 @@ namespace llarp if (not networkConfig.strict_connect.empty()) { const auto& val = networkConfig.strict_connect; + if (is_service_node()) throw std::runtime_error("cannot use strict-connect option as service node"); + if (val.size() < 2) throw std::runtime_error( "Must specify more than one strict-connect router if using strict-connect"); + strictConnectPubkeys.insert(val.begin(), val.end()); log::debug(logcat, "{} strict-connect routers configured", val.size()); } std::vector configRouters = conf.connect.routers; + configRouters.insert( configRouters.end(), conf.bootstrap.files.begin(), conf.bootstrap.files.end()); @@ -595,9 +602,7 @@ namespace llarp if (configRouters.empty() and conf.bootstrap.routers.empty()) { if (fs::exists(defaultBootstrapFile)) - { configRouters.push_back(defaultBootstrapFile); - } } bootstrap_rc_list.clear(); @@ -624,6 +629,7 @@ namespace llarp "No bootstrap routers were loaded. The default bootstrap file {} does not exist, and " "loading fallback bootstrap RCs failed.", defaultBootstrapFile); + throw std::runtime_error("No bootstrap nodes available."); } } @@ -656,11 +662,11 @@ namespace llarp // Init components after relevant config settings loaded _link_manager.init(); - // FIXME: kludge for now, will be part of larger cleanup effort. + // TODO: RC refactor here if (_is_service_node) - InitInboundLinks(); + init_inbounds(); else - InitOutboundLinks(); + init_outbounds(); // profiling _profile_file = conf.router.data_dir / "profiles.dat"; @@ -819,36 +825,51 @@ namespace llarp // (relay-only) if we have fetched the relay list from oxend and // we are registered and funded, we want to gossip our RC periodically auto now_timepoint = std::chrono::system_clock::time_point(now); - if (is_snode and appears_funded() and (now_timepoint > next_rc_gossip)) + + if (is_snode) { - log::info(logcat, "regenerating and gossiping RC"); - router_contact.resign(); - save_rc(); - auto view = router_contact.view(); - _link_manager.gossip_rc( - pubkey(), std::string{reinterpret_cast(view.data()), view.size()}); - last_rc_gossip = now_timepoint; - - // 1min to 5min before "stale time" is next gossip time - auto random_delta = - std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; - next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; - } + if (appears_funded() and now_timepoint > next_rc_gossip) + { + log::info(logcat, "regenerating and gossiping RC"); + + router_contact.resign(); + save_rc(); + + auto view = router_contact.view(); - if (not is_snode) + _link_manager.gossip_rc( + pubkey(), std::string{reinterpret_cast(view.data()), view.size()}); + + last_rc_gossip = now_timepoint; + + // 1min to 5min before "stale time" is next gossip time + auto random_delta = + std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; + + next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; + } + } + else { - // (client-only) periodically fetch updated RCs - if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) + if (needs_initial_fetch) { - node_db()->fetch_rcs(); - last_rc_fetch = now_timepoint; + node_db()->fetch_initial(); } - - // (client-only) periodically fetch updated RouterID list - if (now_timepoint - last_routerid_fetch > ROUTERID_UPDATE_INTERVAL) + else { - node_db()->fetch_router_ids(); - last_routerid_fetch = now_timepoint; + // (client-only) periodically fetch updated RCs + if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) + { + node_db()->fetch_rcs(); + last_rc_fetch = now_timepoint; + } + + // (client-only) periodically fetch updated RouterID list + if (now_timepoint - last_routerid_fetch > ROUTERID_UPDATE_INTERVAL) + { + node_db()->fetch_router_ids(); + last_routerid_fetch = now_timepoint; + } } } @@ -1074,8 +1095,11 @@ namespace llarp log::info(logcat, "Router populated NodeDB with {} routers", _node_db->num_loaded()); _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); + _route_poker->start(); + is_running.store(true); + _started_at = now(); if (is_service_node()) @@ -1321,7 +1345,7 @@ namespace llarp } void - Router::InitInboundLinks() + Router::init_inbounds() { // auto addrs = _config->links.InboundListenAddrs; // if (is_service_node and addrs.empty()) @@ -1370,7 +1394,7 @@ namespace llarp } void - Router::InitOutboundLinks() + Router::init_outbounds() { // auto addrs = config()->links.OutboundLinks; // if (addrs.empty()) diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index cdd6da7ab..90a8a24b4 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -37,15 +37,6 @@ #include #include -/* - TONUKE: - - hidden_service_context - - TODO: - - router should hold DHT nodes container? in either a class or a map - - -*/ - namespace llarp { /// number of routers to publish to @@ -127,15 +118,14 @@ namespace llarp Profiling _router_profiling; fs::path _profile_file; LinkManager _link_manager{*this}; - std::chrono::system_clock::time_point last_rc_gossip{ - std::chrono::system_clock::time_point::min()}; - std::chrono::system_clock::time_point next_rc_gossip{ - std::chrono::system_clock::time_point::min()}; - std::chrono::system_clock::time_point last_rc_fetch{ - std::chrono::system_clock::time_point::min()}; - std::chrono::system_clock::time_point last_routerid_fetch{ + bool needs_initial_fetch{true}; + + std::chrono::system_clock::time_point last_rc_gossip{ std::chrono::system_clock::time_point::min()}; + std::chrono::system_clock::time_point next_rc_gossip{last_rc_gossip}; + std::chrono::system_clock::time_point last_rc_fetch{last_rc_gossip}; + std::chrono::system_clock::time_point last_routerid_fetch{last_rc_gossip}; // should we be sending padded messages every interval? bool send_padding = false; @@ -369,10 +359,10 @@ namespace llarp status_line(); void - InitInboundLinks(); + init_inbounds(); void - InitOutboundLinks(); + init_outbounds(); std::optional GetRandomGoodRouter(); diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index ab13328ca..0cd402802 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -6,7 +6,6 @@ #include "util/bencode.hpp" #include "util/buffer.hpp" #include "util/file.hpp" -#include "util/time.hpp" #include diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index b65aa2c76..c7a04cd89 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -21,8 +22,6 @@ namespace llarp { static auto logcat = log::Cat("RC"); - using rc_time = std::chrono::time_point; - static inline constexpr size_t NETID_SIZE{8}; /// On the wire we encode the data as a dict containing: @@ -298,8 +297,7 @@ namespace llarp void set_systime_timestamp() { - set_timestamp( - std::chrono::time_point_cast(std::chrono::system_clock::now())); + set_timestamp(time_point_now()); } }; diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 0d7eeb05b..8e8188052 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -40,8 +40,9 @@ namespace llarp } catch (const std::exception& e) { - log::warning(logcat, "Failed to parse RemoteRC: {}", e.what()); - throw; + auto err = "Exception caught parsing RemoteRC: {}"_format(e.what()); + log::warning(logcat, err); + throw std::runtime_error{err}; } } diff --git a/llarp/util/time.cpp b/llarp/util/time.cpp index af36a55dd..39fc6e268 100644 --- a/llarp/util/time.cpp +++ b/llarp/util/time.cpp @@ -34,6 +34,12 @@ namespace llarp std::chrono::steady_clock::now() - started_at_steady); } + rc_time + time_point_now() + { + return std::chrono::time_point_cast(std::chrono::system_clock::now()); + } + Duration_t time_now_ms() { diff --git a/llarp/util/time.hpp b/llarp/util/time.hpp index 2a312090d..bba3cf636 100644 --- a/llarp/util/time.hpp +++ b/llarp/util/time.hpp @@ -12,6 +12,11 @@ using namespace std::chrono_literals; namespace llarp { + using rc_time = std::chrono::time_point; + + rc_time + time_point_now(); + /// get time right now as milliseconds, this is monotonic Duration_t time_now_ms(); From 65596178168675833f675e0d3c5498877000d6e9 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 28 Nov 2023 10:14:19 -0800 Subject: [PATCH 02/93] RC/RID fetching logic implemented --- llarp/link/link_manager.cpp | 102 +----------- llarp/link/link_manager.hpp | 8 +- llarp/nodedb.cpp | 311 ++++++++++++++++++++++++++---------- llarp/nodedb.hpp | 24 +-- llarp/router/router.cpp | 11 +- 5 files changed, 258 insertions(+), 198 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 926c1ceac..dda9a666f 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -434,48 +434,9 @@ namespace llarp void LinkManager::fetch_rcs( - const RouterID& source, rc_time since, const std::vector& explicit_ids) + const RouterID& source, std::string payload, std::function func) { - send_control_message( - source, - "fetch_rcs", - RCFetchMessage::serialize(since, explicit_ids), - [this, source = source](oxen::quic::message m) { - if (m.timed_out) - { - // TODO: keep track of this failure for relay quality metrics? - log::info(link_cat, "RC Fetch to {} timed out", source); - return; - } - try - { - oxenc::bt_dict_consumer btdc{m.body()}; - if (not m) - { - auto reason = btdc.require(messages::STATUS_KEY); - log::info(link_cat, "RC Fetch to {} returned error: {}", source, reason); - return; - } - - auto btlc = btdc.require("rcs"sv); - auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; - - std::vector rcs; - - while (not btlc.is_finished()) - { - rcs.emplace_back(btlc.consume_dict_consumer()); - } - - node_db->ingest_rcs(source, std::move(rcs), timestamp); - } - catch (const std::exception& e) - { - // TODO: Inform NodeDB of failure (perhaps just a call to rotate_rc_source()) - log::info(link_cat, "Failed to parse RC Fetch response from {}: {}", source, e.what()); - return; - } - }); + send_control_message(source, "fetch_rcs", std::move(payload), std::move(func)); } void @@ -547,69 +508,16 @@ namespace llarp } void - LinkManager::fetch_router_ids(const RouterID& source, std::function func) + LinkManager::fetch_router_ids( + const RouterID& via, std::string payload, std::function func) { if (ep.conns.empty()) { log::debug(link_cat, "Not attempting to fetch Router IDs: not connected to any relays."); return; } - // TODO: randomize? Also, keep track of successful responses and drop this edge - // if not many come back successfully. - RouterID edge = ep.conns.begin()->first; - send_control_message( - edge, - "fetch_router_ids"s, - RouterIDFetch::serialize(source), - (func) ? std::move(func) : - [this, source = source, edge = std::move(edge)](oxen::quic::message m) { - if (not m) - { - log::info( - link_cat, - "Error fetching RouterIDs from source \"{}\" via edge \"{}\"", - source, - edge); - node_db->ingest_router_ids(edge); // empty response == failure - return; - } - - try - { - oxenc::bt_dict_consumer btdc{m.body()}; - - btdc.required("routers"); - auto router_id_strings = btdc.consume_list>(); - - btdc.require_signature("signature", [&edge](ustring_view msg, ustring_view sig) { - if (sig.size() != 64) - throw std::runtime_error{"Invalid signature: not 64 bytes"}; - if (not crypto::verify(edge, msg, sig)) - throw std::runtime_error{ - "Failed to verify signature for fetch RouterIDs response."}; - }); - - std::vector router_ids; - for (const auto& s : router_id_strings) - { - if (s.size() != RouterID::SIZE) - { - log::warning(link_cat, "Got bad RouterID from edge \"{}\".", edge); - return; - } - router_ids.emplace_back(s.data()); - } - - node_db->ingest_router_ids(edge, std::move(router_ids)); - return; - } - catch (const std::exception& e) - { - log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); - } - node_db->ingest_router_ids(edge); // empty response == failure - }); + send_control_message(via, "fetch_router_ids"s, std::move(payload), std::move(func)); } void diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index cfa4b3c6d..e51f892dd 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -227,13 +227,17 @@ namespace llarp handle_gossip_rc(oxen::quic::message m); void - fetch_rcs(const RouterID& source, rc_time since, const std::vector& explicit_ids); + fetch_rcs( + const RouterID& source, + std::string payload, + std::function func); void handle_fetch_rcs(oxen::quic::message m); void - fetch_router_ids(const RouterID& source, std::function func = nullptr); + fetch_router_ids( + const RouterID& via, std::string payload, std::function func); void handle_fetch_router_ids(oxen::quic::message m); diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 15159f7d6..31834ed50 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -2,6 +2,8 @@ #include "crypto/types.hpp" #include "dht/kademlia.hpp" +#include "messages/rc.hpp" +#include "messages/router_id.hpp" #include "router_contact.hpp" #include "util/time.hpp" @@ -117,7 +119,7 @@ namespace llarp bool NodeDB::rotate_startup_rc_source() { - if (client_known_routers.size() < 13) + if (active_client_routers.size() < 13) { // do something here return false; @@ -126,7 +128,7 @@ namespace llarp RouterID temp = rc_fetch_source; while (temp == rc_fetch_source) - std::sample(client_known_routers.begin(), client_known_routers.end(), &temp, 1, csrng); + std::sample(active_client_routers.begin(), active_client_routers.end(), &temp, 1, csrng); rc_fetch_source = std::move(temp); return true; @@ -145,11 +147,12 @@ namespace llarp throw std::runtime_error{"Called rotate_rc_source with no connections, does not make sense!"}; // We should not be in this function if client_known_routers isn't populated - if (client_known_routers.size() <= 1) + if (active_client_routers.size() <= 1) throw std::runtime_error{"Cannot rotate RC source without RC source(s) to rotate to!"}; RemoteRC new_source{}; _router.link_manager().get_random_connected(new_source); + if (conn_count == 1) { // if we only have one connection, it must be current rc fetch source @@ -171,7 +174,7 @@ namespace llarp while (r == rc_fetch_source) { - std::sample(client_known_routers.begin(), client_known_routers.end(), &r, 1, csrng); + std::sample(active_client_routers.begin(), active_client_routers.end(), &r, 1, csrng); } rc_fetch_source = std::move(r); return; @@ -187,7 +190,7 @@ namespace llarp // TODO: trust model void - NodeDB::ingest_rcs(RouterID source, std::vector rcs, rc_time timestamp) + NodeDB::store_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp) { (void)source; @@ -205,85 +208,35 @@ namespace llarp last_rc_update_relay_timestamp = timestamp; } - // TODO: trust model void - NodeDB::ingest_router_ids(RouterID source, std::vector ids) + NodeDB::ingest_rid_fetch_responses(const RemoteRC& source, std::vector ids) { - router_id_fetch_responses[source] = std::move(ids); + const auto& rid = source.router_id(); - router_id_response_count++; - if (router_id_response_count == router_id_fetch_sources.size()) - { - // TODO: reconcile all the responses, for now just insert all - for (const auto& [rid, responses] : router_id_fetch_responses) - { - // TODO: empty == failure, handle that case - for (const auto& response : responses) - { - client_known_routers.insert(std::move(response)); - } - } - router_id_fetch_in_progress = false; - } + router_id_fetch_responses[rid] = std::move(ids); } - void - NodeDB::fetch_initial() + bool + NodeDB::process_fetched_rids() { - int num_failures = 0; - - [[maybe_unused]] - bool use_bootstrap = false; - - RouterID fetch_src; - - // NodeDB::load_from_disk is called in Router::Run before any calls to Router::Tick are - // made, which trigger this function call. As a result, client_known_routers should be - // populated (if there was anything to populate it with) - auto num_known = client_known_routers.size(); - - if (num_known >= MIN_ACTIVE_RIDS) + for (const auto& [rid, responses] : router_id_fetch_responses) { - std::sample(client_known_routers.begin(), client_known_routers.end(), &fetch_src, 1, csrng); - } - else - { - // DEFAULT TO BOOTSTRAP ROUTERS HERE - use_bootstrap = true; - log::debug( - logcat, - "Insufficient known active RID's to fetch ({}/{}); defaulting to bootstrap", - num_known, - MIN_ACTIVE_RIDS); - - assert(not bootstraps.empty()); - fetch_src = bootstraps.begin()->first; - } - - while (num_failures < MAX_FETCH_ATTEMPTS) - { - - - + // TODO: empty == failure, handle that case + for (const auto& response : responses) + { + active_client_routers.insert(std::move(response)); + } } + router_id_fetch_in_progress = false; - router_id_fetch_in_progress = true; - router_id_response_count = 0; - router_id_fetch_responses.clear(); - } - - bool - NodeDB::fetch_initial_rcs(const RouterID& src) - { - (void)src; return true; } void - NodeDB::fetch_rcs() + NodeDB::fetch_rcs(int n_fails, bool initial) { + int num_failures = n_fails; std::vector needed; - const auto now = time_point_now(); for (const auto& [rid, rc] : known_rcs) @@ -292,15 +245,84 @@ namespace llarp needed.push_back(rid); } - _router.link_manager().fetch_rcs( - rc_fetch_source, last_rc_update_relay_timestamp, std::move(needed)); + RouterID src = (initial) + ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) + : rc_fetch_source; + + while (num_failures < MAX_FETCH_ATTEMPTS) + { + auto success = std::make_shared>(); + auto f = success->get_future(); + + _router.link_manager().fetch_rcs( + src, + RCFetchMessage::serialize(last_rc_update_relay_timestamp, needed), + [this, src, p = std::move(success)](oxen::quic::message m) mutable { + if (m.timed_out) + { + log::info(logcat, "RC fetch to {} timed out", src); + p->set_value(false); + return; + } + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + if (not m) + { + auto reason = btdc.require(messages::STATUS_KEY); + log::info(logcat, "RC fetch to {} returned error: {}", src, reason); + p->set_value(false); + return; + } + + auto btlc = btdc.require("rcs"sv); + auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; + + std::vector rcs; + + while (not btlc.is_finished()) + { + rcs.emplace_back(btlc.consume_dict_consumer()); + } + + store_fetched_rcs(src, std::move(rcs), timestamp); + p->set_value(true); + } + catch (const std::exception& e) + { + log::info(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); + p->set_value(false); + return; + } + }); + + if (f.get()) + { + log::debug(logcat, "Successfully fetched RC's from {}", src); + rc_fetch_source = src; + assert(_router.link_manager().have_connection_to(src)); + break; + } + + ++num_failures; + log::debug( + logcat, + "Unable to fetch RC's from {}; rotating RC source ({}/{} attempts)", + src, + num_failures, + MAX_FETCH_ATTEMPTS); + + src = (initial) + ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) + : std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + } } void - NodeDB::fetch_router_ids() + NodeDB::fetch_router_ids(int n_fails, bool initial) { - if (router_id_fetch_in_progress) - return; + assert(not router_id_fetch_in_progress); + int num_failures = n_fails; if (router_id_fetch_sources.empty()) select_router_id_sources(); @@ -313,19 +335,140 @@ namespace llarp } router_id_fetch_in_progress = true; - router_id_response_count = 0; router_id_fetch_responses.clear(); - for (const auto& rid : router_id_fetch_sources) - _router.link_manager().fetch_router_ids(rid); + RouterID src = (initial) + ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) + : rc_fetch_source; + + std::unordered_set fails; + + while (num_failures < MAX_FETCH_ATTEMPTS) + { + RemoteRC& src_rc = known_rcs[src]; + auto success = std::make_shared>(); + auto f = success->get_future(); + fails.clear(); + + for (const auto& target : router_id_fetch_sources) + { + _router.link_manager().fetch_router_ids( + src, + RouterIDFetch::serialize(target), + [this, src, src_rc, target, p = std::move(success)](oxen::quic::message m) mutable { + if (not m) + { + log::info(link_cat, "RID fetch from {} via {} timed out", src, target); + + ingest_rid_fetch_responses(src_rc); + p->set_value(-1); + return; + } + + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + + btdc.required("routers"); + auto router_id_strings = btdc.consume_list>(); + + btdc.require_signature("signature", [&src](ustring_view msg, ustring_view sig) { + if (sig.size() != 64) + throw std::runtime_error{"Invalid signature: not 64 bytes"}; + if (not crypto::verify(src, msg, sig)) + throw std::runtime_error{ + "Failed to verify signature for fetch RouterIDs response."}; + }); + + std::vector router_ids; + + for (const auto& s : router_id_strings) + { + if (s.size() != RouterID::SIZE) + { + log::warning( + link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); + p->set_value(0); + return; + } + + router_ids.emplace_back(s.data()); + } + + ingest_rid_fetch_responses(src_rc, std::move(router_ids)); + return; + } + catch (const std::exception& e) + { + log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); + p->set_value(0); + } + + ingest_rid_fetch_responses(src_rc); // empty response == failure + }); + + switch (f.get()) + { + case 1: + log::debug(logcat, "Successfully fetched RID's from {} via {}", target, src); + continue; + case 0: + // RC node relayed our fetch routerID request, but the request failed at the target + log::debug(logcat, "Unsuccessfully fetched RID's from {} via {}", target, src); + fails.insert(target); + continue; + default: + // RC node failed to relay our routerID request; re-select RC node and continue + log::debug(logcat, "RC source {} failed to mediate RID fetching from {}", src, target); + src = (initial) + ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) + : std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + ++num_failures; + fetch_rcs(num_failures); + continue; + } + } + + auto n_fails = fails.size(); + + if (n_fails <= MAX_RID_ERRORS) + { + log::debug( + logcat, + "RID fetching was successful ({}/{} acceptable errors)", + fails.size(), + MAX_RID_ERRORS); + rc_fetch_source = src; + assert(_router.link_manager().have_connection_to(src)); + + // this is where the trust model will do verification based on the similarity of the sets + if (process_fetched_rids()) + { + log::debug(logcat, "Accumulated RID's accepted by trust model"); + return; + } + + log::debug( + logcat, "Accumulated RID's rejected by trust model, reselecting all RID sources..."); + select_router_id_sources(router_id_fetch_sources); + ++num_failures; + continue; + } + + // we had 4 or more failed requests, so we will need to rotate our rid sources + log::debug( + logcat, "RID fetching found {} failures; reselecting failed RID sources...", n_fails); + ++num_failures; + select_router_id_sources(fails); + } } void NodeDB::select_router_id_sources(std::unordered_set excluded) { - // TODO: bootstrapping should be finished before this is called, so this - // shouldn't happen; need to make sure that's the case. - if (client_known_routers.empty()) + // bootstrapping should be finished before this is called, so this + // shouldn't happen; need to make sure that's the case. + if (active_client_routers.empty()) return; // keep using any we've been using, but remove `excluded` ones @@ -333,9 +476,9 @@ namespace llarp router_id_fetch_sources.erase(r); // only know so many routers, so no need to randomize - if (client_known_routers.size() <= (ROUTER_ID_SOURCE_COUNT + excluded.size())) + if (active_client_routers.size() <= (ROUTER_ID_SOURCE_COUNT + excluded.size())) { - for (const auto& r : client_known_routers) + for (const auto& r : active_client_routers) { if (excluded.count(r)) continue; @@ -347,7 +490,7 @@ namespace llarp while (router_id_fetch_sources.size() < ROUTER_ID_SOURCE_COUNT) { RouterID r; - std::sample(client_known_routers.begin(), client_known_routers.end(), &r, 1, csrng); + std::sample(active_client_routers.begin(), active_client_routers.end(), &r, 1, csrng); if (excluded.count(r) == 0) router_id_fetch_sources.insert(r); } @@ -457,7 +600,7 @@ namespace llarp // TODO: the list of relays should be maintained and stored separately from // the RCs, as we keep older RCs around in case we go offline and need to // bootstrap, but they shouldn't be in the "good relays" list. - client_known_routers.insert(rid); + active_client_routers.insert(rid); return true; }); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index fc396f4ed..8b6b540dd 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -22,7 +22,10 @@ namespace llarp { struct Router; + inline constexpr size_t ROUTER_ID_SOURCE_COUNT{12}; + inline constexpr size_t MIN_RID_FETCHES{8}; inline constexpr size_t MIN_ACTIVE_RIDS{24}; + inline constexpr size_t MAX_RID_ERRORS{ROUTER_ID_SOURCE_COUNT - MIN_RID_FETCHES}; inline constexpr int MAX_FETCH_ATTEMPTS{10}; class NodeDB @@ -57,23 +60,20 @@ namespace llarp std::unordered_map last_rc_update_times; // Client list of active RouterID's - std::unordered_set client_known_routers; + std::unordered_set active_client_routers; // only ever use to specific edges as path first-hops std::unordered_set pinned_edges; - // rc update info + // rc update info: we only set this upon a SUCCESSFUL fetching RouterID rc_fetch_source; rc_time last_rc_update_relay_timestamp; - static constexpr auto ROUTER_ID_SOURCE_COUNT = 12; - std::unordered_set router_id_fetch_sources; - std::unordered_map> router_id_fetch_responses; // process responses once all are received (or failed/timed out) - size_t router_id_response_count{0}; + std::unordered_map> router_id_fetch_responses; bool router_id_fetch_in_progress{false}; bool @@ -136,22 +136,22 @@ namespace llarp rotate_startup_rc_source(); void - ingest_rcs(RouterID source, std::vector rcs, rc_time timestamp); + store_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); void - ingest_router_ids(RouterID source, std::vector ids = {}); + ingest_rid_fetch_responses(const RemoteRC& source, std::vector ids = {}); - void - fetch_initial(); + bool + process_fetched_rids(); bool fetch_initial_rcs(const RouterID& src); void - fetch_rcs(); + fetch_rcs(int n_fails = 0, bool initial = false); void - fetch_router_ids(); + fetch_router_ids(int n_fails = 0, bool initial = false); void select_router_id_sources(std::unordered_set excluded = {}); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index da98cf31b..6a50d2f7d 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -851,23 +851,28 @@ namespace llarp } else { + int num_failures = 0; + if (needs_initial_fetch) { - node_db()->fetch_initial(); + node_db()->fetch_rcs(num_failures, true); + last_rc_fetch = now_timepoint; + node_db()->fetch_router_ids(num_failures, true); + last_routerid_fetch = now_timepoint; } else { // (client-only) periodically fetch updated RCs if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) { - node_db()->fetch_rcs(); + node_db()->fetch_rcs(num_failures); last_rc_fetch = now_timepoint; } // (client-only) periodically fetch updated RouterID list if (now_timepoint - last_routerid_fetch > ROUTERID_UPDATE_INTERVAL) { - node_db()->fetch_router_ids(); + node_db()->fetch_router_ids(num_failures); last_routerid_fetch = now_timepoint; } } From 3fc798069135234fb6bcd0f9bc1a6dd5a2b6cefe Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 28 Nov 2023 12:05:07 -0800 Subject: [PATCH 03/93] less synchronous for the subsequent fetches --- llarp/link/link_manager.cpp | 20 +- llarp/nodedb.cpp | 416 ++++++++++++++++++++++++++++++------ llarp/nodedb.hpp | 40 +++- llarp/router/router.cpp | 15 +- llarp/router/router.hpp | 15 +- 5 files changed, 407 insertions(+), 99 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index dda9a666f..ccbeb7539 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -457,6 +457,14 @@ namespace llarp auto since_time = rc_time{std::chrono::seconds{btdc.require("since")}}; + std::unordered_set explicit_relays; + + // Initial fetch: give me all the RC's + if (explicit_ids.empty()) + { + // TODO: this + } + if (explicit_ids.size() > (rcs.size() / 4)) { log::info( @@ -465,8 +473,6 @@ namespace llarp return; } - std::unordered_set explicit_relays; - for (auto& sv : explicit_ids) { if (sv.size() != RouterID::SIZE) @@ -478,10 +484,10 @@ namespace llarp explicit_relays.emplace(reinterpret_cast(sv.data())); } - oxenc::bt_dict_producer resp; + oxenc::bt_dict_producer btdp; { - auto rc_bt_list = resp.append_list("rcs"); + auto rc_sublist = btdp.append_list("rcs"); const auto& last_time = node_db->get_last_rc_update_times(); @@ -492,13 +498,13 @@ namespace llarp for (const auto& [_, rc] : rcs) { if (last_time.at(rc.router_id()) > since_time or explicit_relays.count(rc.router_id())) - rc_bt_list.append_encoded(rc.view()); + rc_sublist.append_encoded(rc.view()); } } - resp.append("time", now.time_since_epoch().count()); + btdp.append("time", now.time_since_epoch().count()); - m.respond(std::move(resp).str()); + m.respond(std::move(btdp).str()); } catch (const std::exception& e) { diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 31834ed50..38b930b02 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -125,12 +125,12 @@ namespace llarp return false; } - RouterID temp = rc_fetch_source; + RouterID temp = fetch_source; - while (temp == rc_fetch_source) + while (temp == fetch_source) std::sample(active_client_routers.begin(), active_client_routers.end(), &temp, 1, csrng); - rc_fetch_source = std::move(temp); + fetch_source = std::move(temp); return true; } @@ -156,56 +156,55 @@ namespace llarp if (conn_count == 1) { // if we only have one connection, it must be current rc fetch source - assert(new_source.router_id() == rc_fetch_source); + assert(new_source.router_id() == fetch_source); if (pinned_edges.size() == 1) { // only one pinned edge set, use it even though it gave unsatisfactory RCs - assert(rc_fetch_source == *(pinned_edges.begin())); + assert(fetch_source == *(pinned_edges.begin())); log::warning( logcat, "Single pinned edge {} gave bad RC response; still using it despite this.", - rc_fetch_source); + fetch_source); return; } // only one connection, choose a new relay to connect to for rc fetching - RouterID r = rc_fetch_source; + RouterID r = fetch_source; - while (r == rc_fetch_source) + while (r == fetch_source) { std::sample(active_client_routers.begin(), active_client_routers.end(), &r, 1, csrng); } - rc_fetch_source = std::move(r); + fetch_source = std::move(r); return; } // choose one of our other existing connections to use as the RC fetch source - while (new_source.router_id() == rc_fetch_source) + while (new_source.router_id() == fetch_source) { _router.link_manager().get_random_connected(new_source); } - rc_fetch_source = new_source.router_id(); + + fetch_source = new_source.router_id(); } // TODO: trust model - void - NodeDB::store_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp) + bool + NodeDB::process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp) { - (void)source; + fetch_source = source; - // TODO: if we don't currently have a "trusted" relay we've been fetching from, - // this will be a full list of RCs. We need to first check if it aligns closely - // with our trusted RouterID list, then replace our RCs with the incoming set. + /* + TODO: trust model analyzing returned list of RCs + */ for (auto& rc : rcs) put_rc_if_newer(std::move(rc), timestamp); - // TODO: if we have a "trusted" relay we've been fetching from, this will be - // an incremental update to the RC list, so *after* insertion we check if the - // RCs' RouterIDs closely match our trusted RouterID list. - last_rc_update_relay_timestamp = timestamp; + + return true; } void @@ -213,13 +212,14 @@ namespace llarp { const auto& rid = source.router_id(); - router_id_fetch_responses[rid] = std::move(ids); + fetch_rid_responses[rid] = std::move(ids); } + // TODO: trust model bool NodeDB::process_fetched_rids() { - for (const auto& [rid, responses] : router_id_fetch_responses) + for (const auto& [rid, responses] : fetch_rid_responses) { // TODO: empty == failure, handle that case for (const auto& response : responses) @@ -227,15 +227,36 @@ namespace llarp active_client_routers.insert(std::move(response)); } } - router_id_fetch_in_progress = false; return true; } void - NodeDB::fetch_rcs(int n_fails, bool initial) + NodeDB::fetch_initial() + { + int num_fails = 0; + + // fetch_initial_{rcs,router_ids} return false when num_fails == 0 + if (fetch_initial_rcs(num_fails)) + { + _router.last_rc_fetch = llarp::time_point_now(); + + if (fetch_initial_router_ids(num_fails)) + { + _router.last_rid_fetch = llarp::time_point_now(); + return; + } + } + + // failure case + // TODO: use bootstrap here! + } + + bool + NodeDB::fetch_initial_rcs(int n_fails) { int num_failures = n_fails; + is_fetching_rcs = true; std::vector needed; const auto now = time_point_now(); @@ -245,9 +266,8 @@ namespace llarp needed.push_back(rid); } - RouterID src = (initial) - ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) - : rc_fetch_source; + RouterID src = + *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); while (num_failures < MAX_FETCH_ATTEMPTS) { @@ -285,7 +305,7 @@ namespace llarp rcs.emplace_back(btlc.consume_dict_consumer()); } - store_fetched_rcs(src, std::move(rcs), timestamp); + process_fetched_rcs(src, std::move(rcs), timestamp); p->set_value(true); } catch (const std::exception& e) @@ -299,9 +319,8 @@ namespace llarp if (f.get()) { log::debug(logcat, "Successfully fetched RC's from {}", src); - rc_fetch_source = src; - assert(_router.link_manager().have_connection_to(src)); - break; + fetch_source = src; + return true; } ++num_failures; @@ -312,34 +331,25 @@ namespace llarp num_failures, MAX_FETCH_ATTEMPTS); - src = (initial) - ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) - : std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + src = *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); } + + return false; } - void - NodeDB::fetch_router_ids(int n_fails, bool initial) + bool + NodeDB::fetch_initial_router_ids(int n_fails) { - assert(not router_id_fetch_in_progress); - int num_failures = n_fails; - - if (router_id_fetch_sources.empty()) - select_router_id_sources(); + assert(not is_fetching_rids); - // if we *still* don't have fetch sources, we can't exactly fetch... - if (router_id_fetch_sources.empty()) - { - log::info(logcat, "Attempting to fetch RouterIDs, but have no source from which to do so."); - return; - } + int num_failures = n_fails; + select_router_id_sources(); - router_id_fetch_in_progress = true; - router_id_fetch_responses.clear(); + is_fetching_rids = true; + fetch_rid_responses.clear(); - RouterID src = (initial) - ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) - : rc_fetch_source; + RouterID src = + *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); std::unordered_set fails; @@ -350,7 +360,7 @@ namespace llarp auto f = success->get_future(); fails.clear(); - for (const auto& target : router_id_fetch_sources) + for (const auto& target : rid_sources) { _router.link_manager().fetch_router_ids( src, @@ -420,11 +430,9 @@ namespace llarp default: // RC node failed to relay our routerID request; re-select RC node and continue log::debug(logcat, "RC source {} failed to mediate RID fetching from {}", src, target); - src = (initial) - ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) - : std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + src = *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); ++num_failures; - fetch_rcs(num_failures); + fetch_rcs(); continue; } } @@ -438,19 +446,18 @@ namespace llarp "RID fetching was successful ({}/{} acceptable errors)", fails.size(), MAX_RID_ERRORS); - rc_fetch_source = src; - assert(_router.link_manager().have_connection_to(src)); + fetch_source = src; // this is where the trust model will do verification based on the similarity of the sets if (process_fetched_rids()) { log::debug(logcat, "Accumulated RID's accepted by trust model"); - return; + return true; } log::debug( logcat, "Accumulated RID's rejected by trust model, reselecting all RID sources..."); - select_router_id_sources(router_id_fetch_sources); + select_router_id_sources(rid_sources); ++num_failures; continue; } @@ -461,6 +468,271 @@ namespace llarp ++num_failures; select_router_id_sources(fails); } + + return false; + } + + void + NodeDB::fetch_rcs() + { + auto& num_failures = fetch_failures; + + // base case; this function is called recursively + if (num_failures > MAX_FETCH_ATTEMPTS) + { + fetch_rcs_result(true); + return; + } + + is_fetching_rcs = true; + + std::vector needed; + const auto now = time_point_now(); + + for (const auto& [rid, rc] : known_rcs) + { + if (now - rc.timestamp() > RouterContact::OUTDATED_AGE) + needed.push_back(rid); + } + + RouterID& src = fetch_source; + + _router.link_manager().fetch_rcs( + src, + RCFetchMessage::serialize(last_rc_update_relay_timestamp, needed), + [this, src](oxen::quic::message m) mutable { + if (m.timed_out) + { + log::info(logcat, "RC fetch to {} timed out", src); + fetch_rcs_result(true); + return; + } + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + if (not m) + { + auto reason = btdc.require(messages::STATUS_KEY); + log::info(logcat, "RC fetch to {} returned error: {}", src, reason); + fetch_rcs_result(true); + return; + } + + auto btlc = btdc.require("rcs"sv); + auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; + + std::vector rcs; + + while (not btlc.is_finished()) + { + rcs.emplace_back(btlc.consume_dict_consumer()); + } + + // if process_fetched_rcs returns false, then the trust model rejected the fetched RC's + fetch_rcs_result(not process_fetched_rcs(src, std::move(rcs), timestamp)); + } + catch (const std::exception& e) + { + log::info(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); + fetch_rcs_result(true); + return; + } + }); + } + + void + NodeDB::fetch_rcs_result(bool error) + { + if (error) + { + ++fetch_failures; + + if (fetch_failures > MAX_FETCH_ATTEMPTS) + { + log::info( + logcat, + "Failed {} attempts to fetch RC's from {}; reverting to bootstrap...", + MAX_FETCH_ATTEMPTS, + fetch_source); + // TODO: revert to bootstrap + // set rc_fetch_source to bootstrap and try again! + } + else + // find new non-bootstrap RC fetch source and try again buddy + fetch_source = std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + + fetch_rcs(); + } + else + { + log::debug(logcat, "Successfully fetched RC's from {}", fetch_source); + post_fetch_rcs(); + } + } + + void + NodeDB::post_fetch_rcs() + { + is_fetching_rcs = false; + _router.last_rc_fetch = llarp::time_point_now(); + } + + // TODO: differentiate between errors from the relay node vs errors from the target nodes + void + NodeDB::fetch_router_ids() + { + auto& num_failures = fetch_failures; + + // base case; this function is called recursively + if (num_failures > MAX_FETCH_ATTEMPTS) + { + fetch_rids_result(fetch_source, true); + return; + } + + if (rid_sources.empty()) + select_router_id_sources(); + + // if we *still* don't have fetch sources, we can't exactly fetch... + if (rid_sources.empty()) + { + log::error(logcat, "Attempting to fetch RouterIDs, but have no source from which to do so."); + return; + } + + is_fetching_rids = true; + fetch_rid_responses.clear(); + + RouterID& src = fetch_source; + RemoteRC& src_rc = known_rcs[src]; + + for (const auto& target : rid_sources) + { + _router.link_manager().fetch_router_ids( + src, + RouterIDFetch::serialize(target), + [this, src, src_rc, target](oxen::quic::message m) mutable { + if (not m) + { + log::info(link_cat, "RID fetch from {} via {} timed out", src, target); + + ingest_rid_fetch_responses(src_rc); + fetch_rids_result(src, true); + return; + } + + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + + btdc.required("routers"); + auto router_id_strings = btdc.consume_list>(); + + btdc.require_signature("signature", [&src](ustring_view msg, ustring_view sig) { + if (sig.size() != 64) + throw std::runtime_error{"Invalid signature: not 64 bytes"}; + if (not crypto::verify(src, msg, sig)) + throw std::runtime_error{ + "Failed to verify signature for fetch RouterIDs response."}; + }); + + std::vector router_ids; + + for (const auto& s : router_id_strings) + { + if (s.size() != RouterID::SIZE) + { + log::warning( + link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); + ingest_rid_fetch_responses(src_rc); + fetch_rids_result(src, true); + return; + } + + router_ids.emplace_back(s.data()); + } + + ingest_rid_fetch_responses(src_rc, std::move(router_ids)); + fetch_rids_result(src); + return; + } + catch (const std::exception& e) + { + log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); + ingest_rid_fetch_responses(src_rc); + fetch_rids_result(src, true); + } + }); + } + } + + void + NodeDB::fetch_rids_result(const RouterID& target, bool error) + { + if (error) + { + fail_sources.insert(target); + ++fetch_failures; + + if (fetch_failures > MAX_FETCH_ATTEMPTS) + { + log::info( + logcat, + "Failed {} attempts to fetch RID's from {}; reverting to bootstrap...", + MAX_FETCH_ATTEMPTS, + fetch_source); + // TODO: revert to bootstrap + // set rc_fetch_source to bootstrap and START OVER! + } + else + // find new non-bootstrap RC fetch source and try again buddy + fetch_source = std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + + fetch_router_ids(); + return; + } + + log::debug(logcat, "Successfully fetched RID's from {}", fetch_source); + auto n_fails = fail_sources.size(); + + if (n_fails <= MAX_RID_ERRORS) + { + log::debug( + logcat, "RID fetching was successful ({}/{} acceptable errors)", n_fails, MAX_RID_ERRORS); + + // this is where the trust model will do verification based on the similarity of the sets + if (process_fetched_rids()) + { + log::debug(logcat, "Accumulated RID's accepted by trust model"); + post_fetch_rids(); + return; + } + + log::debug( + logcat, "Accumulated RID's rejected by trust model, reselecting all RID sources..."); + select_router_id_sources(rid_sources); + ++fetch_failures; + } + else + { + // we had 4 or more failed requests, so we will need to rotate our rid sources + log::debug( + logcat, "RID fetching found {} failures; reselecting failed RID sources...", n_fails); + ++fetch_failures; + select_router_id_sources(fail_sources); + } + + fetch_router_ids(); + } + + void + NodeDB::post_fetch_rids() + { + is_fetching_rids = false; + fetch_rid_responses.clear(); + fail_sources.clear(); + fetch_failures = 0; + _router.last_rid_fetch = llarp::time_point_now(); } void @@ -471,9 +743,17 @@ namespace llarp if (active_client_routers.empty()) return; + // in case we pass the entire list + std::unordered_set temp = rid_sources; + // keep using any we've been using, but remove `excluded` ones - for (const auto& r : excluded) - router_id_fetch_sources.erase(r); + if (excluded == rid_sources) + temp.clear(); + else + { + for (const auto& r : excluded) + temp.erase(r); + } // only know so many routers, so no need to randomize if (active_client_routers.size() <= (ROUTER_ID_SOURCE_COUNT + excluded.size())) @@ -482,18 +762,20 @@ namespace llarp { if (excluded.count(r)) continue; - router_id_fetch_sources.insert(r); + temp.insert(r); } } // select at random until we have chosen enough - while (router_id_fetch_sources.size() < ROUTER_ID_SOURCE_COUNT) + while (temp.size() < ROUTER_ID_SOURCE_COUNT) { RouterID r; std::sample(active_client_routers.begin(), active_client_routers.end(), &r, 1, csrng); if (excluded.count(r) == 0) - router_id_fetch_sources.insert(r); + temp.insert(r); } + + rid_sources.swap(temp); } void diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 8b6b540dd..df5fb1ee8 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -66,15 +66,19 @@ namespace llarp std::unordered_set pinned_edges; // rc update info: we only set this upon a SUCCESSFUL fetching - RouterID rc_fetch_source; + RouterID fetch_source; rc_time last_rc_update_relay_timestamp; - std::unordered_set router_id_fetch_sources; + std::unordered_set rid_sources; + + std::unordered_set fail_sources; // process responses once all are received (or failed/timed out) - std::unordered_map> router_id_fetch_responses; - bool router_id_fetch_in_progress{false}; + std::unordered_map> fetch_rid_responses; + + std::atomic is_fetching_rids{false}, is_fetching_rcs{false}; + std::atomic fetch_failures{0}; bool want_rc(const RouterID& rid) const; @@ -135,8 +139,8 @@ namespace llarp bool rotate_startup_rc_source(); - void - store_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); + bool + process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); void ingest_rid_fetch_responses(const RemoteRC& source, std::vector ids = {}); @@ -144,14 +148,32 @@ namespace llarp bool process_fetched_rids(); + void + fetch_initial(); + + bool + fetch_initial_rcs(int n_fails = 0); + bool - fetch_initial_rcs(const RouterID& src); + fetch_initial_router_ids(int n_fails = 0); + + void + fetch_rcs(); + + void + fetch_rcs_result(bool error = false); + + void + fetch_router_ids(); + + void + post_fetch_rcs(); void - fetch_rcs(int n_fails = 0, bool initial = false); + post_fetch_rids(); void - fetch_router_ids(int n_fails = 0, bool initial = false); + fetch_rids_result(const RouterID& target, bool error = false); void select_router_id_sources(std::unordered_set excluded = {}); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 6a50d2f7d..e4f0cafaf 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -851,29 +851,24 @@ namespace llarp } else { - int num_failures = 0; - if (needs_initial_fetch) { - node_db()->fetch_rcs(num_failures, true); - last_rc_fetch = now_timepoint; - node_db()->fetch_router_ids(num_failures, true); - last_routerid_fetch = now_timepoint; + node_db()->fetch_initial(); } else { // (client-only) periodically fetch updated RCs if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) { - node_db()->fetch_rcs(num_failures); + node_db()->fetch_rcs(); last_rc_fetch = now_timepoint; } // (client-only) periodically fetch updated RouterID list - if (now_timepoint - last_routerid_fetch > ROUTERID_UPDATE_INTERVAL) + if (now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) { - node_db()->fetch_router_ids(num_failures); - last_routerid_fetch = now_timepoint; + node_db()->fetch_router_ids(); + last_rid_fetch = now_timepoint; } } } diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 90a8a24b4..44e9f40a9 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -55,6 +55,8 @@ namespace llarp struct Router : std::enable_shared_from_this { + friend class NodeDB; + explicit Router(EventLoop_ptr loop, std::shared_ptr vpnPlatform); ~Router(); @@ -121,12 +123,6 @@ namespace llarp bool needs_initial_fetch{true}; - std::chrono::system_clock::time_point last_rc_gossip{ - std::chrono::system_clock::time_point::min()}; - std::chrono::system_clock::time_point next_rc_gossip{last_rc_gossip}; - std::chrono::system_clock::time_point last_rc_fetch{last_rc_gossip}; - std::chrono::system_clock::time_point last_routerid_fetch{last_rc_gossip}; - // should we be sending padded messages every interval? bool send_padding = false; @@ -149,6 +145,13 @@ namespace llarp bool insufficient_peers() const; + protected: + std::chrono::system_clock::time_point last_rc_gossip{ + std::chrono::system_clock::time_point::min()}; + std::chrono::system_clock::time_point next_rc_gossip{last_rc_gossip}; + std::chrono::system_clock::time_point last_rc_fetch{last_rc_gossip}; + std::chrono::system_clock::time_point last_rid_fetch{last_rc_gossip}; + public: void for_each_connection(std::function func); From b6cc86e25b0e1b652ccde1a0ae9439d172822d3d Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 28 Nov 2023 12:50:07 -0800 Subject: [PATCH 04/93] fetch RID result handler logic --- llarp/link/link_manager.cpp | 10 +---- llarp/nodedb.cpp | 79 +++++++++++++++++-------------------- llarp/nodedb.hpp | 4 +- 3 files changed, 41 insertions(+), 52 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index ccbeb7539..e9070533a 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -167,7 +167,7 @@ namespace llarp _router.loop()->call([this, msg = std::move(m), func = std::move(func)]() mutable { auto body = msg.body_str(); auto respond = [m = std::move(msg)](std::string response) mutable { - m.respond(std::move(response)); + m.respond(std::move(response), not m); }; std::invoke(func, this, body, std::move(respond)); }); @@ -517,12 +517,6 @@ namespace llarp LinkManager::fetch_router_ids( const RouterID& via, std::string payload, std::function func) { - if (ep.conns.empty()) - { - log::debug(link_cat, "Not attempting to fetch Router IDs: not connected to any relays."); - return; - } - send_control_message(via, "fetch_router_ids"s, std::move(payload), std::move(func)); } @@ -571,7 +565,7 @@ namespace llarp [source_rid = std::move(source_rid), orig_mess = std::move(m)](oxen::quic::message m) mutable { if (not m.timed_out) - orig_mess.respond(m.body_str()); + orig_mess.respond(m.body_str(), not m); // on timeout, just silently drop (as original requester will just time out anyway) }); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 38b930b02..a72211727 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -208,11 +208,9 @@ namespace llarp } void - NodeDB::ingest_rid_fetch_responses(const RemoteRC& source, std::vector ids) + NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::vector ids) { - const auto& rid = source.router_id(); - - fetch_rid_responses[rid] = std::move(ids); + fetch_rid_responses[source] = std::move(ids); } // TODO: trust model @@ -355,7 +353,6 @@ namespace llarp while (num_failures < MAX_FETCH_ATTEMPTS) { - RemoteRC& src_rc = known_rcs[src]; auto success = std::make_shared>(); auto f = success->get_future(); fails.clear(); @@ -365,12 +362,12 @@ namespace llarp _router.link_manager().fetch_router_ids( src, RouterIDFetch::serialize(target), - [this, src, src_rc, target, p = std::move(success)](oxen::quic::message m) mutable { + [this, src, target, p = std::move(success)](oxen::quic::message m) mutable { if (not m) { log::info(link_cat, "RID fetch from {} via {} timed out", src, target); - ingest_rid_fetch_responses(src_rc); + ingest_rid_fetch_responses(src); p->set_value(-1); return; } @@ -405,7 +402,7 @@ namespace llarp router_ids.emplace_back(s.data()); } - ingest_rid_fetch_responses(src_rc, std::move(router_ids)); + ingest_rid_fetch_responses(src, std::move(router_ids)); return; } catch (const std::exception& e) @@ -414,7 +411,7 @@ namespace llarp p->set_value(0); } - ingest_rid_fetch_responses(src_rc); // empty response == failure + ingest_rid_fetch_responses(src); // empty response == failure }); switch (f.get()) @@ -581,12 +578,10 @@ namespace llarp void NodeDB::fetch_router_ids() { - auto& num_failures = fetch_failures; - // base case; this function is called recursively - if (num_failures > MAX_FETCH_ATTEMPTS) + if (fetch_failures > MAX_FETCH_ATTEMPTS) { - fetch_rids_result(fetch_source, true); + fetch_rids_result(); return; } @@ -604,20 +599,20 @@ namespace llarp fetch_rid_responses.clear(); RouterID& src = fetch_source; - RemoteRC& src_rc = known_rcs[src]; for (const auto& target : rid_sources) { _router.link_manager().fetch_router_ids( src, RouterIDFetch::serialize(target), - [this, src, src_rc, target](oxen::quic::message m) mutable { - if (not m) + [this, src, target](oxen::quic::message m) mutable { + if (m.timed_out) { log::info(link_cat, "RID fetch from {} via {} timed out", src, target); - ingest_rid_fetch_responses(src_rc); - fetch_rids_result(src, true); + ++fetch_failures; + ingest_rid_fetch_responses(src); + fetch_rids_result(); return; } @@ -644,55 +639,55 @@ namespace llarp { log::warning( link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); - ingest_rid_fetch_responses(src_rc); - fetch_rids_result(src, true); + ingest_rid_fetch_responses(target); + fail_sources.insert(target); + fetch_rids_result(); return; } router_ids.emplace_back(s.data()); } - ingest_rid_fetch_responses(src_rc, std::move(router_ids)); - fetch_rids_result(src); + ingest_rid_fetch_responses(target, std::move(router_ids)); + fetch_rids_result(); // success return; } catch (const std::exception& e) { log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); - ingest_rid_fetch_responses(src_rc); - fetch_rids_result(src, true); + ingest_rid_fetch_responses(target); + fail_sources.insert(target); + fetch_rids_result(); } }); } } void - NodeDB::fetch_rids_result(const RouterID& target, bool error) + NodeDB::fetch_rids_result() { - if (error) + if (fetch_failures > MAX_FETCH_ATTEMPTS) { - fail_sources.insert(target); - ++fetch_failures; + log::info( + logcat, + "Failed {} attempts to fetch RID's from {}; reverting to bootstrap...", + MAX_FETCH_ATTEMPTS, + fetch_source); - if (fetch_failures > MAX_FETCH_ATTEMPTS) - { - log::info( - logcat, - "Failed {} attempts to fetch RID's from {}; reverting to bootstrap...", - MAX_FETCH_ATTEMPTS, - fetch_source); - // TODO: revert to bootstrap - // set rc_fetch_source to bootstrap and START OVER! - } - else - // find new non-bootstrap RC fetch source and try again buddy - fetch_source = std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + // TODO: revert rc_source to bootstrap, start over fetch_router_ids(); return; } - log::debug(logcat, "Successfully fetched RID's from {}", fetch_source); + auto n_responses = fetch_rid_responses.size(); + + if (n_responses < MIN_RID_FETCHES) + { + log::debug(logcat, "Received {}/{} fetch RID requests", n_responses, 12); + return; + } + auto n_fails = fail_sources.size(); if (n_fails <= MAX_RID_ERRORS) diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index df5fb1ee8..fec6d5d51 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -143,7 +143,7 @@ namespace llarp process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); void - ingest_rid_fetch_responses(const RemoteRC& source, std::vector ids = {}); + ingest_rid_fetch_responses(const RouterID& source, std::vector ids = {}); bool process_fetched_rids(); @@ -173,7 +173,7 @@ namespace llarp post_fetch_rids(); void - fetch_rids_result(const RouterID& target, bool error = false); + fetch_rids_result(); void select_router_id_sources(std::unordered_set excluded = {}); From ba3fffb7650df70f8c039c62f23b8217e3f61a27 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 29 Nov 2023 06:03:54 -0800 Subject: [PATCH 05/93] housekeeping pre-trust model --- llarp/link/link_manager.cpp | 246 ++++++++++++++++++------------------ llarp/nodedb.cpp | 8 +- llarp/nodedb.hpp | 68 ++++++---- 3 files changed, 165 insertions(+), 157 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index e9070533a..c679b8a2e 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -395,6 +395,129 @@ namespace llarp }); } + bool + LinkManager::have_connection_to(const RouterID& remote, bool client_only) const + { + return ep.have_conn(remote, client_only); + } + + bool + LinkManager::have_client_connection_to(const RouterID& remote) const + { + return ep.have_conn(remote, true); + } + + void + LinkManager::deregister_peer(RouterID remote) + { + if (auto rv = ep.deregister_peer(remote); rv) + { + persisting_conns.erase(remote); + log::info(logcat, "Peer {} successfully de-registered", remote); + } + else + log::warning(logcat, "Peer {} not found for de-registration!", remote); + } + + void + LinkManager::stop() + { + if (is_stopping) + { + return; + } + + LogInfo("stopping links"); + is_stopping = true; + + quic.reset(); + } + + void + LinkManager::set_conn_persist(const RouterID& remote, llarp_time_t until) + { + if (is_stopping) + return; + + persisting_conns[remote] = std::max(until, persisting_conns[remote]); + if (have_client_connection_to(remote)) + { + // mark this as a client so we don't try to back connect + clients.Upsert(remote); + } + } + + size_t + LinkManager::get_num_connected(bool clients_only) const + { + return ep.num_connected(clients_only); + } + + size_t + LinkManager::get_num_connected_clients() const + { + return get_num_connected(true); + } + + bool + LinkManager::get_random_connected(RemoteRC& router) const + { + return ep.get_random_connection(router); + } + + // TODO: this? perhaps no longer necessary in the same way? + void + LinkManager::check_persisting_conns(llarp_time_t) + { + if (is_stopping) + return; + } + + // TODO: this + util::StatusObject + LinkManager::extract_status() const + { + return {}; + } + + void + LinkManager::init() + { + is_stopping = false; + node_db = _router.node_db(); + } + + void + LinkManager::connect_to_random(int num_conns) + { + std::set exclude; + auto remainder = num_conns; + + do + { + auto filter = [exclude](const auto& rc) -> bool { + return exclude.count(rc.router_id()) == 0; + }; + + if (auto maybe_other = node_db->GetRandom(filter)) + { + exclude.insert(maybe_other->router_id()); + + if (not node_db->is_connection_allowed(maybe_other->router_id())) + continue; + + connect_to(*maybe_other); + --remainder; + } + } while (remainder > 0); + } + + void + LinkManager::recv_data_message(oxen::quic::dgram_interface&, bstring) + { + // TODO: this + } + void LinkManager::gossip_rc(const RouterID& rc_rid, std::string serialized_rc) { @@ -575,129 +698,6 @@ namespace llarp } } - bool - LinkManager::have_connection_to(const RouterID& remote, bool client_only) const - { - return ep.have_conn(remote, client_only); - } - - bool - LinkManager::have_client_connection_to(const RouterID& remote) const - { - return ep.have_conn(remote, true); - } - - void - LinkManager::deregister_peer(RouterID remote) - { - if (auto rv = ep.deregister_peer(remote); rv) - { - persisting_conns.erase(remote); - log::info(logcat, "Peer {} successfully de-registered", remote); - } - else - log::warning(logcat, "Peer {} not found for de-registration!", remote); - } - - void - LinkManager::stop() - { - if (is_stopping) - { - return; - } - - LogInfo("stopping links"); - is_stopping = true; - - quic.reset(); - } - - void - LinkManager::set_conn_persist(const RouterID& remote, llarp_time_t until) - { - if (is_stopping) - return; - - persisting_conns[remote] = std::max(until, persisting_conns[remote]); - if (have_client_connection_to(remote)) - { - // mark this as a client so we don't try to back connect - clients.Upsert(remote); - } - } - - size_t - LinkManager::get_num_connected(bool clients_only) const - { - return ep.num_connected(clients_only); - } - - size_t - LinkManager::get_num_connected_clients() const - { - return get_num_connected(true); - } - - bool - LinkManager::get_random_connected(RemoteRC& router) const - { - return ep.get_random_connection(router); - } - - // TODO: this? perhaps no longer necessary in the same way? - void - LinkManager::check_persisting_conns(llarp_time_t) - { - if (is_stopping) - return; - } - - // TODO: this - util::StatusObject - LinkManager::extract_status() const - { - return {}; - } - - void - LinkManager::init() - { - is_stopping = false; - node_db = _router.node_db(); - } - - void - LinkManager::connect_to_random(int num_conns) - { - std::set exclude; - auto remainder = num_conns; - - do - { - auto filter = [exclude](const auto& rc) -> bool { - return exclude.count(rc.router_id()) == 0; - }; - - if (auto maybe_other = node_db->GetRandom(filter)) - { - exclude.insert(maybe_other->router_id()); - - if (not node_db->is_connection_allowed(maybe_other->router_id())) - continue; - - connect_to(*maybe_other); - --remainder; - } - } while (remainder > 0); - } - - void - LinkManager::recv_data_message(oxen::quic::dgram_interface&, bstring) - { - // TODO: this - } - void LinkManager::handle_find_name(std::string_view body, std::function respond) { diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index a72211727..b7bd34e68 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -47,13 +47,11 @@ namespace llarp } } - constexpr auto FlushInterval = 5min; - NodeDB::NodeDB(fs::path root, std::function)> diskCaller, Router* r) : _router{*r} , _root{std::move(root)} , _disk(std::move(diskCaller)) - , _next_flush_time{time_now_ms() + FlushInterval} + , _next_flush_time{time_now_ms() + FLUSH_INTERVAL} { EnsureSkiplist(_root); } @@ -67,7 +65,7 @@ namespace llarp if (now > _next_flush_time) { _router.loop()->call([this]() { - _next_flush_time += FlushInterval; + _next_flush_time += FLUSH_INTERVAL; // make copy of all rcs std::vector copy; @@ -189,7 +187,6 @@ namespace llarp fetch_source = new_source.router_id(); } - // TODO: trust model bool NodeDB::process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp) { @@ -213,7 +210,6 @@ namespace llarp fetch_rid_responses[source] = std::move(ids); } - // TODO: trust model bool NodeDB::process_fetched_rids() { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index fec6d5d51..95933fd27 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -28,61 +28,73 @@ namespace llarp inline constexpr size_t MAX_RID_ERRORS{ROUTER_ID_SOURCE_COUNT - MIN_RID_FETCHES}; inline constexpr int MAX_FETCH_ATTEMPTS{10}; + inline constexpr auto FLUSH_INTERVAL{5min}; + class NodeDB { - std::unordered_map known_rcs; - Router& _router; const fs::path _root; const std::function)> _disk; llarp_time_t _next_flush_time; - /// asynchronously remove the files for a set of rcs on disk given their public ident key - void - remove_many_from_disk_async(std::unordered_set idents) const; - - /// get filename of an RC file given its public ident key - fs::path - get_path_by_pubkey(RouterID pk) const; + /******** RouterID/RouterContacts ********/ - std::unordered_map bootstraps; + /** RouterID mappings + Both the following are populated in NodeDB startup with RouterID's stored on disk. + - active_client_routers: meant to persist between lokinet sessions, and is only + populated during startup and RouterID fetching. This is meant to represent the + client instance's perspective of the network and which RouterID's are "active" + - known_rcs: populated during startup and when RC's are updated both during gossip + and periodic RC fetching + */ + std::unordered_set active_client_routers; + std::unordered_map known_rcs; - // Router lists for snodes - // whitelist = active routers + /** RouterID lists + - white: active routers + - gray: fully funded, but decommissioned routers + - green: registered, but not fully-staked routers + */ std::unordered_set router_whitelist; - // greylist = fully funded, but decommissioned routers std::unordered_set router_greylist; - // greenlist = registered but not fully-staked routers std::unordered_set router_greenlist; - // all registered relays (snodes) + + // All registered relays (service nodes) std::unordered_set registered_routers; + // timing std::unordered_map last_rc_update_times; - - // Client list of active RouterID's - std::unordered_set active_client_routers; - + rc_time last_rc_update_relay_timestamp; // only ever use to specific edges as path first-hops std::unordered_set pinned_edges; - - // rc update info: we only set this upon a SUCCESSFUL fetching + // source of "truth" for RC updating. This relay will also mediate requests to the + // 12 selected active RID's for RID fetching RouterID fetch_source; - - rc_time last_rc_update_relay_timestamp; - + // set of 12 randomly selected RID's from the set of active client routers std::unordered_set rid_sources; - + // logs the RID's that resulted in an error during RID fetching std::unordered_set fail_sources; - - // process responses once all are received (or failed/timed out) + // stores all RID fetch responses for greedy comprehensive processing std::unordered_map> fetch_rid_responses; + // tracks fetch failures from the RC node performing the initial RC fetch and mediating + // the 12 RID requests to the 12 sources, NOT failures from the 12 sources themselves + std::atomic fetch_failures{0}; std::atomic is_fetching_rids{false}, is_fetching_rcs{false}; - std::atomic fetch_failures{0}; bool want_rc(const RouterID& rid) const; + /// asynchronously remove the files for a set of rcs on disk given their public ident key + void + remove_many_from_disk_async(std::unordered_set idents) const; + + /// get filename of an RC file given its public ident key + fs::path + get_path_by_pubkey(RouterID pk) const; + + std::unordered_map bootstraps; + public: void set_bootstrap_routers(const std::set& rcs); From f6e651caeaf3647d8da0df98dc6998cb89bdd47e Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 29 Nov 2023 06:11:38 -0800 Subject: [PATCH 06/93] move from vectors to unordered_sets - When receiving a request to fetch RouterID's, the remote endpoint fulfilling the request stores them in an unordered set. When the request caller receives that payload, it is loaded into a vector in the same order. However, we should just load it directly into an unordered set to enforce both the order and that none appear twice - The trust model will have to operate on multiple large lists of RouterID's and RC's efficiently, and maintaining a sort order ensures the values are workable immediately after deserialization --- llarp/nodedb.cpp | 10 +++++----- llarp/nodedb.hpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index b7bd34e68..5c898d051 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -205,7 +205,7 @@ namespace llarp } void - NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::vector ids) + NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::unordered_set ids) { fetch_rid_responses[source] = std::move(ids); } @@ -383,7 +383,7 @@ namespace llarp "Failed to verify signature for fetch RouterIDs response."}; }); - std::vector router_ids; + std::unordered_set router_ids; for (const auto& s : router_id_strings) { @@ -395,7 +395,7 @@ namespace llarp return; } - router_ids.emplace_back(s.data()); + router_ids.emplace(s.data()); } ingest_rid_fetch_responses(src, std::move(router_ids)); @@ -627,7 +627,7 @@ namespace llarp "Failed to verify signature for fetch RouterIDs response."}; }); - std::vector router_ids; + std::unordered_set router_ids; for (const auto& s : router_id_strings) { @@ -641,7 +641,7 @@ namespace llarp return; } - router_ids.emplace_back(s.data()); + router_ids.emplace(s.data()); } ingest_rid_fetch_responses(target, std::move(router_ids)); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 95933fd27..9a089ddc5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -75,7 +75,7 @@ namespace llarp // logs the RID's that resulted in an error during RID fetching std::unordered_set fail_sources; // stores all RID fetch responses for greedy comprehensive processing - std::unordered_map> fetch_rid_responses; + std::unordered_map> fetch_rid_responses; // tracks fetch failures from the RC node performing the initial RC fetch and mediating // the 12 RID requests to the 12 sources, NOT failures from the 12 sources themselves std::atomic fetch_failures{0}; @@ -155,7 +155,7 @@ namespace llarp process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); void - ingest_rid_fetch_responses(const RouterID& source, std::vector ids = {}); + ingest_rid_fetch_responses(const RouterID& source, std::unordered_set ids = {}); bool process_fetched_rids(); From 91121ea22b4b6b9145588ed93aec033238c64b35 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 30 Nov 2023 13:53:41 -0800 Subject: [PATCH 07/93] pull yourself up by your bootstraps sonny - initial/subsequent fetching combined for RouterContacts and RouterIDs - bootstraps fallback implemented and looped into fetch logic --- llarp/CMakeLists.txt | 2 +- llarp/bootstrap.cpp | 18 ++ llarp/bootstrap.hpp | 17 ++ llarp/link/link_manager.cpp | 459 ++++++++++++++++------------ llarp/link/link_manager.hpp | 9 + llarp/messages/common.hpp | 1 - llarp/messages/fetch.hpp | 82 +++++ llarp/messages/rc.hpp | 31 -- llarp/messages/router_id.hpp | 17 -- llarp/nodedb.cpp | 564 +++++++++++------------------------ llarp/nodedb.hpp | 176 +++++------ llarp/router/router.cpp | 52 ++-- llarp/router/router.hpp | 18 +- 13 files changed, 703 insertions(+), 743 deletions(-) create mode 100644 llarp/messages/fetch.hpp delete mode 100644 llarp/messages/rc.hpp delete mode 100644 llarp/messages/router_id.hpp diff --git a/llarp/CMakeLists.txt b/llarp/CMakeLists.txt index 657c20cb2..3aff1a3f5 100644 --- a/llarp/CMakeLists.txt +++ b/llarp/CMakeLists.txt @@ -178,7 +178,7 @@ lokinet_add_library(lokinet-nodedb set(BOOTSTRAP_FALLBACKS) foreach(bs IN ITEMS MAINNET TESTNET) if(BOOTSTRAP_FALLBACK_${bs}) - message(STATUS "Building with ${bs} fallback boostrap path \"${BOOTSTRAP_FALLBACK_${bs}}\"") + message(STATUS "Building with ${bs} fallback bootstrap path \"${BOOTSTRAP_FALLBACK_${bs}}\"") file(READ "${BOOTSTRAP_FALLBACK_${bs}}" bs_data HEX) if(bs STREQUAL TESTNET) set(network "gamma") diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index 474052c30..65d53773c 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -25,6 +25,24 @@ namespace llarp return true; } + bool + BootstrapList::contains(const RouterID& rid) + { + for (const auto& it : *this) + { + if (it.router_id() == rid) + return true; + } + + return false; + } + + bool + BootstrapList::contains(const RemoteRC& rc) + { + return count(rc); + } + std::string_view BootstrapList::bt_encode() const { diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index 11e8286d3..bb1a30943 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -11,6 +11,8 @@ namespace llarp { struct BootstrapList final : public std::set { + size_t index; + bool bt_decode(std::string_view buf); @@ -20,6 +22,21 @@ namespace llarp void read_from_file(const fs::path& fpath); + bool + contains(const RouterID& rid); + + // returns a reference to the next index and a boolean that equals true if + // this is the front of the set + std::pair + next() + { + ++index %= this->size(); + return std::make_pair(*std::next(this->begin(), index), index == 0); + } + + bool + contains(const RemoteRC& rc); + void clear_list() { diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index c679b8a2e..d48cc9ba2 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -5,9 +5,8 @@ #include #include +#include #include -#include -#include #include #include #include @@ -537,22 +536,68 @@ namespace llarp void LinkManager::handle_gossip_rc(oxen::quic::message m) { - try + // RemoteRC constructor wraps deserialization in a try/catch + RemoteRC rc{m.body()}; + + if (node_db->put_rc_if_newer(rc)) { - RemoteRC rc{m.body()}; + log::info(link_cat, "Received updated RC, forwarding to relay peers."); + gossip_rc(rc.router_id(), m.body_str()); + } + else + log::debug(link_cat, "Received known or old RC, not storing or forwarding."); + } - if (node_db->put_rc_if_newer(rc)) - { - log::info(link_cat, "Received updated RC, forwarding to relay peers."); - gossip_rc(rc.router_id(), m.body_str()); - } - else - log::debug(link_cat, "Received known or old RC, not storing or forwarding."); + void + LinkManager::fetch_bootstrap_rcs( + const RouterID& source, std::string payload, std::function func) + { + send_control_message(source, "bfetch_rcs", std::move(payload), std::move(func)); + } + + void + LinkManager::handle_fetch_bootstrap_rcs(oxen::quic::message m) + { + // this handler should not be registered for clients + assert(_router.is_service_node()); + + const auto& rcs = node_db->get_rcs(); + size_t quantity; + + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + quantity = btdc.require("quantity"); } catch (const std::exception& e) { - log::info(link_cat, "Recieved invalid RC, dropping on the floor."); + log::info(link_cat, "Exception handling RC Fetch request: {}", e.what()); + m.respond(messages::ERROR_RESPONSE, true); + return; } + + auto rc_size = rcs.size(); + auto now = llarp::time_now_ms(); + size_t i = 0; + + oxenc::bt_dict_producer btdp; + + { + auto sublist = btdp.append_list("rcs"); + + while (i < quantity) + { + auto& next_rc = std::next(rcs.begin(), csrng() % rc_size)->second; + + if (next_rc.is_expired(now)) + continue; + + sublist.append_encoded(next_rc.view()); + ++i; + } + } + + m.respond(std::move(btdp).str()); } void @@ -571,69 +616,64 @@ namespace llarp const auto& rcs = node_db->get_rcs(); const auto now = time_point_now(); + std::vector explicit_ids; + rc_time since_time; + try { oxenc::bt_dict_consumer btdc{m.body()}; btdc.required("explicit_ids"); - auto explicit_ids = btdc.consume_list>(); + explicit_ids = btdc.consume_list>(); - auto since_time = rc_time{std::chrono::seconds{btdc.require("since")}}; + since_time = rc_time{std::chrono::seconds{btdc.require("since")}}; + } + catch (const std::exception& e) + { + log::info(link_cat, "Exception handling RC Fetch request: {}", e.what()); + m.respond(messages::ERROR_RESPONSE, true); + return; + } - std::unordered_set explicit_relays; + // Initial fetch: give me all the RC's + if (explicit_ids.empty()) + { + // TODO: this + } - // Initial fetch: give me all the RC's - if (explicit_ids.empty()) - { - // TODO: this - } + std::unordered_set explicit_relays; - if (explicit_ids.size() > (rcs.size() / 4)) + for (auto& sv : explicit_ids) + { + if (sv.size() != RouterID::SIZE) { - log::info( - link_cat, "Remote requested too many relay IDs (greater than 1/4 of what we have)."); m.respond(RCFetchMessage::INVALID_REQUEST, true); return; } - for (auto& sv : explicit_ids) - { - if (sv.size() != RouterID::SIZE) - { - m.respond(RCFetchMessage::INVALID_REQUEST, true); - return; - } - - explicit_relays.emplace(reinterpret_cast(sv.data())); - } - - oxenc::bt_dict_producer btdp; + explicit_relays.emplace(reinterpret_cast(sv.data())); + } - { - auto rc_sublist = btdp.append_list("rcs"); + oxenc::bt_dict_producer btdp; + const auto& last_time = node_db->get_last_rc_update_times(); - const auto& last_time = node_db->get_last_rc_update_times(); + { + auto sublist = btdp.append_list("rcs"); - // if since_time isn't epoch start, subtract a bit for buffer - if (since_time != decltype(since_time)::min()) - since_time -= 5s; + // if since_time isn't epoch start, subtract a bit for buffer + if (since_time != decltype(since_time)::min()) + since_time -= 5s; - for (const auto& [_, rc] : rcs) - { - if (last_time.at(rc.router_id()) > since_time or explicit_relays.count(rc.router_id())) - rc_sublist.append_encoded(rc.view()); - } + for (const auto& [_, rc] : rcs) + { + if (last_time.at(rc.router_id()) > since_time or explicit_relays.count(rc.router_id())) + sublist.append_encoded(rc.view()); } + } - btdp.append("time", now.time_since_epoch().count()); + btdp.append("time", now.time_since_epoch().count()); - m.respond(std::move(btdp).str()); - } - catch (const std::exception& e) - { - log::info(link_cat, "Exception handling RC Fetch request: {}", e.what()); - m.respond(messages::ERROR_RESPONSE, true); - } + m.respond(std::move(btdp).str()); } void @@ -713,6 +753,7 @@ namespace llarp { log::warning(link_cat, "Exception: {}", e.what()); respond(messages::ERROR_RESPONSE); + return; } _router.rpc_client()->lookup_ons_hash( @@ -1305,33 +1346,20 @@ namespace llarp void LinkManager::handle_obtain_exit(oxen::quic::message m) { + uint64_t flag; + ustring_view pubkey, sig; + std::string_view tx_id, dict_data; + try { - uint64_t flag; - ustring_view pubkey, sig; - std::string_view tx_id; - oxenc::bt_list_consumer btlc{m.body()}; - auto dict_data = btlc.consume_dict_data(); + dict_data = btlc.consume_dict_data(); oxenc::bt_dict_consumer btdc{dict_data}; sig = to_usv(btlc.consume_string_view()); flag = btdc.require("E"); pubkey = btdc.require("I"); tx_id = btdc.require("T"); - - RouterID target{pubkey.data()}; - auto transit_hop = - _router.path_context().GetTransitHop(target, PathID_t{to_usv(tx_id).data()}); - - const auto rx_id = transit_hop->info.rxID; - - auto success = - (crypto::verify(pubkey, to_usv(dict_data), sig) - and _router.exitContext().ObtainNewExit(PubKey{pubkey.data()}, rx_id, flag != 0)); - - m.respond( - ObtainExitMessage::sign_and_serialize_response(_router.identity(), tx_id), not success); } catch (const std::exception& e) { @@ -1339,6 +1367,18 @@ namespace llarp m.respond(messages::ERROR_RESPONSE, true); throw; } + + RouterID target{pubkey.data()}; + auto transit_hop = _router.path_context().GetTransitHop(target, PathID_t{to_usv(tx_id).data()}); + + const auto rx_id = transit_hop->info.rxID; + + auto success = + (crypto::verify(pubkey, to_usv(dict_data), sig) + and _router.exitContext().ObtainNewExit(PubKey{pubkey.data()}, rx_id, flag != 0)); + + m.respond( + ObtainExitMessage::sign_and_serialize_response(_router.identity(), tx_id), not success); } void @@ -1354,62 +1394,45 @@ namespace llarp // TODO: what to do here } + std::string_view tx_id, dict_data; + ustring_view sig; + try { - std::string_view tx_id; - ustring_view sig; - oxenc::bt_list_consumer btlc{m.body()}; - auto dict_data = btlc.consume_dict_data(); + dict_data = btlc.consume_dict_data(); oxenc::bt_dict_consumer btdc{dict_data}; sig = to_usv(btlc.consume_string_view()); tx_id = btdc.require("T"); - - auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); - - if (crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) - path_ptr->enable_exit_traffic(); } catch (const std::exception& e) { log::warning(link_cat, "Exception: {}", e.what()); throw; } + + auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); + + if (crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) + path_ptr->enable_exit_traffic(); } void LinkManager::handle_update_exit(oxen::quic::message m) { + std::string_view path_id, tx_id, dict_data; + ustring_view sig; + try { - std::string_view path_id, tx_id; - ustring_view sig; - oxenc::bt_list_consumer btlc{m.body()}; - auto dict_data = btlc.consume_dict_data(); + dict_data = btlc.consume_dict_data(); oxenc::bt_dict_consumer btdc{dict_data}; sig = to_usv(btlc.consume_string_view()); path_id = btdc.require("P"); tx_id = btdc.require("T"); - - auto transit_hop = - _router.path_context().GetTransitHop(_router.pubkey(), PathID_t{to_usv(tx_id).data()}); - - if (auto exit_ep = - _router.exitContext().FindEndpointForPath(PathID_t{to_usv(path_id).data()})) - { - if (crypto::verify(exit_ep->PubKey().data(), to_usv(dict_data), sig)) - { - (exit_ep->UpdateLocalPath(transit_hop->info.rxID)) - ? m.respond(UpdateExitMessage::sign_and_serialize_response(_router.identity(), tx_id)) - : m.respond( - serialize_response({{messages::STATUS_KEY, UpdateExitMessage::UPDATE_FAILED}}), - true); - } - // If we fail to verify the message, no-op - } } catch (const std::exception& e) { @@ -1417,6 +1440,22 @@ namespace llarp m.respond(messages::ERROR_RESPONSE, true); return; } + + auto transit_hop = + _router.path_context().GetTransitHop(_router.pubkey(), PathID_t{to_usv(tx_id).data()}); + + if (auto exit_ep = _router.exitContext().FindEndpointForPath(PathID_t{to_usv(path_id).data()})) + { + if (crypto::verify(exit_ep->PubKey().data(), to_usv(dict_data), sig)) + { + (exit_ep->UpdateLocalPath(transit_hop->info.rxID)) + ? m.respond(UpdateExitMessage::sign_and_serialize_response(_router.identity(), tx_id)) + : m.respond( + serialize_response({{messages::STATUS_KEY, UpdateExitMessage::UPDATE_FAILED}}), + true); + } + // If we fail to verify the message, no-op + } } void @@ -1432,69 +1471,53 @@ namespace llarp // TODO: what to do here } + std::string tx_id; + std::string_view dict_data; + ustring_view sig; + try { - std::string tx_id; - ustring_view sig; - oxenc::bt_list_consumer btlc{m.body()}; - auto dict_data = btlc.consume_dict_data(); + dict_data = btlc.consume_dict_data(); oxenc::bt_dict_consumer btdc{dict_data}; sig = to_usv(btlc.consume_string_view()); tx_id = btdc.require("T"); - - auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); - - if (crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) - { - if (path_ptr->update_exit(std::stoul(tx_id))) - { - // TODO: talk to tom and Jason about how this stupid shit was a no-op originally - // see Path::HandleUpdateExitVerifyMessage - } - else - {} - } } catch (const std::exception& e) { log::warning(link_cat, "Exception: {}", e.what()); return; } + + auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); + + if (crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) + { + if (path_ptr->update_exit(std::stoul(tx_id))) + { + // TODO: talk to tom and Jason about how this stupid shit was a no-op originally + // see Path::HandleUpdateExitVerifyMessage + } + else + {} + } } void LinkManager::handle_close_exit(oxen::quic::message m) { + std::string_view tx_id, dict_data; + ustring_view sig; + try { - std::string_view tx_id; - ustring_view sig; - oxenc::bt_list_consumer btlc{m.body()}; - auto dict_data = btlc.consume_dict_data(); + dict_data = btlc.consume_dict_data(); oxenc::bt_dict_consumer btdc{dict_data}; sig = to_usv(btlc.consume_string_view()); tx_id = btdc.require("T"); - - auto transit_hop = - _router.path_context().GetTransitHop(_router.pubkey(), PathID_t{to_usv(tx_id).data()}); - - const auto rx_id = transit_hop->info.rxID; - - if (auto exit_ep = router().exitContext().FindEndpointForPath(rx_id)) - { - if (crypto::verify(exit_ep->PubKey().data(), to_usv(dict_data), sig)) - { - exit_ep->Close(); - m.respond(CloseExitMessage::sign_and_serialize_response(_router.identity(), tx_id)); - } - } - - m.respond( - serialize_response({{messages::STATUS_KEY, CloseExitMessage::UPDATE_FAILED}}), true); } catch (const std::exception& e) { @@ -1502,6 +1525,22 @@ namespace llarp m.respond(messages::ERROR_RESPONSE, true); return; } + + auto transit_hop = + _router.path_context().GetTransitHop(_router.pubkey(), PathID_t{to_usv(tx_id).data()}); + + const auto rx_id = transit_hop->info.rxID; + + if (auto exit_ep = router().exitContext().FindEndpointForPath(rx_id)) + { + if (crypto::verify(exit_ep->PubKey().data(), to_usv(dict_data), sig)) + { + exit_ep->Close(); + m.respond(CloseExitMessage::sign_and_serialize_response(_router.identity(), tx_id)); + } + } + + m.respond(serialize_response({{messages::STATUS_KEY, CloseExitMessage::UPDATE_FAILED}}), true); } void @@ -1517,95 +1556,127 @@ namespace llarp // TODO: what to do here } + std::string_view nonce, tx_id, dict_data; + ustring_view sig; + try { - std::string_view nonce, tx_id; - ustring_view sig; - oxenc::bt_list_consumer btlc{m.body()}; - auto dict_data = btlc.consume_dict_data(); + dict_data = btlc.consume_dict_data(); oxenc::bt_dict_consumer btdc{dict_data}; sig = to_usv(btlc.consume_string_view()); tx_id = btdc.require("T"); nonce = btdc.require("Y"); - - auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); - - if (path_ptr->SupportsAnyRoles(path::ePathRoleExit | path::ePathRoleSVC) - and crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) - path_ptr->mark_exit_closed(); } catch (const std::exception& e) { log::warning(link_cat, "Exception: {}", e.what()); return; } + + auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); + + if (path_ptr->SupportsAnyRoles(path::ePathRoleExit | path::ePathRoleSVC) + and crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) + path_ptr->mark_exit_closed(); } void LinkManager::handle_path_control(oxen::quic::message m, const RouterID& from) { + ustring_view nonce, path_id_str; + std::string payload; + try { oxenc::bt_dict_consumer btdc{m.body()}; - auto nonce = SymmNonce{btdc.require("NONCE").data()}; - auto path_id_str = btdc.require("PATHID"); - auto payload = btdc.require("PAYLOAD"); - auto path_id = PathID_t{path_id_str.data()}; - auto hop = _router.path_context().GetTransitHop(from, path_id); - - // TODO: use "path_control" for both directions? If not, drop message on - // floor if we don't have the path_id in question; if we decide to make this - // bidirectional, will need to check if we have a Path with path_id. - if (not hop) - return; - - // if terminal hop, payload should contain a request (e.g. "find_name"); handle and respond. - if (hop->terminal_hop) - { - hop->onion(payload, nonce, false); - handle_inner_request(std::move(m), std::move(payload), std::move(hop)); - return; - } - - auto next_id = path_id == hop->info.rxID ? hop->info.txID : hop->info.rxID; - auto next_router = path_id == hop->info.rxID ? hop->info.upstream : hop->info.downstream; - auto new_payload = hop->onion_and_payload(payload, next_id, nonce); - send_control_message( - next_router, - "path_control"s, - std::move(new_payload), - [hop_weak = hop->weak_from_this(), path_id, prev_message = std::move(m)]( - oxen::quic::message response) mutable { - auto hop = hop_weak.lock(); - if (not hop) - return; - - oxenc::bt_dict_consumer resp_btdc{response.body()}; - auto nonce = SymmNonce{resp_btdc.require("NONCE").data()}; - auto payload = resp_btdc.require("PAYLOAD"); - auto resp_payload = hop->onion_and_payload(payload, path_id, nonce); - prev_message.respond(std::move(resp_payload), false); - }); + nonce = btdc.require("NONCE"); + path_id_str = btdc.require("PATHID"); + payload = btdc.require("PAYLOAD"); } catch (const std::exception& e) { log::warning(link_cat, "Exception: {}", e.what()); return; } + + auto symnonce = SymmNonce{nonce.data()}; + auto path_id = PathID_t{path_id_str.data()}; + auto hop = _router.path_context().GetTransitHop(from, path_id); + + // TODO: use "path_control" for both directions? If not, drop message on + // floor if we don't have the path_id in question; if we decide to make this + // bidirectional, will need to check if we have a Path with path_id. + if (not hop) + return; + + // if terminal hop, payload should contain a request (e.g. "find_name"); handle and respond. + if (hop->terminal_hop) + { + hop->onion(payload, symnonce, false); + handle_inner_request(std::move(m), std::move(payload), std::move(hop)); + return; + } + + auto& next_id = path_id == hop->info.rxID ? hop->info.txID : hop->info.rxID; + auto& next_router = path_id == hop->info.rxID ? hop->info.upstream : hop->info.downstream; + + std::string new_payload = hop->onion_and_payload(payload, next_id, symnonce); + + send_control_message( + next_router, + "path_control"s, + std::move(new_payload), + [hop_weak = hop->weak_from_this(), path_id, prev_message = std::move(m)]( + oxen::quic::message response) mutable { + auto hop = hop_weak.lock(); + + if (not hop) + return; + + ustring_view nonce; + std::string payload, response_body; + + try + { + oxenc::bt_dict_consumer btdc{response.body()}; + nonce = btdc.require("NONCE"); + payload = btdc.require("PAYLOAD"); + } + catch (const std::exception& e) + { + log::warning(link_cat, "Exception: {}", e.what()); + return; + } + + auto symnonce = SymmNonce{nonce.data()}; + auto resp_payload = hop->onion_and_payload(payload, path_id, symnonce); + prev_message.respond(std::move(resp_payload), false); + }); } void LinkManager::handle_inner_request( oxen::quic::message m, std::string payload, std::shared_ptr hop) { - oxenc::bt_dict_consumer btdc{payload}; - auto body = btdc.require("BODY"); - auto method = btdc.require("METHOD"); + std::string_view body, method; + + try + { + oxenc::bt_dict_consumer btdc{payload}; + body = btdc.require("BODY"); + method = btdc.require("METHOD"); + } + catch (const std::exception& e) + { + log::warning(link_cat, "Exception: {}", e.what()); + return; + } // If a handler exists for "method", call it; else drop request on the floor. auto itr = path_requests.find(method); + if (itr == path_requests.end()) { log::info(link_cat, "Received path control request \"{}\", which has no handler.", method); @@ -1634,7 +1705,9 @@ namespace llarp } try - {} + { + // + } catch (const std::exception& e) { log::warning(link_cat, "Exception: {}", e.what()); diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index e51f892dd..b305fa6ef 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -242,6 +242,15 @@ namespace llarp void handle_fetch_router_ids(oxen::quic::message m); + void + fetch_bootstrap_rcs( + const RouterID& source, + std::string payload, + std::function func); + + void + handle_fetch_bootstrap_rcs(oxen::quic::message m); + bool have_connection_to(const RouterID& remote, bool client_only = false) const; diff --git a/llarp/messages/common.hpp b/llarp/messages/common.hpp index f15a5184d..9d774aee2 100644 --- a/llarp/messages/common.hpp +++ b/llarp/messages/common.hpp @@ -20,7 +20,6 @@ namespace llarp { namespace messages { - inline std::string serialize_response(oxenc::bt_dict supplement = {}) { diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp new file mode 100644 index 000000000..35e9d18ec --- /dev/null +++ b/llarp/messages/fetch.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include "common.hpp" + +namespace llarp +{ + namespace RCFetchMessage + { + inline const auto INVALID_REQUEST = + messages::serialize_response({{messages::STATUS_KEY, "Invalid relay ID requested"}}); + + inline static std::string + serialize( + std::chrono::system_clock::time_point since, const std::vector& explicit_ids) + { + oxenc::bt_dict_producer btdp; + + try + { + { + auto sublist = btdp.append_list("explicit_ids"); + + for (const auto& rid : explicit_ids) + sublist.append(rid.ToView()); + } + + btdp.append("since", since.time_since_epoch() / 1s); + } + catch (...) + { + log::error(link_cat, "Error: RCFetchMessage failed to bt encode contents!"); + } + + return std::move(btdp).str(); + } + } // namespace RCFetchMessage + + namespace BootstrapFetchMessage + { + inline static std::string + serialize(size_t quantity) + { + oxenc::bt_dict_producer btdp; + btdp.append("quantity", quantity); + return std::move(btdp).str(); + } + + inline static std::string + serialize_response(const std::vector& explicit_ids) + { + oxenc::bt_dict_producer btdp; + + try + { + auto sublist = btdp.append_list("explicit_ids"); + + for (const auto& rid : explicit_ids) + sublist.append(rid.ToView()); + } + catch (...) + { + log::error(link_cat, "Error: BootstrapFetchMessage failed to bt encode contents!"); + } + + return std::move(btdp).str(); + } + } // namespace BootstrapFetchMessage + + namespace FetchRIDMessage + { + inline constexpr auto INVALID_REQUEST = "Invalid relay ID requested to relay response from."sv; + + inline static std::string + serialize(const RouterID& source) + { + // serialize_response is a bit weird here, and perhaps could have a sister function + // with the same purpose but as a request, but...it works. + return messages::serialize_response({{"source", source.ToView()}}); + } + } // namespace FetchRIDMessage + +} // namespace llarp diff --git a/llarp/messages/rc.hpp b/llarp/messages/rc.hpp deleted file mode 100644 index 633b540d2..000000000 --- a/llarp/messages/rc.hpp +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include "common.hpp" - -namespace llarp::RCFetchMessage -{ - inline const auto INVALID_REQUEST = - messages::serialize_response({{messages::STATUS_KEY, "Invalid relay ID requested"}}); - - inline static std::string - serialize(std::chrono::system_clock::time_point since, const std::vector& explicit_ids) - { - oxenc::bt_dict_producer btdp; - - try - { - btdp.append("since", since.time_since_epoch() / 1s); - { - auto id_list = btdp.append_list("explicit_ids"); - for (const auto& rid : explicit_ids) - id_list.append(rid.ToView()); - } - } - catch (...) - { - log::error(link_cat, "Error: RCFetchMessage failed to bt encode contents!"); - } - - return std::move(btdp).str(); - } -} // namespace llarp::RCFetchMessage diff --git a/llarp/messages/router_id.hpp b/llarp/messages/router_id.hpp deleted file mode 100644 index dbdd897de..000000000 --- a/llarp/messages/router_id.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "common.hpp" - -namespace llarp::RouterIDFetch -{ - inline constexpr auto INVALID_REQUEST = "Invalid relay ID requested to relay response from."sv; - - inline static std::string - serialize(const RouterID& source) - { - // serialize_response is a bit weird here, and perhaps could have a sister function - // with the same purpose but as a request, but...it works. - return messages::serialize_response({{"source", source.ToView()}}); - } - -} // namespace llarp::RouterIDFetch diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 5c898d051..dfea702bd 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -2,8 +2,7 @@ #include "crypto/types.hpp" #include "dht/kademlia.hpp" -#include "messages/rc.hpp" -#include "messages/router_id.hpp" +#include "messages/fetch.hpp" #include "router_contact.hpp" #include "util/time.hpp" @@ -105,86 +104,13 @@ namespace llarp } void - NodeDB::set_bootstrap_routers(const std::set& rcs) + NodeDB::set_bootstrap_routers(std::unique_ptr from_router) { - bootstraps.clear(); // this function really shouldn't be called more than once, but... - for (const auto& rc : rcs) - { - bootstraps.emplace(rc.router_id(), rc); - } - } - - bool - NodeDB::rotate_startup_rc_source() - { - if (active_client_routers.size() < 13) - { - // do something here - return false; - } + // TODO: if this needs to be called more than once (ex: drastic failures), then + // change this assert to a bootstraps.clear() call + assert(_bootstraps->empty()); - RouterID temp = fetch_source; - - while (temp == fetch_source) - std::sample(active_client_routers.begin(), active_client_routers.end(), &temp, 1, csrng); - - fetch_source = std::move(temp); - return true; - } - - /// Called in normal operation when the relay we fetched RCs from gives either a "bad" - /// response or a timeout. Attempts to switch to a new relay as our RC source, using - /// existing connections if possible, and respecting pinned edges. - void - NodeDB::rotate_rc_source() - { - auto conn_count = _router.link_manager().get_num_connected(); - - // This function makes no sense to be called if we have no connections... - if (conn_count == 0) - throw std::runtime_error{"Called rotate_rc_source with no connections, does not make sense!"}; - - // We should not be in this function if client_known_routers isn't populated - if (active_client_routers.size() <= 1) - throw std::runtime_error{"Cannot rotate RC source without RC source(s) to rotate to!"}; - - RemoteRC new_source{}; - _router.link_manager().get_random_connected(new_source); - - if (conn_count == 1) - { - // if we only have one connection, it must be current rc fetch source - assert(new_source.router_id() == fetch_source); - - if (pinned_edges.size() == 1) - { - // only one pinned edge set, use it even though it gave unsatisfactory RCs - assert(fetch_source == *(pinned_edges.begin())); - log::warning( - logcat, - "Single pinned edge {} gave bad RC response; still using it despite this.", - fetch_source); - return; - } - - // only one connection, choose a new relay to connect to for rc fetching - RouterID r = fetch_source; - - while (r == fetch_source) - { - std::sample(active_client_routers.begin(), active_client_routers.end(), &r, 1, csrng); - } - fetch_source = std::move(r); - return; - } - - // choose one of our other existing connections to use as the RC fetch source - while (new_source.router_id() == fetch_source) - { - _router.link_manager().get_random_connected(new_source); - } - - fetch_source = new_source.router_id(); + _bootstraps = std::move(from_router); } bool @@ -207,12 +133,33 @@ namespace llarp void NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::unordered_set ids) { + if (ids.empty()) + fail_sources.insert(source); + fetch_rid_responses[source] = std::move(ids); } + /** We only call into this function after ensuring two conditions: + 1) We have received at least 8 of 12 responses from the queried RouterID sources + 2) Of those reponses, less than 4 were errors of any sorts + + Logically, this function performs the following basic analysis of the returned RIDs. + */ bool NodeDB::process_fetched_rids() { + std::unordered_set union_set; + + for (const auto& [rid, responses] : fetch_rid_responses) + { + std::merge( + union_set.begin(), + union_set.end(), + responses.begin(), + responses.end(), + std::inserter(union_set, union_set.begin())); + } + for (const auto& [rid, responses] : fetch_rid_responses) { // TODO: empty == failure, handle that case @@ -228,256 +175,26 @@ namespace llarp void NodeDB::fetch_initial() { - int num_fails = 0; - - // fetch_initial_{rcs,router_ids} return false when num_fails == 0 - if (fetch_initial_rcs(num_fails)) - { - _router.last_rc_fetch = llarp::time_point_now(); - - if (fetch_initial_router_ids(num_fails)) - { - _router.last_rid_fetch = llarp::time_point_now(); - return; - } - } - - // failure case - // TODO: use bootstrap here! - } - - bool - NodeDB::fetch_initial_rcs(int n_fails) - { - int num_failures = n_fails; - is_fetching_rcs = true; - std::vector needed; - const auto now = time_point_now(); - - for (const auto& [rid, rc] : known_rcs) - { - if (now - rc.timestamp() > RouterContact::OUTDATED_AGE) - needed.push_back(rid); - } - - RouterID src = - *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); - - while (num_failures < MAX_FETCH_ATTEMPTS) - { - auto success = std::make_shared>(); - auto f = success->get_future(); - - _router.link_manager().fetch_rcs( - src, - RCFetchMessage::serialize(last_rc_update_relay_timestamp, needed), - [this, src, p = std::move(success)](oxen::quic::message m) mutable { - if (m.timed_out) - { - log::info(logcat, "RC fetch to {} timed out", src); - p->set_value(false); - return; - } - try - { - oxenc::bt_dict_consumer btdc{m.body()}; - if (not m) - { - auto reason = btdc.require(messages::STATUS_KEY); - log::info(logcat, "RC fetch to {} returned error: {}", src, reason); - p->set_value(false); - return; - } - - auto btlc = btdc.require("rcs"sv); - auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; - - std::vector rcs; - - while (not btlc.is_finished()) - { - rcs.emplace_back(btlc.consume_dict_consumer()); - } - - process_fetched_rcs(src, std::move(rcs), timestamp); - p->set_value(true); - } - catch (const std::exception& e) - { - log::info(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); - p->set_value(false); - return; - } - }); - - if (f.get()) - { - log::debug(logcat, "Successfully fetched RC's from {}", src); - fetch_source = src; - return true; - } - - ++num_failures; - log::debug( - logcat, - "Unable to fetch RC's from {}; rotating RC source ({}/{} attempts)", - src, - num_failures, - MAX_FETCH_ATTEMPTS); - - src = *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); - } - - return false; - } - - bool - NodeDB::fetch_initial_router_ids(int n_fails) - { - assert(not is_fetching_rids); - - int num_failures = n_fails; - select_router_id_sources(); - - is_fetching_rids = true; - fetch_rid_responses.clear(); - - RouterID src = + // Set fetch source as random selection of known active client routers + fetch_source = *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); - std::unordered_set fails; - - while (num_failures < MAX_FETCH_ATTEMPTS) - { - auto success = std::make_shared>(); - auto f = success->get_future(); - fails.clear(); - - for (const auto& target : rid_sources) - { - _router.link_manager().fetch_router_ids( - src, - RouterIDFetch::serialize(target), - [this, src, target, p = std::move(success)](oxen::quic::message m) mutable { - if (not m) - { - log::info(link_cat, "RID fetch from {} via {} timed out", src, target); - - ingest_rid_fetch_responses(src); - p->set_value(-1); - return; - } - - try - { - oxenc::bt_dict_consumer btdc{m.body()}; - - btdc.required("routers"); - auto router_id_strings = btdc.consume_list>(); - - btdc.require_signature("signature", [&src](ustring_view msg, ustring_view sig) { - if (sig.size() != 64) - throw std::runtime_error{"Invalid signature: not 64 bytes"}; - if (not crypto::verify(src, msg, sig)) - throw std::runtime_error{ - "Failed to verify signature for fetch RouterIDs response."}; - }); - - std::unordered_set router_ids; - - for (const auto& s : router_id_strings) - { - if (s.size() != RouterID::SIZE) - { - log::warning( - link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); - p->set_value(0); - return; - } - - router_ids.emplace(s.data()); - } - - ingest_rid_fetch_responses(src, std::move(router_ids)); - return; - } - catch (const std::exception& e) - { - log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); - p->set_value(0); - } - - ingest_rid_fetch_responses(src); // empty response == failure - }); - - switch (f.get()) - { - case 1: - log::debug(logcat, "Successfully fetched RID's from {} via {}", target, src); - continue; - case 0: - // RC node relayed our fetch routerID request, but the request failed at the target - log::debug(logcat, "Unsuccessfully fetched RID's from {} via {}", target, src); - fails.insert(target); - continue; - default: - // RC node failed to relay our routerID request; re-select RC node and continue - log::debug(logcat, "RC source {} failed to mediate RID fetching from {}", src, target); - src = *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); - ++num_failures; - fetch_rcs(); - continue; - } - } - - auto n_fails = fails.size(); - - if (n_fails <= MAX_RID_ERRORS) - { - log::debug( - logcat, - "RID fetching was successful ({}/{} acceptable errors)", - fails.size(), - MAX_RID_ERRORS); - fetch_source = src; - - // this is where the trust model will do verification based on the similarity of the sets - if (process_fetched_rids()) - { - log::debug(logcat, "Accumulated RID's accepted by trust model"); - return true; - } - - log::debug( - logcat, "Accumulated RID's rejected by trust model, reselecting all RID sources..."); - select_router_id_sources(rid_sources); - ++num_failures; - continue; - } - - // we had 4 or more failed requests, so we will need to rotate our rid sources - log::debug( - logcat, "RID fetching found {} failures; reselecting failed RID sources...", n_fails); - ++num_failures; - select_router_id_sources(fails); - } - - return false; + fetch_rcs(true); } void - NodeDB::fetch_rcs() + NodeDB::fetch_rcs(bool initial) { auto& num_failures = fetch_failures; // base case; this function is called recursively if (num_failures > MAX_FETCH_ATTEMPTS) { - fetch_rcs_result(true); + fetch_rcs_result(initial, true); return; } - is_fetching_rcs = true; + is_fetching_rcs = true; // TOTHINK: do these booleans do anything? std::vector needed; const auto now = time_point_now(); @@ -493,21 +210,22 @@ namespace llarp _router.link_manager().fetch_rcs( src, RCFetchMessage::serialize(last_rc_update_relay_timestamp, needed), - [this, src](oxen::quic::message m) mutable { + [this, src, initial](oxen::quic::message m) mutable { if (m.timed_out) { log::info(logcat, "RC fetch to {} timed out", src); - fetch_rcs_result(true); + fetch_rcs_result(initial, true); return; } try { oxenc::bt_dict_consumer btdc{m.body()}; + if (not m) { auto reason = btdc.require(messages::STATUS_KEY); log::info(logcat, "RC fetch to {} returned error: {}", src, reason); - fetch_rcs_result(true); + fetch_rcs_result(initial, true); return; } @@ -517,75 +235,34 @@ namespace llarp std::vector rcs; while (not btlc.is_finished()) - { rcs.emplace_back(btlc.consume_dict_consumer()); - } // if process_fetched_rcs returns false, then the trust model rejected the fetched RC's - fetch_rcs_result(not process_fetched_rcs(src, std::move(rcs), timestamp)); + fetch_rcs_result(initial, not process_fetched_rcs(src, std::move(rcs), timestamp)); } catch (const std::exception& e) { log::info(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); - fetch_rcs_result(true); + fetch_rcs_result(initial, true); return; } }); } void - NodeDB::fetch_rcs_result(bool error) - { - if (error) - { - ++fetch_failures; - - if (fetch_failures > MAX_FETCH_ATTEMPTS) - { - log::info( - logcat, - "Failed {} attempts to fetch RC's from {}; reverting to bootstrap...", - MAX_FETCH_ATTEMPTS, - fetch_source); - // TODO: revert to bootstrap - // set rc_fetch_source to bootstrap and try again! - } - else - // find new non-bootstrap RC fetch source and try again buddy - fetch_source = std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; - - fetch_rcs(); - } - else - { - log::debug(logcat, "Successfully fetched RC's from {}", fetch_source); - post_fetch_rcs(); - } - } - - void - NodeDB::post_fetch_rcs() - { - is_fetching_rcs = false; - _router.last_rc_fetch = llarp::time_point_now(); - } - - // TODO: differentiate between errors from the relay node vs errors from the target nodes - void - NodeDB::fetch_router_ids() + NodeDB::fetch_rids(bool initial) { // base case; this function is called recursively if (fetch_failures > MAX_FETCH_ATTEMPTS) { - fetch_rids_result(); + fetch_rids_result(initial); return; } if (rid_sources.empty()) select_router_id_sources(); - // if we *still* don't have fetch sources, we can't exactly fetch... - if (rid_sources.empty()) + if (not initial and rid_sources.empty()) { log::error(logcat, "Attempting to fetch RouterIDs, but have no source from which to do so."); return; @@ -600,15 +277,13 @@ namespace llarp { _router.link_manager().fetch_router_ids( src, - RouterIDFetch::serialize(target), - [this, src, target](oxen::quic::message m) mutable { - if (m.timed_out) + FetchRIDMessage::serialize(target), + [this, src, target, initial](oxen::quic::message m) mutable { + if (not m) { log::info(link_cat, "RID fetch from {} via {} timed out", src, target); - - ++fetch_failures; - ingest_rid_fetch_responses(src); - fetch_rids_result(); + ingest_rid_fetch_responses(target); + fetch_rids_result(initial); return; } @@ -636,8 +311,7 @@ namespace llarp log::warning( link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); ingest_rid_fetch_responses(target); - fail_sources.insert(target); - fetch_rids_result(); + fetch_rids_result(initial); return; } @@ -645,22 +319,54 @@ namespace llarp } ingest_rid_fetch_responses(target, std::move(router_ids)); - fetch_rids_result(); // success + fetch_rids_result(initial); // success return; } catch (const std::exception& e) { log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); ingest_rid_fetch_responses(target); - fail_sources.insert(target); - fetch_rids_result(); + fetch_rids_result(initial); } }); } } void - NodeDB::fetch_rids_result() + NodeDB::fetch_rcs_result(bool initial, bool error) + { + if (error) + { + ++fetch_failures; + + if (fetch_failures > MAX_FETCH_ATTEMPTS) + { + log::info( + logcat, + "Failed {} attempts to fetch RC's from {}; reverting to bootstrap...", + MAX_FETCH_ATTEMPTS, + fetch_source); + + fallback_to_bootstrap(); + return; + } + + // find new non-bootstrap RC fetch source and try again buddy + fetch_source = (initial) + ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) + : std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + + fetch_rcs(initial); + } + else + { + log::debug(logcat, "Successfully fetched RC's from {}", fetch_source); + post_fetch_rcs(initial); + } + } + + void + NodeDB::fetch_rids_result(bool initial) { if (fetch_failures > MAX_FETCH_ATTEMPTS) { @@ -670,17 +376,15 @@ namespace llarp MAX_FETCH_ATTEMPTS, fetch_source); - // TODO: revert rc_source to bootstrap, start over - - fetch_router_ids(); + fallback_to_bootstrap(); return; } auto n_responses = fetch_rid_responses.size(); - if (n_responses < MIN_RID_FETCHES) + if (n_responses < ROUTER_ID_SOURCE_COUNT) { - log::debug(logcat, "Received {}/{} fetch RID requests", n_responses, 12); + log::debug(logcat, "Received {}/{} fetch RID requests", n_responses, ROUTER_ID_SOURCE_COUNT); return; } @@ -695,7 +399,7 @@ namespace llarp if (process_fetched_rids()) { log::debug(logcat, "Accumulated RID's accepted by trust model"); - post_fetch_rids(); + post_fetch_rids(initial); return; } @@ -713,17 +417,100 @@ namespace llarp select_router_id_sources(fail_sources); } - fetch_router_ids(); + fetch_rids(true); + } + + void + NodeDB::post_fetch_rcs(bool initial) + { + is_fetching_rcs = false; + _router.last_rc_fetch = llarp::time_point_now(); + + if (initial) + fetch_rids(initial); } void - NodeDB::post_fetch_rids() + NodeDB::post_fetch_rids(bool initial) { is_fetching_rids = false; fetch_rid_responses.clear(); fail_sources.clear(); fetch_failures = 0; _router.last_rid_fetch = llarp::time_point_now(); + + if (initial) + _router.initial_fetch_completed(); + } + + void + NodeDB::fallback_to_bootstrap() + { + if (bootstrap_failures >= MAX_BOOTSTRAP_FETCH_ATTEMPTS) + { + log::info(logcat, "Current bootstrap failed... cycling to next bootstrap..."); + + bootstrap_failures = 0; + auto [rc, is_front] = _bootstraps->next(); + + // Base case: if we have returned to the front of the bootstrap list, we're in a + // bad spot + if (using_bootstrap_fallback && is_front) + { + auto err = fmt::format("ERROR: ALL BOOTSTRAPS ARE BAD"); + log::error(logcat, err); + throw std::runtime_error{err}; + } + + using_bootstrap_fallback = true; + fetch_source = rc.router_id(); + } + + _router.link_manager().fetch_bootstrap_rcs( + fetch_source, + BootstrapFetchMessage::serialize(BOOTSTRAP_SOURCE_COUNT), + [this](oxen::quic::message m) mutable { + if (not m) + { + ++bootstrap_failures; + fallback_to_bootstrap(); + return; + } + + std::unordered_set rids; + + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + + { + auto btlc = btdc.require("rcs"sv); + + while (not btlc.is_finished()) + { + auto rc = RemoteRC{btlc.consume_dict_consumer()}; + rids.emplace(rc.router_id()); + } + } + } + catch (const std::exception& e) + { + log::info( + logcat, + "Failed to parse BootstrapRC fetch response from {}: {}", + fetch_source, + e.what()); + ++bootstrap_failures; + fallback_to_bootstrap(); + return; + } + + rid_sources.swap(rids); + // if this result is bad, we won't try this bootstrap again + bootstrap_failures = MAX_BOOTSTRAP_FETCH_ATTEMPTS; + using_bootstrap_fallback = false; + fetch_initial(); + }); } void @@ -809,7 +596,8 @@ namespace llarp bool NodeDB::is_connection_allowed(const RouterID& remote) const { - if (pinned_edges.size() && pinned_edges.count(remote) == 0 && bootstraps.count(remote) == 0) + if (_pinned_edges.size() && _pinned_edges.count(remote) == 0 + && not _bootstraps->contains(remote)) { return false; } @@ -823,7 +611,7 @@ namespace llarp bool NodeDB::is_first_hop_allowed(const RouterID& remote) const { - if (pinned_edges.size() && pinned_edges.count(remote) == 0) + if (_pinned_edges.size() && _pinned_edges.count(remote) == 0) return false; return true; } diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 9a089ddc5..a23d02904 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -27,6 +27,8 @@ namespace llarp inline constexpr size_t MIN_ACTIVE_RIDS{24}; inline constexpr size_t MAX_RID_ERRORS{ROUTER_ID_SOURCE_COUNT - MIN_RID_FETCHES}; inline constexpr int MAX_FETCH_ATTEMPTS{10}; + inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{3}; + inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; inline constexpr auto FLUSH_INTERVAL{5min}; @@ -66,21 +68,27 @@ namespace llarp std::unordered_map last_rc_update_times; rc_time last_rc_update_relay_timestamp; // only ever use to specific edges as path first-hops - std::unordered_set pinned_edges; + std::unordered_set _pinned_edges; // source of "truth" for RC updating. This relay will also mediate requests to the // 12 selected active RID's for RID fetching RouterID fetch_source; - // set of 12 randomly selected RID's from the set of active client routers - std::unordered_set rid_sources; + // set of 12 randomly selected RID's from the client's set of routers + std::unordered_set rid_sources{}; // logs the RID's that resulted in an error during RID fetching - std::unordered_set fail_sources; + std::unordered_set fail_sources{}; // stores all RID fetch responses for greedy comprehensive processing std::unordered_map> fetch_rid_responses; - // tracks fetch failures from the RC node performing the initial RC fetch and mediating - // the 12 RID requests to the 12 sources, NOT failures from the 12 sources themselves - std::atomic fetch_failures{0}; + /** Failure counters: + - fetch_failures: tracks errors fetching RC's from the RC node and requesting RID's + from the 12 RID sources. Errors in the individual RID sets are NOT counted towards + this, their performance as a group is evaluated wholistically + - bootstrap_failures: tracks errors fetching both RC's from bootstrasps and RID requests + they mediate. This is a different counter as we only bootstrap in problematic cases + */ + std::atomic fetch_failures{0}, bootstrap_failures{0}; - std::atomic is_fetching_rids{false}, is_fetching_rcs{false}; + std::atomic is_fetching_rids{false}, is_fetching_rcs{false}, + using_bootstrap_fallback{false}; bool want_rc(const RouterID& rid) const; @@ -93,63 +101,14 @@ namespace llarp fs::path get_path_by_pubkey(RouterID pk) const; - std::unordered_map bootstraps; + std::unique_ptr _bootstraps; public: - void - set_bootstrap_routers(const std::set& rcs); - - const std::unordered_set& - whitelist() const - { - return router_whitelist; - } - - const std::unordered_set& - greylist() const - { - return router_greylist; - } - - const std::unordered_set& - get_registered_routers() const - { - return registered_routers; - } - - const std::unordered_map& - get_rcs() const - { - return known_rcs; - } - - const std::unordered_map& - get_last_rc_update_times() const - { - return last_rc_update_times; - } + explicit NodeDB( + fs::path rootdir, std::function)> diskCaller, Router* r); - /// If we receive a bad set of RCs from our current RC source relay, we consider - /// that relay to be a bad source of RCs and we randomly choose a new one. - /// - /// When using a new RC fetch relay, we first re-fetch the full RC list and, if - /// that aligns with our RouterID list, we go back to periodic updates from that relay. - /// - /// This will respect edge-pinning and attempt to use a relay we already have - /// a connection with. - void - rotate_rc_source(); - - /// This function is called during startup and initial fetching. When a lokinet client - /// instance performs its initial RC/RID fetching, it may need to randomly select a - /// node from its list of stale RC's to relay its requests. If there is a failure in - /// mediating these request, the client will randomly select another RC source - /// - /// Returns: - /// true - a new startup RC source was selected - /// false - a new startup RC source was NOT selected - bool - rotate_startup_rc_source(); + /// in memory nodedb + NodeDB(); bool process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); @@ -163,33 +122,51 @@ namespace llarp void fetch_initial(); - bool - fetch_initial_rcs(int n_fails = 0); - - bool - fetch_initial_router_ids(int n_fails = 0); - + // RouterContact fetching void - fetch_rcs(); - + fetch_rcs(bool initial = false); void - fetch_rcs_result(bool error = false); - + post_fetch_rcs(bool initial = false); void - fetch_router_ids(); + fetch_rcs_result(bool initial = false, bool error = false); + // RouterID fetching void - post_fetch_rcs(); - + fetch_rids(bool initial = false); void - post_fetch_rids(); + post_fetch_rids(bool initial = false); + void + fetch_rids_result(bool initial = false); + // Bootstrap fallback void - fetch_rids_result(); + fallback_to_bootstrap(); void select_router_id_sources(std::unordered_set excluded = {}); + // /// If we receive a bad set of RCs from our current RC source relay, we consider + // /// that relay to be a bad source of RCs and we randomly choose a new one. + // /// + // /// When using a new RC fetch relay, we first re-fetch the full RC list and, if + // /// that aligns with our RouterID list, we go back to periodic updates from that relay. + // /// + // /// This will respect edge-pinning and attempt to use a relay we already have + // /// a connection with. + // void + // rotate_rc_source(); + + // /// This function is called during startup and initial fetching. When a lokinet client + // /// instance performs its initial RC/RID fetching, it may need to randomly select a + // /// node from its list of stale RC's to relay its requests. If there is a failure in + // /// mediating these request, the client will randomly select another RC source + // /// + // /// Returns: + // /// true - a new startup RC source was selected + // /// false - a new startup RC source was NOT selected + // bool + // rotate_startup_rc_source(); + void set_router_whitelist( const std::vector& whitelist, @@ -226,17 +203,50 @@ namespace llarp bool is_first_hop_allowed(const RouterID& remote) const; + std::unordered_set& + pinned_edges() + { + return _pinned_edges; + } + + std::unique_ptr& + bootstrap_list() + { + return _bootstraps; + } + + void + set_bootstrap_routers(std::unique_ptr from_router); + const std::unordered_set& - get_pinned_edges() const + whitelist() const { - return pinned_edges; + return router_whitelist; } - explicit NodeDB( - fs::path rootdir, std::function)> diskCaller, Router* r); + const std::unordered_set& + greylist() const + { + return router_greylist; + } - /// in memory nodedb - NodeDB(); + const std::unordered_set& + get_registered_routers() const + { + return registered_routers; + } + + const std::unordered_map& + get_rcs() const + { + return known_rcs; + } + + const std::unordered_map& + get_last_rc_update_times() const + { + return last_rc_update_times; + } /// load all known_rcs from disk syncrhonously void diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index e4f0cafaf..cc24a8dc9 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -464,19 +464,21 @@ namespace llarp bool Router::appears_decommed() const { - return have_snode_whitelist() and node_db()->greylist().count(pubkey()); + return _is_service_node and have_snode_whitelist() and node_db()->greylist().count(pubkey()); } bool Router::appears_funded() const { - return have_snode_whitelist() and node_db()->is_connection_allowed(pubkey()); + return _is_service_node and have_snode_whitelist() + and node_db()->is_connection_allowed(pubkey()); } bool Router::appears_registered() const { - return have_snode_whitelist() and node_db()->get_registered_routers().count(pubkey()); + return _is_service_node and have_snode_whitelist() + and node_db()->get_registered_routers().count(pubkey()); } bool @@ -573,8 +575,6 @@ namespace llarp auto& networkConfig = conf.network; /// build a set of strictConnectPubkeys - std::unordered_set strictConnectPubkeys; - if (not networkConfig.strict_connect.empty()) { const auto& val = networkConfig.strict_connect; @@ -586,7 +586,7 @@ namespace llarp throw std::runtime_error( "Must specify more than one strict-connect router if using strict-connect"); - strictConnectPubkeys.insert(val.begin(), val.end()); + _node_db->pinned_edges().insert(val.begin(), val.end()); log::debug(logcat, "{} strict-connect routers configured", val.size()); } @@ -599,29 +599,31 @@ namespace llarp // /bootstrap.signed. If this isn't present, leave a useful error message // TODO: use constant fs::path defaultBootstrapFile = conf.router.data_dir / "bootstrap.signed"; + if (configRouters.empty() and conf.bootstrap.routers.empty()) { if (fs::exists(defaultBootstrapFile)) configRouters.push_back(defaultBootstrapFile); } - bootstrap_rc_list.clear(); + auto _bootstrap_rc_list = std::make_unique(); + for (const auto& router : configRouters) { log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); - bootstrap_rc_list.read_from_file(router); + _bootstrap_rc_list->read_from_file(router); } for (const auto& rc : conf.bootstrap.routers) { - bootstrap_rc_list.emplace(rc); + _bootstrap_rc_list->emplace(rc); } - if (bootstrap_rc_list.empty() and not conf.bootstrap.seednode) + if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode) { auto fallbacks = llarp::load_bootstrap_fallbacks(); - if (bootstrap_rc_list.empty() and not conf.bootstrap.seednode) + if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode) { // empty after trying fallback, if set log::error( @@ -636,10 +638,10 @@ namespace llarp // in case someone has an old bootstrap file and is trying to use a bootstrap // that no longer exists - for (auto it = bootstrap_rc_list.begin(); it != bootstrap_rc_list.end();) + for (auto it = _bootstrap_rc_list->begin(); it != _bootstrap_rc_list->end();) { if (it->is_obsolete_bootstrap()) - log::warning(logcat, "ignoring obsolete boostrap RC: {}", it->router_id()); + log::warning(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); else if (not it->verify()) log::warning(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); else @@ -649,15 +651,15 @@ namespace llarp } // we are in one of the above error cases that we warned about: - it = bootstrap_rc_list.erase(it); + it = _bootstrap_rc_list->erase(it); } - node_db()->set_bootstrap_routers(bootstrap_rc_list); + node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); if (conf.bootstrap.seednode) LogInfo("we are a seed node"); else - LogInfo("Loaded ", bootstrap_rc_list.size(), " bootstrap routers"); + LogInfo("Loaded ", _bootstrap_rc_list->size(), " bootstrap routers"); // Init components after relevant config settings loaded _link_manager.init(); @@ -703,9 +705,10 @@ namespace llarp bool Router::IsBootstrapNode(const RouterID r) const { + const auto& b = _node_db->bootstrap_list(); return std::count_if( - bootstrap_rc_list.begin(), - bootstrap_rc_list.end(), + b->begin(), + b->end(), [r](const RemoteRC& rc) -> bool { return rc.router_id() == r; }) > 0; } @@ -725,7 +728,7 @@ namespace llarp logcat, "{} RCs loaded with {} bootstrap peers and {} router connections!", node_db()->num_loaded(), - bootstrap_rc_list.size(), + _node_db->bootstrap_list()->size(), NumberOfConnectedRouters()); if (is_service_node()) @@ -851,7 +854,7 @@ namespace llarp } else { - if (needs_initial_fetch) + if (_needs_initial_fetch) { node_db()->fetch_initial(); } @@ -861,14 +864,12 @@ namespace llarp if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) { node_db()->fetch_rcs(); - last_rc_fetch = now_timepoint; } // (client-only) periodically fetch updated RouterID list if (now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) { - node_db()->fetch_router_ids(); - last_rid_fetch = now_timepoint; + node_db()->fetch_rids(); } } } @@ -946,8 +947,9 @@ namespace llarp size_t connected = NumberOfConnectedRouters(); size_t connectToNum = _link_manager.min_connected_routers; - const auto& pinned_edges = _node_db->get_pinned_edges(); + const auto& pinned_edges = _node_db->pinned_edges(); const auto pinned_count = pinned_edges.size(); + if (pinned_count > 0 && connectToNum > pinned_count) { connectToNum = pinned_count; @@ -1085,7 +1087,7 @@ namespace llarp _contacts = std::make_shared(llarp::dht::Key_t(pubkey()), *this); - for (const auto& rc : bootstrap_rc_list) + for (const auto& rc : *_node_db->bootstrap_list()) { node_db()->put_rc(rc); _contacts->rc_nodes()->PutNode(rc); diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 44e9f40a9..3a15cc85a 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -63,8 +63,6 @@ namespace llarp private: std::shared_ptr _route_poker; - /// bootstrap RCs - BootstrapList bootstrap_rc_list; std::chrono::steady_clock::time_point _next_explore_at; llarp_time_t last_pump = 0s; // transient iwp encryption key @@ -121,8 +119,6 @@ namespace llarp fs::path _profile_file; LinkManager _link_manager{*this}; - bool needs_initial_fetch{true}; - // should we be sending padded messages every interval? bool send_padding = false; @@ -146,6 +142,8 @@ namespace llarp insufficient_peers() const; protected: + bool _needs_initial_fetch{true}; + std::chrono::system_clock::time_point last_rc_gossip{ std::chrono::system_clock::time_point::min()}; std::chrono::system_clock::time_point next_rc_gossip{last_rc_gossip}; @@ -153,6 +151,18 @@ namespace llarp std::chrono::system_clock::time_point last_rid_fetch{last_rc_gossip}; public: + bool + needs_initial_fetch() const + { + return _needs_initial_fetch; + } + + void + initial_fetch_completed() + { + _needs_initial_fetch = false; + } + void for_each_connection(std::function func); From 3508dee8d84e7e5d6ba3bd0d72fe1f78da04732c Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 1 Dec 2023 09:19:07 -0800 Subject: [PATCH 08/93] trust model - greedy evaluation of returned rid's, simplifying post-processing logic to simple frequency comparison per rid against a constant threshold - tidied up link_manager request/response handling - TODO: - review and decide thresholds - evaluate necessity and potential implementation of rc comparison --- llarp/bootstrap.hpp | 18 ++- llarp/link/link_manager.cpp | 4 +- llarp/nodedb.cpp | 315 ++++++++++++++++++++---------------- llarp/nodedb.hpp | 133 +++++++++------ llarp/router_contact.hpp | 22 ++- 5 files changed, 298 insertions(+), 194 deletions(-) diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index bb1a30943..f74faa3cb 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -2,6 +2,7 @@ #include "router_contact.hpp" +#include #include #include @@ -12,6 +13,7 @@ namespace llarp struct BootstrapList final : public std::set { size_t index; + std::set::iterator current; bool bt_decode(std::string_view buf); @@ -27,16 +29,26 @@ namespace llarp // returns a reference to the next index and a boolean that equals true if // this is the front of the set - std::pair + const RemoteRC& next() { - ++index %= this->size(); - return std::make_pair(*std::next(this->begin(), index), index == 0); + ++current; + + if (current == this->end()) + current = this->begin(); + + return *current; } bool contains(const RemoteRC& rc); + void + randomize() + { + current = std::next(begin(), std::uniform_int_distribution{0, size() - 1}(csrng)); + } + void clear_list() { diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index d48cc9ba2..5b361204c 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -587,7 +587,7 @@ namespace llarp while (i < quantity) { - auto& next_rc = std::next(rcs.begin(), csrng() % rc_size)->second; + auto& next_rc = *std::next(rcs.begin(), csrng() % rc_size); if (next_rc.is_expired(now)) continue; @@ -664,7 +664,7 @@ namespace llarp if (since_time != decltype(since_time)::min()) since_time -= 5s; - for (const auto& [_, rc] : rcs) + for (const auto& rc : rcs) { if (last_time.at(rc.router_id()) > since_time or explicit_relays.count(rc.router_id())) sublist.append_encoded(rc.view()); diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index dfea702bd..08d84820e 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -53,6 +53,7 @@ namespace llarp , _next_flush_time{time_now_ms() + FLUSH_INTERVAL} { EnsureSkiplist(_root); + fetch_counters.clear(); } void @@ -68,7 +69,7 @@ namespace llarp // make copy of all rcs std::vector copy; - for (const auto& item : known_rcs) + for (const auto& item : rc_lookup) copy.push_back(item.second); // flush them to disk in one big job @@ -111,75 +112,95 @@ namespace llarp assert(_bootstraps->empty()); _bootstraps = std::move(from_router); + _bootstraps->randomize(); } bool - NodeDB::process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp) + NodeDB::process_fetched_rcs(RouterID, std::vector rcs, rc_time timestamp) { - fetch_source = source; + std::unordered_set union_set; - /* - TODO: trust model analyzing returned list of RCs - */ + // if we are not bootstrapping, we should check the rc's against the ones we currently hold + // if (not using_bootstrap_fallback) + // { + // const auto local_count = static_cast(known_rcs.size()); + // const auto& recv_count = rcs.size(); + + // std::set_intersection( + // known_rcs.begin(), + // known_rcs.end(), + // rcs.begin(), + // rcs.end(), + // std::inserter(union_set, union_set.begin())); + + // const auto union_size = static_cast(union_set.size()); + // } for (auto& rc : rcs) put_rc_if_newer(std::move(rc), timestamp); - last_rc_update_relay_timestamp = timestamp; - return true; } void - NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::unordered_set ids) + NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::unordered_set rids) { - if (ids.empty()) + if (rids.empty()) + { fail_sources.insert(source); + return; + } - fetch_rid_responses[source] = std::move(ids); + for (const auto& rid : rids) + fetch_counters[rid] += 1; } /** We only call into this function after ensuring two conditions: - 1) We have received at least 8 of 12 responses from the queried RouterID sources - 2) Of those reponses, less than 4 were errors of any sorts - - Logically, this function performs the following basic analysis of the returned RIDs. + 1) We have received all 12 responses from the queried RouterID sources, whether that + response was a timeout or not + 2) Of those responses, less than 4 were errors of any sorts + + Upon receiving each response from the rid fetch sources, the returned rid's are incremented + in fetch_counters. This greatly simplifies the analysis required by this function to the + determine success or failure: + - If the frequency of each rid is above a threshold, it is accepted + - If the number of accepted rids is below a certain amount, the set is rejected + + Logically, this function performs the following basic analysis of the returned RIDs: + 1) All responses are coalesced into a union set with no repetitions + 2) If we are bootstrapping: + - The routerID's returned */ bool NodeDB::process_fetched_rids() { std::unordered_set union_set; - for (const auto& [rid, responses] : fetch_rid_responses) + for (const auto& [rid, count] : fetch_counters) { - std::merge( - union_set.begin(), - union_set.end(), - responses.begin(), - responses.end(), - std::inserter(union_set, union_set.begin())); + if (count > MIN_RID_FETCH_FREQ) + union_set.insert(rid); } - for (const auto& [rid, responses] : fetch_rid_responses) - { - // TODO: empty == failure, handle that case - for (const auto& response : responses) - { - active_client_routers.insert(std::move(response)); - } - } + const auto num_rids = static_cast(known_rids.size()); + const auto union_size = static_cast(union_set.size()); - return true; + return (union_size / num_rids) > GOOD_RID_FETCH_THRESHOLD and union_size > MIN_RID_FETCH_TOTAL; } void NodeDB::fetch_initial() { - // Set fetch source as random selection of known active client routers - fetch_source = - *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()); - - fetch_rcs(true); + if (known_rids.empty()) + { + fallback_to_bootstrap(); + } + else + { + // Set fetch source as random selection of known active client routers + fetch_source = *std::next(known_rids.begin(), csrng() % known_rids.size()); + fetch_rcs(true); + } } void @@ -194,12 +215,12 @@ namespace llarp return; } - is_fetching_rcs = true; // TOTHINK: do these booleans do anything? + is_fetching_rcs = true; // DISCUSS: do these booleans do anything? std::vector needed; const auto now = time_point_now(); - for (const auto& [rid, rc] : known_rcs) + for (const auto& [rid, rc] : rc_lookup) { if (now - rc.timestamp() > RouterContact::OUTDATED_AGE) needed.push_back(rid); @@ -209,7 +230,7 @@ namespace llarp _router.link_manager().fetch_rcs( src, - RCFetchMessage::serialize(last_rc_update_relay_timestamp, needed), + RCFetchMessage::serialize(_router.last_rc_fetch, needed), [this, src, initial](oxen::quic::message m) mutable { if (m.timed_out) { @@ -260,7 +281,9 @@ namespace llarp } if (rid_sources.empty()) - select_router_id_sources(); + { + reselect_router_id_sources(rid_sources); + } if (not initial and rid_sources.empty()) { @@ -269,7 +292,8 @@ namespace llarp } is_fetching_rids = true; - fetch_rid_responses.clear(); + fetch_counters.clear(); + // fetch_rid_responses.clear(); RouterID& src = fetch_source; @@ -337,24 +361,31 @@ namespace llarp { if (error) { - ++fetch_failures; - - if (fetch_failures > MAX_FETCH_ATTEMPTS) + auto& fail_count = (using_bootstrap_fallback) ? bootstrap_failures : fetch_failures; + auto& THRESHOLD = + (using_bootstrap_fallback) ? MAX_BOOTSTRAP_FETCH_ATTEMPTS : MAX_FETCH_ATTEMPTS; + + // This catches three different failure cases; + // 1) bootstrap fetching and over failure threshold + // 2) bootstrap fetching and more failures to go + // 3) standard fetching and over threshold + if (++fail_count >= THRESHOLD || using_bootstrap_fallback) { log::info( logcat, - "Failed {} attempts to fetch RC's from {}; reverting to bootstrap...", - MAX_FETCH_ATTEMPTS, - fetch_source); + "RC fetching from {} reached failure threshold ({}); falling back to bootstrap...", + fetch_source, + THRESHOLD); fallback_to_bootstrap(); return; } + // If we have passed the last last conditional, then it means we are not bootstrapping + // and the current fetch_source has more attempts before being rotated. As a result, we // find new non-bootstrap RC fetch source and try again buddy - fetch_source = (initial) - ? *std::next(active_client_routers.begin(), csrng() % active_client_routers.size()) - : std::next(known_rcs.begin(), csrng() % known_rcs.size())->first; + fetch_source = (initial) ? *std::next(known_rids.begin(), csrng() % known_rids.size()) + : std::next(rc_lookup.begin(), csrng() % rc_lookup.size())->first; fetch_rcs(initial); } @@ -380,11 +411,11 @@ namespace llarp return; } - auto n_responses = fetch_rid_responses.size(); + auto n_responses = RID_SOURCE_COUNT - fail_sources.size(); - if (n_responses < ROUTER_ID_SOURCE_COUNT) + if (n_responses < RID_SOURCE_COUNT) { - log::debug(logcat, "Received {}/{} fetch RID requests", n_responses, ROUTER_ID_SOURCE_COUNT); + log::debug(logcat, "Received {}/{} fetch RID requests", n_responses, RID_SOURCE_COUNT); return; } @@ -405,7 +436,7 @@ namespace llarp log::debug( logcat, "Accumulated RID's rejected by trust model, reselecting all RID sources..."); - select_router_id_sources(rid_sources); + reselect_router_id_sources(rid_sources); ++fetch_failures; } else @@ -414,7 +445,7 @@ namespace llarp log::debug( logcat, "RID fetching found {} failures; reselecting failed RID sources...", n_fails); ++fetch_failures; - select_router_id_sources(fail_sources); + reselect_router_id_sources(fail_sources); } fetch_rids(true); @@ -434,10 +465,11 @@ namespace llarp NodeDB::post_fetch_rids(bool initial) { is_fetching_rids = false; - fetch_rid_responses.clear(); + // fetch_rid_responses.clear(); fail_sources.clear(); fetch_failures = 0; _router.last_rid_fetch = llarp::time_point_now(); + fetch_counters.clear(); if (initial) _router.initial_fetch_completed(); @@ -446,26 +478,40 @@ namespace llarp void NodeDB::fallback_to_bootstrap() { - if (bootstrap_failures >= MAX_BOOTSTRAP_FETCH_ATTEMPTS) + auto at_max_failures = bootstrap_failures >= MAX_BOOTSTRAP_FETCH_ATTEMPTS; + + // base case: we have failed to query a bootstrap relay 3 times, or we received a + // sample of the network, but the sample was unusable or unreachable (immediately + // counts as 3 strikes) + // We will also enter this if we are on our first fallback to bootstrap so we can + // set the fetch_source (by checking not using_bootstrap_fallback) + if (at_max_failures || not using_bootstrap_fallback) { - log::info(logcat, "Current bootstrap failed... cycling to next bootstrap..."); + if (at_max_failures) + log::error(logcat, "All bootstraps failed... reattempting in {}...", REBOOTSTRAP_INTERVAL); bootstrap_failures = 0; - auto [rc, is_front] = _bootstraps->next(); + auto rc = _bootstraps->next(); - // Base case: if we have returned to the front of the bootstrap list, we're in a - // bad spot - if (using_bootstrap_fallback && is_front) + // Fail case: if we have returned to the front of the bootstrap list, we're in a + // bad spot; we are unable to do anything + if (using_bootstrap_fallback) { auto err = fmt::format("ERROR: ALL BOOTSTRAPS ARE BAD"); log::error(logcat, err); - throw std::runtime_error{err}; + + /* + TODO: + trigger bootstrap cooldown timer in router + */ } - using_bootstrap_fallback = true; fetch_source = rc.router_id(); } + // By passing the last conditional, we ensure this is set to true + using_bootstrap_fallback = true; + _router.link_manager().fetch_bootstrap_rcs( fetch_source, BootstrapFetchMessage::serialize(BOOTSTRAP_SOURCE_COUNT), @@ -473,6 +519,12 @@ namespace llarp if (not m) { ++bootstrap_failures; + log::warning( + logcat, + "BootstrapRC fetch request to {} failed (error {}/{})", + fetch_source, + bootstrap_failures, + MAX_BOOTSTRAP_FETCH_ATTEMPTS); fallback_to_bootstrap(); return; } @@ -495,65 +547,47 @@ namespace llarp } catch (const std::exception& e) { - log::info( + ++bootstrap_failures; + log::warning( logcat, - "Failed to parse BootstrapRC fetch response from {}: {}", + "Failed to parse BootstrapRC fetch response from {} (error {}/{}): {}", fetch_source, + bootstrap_failures, + MAX_BOOTSTRAP_FETCH_ATTEMPTS, e.what()); - ++bootstrap_failures; fallback_to_bootstrap(); return; } - rid_sources.swap(rids); - // if this result is bad, we won't try this bootstrap again + // We set this to the max allowable value because if this result is bad, we won't + // try this bootstrap again. If this result is undersized, we roll right into the + // next call to fallback_to_bootstrap() and hit the base case, rotating sources bootstrap_failures = MAX_BOOTSTRAP_FETCH_ATTEMPTS; - using_bootstrap_fallback = false; - fetch_initial(); + + if (rids.size() == BOOTSTRAP_SOURCE_COUNT) + { + known_rids.swap(rids); + fetch_initial(); + } + else + { + ++bootstrap_failures; + log::warning( + logcat, + "BootstrapRC fetch response from {} returned insufficient number of RC's (error " + "{}/{})", + fetch_source, + bootstrap_failures, + MAX_BOOTSTRAP_FETCH_ATTEMPTS); + fallback_to_bootstrap(); + } }); } void - NodeDB::select_router_id_sources(std::unordered_set excluded) + NodeDB::reselect_router_id_sources(std::unordered_set specific) { - // bootstrapping should be finished before this is called, so this - // shouldn't happen; need to make sure that's the case. - if (active_client_routers.empty()) - return; - - // in case we pass the entire list - std::unordered_set temp = rid_sources; - - // keep using any we've been using, but remove `excluded` ones - if (excluded == rid_sources) - temp.clear(); - else - { - for (const auto& r : excluded) - temp.erase(r); - } - - // only know so many routers, so no need to randomize - if (active_client_routers.size() <= (ROUTER_ID_SOURCE_COUNT + excluded.size())) - { - for (const auto& r : active_client_routers) - { - if (excluded.count(r)) - continue; - temp.insert(r); - } - } - - // select at random until we have chosen enough - while (temp.size() < ROUTER_ID_SOURCE_COUNT) - { - RouterID r; - std::sample(active_client_routers.begin(), active_client_routers.end(), &r, 1, csrng); - if (excluded.count(r) == 0) - temp.insert(r); - } - - rid_sources.swap(temp); + replace_subset(rid_sources, specific, known_rids, RID_SOURCE_COUNT, csrng); } void @@ -598,9 +632,7 @@ namespace llarp { if (_pinned_edges.size() && _pinned_edges.count(remote) == 0 && not _bootstraps->contains(remote)) - { return false; - } if (not _router.is_service_node()) return true; @@ -613,6 +645,7 @@ namespace llarp { if (_pinned_edges.size() && _pinned_edges.count(remote) == 0) return false; + return true; } @@ -630,6 +663,7 @@ namespace llarp { if (!ch) continue; + std::string p; p += ch; fs::path sub = _root / p; @@ -657,11 +691,9 @@ namespace llarp const auto& rid = rc.router_id(); - known_rcs.emplace(rid, rc); - // TODO: the list of relays should be maintained and stored separately from - // the RCs, as we keep older RCs around in case we go offline and need to - // bootstrap, but they shouldn't be in the "good relays" list. - active_client_routers.insert(rid); + auto [itr, b] = known_rcs.emplace(std::move(rc)); + rc_lookup.emplace(rid, *itr); + known_rids.insert(rid); return true; }); @@ -683,33 +715,33 @@ namespace llarp return; _router.loop()->call([this]() { - for (const auto& item : known_rcs) - item.second.write(get_path_by_pubkey(item.first)); + for (const auto& rc : rc_lookup) + { + rc.second.write(get_path_by_pubkey(rc.first)); + } }); } bool NodeDB::has_rc(RouterID pk) const { - return known_rcs.count(pk); + return rc_lookup.count(pk); } std::optional NodeDB::get_rc(RouterID pk) const { - const auto itr = known_rcs.find(pk); + if (auto itr = rc_lookup.find(pk); itr != rc_lookup.end()) + return itr->second; - if (itr == known_rcs.end()) - return std::nullopt; - - return itr->second; + return std::nullopt; } void NodeDB::remove_router(RouterID pk) { _router.loop()->call([this, pk]() { - known_rcs.erase(pk); + rc_lookup.erase(pk); remove_many_from_disk_async({pk}); }); } @@ -721,12 +753,13 @@ namespace llarp cutoff_time -= _router.is_service_node() ? RouterContact::OUTDATED_AGE : RouterContact::LIFETIME; - for (auto itr = known_rcs.begin(); itr != known_rcs.end();) + + for (auto itr = rc_lookup.begin(); itr != rc_lookup.end();) { if (cutoff_time > itr->second.timestamp()) { log::info(logcat, "Pruning RC for {}, as it is too old to keep.", itr->first); - known_rcs.erase(itr); + rc_lookup.erase(itr); continue; } itr++; @@ -737,10 +770,16 @@ namespace llarp NodeDB::put_rc(RemoteRC rc, rc_time now) { const auto& rid = rc.router_id(); + if (not want_rc(rid)) return false; - known_rcs.erase(rid); - known_rcs.emplace(rid, std::move(rc)); + + rc_lookup.erase(rid); + + auto [itr, b] = known_rcs.emplace(std::move(rc)); + rc_lookup.emplace(rid, *itr); + known_rids.insert(rid); + last_rc_update_times[rid] = now; return true; } @@ -754,11 +793,10 @@ namespace llarp bool NodeDB::put_rc_if_newer(RemoteRC rc, rc_time now) { - auto itr = known_rcs.find(rc.router_id()); - if (itr == known_rcs.end() or itr->second.other_is_newer(rc)) - { + if (auto itr = rc_lookup.find(rc.router_id()); + itr == rc_lookup.end() or itr->second.other_is_newer(rc)) return put_rc(std::move(rc), now); - } + return false; } @@ -767,12 +805,13 @@ namespace llarp { if (_root.empty()) return; + // build file list std::set files; + for (auto id : remove) - { files.emplace(get_path_by_pubkey(std::move(id))); - } + // remove them from the disk via the diskio thread _disk([files]() { for (auto fpath : files) @@ -807,12 +846,14 @@ namespace llarp std::vector all; all.reserve(known_rcs.size()); - for (auto& entry : known_rcs) + + for (auto& entry : rc_lookup) { all.push_back(&entry.second); } auto it_mid = numRouters < all.size() ? all.begin() + numRouters : all.end(); + std::partial_sort( all.begin(), it_mid, all.end(), [compare = dht::XorMetric{location}](auto* a, auto* b) { return compare(*a, *b); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index a23d02904..2a361f1d7 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -12,9 +12,9 @@ #include #include +#include #include #include -#include #include #include @@ -22,13 +22,24 @@ namespace llarp { struct Router; - inline constexpr size_t ROUTER_ID_SOURCE_COUNT{12}; - inline constexpr size_t MIN_RID_FETCHES{8}; + inline constexpr size_t RID_SOURCE_COUNT{12}; + inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; + inline constexpr size_t MIN_ACTIVE_RIDS{24}; - inline constexpr size_t MAX_RID_ERRORS{ROUTER_ID_SOURCE_COUNT - MIN_RID_FETCHES}; + + inline constexpr size_t MAX_RID_ERRORS{4}; + + // when fetching rids, each returned rid must appear this number of times across + inline constexpr int MIN_RID_FETCH_FREQ{6}; + // when fetching rids, the total number of accepted returned rids should be above this number + inline constexpr int MIN_RID_FETCH_TOTAL{}; + // when fetching rids, the ratio of accepted:rejected rids must be above this ratio + inline constexpr double GOOD_RID_FETCH_THRESHOLD{}; + inline constexpr int MAX_FETCH_ATTEMPTS{10}; - inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{3}; - inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; + inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{5}; + + inline constexpr auto REBOOTSTRAP_INTERVAL{1min}; inline constexpr auto FLUSH_INTERVAL{5min}; @@ -44,14 +55,18 @@ namespace llarp /** RouterID mappings Both the following are populated in NodeDB startup with RouterID's stored on disk. - - active_client_routers: meant to persist between lokinet sessions, and is only + - known_rids: meant to persist between lokinet sessions, and is only populated during startup and RouterID fetching. This is meant to represent the - client instance's perspective of the network and which RouterID's are "active" + client instance's most recent perspective of the network, and record which RouterID's + were recently "active" and connected to - known_rcs: populated during startup and when RC's are updated both during gossip and periodic RC fetching + - rc_lookup: holds all the same rc's as known_rcs, but can be used to look them up by + their rid. Deleting an rid key deletes the corresponding rc in known_rcs */ - std::unordered_set active_client_routers; - std::unordered_map known_rcs; + std::unordered_set known_rids; + std::unordered_set known_rcs; + std::unordered_map rc_lookup; /** RouterID lists - white: active routers @@ -64,10 +79,9 @@ namespace llarp // All registered relays (service nodes) std::unordered_set registered_routers; - // timing + // timing (note: Router holds the variables for last rc and rid request times) std::unordered_map last_rc_update_times; - rc_time last_rc_update_relay_timestamp; - // only ever use to specific edges as path first-hops + // if populated from a config file, lists specific exclusively used as path first-hops std::unordered_set _pinned_edges; // source of "truth" for RC updating. This relay will also mediate requests to the // 12 selected active RID's for RID fetching @@ -76,8 +90,11 @@ namespace llarp std::unordered_set rid_sources{}; // logs the RID's that resulted in an error during RID fetching std::unordered_set fail_sources{}; + // tracks the number of times each rid appears in the above responses + std::unordered_map fetch_counters{}; // stores all RID fetch responses for greedy comprehensive processing - std::unordered_map> fetch_rid_responses; + // std::unordered_map> fetch_rid_responses; + /** Failure counters: - fetch_failures: tracks errors fetching RC's from the RC node and requesting RID's from the 12 RID sources. Errors in the individual RID sets are NOT counted towards @@ -142,30 +159,11 @@ namespace llarp void fallback_to_bootstrap(); + // Populate rid_sources with random sample from known_rids. A set of rids is passed + // if only specific RID's need to be re-selected; to re-select all, pass the member + // variable ::known_rids void - select_router_id_sources(std::unordered_set excluded = {}); - - // /// If we receive a bad set of RCs from our current RC source relay, we consider - // /// that relay to be a bad source of RCs and we randomly choose a new one. - // /// - // /// When using a new RC fetch relay, we first re-fetch the full RC list and, if - // /// that aligns with our RouterID list, we go back to periodic updates from that relay. - // /// - // /// This will respect edge-pinning and attempt to use a relay we already have - // /// a connection with. - // void - // rotate_rc_source(); - - // /// This function is called during startup and initial fetching. When a lokinet client - // /// instance performs its initial RC/RID fetching, it may need to randomly select a - // /// node from its list of stale RC's to relay its requests. If there is a failure in - // /// mediating these request, the client will randomly select another RC source - // /// - // /// Returns: - // /// true - a new startup RC source was selected - // /// false - a new startup RC source was NOT selected - // bool - // rotate_startup_rc_source(); + reselect_router_id_sources(std::unordered_set specific); void set_router_whitelist( @@ -236,12 +234,18 @@ namespace llarp return registered_routers; } - const std::unordered_map& + const std::unordered_set& get_rcs() const { return known_rcs; } + // const std::unordered_map& + // get_rcs() const + // { + // return known_rcs; + // } + const std::unordered_map& get_last_rc_update_times() const { @@ -284,23 +288,49 @@ namespace llarp std::optional GetRandom(Filter visit) const { - return _router.loop()->call_get([visit]() -> std::optional { - std::vector known_rcs; - for (const auto& entry : known_rcs) - known_rcs.push_back(entry); + return _router.loop()->call_get([visit, this]() mutable -> std::optional { + std::vector rcs{known_rcs.begin(), known_rcs.end()}; - std::shuffle(known_rcs.begin(), known_rcs.end(), llarp::csrng); + std::shuffle(rcs.begin(), rcs.end(), llarp::csrng); - for (const auto entry : known_rcs) + for (const auto& entry : known_rcs) { - if (visit(entry->second)) - return entry->second; + if (visit(entry)) + return entry; } return std::nullopt; }); } + // Updates `current` to not contain any of the elements of `replace` and resamples (up to + // `target_size`) from population to refill it. + template + void + replace_subset( + std::unordered_set& current, + const std::unordered_set& replace, + std::unordered_set population, + size_t target_size, + RNG&& rng) + { + // Remove the ones we are replacing from current: + current.erase(replace.begin(), replace.end()); + + // Remove ones we are replacing, and ones we already have, from the population so that we + // won't reselect them: + population.erase(replace.begin(), replace.end()); + population.erase(current.begin(), current.end()); + + if (current.size() < target_size) + std::sample( + population.begin(), + population.end(), + std::inserter(current, current.end()), + target_size - current.size(), + rng); + } + /// visit all known_rcs template void @@ -308,7 +338,7 @@ namespace llarp { _router.loop()->call([this, visit]() { for (const auto& item : known_rcs) - visit(item.second); + visit(item); }); } @@ -323,17 +353,18 @@ namespace llarp { _router.loop()->call([this, visit]() { std::unordered_set removed; - auto itr = known_rcs.begin(); - while (itr != known_rcs.end()) + + for (auto itr = rc_lookup.begin(); itr != rc_lookup.end();) { if (visit(itr->second)) { - removed.insert(itr->second.router_id()); - itr = known_rcs.erase(itr); + removed.insert(itr->first); + itr = rc_lookup.erase(itr); } else ++itr; } + if (not removed.empty()) remove_many_from_disk_async(std::move(removed)); }); diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index c7a04cd89..e3cb4ca1a 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -358,10 +358,30 @@ namespace std template <> struct hash { - size_t + virtual size_t operator()(const llarp::RouterContact& r) const { return std::hash{}(r.router_id()); } }; + + template <> + struct hash final : public hash + { + size_t + operator()(const llarp::RouterContact& r) const override + { + return std::hash{}(r.router_id()); + } + }; + + template <> + struct hash final : public hash + { + size_t + operator()(const llarp::RouterContact& r) const override + { + return std::hash{}(r.router_id()); + } + }; } // namespace std From 70e9c1ae25457401332afe313572603bf45270c7 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 4 Dec 2023 07:27:42 -0800 Subject: [PATCH 09/93] trust model fetch fails - bootstrap cooldown implemented with 1min timer in case all bootstraps fail - set comparison implemented in non-initial and non-bootstrap rc fetching; set comparison in rid fetching is done every fetch - nodedb get_random functions refactored into conditional/non-conditional methods. Conditional search implements reservoir sampling for one-pass accumulation of n random rcs --- llarp/link/link_manager.cpp | 27 ++--- llarp/nodedb.cpp | 224 ++++++++++++++++++++++++++++-------- llarp/nodedb.hpp | 89 +++++++++++--- llarp/path/pathbuilder.cpp | 11 +- llarp/router/router.cpp | 21 +++- llarp/router/router.hpp | 15 +-- llarp/service/endpoint.cpp | 18 +-- 7 files changed, 297 insertions(+), 108 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 5b361204c..f8d999ef0 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -492,23 +492,20 @@ namespace llarp std::set exclude; auto remainder = num_conns; - do - { - auto filter = [exclude](const auto& rc) -> bool { - return exclude.count(rc.router_id()) == 0; - }; - - if (auto maybe_other = node_db->GetRandom(filter)) - { - exclude.insert(maybe_other->router_id()); + auto filter = [exclude](const RemoteRC& rc) -> bool { + return exclude.count(rc.router_id()) == 0; + }; - if (not node_db->is_connection_allowed(maybe_other->router_id())) - continue; + if (auto maybe = node_db->get_n_random_rcs_conditional(remainder, filter)) + { + std::vector& rcs = *maybe; - connect_to(*maybe_other); - --remainder; - } - } while (remainder > 0); + for (const auto& rc : rcs) + connect_to(rc); + } + else + log::warning( + logcat, "NodeDB query for {} random RCs for connection returned none", num_conns); } void diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 08d84820e..fb763b7fa 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -56,6 +56,83 @@ namespace llarp fetch_counters.clear(); } + std::optional + NodeDB::get_random_rc() const + { + std::optional rand = std::nullopt; + + std::sample(known_rcs.begin(), known_rcs.end(), &*rand, 1, csrng); + return rand; + } + + std::optional> + NodeDB::get_n_random_rcs(size_t n) const + { + std::vector rand{}; + + std::sample(known_rcs.begin(), known_rcs.end(), std::back_inserter(rand), n, csrng); + return rand.empty() ? std::nullopt : std::make_optional(rand); + } + + std::optional + NodeDB::get_random_rc_conditional(std::function hook) const + { + std::optional rand = get_random_rc(); + + if (rand and hook(*rand)) + return rand; + + size_t i = 0; + + for (const auto& rc : known_rcs) + { + if (not hook(rc)) + continue; + + if (++i <= 1) + { + rand = rc; + continue; + } + + size_t x = csrng() % (i + 1); + if (x <= 1) + rand = rc; + } + + return rand; + } + + std::optional> + NodeDB::get_n_random_rcs_conditional(size_t n, std::function hook) const + { + std::vector selected; + selected.reserve(n); + + size_t i = 0; + + for (const auto& rc : known_rcs) + { + // ignore any RC's that do not pass the condition + if (not hook(rc)) + continue; + + // load the first n RC's that pass the condition into selected + if (++i <= n) + { + selected.push_back(rc); + continue; + } + + // replace selections with decreasing probability per iteration + size_t x = csrng() % (i + 1); + if (x < n) + selected[x] = rc; + } + + return selected.size() == n ? std::make_optional(selected) : std::nullopt; + } + void NodeDB::Tick(llarp_time_t now) { @@ -116,25 +193,43 @@ namespace llarp } bool - NodeDB::process_fetched_rcs(RouterID, std::vector rcs, rc_time timestamp) + NodeDB::process_fetched_rcs(std::vector& rcs) { - std::unordered_set union_set; + std::unordered_set inter_set; - // if we are not bootstrapping, we should check the rc's against the ones we currently hold - // if (not using_bootstrap_fallback) - // { - // const auto local_count = static_cast(known_rcs.size()); - // const auto& recv_count = rcs.size(); + std::set_intersection( + known_rcs.begin(), + known_rcs.end(), + rcs.begin(), + rcs.end(), + std::inserter(inter_set, inter_set.begin())); - // std::set_intersection( - // known_rcs.begin(), - // known_rcs.end(), - // rcs.begin(), - // rcs.end(), - // std::inserter(union_set, union_set.begin())); + // the total number of rcs received + const auto num_received = static_cast(rcs.size()); + // the number of returned "good" rcs (that are also found locally) + const auto inter_size = inter_set.size(); + // the number of rcs currently held locally + const auto local_count = static_cast(known_rcs.size()); + + const auto fetch_threshold = (double)inter_size / num_received; + const auto local_alignment = (double)inter_size / local_count; + + /** We are checking 3 things here: + 1) The number of "good" rcs is above MIN_GOOD_RC_FETCH_TOTAL + 2) The ratio of "good" rcs to total received is above MIN_GOOD_RC_FETCH_THRESHOLD + 3) The ratio of received and found locally to total found locally is above + LOCAL_RC_ALIGNMENT_THRESHOLD + */ + return inter_size > MIN_GOOD_RC_FETCH_TOTAL and fetch_threshold > MIN_GOOD_RC_FETCH_THRESHOLD + and local_alignment > LOCAL_RC_ALIGNMENT_THRESHOLD; + } - // const auto union_size = static_cast(union_set.size()); - // } + bool + NodeDB::ingest_fetched_rcs(std::vector rcs, rc_time timestamp) + { + // if we are not bootstrapping, we should check the rc's against the ones we currently hold + if (not _using_bootstrap_fallback) + {} for (auto& rc : rcs) put_rc_if_newer(std::move(rc), timestamp); @@ -174,7 +269,7 @@ namespace llarp bool NodeDB::process_fetched_rids() { - std::unordered_set union_set; + std::unordered_set union_set, intersection_set; for (const auto& [rid, count] : fetch_counters) { @@ -182,10 +277,39 @@ namespace llarp union_set.insert(rid); } - const auto num_rids = static_cast(known_rids.size()); - const auto union_size = static_cast(union_set.size()); - - return (union_size / num_rids) > GOOD_RID_FETCH_THRESHOLD and union_size > MIN_RID_FETCH_TOTAL; + // get the intersection of accepted rids and local rids + std::set_intersection( + known_rids.begin(), + known_rids.end(), + union_set.begin(), + union_set.end(), + std::inserter(intersection_set, intersection_set.begin())); + + // the total number of rids received + const auto num_received = (double)fetch_counters.size(); + // the total number of received AND accepted rids + const auto union_size = union_set.size(); + // the number of rids currently held locally + const auto local_count = (double)known_rids.size(); + // the number of accepted rids that are also found locally + const auto inter_size = (double)intersection_set.size(); + + const auto fetch_threshold = (double)union_size / num_received; + const auto local_alignment = (double)inter_size / local_count; + + /** We are checking 2, potentially 3 things here: + 1) The ratio of received/accepted to total received is above GOOD_RID_FETCH_THRESHOLD. + This tells us how well the rid source's sets of rids "agree" with one another + 2) The total number received is above MIN_RID_FETCH_TOTAL. This ensures that we are + receiving a sufficient amount to make a comparison of any sorts + 3) If we are not bootstrapping, then the ratio of received/accepted found locally to + the total number locally held is above LOCAL_RID_ALIGNMENT_THRESHOLD. This gives us + an estimate of how "aligned" the rid source's set of rid's is to ours + */ + return (fetch_threshold > GOOD_RID_FETCH_THRESHOLD) and (union_size > MIN_GOOD_RID_FETCH_TOTAL) + and (not _using_bootstrap_fallback) + ? local_alignment > LOCAL_RID_ALIGNMENT_THRESHOLD + : true; } void @@ -215,8 +339,6 @@ namespace llarp return; } - is_fetching_rcs = true; // DISCUSS: do these booleans do anything? - std::vector needed; const auto now = time_point_now(); @@ -259,7 +381,7 @@ namespace llarp rcs.emplace_back(btlc.consume_dict_consumer()); // if process_fetched_rcs returns false, then the trust model rejected the fetched RC's - fetch_rcs_result(initial, not process_fetched_rcs(src, std::move(rcs), timestamp)); + fetch_rcs_result(initial, not ingest_fetched_rcs(std::move(rcs), timestamp)); } catch (const std::exception& e) { @@ -288,12 +410,11 @@ namespace llarp if (not initial and rid_sources.empty()) { log::error(logcat, "Attempting to fetch RouterIDs, but have no source from which to do so."); + fallback_to_bootstrap(); return; } - is_fetching_rids = true; fetch_counters.clear(); - // fetch_rid_responses.clear(); RouterID& src = fetch_source; @@ -361,15 +482,15 @@ namespace llarp { if (error) { - auto& fail_count = (using_bootstrap_fallback) ? bootstrap_failures : fetch_failures; + auto& fail_count = (_using_bootstrap_fallback) ? bootstrap_failures : fetch_failures; auto& THRESHOLD = - (using_bootstrap_fallback) ? MAX_BOOTSTRAP_FETCH_ATTEMPTS : MAX_FETCH_ATTEMPTS; + (_using_bootstrap_fallback) ? MAX_BOOTSTRAP_FETCH_ATTEMPTS : MAX_FETCH_ATTEMPTS; // This catches three different failure cases; // 1) bootstrap fetching and over failure threshold // 2) bootstrap fetching and more failures to go // 3) standard fetching and over threshold - if (++fail_count >= THRESHOLD || using_bootstrap_fallback) + if (++fail_count >= THRESHOLD || _using_bootstrap_fallback) { log::info( logcat, @@ -454,7 +575,6 @@ namespace llarp void NodeDB::post_fetch_rcs(bool initial) { - is_fetching_rcs = false; _router.last_rc_fetch = llarp::time_point_now(); if (initial) @@ -464,15 +584,14 @@ namespace llarp void NodeDB::post_fetch_rids(bool initial) { - is_fetching_rids = false; - // fetch_rid_responses.clear(); fail_sources.clear(); fetch_failures = 0; _router.last_rid_fetch = llarp::time_point_now(); fetch_counters.clear(); + _needs_rebootstrap = false; if (initial) - _router.initial_fetch_completed(); + _needs_initial_fetch = false; } void @@ -480,37 +599,33 @@ namespace llarp { auto at_max_failures = bootstrap_failures >= MAX_BOOTSTRAP_FETCH_ATTEMPTS; - // base case: we have failed to query a bootstrap relay 3 times, or we received a - // sample of the network, but the sample was unusable or unreachable (immediately - // counts as 3 strikes) - // We will also enter this if we are on our first fallback to bootstrap so we can - // set the fetch_source (by checking not using_bootstrap_fallback) - if (at_max_failures || not using_bootstrap_fallback) + // base case: we have failed to query all bootstraps, or we received a sample of + // the network, but the sample was unusable or unreachable. We will also enter this + // if we are on our first fallback to bootstrap so we can set the fetch_source (by + // checking not using_bootstrap_fallback) + if (at_max_failures || not _using_bootstrap_fallback) { - if (at_max_failures) - log::error(logcat, "All bootstraps failed... reattempting in {}...", REBOOTSTRAP_INTERVAL); - bootstrap_failures = 0; - auto rc = _bootstraps->next(); // Fail case: if we have returned to the front of the bootstrap list, we're in a // bad spot; we are unable to do anything - if (using_bootstrap_fallback) + if (_using_bootstrap_fallback) { - auto err = fmt::format("ERROR: ALL BOOTSTRAPS ARE BAD"); + auto err = fmt::format( + "ERROR: ALL BOOTSTRAPS ARE BAD... REATTEMPTING IN {}...", BOOTSTRAP_COOLDOWN); log::error(logcat, err); - /* - TODO: - trigger bootstrap cooldown timer in router - */ + bootstrap_cooldown(); + return; } + auto rc = _bootstraps->next(); fetch_source = rc.router_id(); } // By passing the last conditional, we ensure this is set to true - using_bootstrap_fallback = true; + _using_bootstrap_fallback = true; + _needs_rebootstrap = false; _router.link_manager().fetch_bootstrap_rcs( fetch_source, @@ -584,6 +699,13 @@ namespace llarp }); } + void + NodeDB::bootstrap_cooldown() + { + _needs_rebootstrap = true; + _router.next_bootstrap_attempt = llarp::time_point_now() + BOOTSTRAP_COOLDOWN; + } + void NodeDB::reselect_router_id_sources(std::unordered_set specific) { @@ -759,6 +881,7 @@ namespace llarp if (cutoff_time > itr->second.timestamp()) { log::info(logcat, "Pruning RC for {}, as it is too old to keep.", itr->first); + known_rcs.erase(itr->second); rc_lookup.erase(itr); continue; } @@ -774,6 +897,7 @@ namespace llarp if (not want_rc(rid)) return false; + known_rcs.erase(rc); rc_lookup.erase(rid); auto [itr, b] = known_rcs.emplace(std::move(rc)); @@ -823,7 +947,7 @@ namespace llarp NodeDB::find_closest_to(llarp::dht::Key_t location) const { return _router.loop()->call_get([this, location]() -> RemoteRC { - RemoteRC rc; + RemoteRC rc{}; const llarp::dht::XorMetric compare(location); VisitAll([&rc, compare](const auto& otherRC) { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 2a361f1d7..d476298f3 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -22,24 +22,38 @@ namespace llarp { struct Router; - inline constexpr size_t RID_SOURCE_COUNT{12}; - inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; - + /* RC Fetch Constants */ + // max number of attempts we make in non-bootstrap fetch requests + inline constexpr int MAX_FETCH_ATTEMPTS{10}; + // the total number of returned rcs that are held locally should be at least this + inline constexpr size_t MIN_GOOD_RC_FETCH_TOTAL{}; + // the ratio of returned rcs found locally to to total returned should be above this ratio + inline constexpr double MIN_GOOD_RC_FETCH_THRESHOLD{}; + // the ratio of returned rcs that are found locally to total held locally should above this ratio + inline constexpr double LOCAL_RC_ALIGNMENT_THRESHOLD{}; + + /* RID Fetch Constants */ inline constexpr size_t MIN_ACTIVE_RIDS{24}; - + // the number of rid sources that we make rid fetch requests to + inline constexpr size_t RID_SOURCE_COUNT{12}; + // upper limit on how many rid fetch requests to rid sources can fail inline constexpr size_t MAX_RID_ERRORS{4}; - - // when fetching rids, each returned rid must appear this number of times across + // each returned rid must appear this number of times across all responses inline constexpr int MIN_RID_FETCH_FREQ{6}; - // when fetching rids, the total number of accepted returned rids should be above this number - inline constexpr int MIN_RID_FETCH_TOTAL{}; - // when fetching rids, the ratio of accepted:rejected rids must be above this ratio + // the total number of accepted returned rids should be above this number + inline constexpr size_t MIN_GOOD_RID_FETCH_TOTAL{}; + // the ratio of accepted:rejected rids must be above this ratio inline constexpr double GOOD_RID_FETCH_THRESHOLD{}; + // if we are not bootstrapping, the ratio of accepted:local rids must be above this ratio + inline constexpr double LOCAL_RID_ALIGNMENT_THRESHOLD{}; - inline constexpr int MAX_FETCH_ATTEMPTS{10}; + /* Bootstrap Constants */ + // the number of rc's we query the bootstrap for + inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; + // the maximum number of fetch requests we make across all bootstraps inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{5}; - - inline constexpr auto REBOOTSTRAP_INTERVAL{1min}; + // if all bootstraps fail, router will trigger re-bootstrapping after this cooldown + inline constexpr auto BOOTSTRAP_COOLDOWN{1min}; inline constexpr auto FLUSH_INTERVAL{5min}; @@ -66,7 +80,7 @@ namespace llarp */ std::unordered_set known_rids; std::unordered_set known_rcs; - std::unordered_map rc_lookup; + std::unordered_map rc_lookup; // TODO: look into this again /** RouterID lists - white: active routers @@ -92,8 +106,6 @@ namespace llarp std::unordered_set fail_sources{}; // tracks the number of times each rid appears in the above responses std::unordered_map fetch_counters{}; - // stores all RID fetch responses for greedy comprehensive processing - // std::unordered_map> fetch_rid_responses; /** Failure counters: - fetch_failures: tracks errors fetching RC's from the RC node and requesting RID's @@ -104,8 +116,8 @@ namespace llarp */ std::atomic fetch_failures{0}, bootstrap_failures{0}; - std::atomic is_fetching_rids{false}, is_fetching_rcs{false}, - using_bootstrap_fallback{false}; + std::atomic _using_bootstrap_fallback{false}, _needs_rebootstrap{false}, + _needs_initial_fetch{false}; bool want_rc(const RouterID& rid) const; @@ -128,7 +140,22 @@ namespace llarp NodeDB(); bool - process_fetched_rcs(RouterID source, std::vector rcs, rc_time timestamp); + needs_initial_fetch() const + { + return _needs_initial_fetch; + } + + bool + needs_rebootstrap() const + { + return _needs_rebootstrap; + } + + bool + ingest_fetched_rcs(std::vector rcs, rc_time timestamp); + + bool + process_fetched_rcs(std::vector& rcs); void ingest_rid_fetch_responses(const RouterID& source, std::unordered_set ids = {}); @@ -158,6 +185,8 @@ namespace llarp // Bootstrap fallback void fallback_to_bootstrap(); + void + bootstrap_cooldown(); // Populate rid_sources with random sample from known_rids. A set of rids is passed // if only specific RID's need to be re-selected; to re-select all, pass the member @@ -284,6 +313,29 @@ namespace llarp std::optional get_rc(RouterID pk) const; + std::optional + get_random_rc() const; + + std::optional> + get_n_random_rcs(size_t n) const; + + /** The following random conditional functions utilize a simple implementation of reservoir + sampling to return either 1 or n random RC's using only one pass through the set of RC's. + + Pseudocode: + - begin iterating through the set + - load the first n (or 1) that pass hook(n) into a list Selected[] + - for all that pass the hook, increment i, tracking the number seen thus far + - generate a random integer x from 0 to i + - x < n ? Selected[x] = current : continue; + */ + + std::optional + get_random_rc_conditional(std::function hook) const; + + std::optional> + get_n_random_rcs_conditional(size_t n, std::function hook) const; + template std::optional GetRandom(Filter visit) const @@ -359,6 +411,7 @@ namespace llarp if (visit(itr->second)) { removed.insert(itr->first); + known_rcs.erase(itr->second); itr = rc_lookup.erase(itr); } else diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index 49b080cd0..9ba1392c8 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -244,13 +244,13 @@ namespace llarp std::optional> Builder::GetHopsForBuild() { - auto filter = [r = router](const auto& rc) -> bool { + auto filter = [r = router](const RemoteRC& rc) -> bool { return not r->router_profiling().IsBadForPath(rc.router_id(), 1); }; - if (const auto maybe = router->node_db()->GetRandom(filter)) - { + + if (auto maybe = router->node_db()->get_random_rc_conditional(filter)) return GetHopsAlignedToForBuild(maybe->router_id()); - } + return std::nullopt; } @@ -359,6 +359,7 @@ namespace llarp if (r->router_profiling().IsBadForPath(rid, 1)) return false; + for (const auto& hop : hopsSet) { if (hop.router_id() == rid) @@ -373,7 +374,7 @@ namespace llarp return rc.router_id() != endpointRC.router_id(); }; - if (const auto maybe = router->node_db()->GetRandom(filter)) + if (auto maybe = router->node_db()->get_random_rc_conditional(filter)) hops.emplace_back(*maybe); else return std::nullopt; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index cc24a8dc9..4d2374631 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -193,6 +193,18 @@ namespace llarp return stats; } + bool + Router::needs_initial_fetch() const + { + return _node_db->needs_initial_fetch(); + } + + bool + Router::needs_rebootstrap() const + { + return _node_db->needs_rebootstrap(); + } + void Router::Freeze() { @@ -235,10 +247,11 @@ namespace llarp return node_db()->get_random_whitelist_router(); } - if (auto maybe = node_db()->GetRandom([](const auto&) -> bool { return true; })) + if (auto maybe = node_db()->get_random_rc()) { return maybe->router_id(); } + return std::nullopt; } @@ -854,10 +867,14 @@ namespace llarp } else { - if (_needs_initial_fetch) + if (needs_initial_fetch()) { node_db()->fetch_initial(); } + else if (needs_rebootstrap() and next_bootstrap_attempt > now_timepoint) + { + node_db()->fallback_to_bootstrap(); + } else { // (client-only) periodically fetch updated RCs diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 3a15cc85a..ac1935d8f 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -142,26 +142,21 @@ namespace llarp insufficient_peers() const; protected: - bool _needs_initial_fetch{true}; + // bool _needs_initial_fetch{true}; std::chrono::system_clock::time_point last_rc_gossip{ std::chrono::system_clock::time_point::min()}; std::chrono::system_clock::time_point next_rc_gossip{last_rc_gossip}; std::chrono::system_clock::time_point last_rc_fetch{last_rc_gossip}; std::chrono::system_clock::time_point last_rid_fetch{last_rc_gossip}; + std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip}; public: bool - needs_initial_fetch() const - { - return _needs_initial_fetch; - } + needs_initial_fetch() const; - void - initial_fetch_completed() - { - _needs_initial_fetch = false; - } + bool + needs_rebootstrap() const; void for_each_connection(std::function func); diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index dabc654da..ac1969420 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -697,14 +697,16 @@ namespace llarp::service { std::unordered_set exclude; ForEachPath([&exclude](auto path) { exclude.insert(path->Endpoint()); }); - const auto maybe = - router()->node_db()->GetRandom([exclude, r = router()](const RemoteRC& rc) -> bool { - const auto& rid = rc.router_id(); - return exclude.count(rid) == 0 and not r->router_profiling().IsBadForPath(rid); - }); - if (not maybe.has_value()) - return std::nullopt; - return GetHopsForBuildWithEndpoint(maybe->router_id()); + + auto hook = [exclude, r = router()](const RemoteRC& rc) -> bool { + const auto& rid = rc.router_id(); + return not(exclude.count(rid) || r->router_profiling().IsBadForPath(rid)); + }; + + if (auto maybe = router()->node_db()->get_random_rc_conditional(hook)) + return GetHopsForBuildWithEndpoint(maybe->router_id()); + + return std::nullopt; } std::optional> From 62c37825b0c04435354ee5ef7d523fd9a3af7181 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 4 Dec 2023 10:26:58 -0800 Subject: [PATCH 10/93] testnet prep - disable reachability testing with config option; required to be done on testnet - reachability testing pipeline through link_manager executes pings similar to storage server. connection established hook reports successful reachability, while connection closed callback (with non-default error code) reports unsuccessful testing --- daemon/lokinet-bootstrap.cpp | 4 +- llarp/CMakeLists.txt | 2 +- llarp/config/config.cpp | 43 ++++++++----- llarp/config/config.hpp | 3 +- llarp/constants/version.cpp.in | 1 + llarp/constants/version.hpp | 1 + llarp/link/link_manager.cpp | 28 ++++++--- llarp/link/link_manager.hpp | 10 ++- llarp/router/router.cpp | 112 ++++++++++++++------------------- llarp/router/router.hpp | 16 ++--- llarp/router_contact.hpp | 2 +- llarp/service/endpoint.cpp | 2 +- 12 files changed, 114 insertions(+), 110 deletions(-) diff --git a/daemon/lokinet-bootstrap.cpp b/daemon/lokinet-bootstrap.cpp index afdecec72..22ecb9d6f 100644 --- a/daemon/lokinet-bootstrap.cpp +++ b/daemon/lokinet-bootstrap.cpp @@ -46,10 +46,8 @@ int main(int argc, char* argv[]) { const std::unordered_map bootstrap_urls = { - {"mainnet", "https://seed.lokinet.org/lokinet.signed"}, {"lokinet", "https://seed.lokinet.org/lokinet.signed"}, - {"testnet", "https://seed.lokinet.org/testnet.signed"}, - {"gamma", "https://seed.lokinet.org/testnet.signed"}}; + {"testnet", "https://seed.lokinet.org/testnet.signed"}}; std::string bootstrap_url = bootstrap_urls.at("lokinet"); fs::path outputfile{llarp::GetDefaultBootstrap()}; diff --git a/llarp/CMakeLists.txt b/llarp/CMakeLists.txt index 3aff1a3f5..790c92351 100644 --- a/llarp/CMakeLists.txt +++ b/llarp/CMakeLists.txt @@ -181,7 +181,7 @@ foreach(bs IN ITEMS MAINNET TESTNET) message(STATUS "Building with ${bs} fallback bootstrap path \"${BOOTSTRAP_FALLBACK_${bs}}\"") file(READ "${BOOTSTRAP_FALLBACK_${bs}}" bs_data HEX) if(bs STREQUAL TESTNET) - set(network "gamma") + set(network "testnet") elseif(bs STREQUAL MAINNET) set(network "lokinet") else() diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index 06e95594a..52baa872a 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -61,8 +61,8 @@ namespace llarp "netid", Default{llarp::LOKINET_DEFAULT_NETID}, Comment{ - "Network ID; this is '"s + llarp::LOKINET_DEFAULT_NETID - + "' for mainnet, 'gamma' for testnet.", + "Network ID; this is '"s + llarp::LOKINET_DEFAULT_NETID + "' for mainnet, "s + + llarp::LOKINET_TESTNET_NETID + "for testnet."s, }, [this](std::string arg) { if (arg.size() > NETID_SIZE) @@ -1007,7 +1007,7 @@ namespace llarp }, [this, parse_addr_for_link](const std::string& arg) { if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) - addr = a; + addr = *a; else addr = oxen::quic::Address{""s, DEFAULT_LISTEN_PORT}; @@ -1021,7 +1021,8 @@ namespace llarp MultiValue, Hidden, Comment{ - "********** DEPRECATED **********", + "********** THIS PARAMETER IS DEPRECATED -- USE 'LISTEN' INSTEAD **********", + "", "Note: the new API dictates the lokinet bind address through the 'listen' config", "parameter. Only ONE address will be read (no more lists of inbounds). Any address", "passed to `listen` will supersede the", @@ -1046,7 +1047,7 @@ namespace llarp throw std::runtime_error{"USE THE NEW API -- SPECIFY LOCAL ADDRESS UNDER [LISTEN]"}; if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) - addr = a; + addr = *a; else addr = oxen::quic::Address{""s, DEFAULT_LISTEN_PORT}; }); @@ -1096,7 +1097,7 @@ namespace llarp throw std::runtime_error{"USE THE NEW API -- SPECIFY LOCAL ADDRESS UNDER [LISTEN]"}; if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) - addr = a; + addr = *a; else addr = oxen::quic::Address{""s, DEFAULT_LISTEN_PORT}; }); @@ -1221,17 +1222,16 @@ namespace llarp LokidConfig::define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params) { (void)params; - - conf.define_option("lokid", "enabled", RelayOnly, Deprecated); - - conf.define_option("lokid", "jsonrpc", RelayOnly, [](std::string arg) { - if (arg.empty()) - return; - throw std::invalid_argument( - "the [lokid]:jsonrpc option is no longer supported; please use the [lokid]:rpc config " - "option instead with oxend's lmq-local-control address -- typically a value such as " - "rpc=ipc:///var/lib/oxen/oxend.sock or rpc=ipc:///home/snode/.oxen/oxend.sock"); - }); + conf.define_option( + "lokid", + "disable-testing", + Default{true}, + Hidden, + RelayOnly, + Comment{ + "Development option: set to true to disable reachability testing when using ", + "testnet"}, + assignment_acceptor(disable_testing)); conf.define_option( "lokid", @@ -1250,6 +1250,15 @@ namespace llarp [this](std::string arg) { rpc_addr = oxenmq::address(arg); }); // Deprecated options: + conf.define_option("lokid", "jsonrpc", RelayOnly, Deprecated, [](std::string arg) { + if (arg.empty()) + return; + throw std::invalid_argument( + "the [lokid]:jsonrpc option is no longer supported; please use the [lokid]:rpc config " + "option instead with oxend's lmq-local-control address -- typically a value such as " + "rpc=ipc:///var/lib/oxen/oxend.sock or rpc=ipc:///home/snode/.oxen/oxend.sock"); + }); + conf.define_option("lokid", "enabled", RelayOnly, Deprecated); conf.define_option("lokid", "username", Deprecated); conf.define_option("lokid", "password", Deprecated); conf.define_option("lokid", "service-node-seed", Deprecated); diff --git a/llarp/config/config.hpp b/llarp/config/config.hpp index 624b22cb5..17d3acb8e 100644 --- a/llarp/config/config.hpp +++ b/llarp/config/config.hpp @@ -173,7 +173,7 @@ namespace llarp std::optional public_addr; std::optional public_port; - std::optional addr; + oxen::quic::Address addr; bool using_new_api = false; void @@ -202,6 +202,7 @@ namespace llarp { fs::path id_keyfile; oxenmq::address rpc_addr; + bool disable_testing = true; void define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params); diff --git a/llarp/constants/version.cpp.in b/llarp/constants/version.cpp.in index 60d3af6ae..432f81e8e 100644 --- a/llarp/constants/version.cpp.in +++ b/llarp/constants/version.cpp.in @@ -10,5 +10,6 @@ namespace llarp const char* const LOKINET_RELEASE_MOTTO = "@RELEASE_MOTTO@"; const char* const LOKINET_DEFAULT_NETID = "lokinet"; + const char* const LOKINET_TESTNET_NETID = "testnet"; // clang-format on } // namespace llarp diff --git a/llarp/constants/version.hpp b/llarp/constants/version.hpp index f79b842e2..75b764cd0 100644 --- a/llarp/constants/version.hpp +++ b/llarp/constants/version.hpp @@ -12,4 +12,5 @@ namespace llarp extern const char* const LOKINET_RELEASE_MOTTO; extern const char* const LOKINET_DEFAULT_NETID; + extern const char* const LOKINET_TESTNET_NETID; } // namespace llarp diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index f8d999ef0..46afb055b 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -185,7 +185,7 @@ namespace llarp - will return a BTRequestStream on the first call to get_new_stream */ auto ep = quic->endpoint( - _router.public_ip(), + _router.local_addr(), [this](oxen::quic::connection_interface& ci) { return on_conn_open(ci); }, [this](oxen::quic::connection_interface& ci, uint64_t ec) { return on_conn_closed(ci, ec); @@ -298,20 +298,29 @@ namespace llarp } void - LinkManager::connect_to(const RouterID& rid) + LinkManager::test_reachability( + const RouterID& rid, conn_open_hook on_open, conn_closed_hook on_close) { - auto rc = node_db->get_rc(rid); - if (rc) + if (auto rc = node_db->get_rc(rid)) { - connect_to(*rc); + connect_to(*rc, std::move(on_open), std::move(on_close)); } else - log::warning(quic_cat, "Do something intelligent here for error handling"); + log::warning(quic_cat, "Could not find RouterContact for connection to rid:{}", rid); + } + + void + LinkManager::connect_to(const RouterID& rid, conn_open_hook hook) + { + if (auto rc = node_db->get_rc(rid)) + connect_to(*rc, std::move(hook)); + else + log::warning(quic_cat, "Could not find RouterContact for connection to rid:{}", rid); } // This function assumes the RC has already had its signature verified and connection is allowed. void - LinkManager::connect_to(const RemoteRC& rc) + LinkManager::connect_to(const RemoteRC& rc, conn_open_hook on_open, conn_closed_hook on_close) { if (auto conn = ep.get_conn(rc.router_id()); conn) { @@ -325,7 +334,10 @@ namespace llarp // TODO: confirm remote end is using the expected pubkey (RouterID). // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) if (auto rv = ep.establish_connection( - oxen::quic::RemoteAddress{rc.router_id().ToView(), remote_addr}, rc); + oxen::quic::RemoteAddress{rc.router_id().ToView(), remote_addr}, + rc, + std::move(on_open), + std::move(on_close)); rv) { log::info(quic_cat, "Connection to {} successfully established!", remote_addr); diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index b305fa6ef..47e148054 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -28,6 +28,9 @@ namespace llarp struct LinkManager; class NodeDB; + using conn_open_hook = oxen::quic::connection_established_callback; + using conn_closed_hook = oxen::quic::connection_closed_callback; + namespace link { struct Connection; @@ -261,10 +264,13 @@ namespace llarp deregister_peer(RouterID remote); void - connect_to(const RouterID& router); + test_reachability(const RouterID& rid, conn_open_hook, conn_closed_hook); + + void + connect_to(const RouterID& router, conn_open_hook = nullptr); void - connect_to(const RemoteRC& rc); + connect_to(const RemoteRC& rc, conn_open_hook = nullptr, conn_closed_hook = nullptr); void close_connection(RouterID rid); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 4d2374631..2c9b86802 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -78,7 +78,6 @@ namespace llarp llarp::LogTrace("Router::PumpLL() end"); } - // TOFIX: this util::StatusObject Router::ExtractStatus() const { @@ -91,8 +90,7 @@ namespace llarp {"contacts", _contacts->ExtractStatus()}, {"services", _hidden_service_context.ExtractStatus()}, {"exit", _exit_context.ExtractStatus()}, - {"links", _link_manager.extract_status()}, - /* {"outboundMessages", _outboundMessageHandler.ExtractStatus()} */}; + {"links", _link_manager.extract_status()}}; } // TODO: investigate changes needed for libquic integration @@ -417,6 +415,7 @@ namespace llarp throw std::runtime_error("KeyManager failed to initialize"); log::debug(logcat, "Initializing from configuration"); + if (!from_config(conf)) throw std::runtime_error("FromConfig() failed"); @@ -460,14 +459,6 @@ namespace llarp is_running.store(false); } - bool - Router::ParseRoutingMessageBuffer( - const llarp_buffer_t&, path::AbstractHopHandler&, const PathID_t&) - { - // TODO: will go away with the removal of flush upstream/downstream - return false; - } - bool Router::have_snode_whitelist() const { @@ -497,7 +488,7 @@ namespace llarp bool Router::can_test_routers() const { - return appears_funded(); + return appears_funded() and not _testing_disabled; } bool @@ -542,18 +533,22 @@ namespace llarp // Set netid before anything else log::debug(logcat, "Network ID set to {}", conf.router.net_id); - if (!conf.router.net_id.empty() - && strcmp(conf.router.net_id.c_str(), llarp::LOKINET_DEFAULT_NETID) != 0) + const auto& netid = conf.router.net_id; + + if (not netid.empty() and netid != llarp::LOKINET_DEFAULT_NETID) { - const auto& netid = conf.router.net_id; - llarp::LogWarn( - "!!!! you have manually set netid to be '", + log::critical( + logcat, + "Network ID set to {}, which is not {}! Lokinet will attempt to run on the specified " + "network", netid, - "' which does not equal '", - llarp::LOKINET_DEFAULT_NETID, - "' you will run as a different network, good luck " - "and don't forget: something something traffic shape " - "correlation!!"); + llarp::LOKINET_DEFAULT_NETID); + + _testnet = netid == llarp::LOKINET_TESTNET_NETID; + _testing_disabled = conf.lokid.disable_testing; + + if (_testing_disabled and not _testnet) + throw std::runtime_error{"Error: reachability testing can only be disabled on testnet!"}; } // Router config @@ -583,6 +578,8 @@ namespace llarp else log::debug(logcat, "No explicit public address given; will auto-detect during link setup"); + _local_addr = conf.links.addr; + RouterContact::BLOCK_BOGONS = conf.router.block_bogons; auto& networkConfig = conf.network; @@ -1060,7 +1057,9 @@ namespace llarp if (is_running || is_stopping) return false; - router_contact = LocalRC::make(identity(), public_ip()); + // TODO: look at _ourAddress + + router_contact = LocalRC::make(identity(), local_addr()); if (is_service_node() and not router_contact.is_public_router()) { @@ -1133,7 +1132,9 @@ namespace llarp // yet when we expect to have one. if (not can_test_routers()) return; + auto tests = router_testing.get_failing(); + if (auto maybe = router_testing.next_random(this)) { tests.emplace_back(*maybe, 0); @@ -1154,51 +1155,30 @@ namespace llarp // try to make a session to this random router // this will do a dht lookup if needed - _link_manager.connect_to(router); - - /* - * TODO: container of pending snode test routers to be queried on - * connection success/failure, then do this stuff there. - _outboundSessionMaker.CreateSessionTo( - router, [previous_fails = fails, this](const auto& router, const auto result) { - auto rpc = RpcClient(); - - if (result != SessionResult::Establish) - { - // failed connection mark it as so - m_routerTesting.add_failing_node(router, previous_fails); - LogInfo( - "FAILED SN connection test to ", - router, - " (", - previous_fails + 1, - " consecutive failures) result=", - result); - } - else - { - m_routerTesting.remove_node_from_failing(router); - if (previous_fails > 0) - { - LogInfo( - "Successful SN connection test to ", - router, - " after ", - previous_fails, - " failures"); - } - else - { - LogDebug("Successful SN connection test to ", router); - } - } - if (rpc) + _link_manager.test_reachability( + router, + [this, rid = router, previous = fails](oxen::quic::connection_interface& conn) { + log::info( + logcat, + "Successful SN reachability test to {}{}", + rid, + previous ? "after {} previous failures"_format(previous) : ""); + router_testing.remove_node_from_failing(rid); + _rpc_client->InformConnection(rid, true); + conn.close_connection(); + }, + [this, rid = router, previous = fails]( + oxen::quic::connection_interface&, uint64_t ec) { + if (ec != 0) { - // inform as needed - rpc->InformConnection(router, result == SessionResult::Establish); + log::info( + logcat, + "Unsuccessful SN reachability test to {} after {} previous failures", + rid, + previous); + router_testing.add_failing_node(rid, previous); } }); - */ } }); } @@ -1358,7 +1338,7 @@ namespace llarp } oxen::quic::Address - Router::public_ip() const + Router::local_addr() const { return _local_addr; } diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index ac1935d8f..08a8891a2 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -86,6 +86,11 @@ namespace llarp int _outbound_udp_socket = -1; bool _is_service_node = false; + bool _testnet = false; + bool _testing_disabled = false; + + consensus::reachability_testing router_testing; + std::optional _ourAddress; oxen::quic::Address _local_addr; @@ -124,8 +129,6 @@ namespace llarp service::Context _hidden_service_context; - consensus::reachability_testing router_testing; - bool should_report_stats(llarp_time_t now) const; @@ -273,7 +276,7 @@ namespace llarp } oxen::quic::Address - public_ip() const; + local_addr() const; util::StatusObject ExtractStatus() const; @@ -476,13 +479,6 @@ namespace llarp return llarp::time_now_ms(); } - /// parse a routing message in a buffer and handle it with a handler if - /// successful parsing return true on parse and handle success otherwise - /// return false - bool - ParseRoutingMessageBuffer( - const llarp_buffer_t& buf, path::AbstractHopHandler& p, const PathID_t& rxid); - void ConnectToRandomRouters(int N); diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index e3cb4ca1a..1812f76dc 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -33,7 +33,7 @@ namespace llarp /// "6" -- optional 18 byte IPv6 address & port: 16 byte raw IPv6 address followed by 2 bytes /// of port in network order. /// "i" -- optional network ID string of up to 8 bytes; this is omitted for the default network - /// ID ("lokinet") but included for others (such as "gamma" for testnet). + /// ID ("lokinet") but included for others (such as "testnet" for testnet). /// "p" -- 32-byte router pubkey /// "t" -- timestamp when this RC record was created (which also implicitly determines when it /// goes stale and when it expires). diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index ac1969420..6f7ea08a0 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1398,7 +1398,7 @@ namespace llarp::service while (not _inbound_queue.empty()) { - // succ it out + // suck it out queue.emplace(std::move(*_inbound_queue.popFront())); } From c9268dceba4a2ec28dca3c812f1b5adaadf2cd0d Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 5 Dec 2023 12:04:24 -0800 Subject: [PATCH 11/93] trust model edge case handling - Once we have our set of returned rc's and accepted rid's (ones that were found locally), the remainder are placed in an "unconfirmed" state - Once there, they have five subsequent successful fetches to be found in request response, at which point their verification counter is incremented and their attempt counter is reset - If they appear three times, they are "promoted" and moved to our "known_{rid,rc}" list --- llarp/nodedb.cpp | 110 ++++++++++++++----------- llarp/nodedb.hpp | 174 +++++++++++++++++++++++++++++---------- llarp/router_contact.hpp | 18 +--- 3 files changed, 197 insertions(+), 105 deletions(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index fb763b7fa..700cdc35c 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -193,43 +193,60 @@ namespace llarp } bool - NodeDB::process_fetched_rcs(std::vector& rcs) + NodeDB::process_fetched_rcs(std::set& rcs) { - std::unordered_set inter_set; + std::set confirmed_set, unconfirmed_set; + // the intersection of local RC's and received RC's is our confirmed set std::set_intersection( known_rcs.begin(), known_rcs.end(), rcs.begin(), rcs.end(), - std::inserter(inter_set, inter_set.begin())); + std::inserter(confirmed_set, confirmed_set.begin())); + + // the intersection of the confirmed set and received RC's is our unconfirmed set + std::set_intersection( + rcs.begin(), + rcs.end(), + confirmed_set.begin(), + confirmed_set.end(), + std::inserter(unconfirmed_set, unconfirmed_set.begin())); // the total number of rcs received const auto num_received = static_cast(rcs.size()); // the number of returned "good" rcs (that are also found locally) - const auto inter_size = inter_set.size(); - // the number of rcs currently held locally - const auto local_count = static_cast(known_rcs.size()); + const auto inter_size = confirmed_set.size(); const auto fetch_threshold = (double)inter_size / num_received; - const auto local_alignment = (double)inter_size / local_count; - /** We are checking 3 things here: + /** We are checking 2 things here: 1) The number of "good" rcs is above MIN_GOOD_RC_FETCH_TOTAL 2) The ratio of "good" rcs to total received is above MIN_GOOD_RC_FETCH_THRESHOLD - 3) The ratio of received and found locally to total found locally is above - LOCAL_RC_ALIGNMENT_THRESHOLD */ - return inter_size > MIN_GOOD_RC_FETCH_TOTAL and fetch_threshold > MIN_GOOD_RC_FETCH_THRESHOLD - and local_alignment > LOCAL_RC_ALIGNMENT_THRESHOLD; + bool success = false; + if (success = + inter_size > MIN_GOOD_RC_FETCH_TOTAL and fetch_threshold > MIN_GOOD_RC_FETCH_THRESHOLD; + success) + { + // set rcs to be intersection set + rcs = std::move(confirmed_set); + + process_results(std::move(unconfirmed_set), unconfirmed_rcs, known_rcs); + } + + return success; } bool - NodeDB::ingest_fetched_rcs(std::vector rcs, rc_time timestamp) + NodeDB::ingest_fetched_rcs(std::set rcs, rc_time timestamp) { // if we are not bootstrapping, we should check the rc's against the ones we currently hold if (not _using_bootstrap_fallback) - {} + { + if (not process_fetched_rcs(rcs)) + return false; + } for (auto& rc : rcs) put_rc_if_newer(std::move(rc), timestamp); @@ -237,19 +254,6 @@ namespace llarp return true; } - void - NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::unordered_set rids) - { - if (rids.empty()) - { - fail_sources.insert(source); - return; - } - - for (const auto& rid : rids) - fetch_counters[rid] += 1; - } - /** We only call into this function after ensuring two conditions: 1) We have received all 12 responses from the queried RouterID sources, whether that response was a timeout or not @@ -269,12 +273,14 @@ namespace llarp bool NodeDB::process_fetched_rids() { - std::unordered_set union_set, intersection_set; + std::set union_set, confirmed_set, unconfirmed_set; for (const auto& [rid, count] : fetch_counters) { if (count > MIN_RID_FETCH_FREQ) union_set.insert(rid); + else + unconfirmed_set.insert(rid); } // get the intersection of accepted rids and local rids @@ -283,33 +289,45 @@ namespace llarp known_rids.end(), union_set.begin(), union_set.end(), - std::inserter(intersection_set, intersection_set.begin())); + std::inserter(confirmed_set, confirmed_set.begin())); // the total number of rids received const auto num_received = (double)fetch_counters.size(); // the total number of received AND accepted rids const auto union_size = union_set.size(); - // the number of rids currently held locally - const auto local_count = (double)known_rids.size(); - // the number of accepted rids that are also found locally - const auto inter_size = (double)intersection_set.size(); const auto fetch_threshold = (double)union_size / num_received; - const auto local_alignment = (double)inter_size / local_count; /** We are checking 2, potentially 3 things here: 1) The ratio of received/accepted to total received is above GOOD_RID_FETCH_THRESHOLD. This tells us how well the rid source's sets of rids "agree" with one another 2) The total number received is above MIN_RID_FETCH_TOTAL. This ensures that we are receiving a sufficient amount to make a comparison of any sorts - 3) If we are not bootstrapping, then the ratio of received/accepted found locally to - the total number locally held is above LOCAL_RID_ALIGNMENT_THRESHOLD. This gives us - an estimate of how "aligned" the rid source's set of rid's is to ours */ - return (fetch_threshold > GOOD_RID_FETCH_THRESHOLD) and (union_size > MIN_GOOD_RID_FETCH_TOTAL) - and (not _using_bootstrap_fallback) - ? local_alignment > LOCAL_RID_ALIGNMENT_THRESHOLD - : true; + bool success = false; + if (success = (fetch_threshold > GOOD_RID_FETCH_THRESHOLD) + and (union_size > MIN_GOOD_RID_FETCH_TOTAL); + success) + { + process_results(std::move(unconfirmed_set), unconfirmed_rids, known_rids); + + known_rids.merge(confirmed_set); + } + + return success; + } + + void + NodeDB::ingest_rid_fetch_responses(const RouterID& source, std::set rids) + { + if (rids.empty()) + { + fail_sources.insert(source); + return; + } + + for (const auto& rid : rids) + fetch_counters[rid] += 1; } void @@ -375,10 +393,10 @@ namespace llarp auto btlc = btdc.require("rcs"sv); auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; - std::vector rcs; + std::set rcs; while (not btlc.is_finished()) - rcs.emplace_back(btlc.consume_dict_consumer()); + rcs.emplace(btlc.consume_dict_consumer()); // if process_fetched_rcs returns false, then the trust model rejected the fetched RC's fetch_rcs_result(initial, not ingest_fetched_rcs(std::move(rcs), timestamp)); @@ -447,7 +465,7 @@ namespace llarp "Failed to verify signature for fetch RouterIDs response."}; }); - std::unordered_set router_ids; + std::set router_ids; for (const auto& s : router_id_strings) { @@ -644,7 +662,7 @@ namespace llarp return; } - std::unordered_set rids; + std::set rids; try { @@ -707,7 +725,7 @@ namespace llarp } void - NodeDB::reselect_router_id_sources(std::unordered_set specific) + NodeDB::reselect_router_id_sources(std::set specific) { replace_subset(rid_sources, specific, known_rids, RID_SOURCE_COUNT, csrng); } diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index d476298f3..0cd66e74a 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -29,8 +29,6 @@ namespace llarp inline constexpr size_t MIN_GOOD_RC_FETCH_TOTAL{}; // the ratio of returned rcs found locally to to total returned should be above this ratio inline constexpr double MIN_GOOD_RC_FETCH_THRESHOLD{}; - // the ratio of returned rcs that are found locally to total held locally should above this ratio - inline constexpr double LOCAL_RC_ALIGNMENT_THRESHOLD{}; /* RID Fetch Constants */ inline constexpr size_t MIN_ACTIVE_RIDS{24}; @@ -39,14 +37,11 @@ namespace llarp // upper limit on how many rid fetch requests to rid sources can fail inline constexpr size_t MAX_RID_ERRORS{4}; // each returned rid must appear this number of times across all responses - inline constexpr int MIN_RID_FETCH_FREQ{6}; + inline constexpr int MIN_RID_FETCH_FREQ{RID_SOURCE_COUNT - MAX_RID_ERRORS - 1}; // the total number of accepted returned rids should be above this number inline constexpr size_t MIN_GOOD_RID_FETCH_TOTAL{}; // the ratio of accepted:rejected rids must be above this ratio inline constexpr double GOOD_RID_FETCH_THRESHOLD{}; - // if we are not bootstrapping, the ratio of accepted:local rids must be above this ratio - inline constexpr double LOCAL_RID_ALIGNMENT_THRESHOLD{}; - /* Bootstrap Constants */ // the number of rc's we query the bootstrap for inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; @@ -55,8 +50,53 @@ namespace llarp // if all bootstraps fail, router will trigger re-bootstrapping after this cooldown inline constexpr auto BOOTSTRAP_COOLDOWN{1min}; + /* Other Constants */ + // the maximum number of RC/RID fetches that can pass w/o an unconfirmed rc/rid appearing + inline constexpr int MAX_CONFIRMATION_ATTEMPTS{5}; + // threshold amount of verifications to promote an unconfirmed rc/rid + inline constexpr int CONFIRMATION_THRESHOLD{3}; + inline constexpr auto FLUSH_INTERVAL{5min}; + template < + typename ID_t, + std::enable_if_t || std::is_same_v, int> = 0> + struct Unconfirmed + { + const ID_t id; + int attempts = 0; + int verifications = 0; + + Unconfirmed() = delete; + Unconfirmed(const ID_t& obj) : id{obj} + {} + Unconfirmed(ID_t&& obj) : id{std::move(obj)} + {} + + int + strikes() const + { + return attempts; + } + + operator bool() const + { + return verifications == CONFIRMATION_THRESHOLD; + } + + bool + operator==(const Unconfirmed& other) const + { + return id == other.id; + } + + bool + operator<(const Unconfirmed& other) const + { + return id < other.id; + } + }; + class NodeDB { Router& _router; @@ -73,14 +113,22 @@ namespace llarp populated during startup and RouterID fetching. This is meant to represent the client instance's most recent perspective of the network, and record which RouterID's were recently "active" and connected to + - unconfirmed_rids: holds new rids returned in fetch requests to be verified by subsequent + fetch requests - known_rcs: populated during startup and when RC's are updated both during gossip and periodic RC fetching + - unconfirmed_rcs: holds new rcs to be verified by subsequent fetch requests, similar to + the unknown_rids container - rc_lookup: holds all the same rc's as known_rcs, but can be used to look them up by - their rid. Deleting an rid key deletes the corresponding rc in known_rcs + their rid */ - std::unordered_set known_rids; - std::unordered_set known_rcs; - std::unordered_map rc_lookup; // TODO: look into this again + std::set known_rids; + std::set> unconfirmed_rids; + + std::set known_rcs; + std::set> unconfirmed_rcs; + + std::map rc_lookup; /** RouterID lists - white: active routers @@ -92,18 +140,18 @@ namespace llarp std::unordered_set router_greenlist; // All registered relays (service nodes) - std::unordered_set registered_routers; + std::set registered_routers; // timing (note: Router holds the variables for last rc and rid request times) std::unordered_map last_rc_update_times; // if populated from a config file, lists specific exclusively used as path first-hops - std::unordered_set _pinned_edges; + std::set _pinned_edges; // source of "truth" for RC updating. This relay will also mediate requests to the // 12 selected active RID's for RID fetching RouterID fetch_source; // set of 12 randomly selected RID's from the client's set of routers - std::unordered_set rid_sources{}; + std::set rid_sources{}; // logs the RID's that resulted in an error during RID fetching - std::unordered_set fail_sources{}; + std::set fail_sources{}; // tracks the number of times each rid appears in the above responses std::unordered_map fetch_counters{}; @@ -152,13 +200,13 @@ namespace llarp } bool - ingest_fetched_rcs(std::vector rcs, rc_time timestamp); + ingest_fetched_rcs(std::set rcs, rc_time timestamp); bool - process_fetched_rcs(std::vector& rcs); + process_fetched_rcs(std::set& rcs); void - ingest_rid_fetch_responses(const RouterID& source, std::unordered_set ids = {}); + ingest_rid_fetch_responses(const RouterID& source, std::set ids = {}); bool process_fetched_rids(); @@ -182,7 +230,7 @@ namespace llarp void fetch_rids_result(bool initial = false); - // Bootstrap fallback + // Bootstrap fallback fetching void fallback_to_bootstrap(); void @@ -192,7 +240,7 @@ namespace llarp // if only specific RID's need to be re-selected; to re-select all, pass the member // variable ::known_rids void - reselect_router_id_sources(std::unordered_set specific); + reselect_router_id_sources(std::set specific); void set_router_whitelist( @@ -230,7 +278,7 @@ namespace llarp bool is_first_hop_allowed(const RouterID& remote) const; - std::unordered_set& + std::set& pinned_edges() { return _pinned_edges; @@ -257,13 +305,13 @@ namespace llarp return router_greylist; } - const std::unordered_set& + const std::set& get_registered_routers() const { return registered_routers; } - const std::unordered_set& + const std::set& get_rcs() const { return known_rcs; @@ -336,33 +384,14 @@ namespace llarp std::optional> get_n_random_rcs_conditional(size_t n, std::function hook) const; - template - std::optional - GetRandom(Filter visit) const - { - return _router.loop()->call_get([visit, this]() mutable -> std::optional { - std::vector rcs{known_rcs.begin(), known_rcs.end()}; - - std::shuffle(rcs.begin(), rcs.end(), llarp::csrng); - - for (const auto& entry : known_rcs) - { - if (visit(entry)) - return entry; - } - - return std::nullopt; - }); - } - // Updates `current` to not contain any of the elements of `replace` and resamples (up to // `target_size`) from population to refill it. template void replace_subset( - std::unordered_set& current, - const std::unordered_set& replace, - std::unordered_set population, + std::set& current, + const std::set& replace, + std::set population, size_t target_size, RNG&& rng) { @@ -423,6 +452,52 @@ namespace llarp }); } + template < + typename ID_t, + std::enable_if_t || std::is_same_v, int> = 0> + void + process_results( + std::set unconfirmed, std::set>& container, std::set& known) + { + // before we add the unconfirmed set, we check to see if our local set of unconfirmed + // rcs/rids appeared in the latest unconfirmed set; if so, we will increment their number + // of verifications and reset the attempts counter. Once appearing in 3 different requests, + // the rc/rid will be "verified" and promoted to the known_{rcs,rids} container + for (auto itr = container.begin(); itr != container.end();) + { + auto& id = itr->id; + auto& count = const_cast(itr->attempts); + auto& verifications = const_cast(itr->verifications); + + if (auto found = unconfirmed.find(id); found != unconfirmed.end()) + { + if (++verifications >= CONFIRMATION_THRESHOLD) + { + if constexpr (std::is_same_v) + put_rc_if_newer(id); + else + known.emplace(id); + itr = container.erase(itr); + } + else + { + // reset attempt counter and continue + count = 0; + ++itr; + } + + unconfirmed.erase(found); + } + + itr = (++count >= MAX_CONFIRMATION_ATTEMPTS) ? container.erase(itr) : ++itr; + } + + for (auto& id : unconfirmed) + { + container.emplace(std::move(id)); + } + } + /// remove rcs that are older than we want to keep. For relays, this is when /// they become "outdated" (i.e. 12hrs). Clients will hang on to them until /// they are fully "expired" (i.e. 30 days), as the client may go offline for @@ -441,3 +516,14 @@ namespace llarp put_rc_if_newer(RemoteRC rc, rc_time now = time_point_now()); }; } // namespace llarp + +namespace std +{ + template <> + struct hash> : public hash + {}; + + template <> + struct hash> : hash + {}; +} // namespace std diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 1812f76dc..ce99d3822 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -366,22 +366,10 @@ namespace std }; template <> - struct hash final : public hash - { - size_t - operator()(const llarp::RouterContact& r) const override - { - return std::hash{}(r.router_id()); - } - }; + struct hash : public hash + {}; template <> struct hash final : public hash - { - size_t - operator()(const llarp::RouterContact& r) const override - { - return std::hash{}(r.router_id()); - } - }; + {}; } // namespace std From ed6bd28a357eefe134a0407768f56b1b063a13db Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 6 Dec 2023 11:34:37 -0800 Subject: [PATCH 12/93] testnet prep - redoing link_manager functions again to implement previously ignored review comments on several PRs - conceptually merging "whitelist_routers" and new "known_{rids,rcs}", s.t. we can completely eliminate white/red/gray/green/etc lists in favor of something that isn't dumb --- llarp/config/config.cpp | 79 ++------------- llarp/config/config.hpp | 2 +- llarp/consensus/reachability_testing.cpp | 13 ++- llarp/handlers/exit.cpp | 2 +- llarp/handlers/tun.cpp | 6 +- llarp/link/link_manager.cpp | 124 +++++++++++------------ llarp/messages/fetch.hpp | 4 +- llarp/nodedb.cpp | 34 +++++-- llarp/nodedb.hpp | 23 ++++- llarp/router/router.cpp | 2 +- llarp/router/router.hpp | 8 +- llarp/router_id.cpp | 2 +- llarp/router_id.hpp | 3 +- llarp/rpc/lokid_rpc_client.cpp | 1 + llarp/rpc/rpc_server.cpp | 2 +- llarp/service/address.cpp | 2 +- llarp/service/intro.cpp | 2 +- 17 files changed, 140 insertions(+), 169 deletions(-) diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index 52baa872a..77e6d8206 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -61,8 +61,8 @@ namespace llarp "netid", Default{llarp::LOKINET_DEFAULT_NETID}, Comment{ - "Network ID; this is '"s + llarp::LOKINET_DEFAULT_NETID + "' for mainnet, "s - + llarp::LOKINET_TESTNET_NETID + "for testnet."s, + "Network ID; this is '"s + llarp::LOKINET_DEFAULT_NETID + "' for mainnet, '"s + + llarp::LOKINET_TESTNET_NETID + "' for testnet."s, }, [this](std::string arg) { if (arg.size() > NETID_SIZE) @@ -290,7 +290,7 @@ namespace llarp MultiValue, [this](std::string value) { RouterID router; - if (not router.FromString(value)) + if (not router.from_string(value)) throw std::invalid_argument{"bad snode value: " + value}; if (not strict_connect.insert(router).second) throw std::invalid_argument{"duplicate strict connect snode: " + value}; @@ -710,7 +710,7 @@ namespace llarp }, [this](std::string arg) { RouterID id; - if (not id.FromString(arg)) + if (not id.from_string(arg)) throw std::invalid_argument{fmt::format("Invalid RouterID: {}", arg)}; auto itr = snode_blacklist.emplace(std::move(id)); @@ -923,6 +923,7 @@ namespace llarp SockAddr pubaddr{arg}; public_addr = pubaddr.getIP(); }); + conf.define_option( "bind", "public-port", @@ -980,10 +981,7 @@ namespace llarp conf.define_option( "bind", "listen", - Required, Comment{ - "********** NEW API OPTION (see note) **********", - "", "IP and/or port for lokinet to bind to for inbound/outbound connections.", "", "If IP is omitted then lokinet will search for a local network interface with a", @@ -1009,7 +1007,7 @@ namespace llarp if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) addr = *a; else - addr = oxen::quic::Address{""s, DEFAULT_LISTEN_PORT}; + throw std::invalid_argument{"Could not parse listen address!"}; using_new_api = true; }); @@ -1020,86 +1018,25 @@ namespace llarp RelayOnly, MultiValue, Hidden, - Comment{ - "********** THIS PARAMETER IS DEPRECATED -- USE 'LISTEN' INSTEAD **********", - "", - "Note: the new API dictates the lokinet bind address through the 'listen' config", - "parameter. Only ONE address will be read (no more lists of inbounds). Any address", - "passed to `listen` will supersede the", - "", - "IP and/or port to listen on for incoming connections.", - "", - "If IP is omitted then lokinet will search for a local network interface with a", - "public IP address and use that IP (and will exit with an error if no such IP is found", - "on the system). If port is omitted then lokinet defaults to 1090.", - "", - "Examples:", - " inbound=15.5.29.5:443", - " inbound=10.0.2.2", - " inbound=:1234", - "", - "Using a private range IP address (like the second example entry) will require using", - "the public-ip= and public-port= to specify the public IP address at which this", - "router can be reached.", - }, [this, parse_addr_for_link](const std::string& arg) { if (using_new_api) throw std::runtime_error{"USE THE NEW API -- SPECIFY LOCAL ADDRESS UNDER [LISTEN]"}; if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) addr = *a; - else - addr = oxen::quic::Address{""s, DEFAULT_LISTEN_PORT}; }); conf.define_option( "bind", "outbound", MultiValue, - params.is_relay ? Comment{ - "********** THIS PARAMETER IS DEPRECATED -- USE 'LISTEN' INSTEAD **********", - "", - "IP and/or port to use for outbound socket connections to other lokinet routers.", - "", - "If no outbound bind IP is configured, or the 0.0.0.0 wildcard IP is given, then", - "lokinet will bind to the same IP being used for inbound connections (either an", - "explicit inbound= provided IP, or the default). If no port is given, or port is", - "given as 0, then a random high port will be used.", - "", - "If using multiple inbound= addresses then you *must* provide an explicit oubound= IP.", - "", - "Examples:", - " outbound=1.2.3.4:5678", - " outbound=:9000", - " outbound=8.9.10.11", - "", - "The second example binds on the default incoming IP using port 9000; the third", - "example binds on the given IP address using a random high port.", - } : Comment{ - "********** DEPRECATED **********", - "", - "IP and/or port to use for outbound socket connections to lokinet routers.", - "", - "If no outbound bind IP is configured then lokinet will use a wildcard IP address", - "(equivalent to specifying 0.0.0.0). If no port is given then a random high port", - "will be used.", - "", - "Examples:", - " outbound=1.2.3.4:5678", - " outbound=:9000", - " outbound=8.9.10.11", - "", - "The second example binds on the wildcard address using port 9000; the third example", - "binds on the given IP address using a random high port.", - }, + Hidden, [this, parse_addr_for_link](const std::string& arg) { if (using_new_api) throw std::runtime_error{"USE THE NEW API -- SPECIFY LOCAL ADDRESS UNDER [LISTEN]"}; if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) addr = *a; - else - addr = oxen::quic::Address{""s, DEFAULT_LISTEN_PORT}; }); conf.add_undeclared_handler( @@ -1250,7 +1187,7 @@ namespace llarp [this](std::string arg) { rpc_addr = oxenmq::address(arg); }); // Deprecated options: - conf.define_option("lokid", "jsonrpc", RelayOnly, Deprecated, [](std::string arg) { + conf.define_option("lokid", "jsonrpc", RelayOnly, Hidden, [](std::string arg) { if (arg.empty()) return; throw std::invalid_argument( diff --git a/llarp/config/config.hpp b/llarp/config/config.hpp index 17d3acb8e..f47c92230 100644 --- a/llarp/config/config.hpp +++ b/llarp/config/config.hpp @@ -173,7 +173,7 @@ namespace llarp std::optional public_addr; std::optional public_port; - oxen::quic::Address addr; + oxen::quic::Address addr{""s, DEFAULT_LISTEN_PORT}; bool using_new_api = false; void diff --git a/llarp/consensus/reachability_testing.cpp b/llarp/consensus/reachability_testing.cpp index 1f5068255..9274bff2e 100644 --- a/llarp/consensus/reachability_testing.cpp +++ b/llarp/consensus/reachability_testing.cpp @@ -82,29 +82,34 @@ namespace llarp::consensus // Pull the next element off the queue, but skip ourself, any that are no longer registered, and // any that are currently known to be failing (those are queued for testing separately). - RouterID my_pk{router->pubkey()}; + auto local_pk = router->local_rid(); + while (!testing_queue.empty()) { auto& pk = testing_queue.back(); std::optional sn; - if (pk != my_pk && !failing.count(pk)) + + if (pk != local_pk && !failing.count(pk)) sn = pk; + testing_queue.pop_back(); + if (sn) return sn; } + if (!requeue) return std::nullopt; // FIXME: when a *new* node comes online we need to inject it into a random position in the SN // list with probability (L/N) [L = current list size, N = potential list size] // - // (FIXME: put this FIXME in a better place ;-) ) // We exhausted the queue so repopulate it and try again testing_queue.clear(); - const auto all = router->get_whitelist(); + const auto& all = router->get_whitelist(); + testing_queue.insert(testing_queue.begin(), all.begin(), all.end()); std::shuffle(testing_queue.begin(), testing_queue.end(), llarp::csrng); diff --git a/llarp/handlers/exit.cpp b/llarp/handlers/exit.cpp index ffab0a37c..54555ba13 100644 --- a/llarp/handlers/exit.cpp +++ b/llarp/handlers/exit.cpp @@ -281,7 +281,7 @@ namespace llarp::handlers } // forward dns for snode RouterID r; - if (r.FromString(msg.questions[0].Name())) + if (r.from_string(msg.questions[0].Name())) { huint128_t ip; PubKey pubKey(r); diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp index 15db0ebc7..6e1cc705b 100644 --- a/llarp/handlers/tun.cpp +++ b/llarp/handlers/tun.cpp @@ -535,7 +535,7 @@ namespace llarp::handlers if (auto saddr = service::Address(); saddr.FromString(name)) ReplyToLokiDNSWhenReady(saddr, msg, isV6); - if (auto rid = RouterID(); rid.FromString(name)) + if (auto rid = RouterID(); rid.from_string(name)) ReplyToSNodeDNSWhenReady(rid, msg, isV6); }; @@ -568,7 +568,7 @@ namespace llarp::handlers if (not qname) return false; RouterID addr; - if (not addr.FromString(*qname)) + if (not addr.from_string(*qname)) return false; auto replyMsg = std::make_shared(clear_dns_message(msg)); return ReplyToSNodeDNSWhenReady(addr, std::move(replyMsg), false); @@ -604,7 +604,7 @@ namespace llarp::handlers if (msg.questions[0].qtype == dns::qTypeTXT) { RouterID snode; - if (snode.FromString(qname)) + if (snode.from_string(qname)) { if (auto rc = router()->node_db()->get_rc(snode)) msg.AddTXTReply(std::string{rc->view()}); diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 46afb055b..2f61820b9 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -622,18 +622,17 @@ namespace llarp // this handler should not be registered for clients assert(_router.is_service_node()); - const auto& rcs = node_db->get_rcs(); - const auto now = time_point_now(); - - std::vector explicit_ids; + std::set explicit_ids; rc_time since_time; try { oxenc::bt_dict_consumer btdc{m.body()}; - btdc.required("explicit_ids"); - explicit_ids = btdc.consume_list>(); + auto btlc = btdc.require("explicit_ids"); + + while (not btlc.is_finished()) + explicit_ids.emplace(btlc.consume().data()); since_time = rc_time{std::chrono::seconds{btdc.require("since")}}; } @@ -644,24 +643,8 @@ namespace llarp return; } - // Initial fetch: give me all the RC's - if (explicit_ids.empty()) - { - // TODO: this - } - - std::unordered_set explicit_relays; - - for (auto& sv : explicit_ids) - { - if (sv.size() != RouterID::SIZE) - { - m.respond(RCFetchMessage::INVALID_REQUEST, true); - return; - } - - explicit_relays.emplace(reinterpret_cast(sv.data())); - } + const auto& rcs = node_db->get_rcs(); + const auto now = time_point_now(); oxenc::bt_dict_producer btdp; const auto& last_time = node_db->get_last_rc_update_times(); @@ -673,10 +656,22 @@ namespace llarp if (since_time != decltype(since_time)::min()) since_time -= 5s; - for (const auto& rc : rcs) + // Initial fetch: give me all the RC's + if (explicit_ids.empty()) { - if (last_time.at(rc.router_id()) > since_time or explicit_relays.count(rc.router_id())) - sublist.append_encoded(rc.view()); + for (const auto& rc : rcs) + { + if (last_time.at(rc.router_id()) > since_time) + sublist.append_encoded(rc.view()); + } + } + else + { + for (const auto& rid : explicit_ids) + { + if (auto maybe_rc = node_db->get_rc_by_rid(rid)) + sublist.append_encoded(maybe_rc->view()); + } } } @@ -695,56 +690,57 @@ namespace llarp void LinkManager::handle_fetch_router_ids(oxen::quic::message m) { + RouterID source; + RouterID local = router().local_rid(); + try { oxenc::bt_dict_consumer btdc{m.body()}; - auto source = btdc.require("source"); - - // if bad request, silently fail - if (source.size() != RouterID::SIZE) - return; - - const auto source_rid = RouterID{reinterpret_cast(source.data())}; - const auto our_rid = RouterID{router().pubkey()}; - - if (source_rid == our_rid) - { - oxenc::bt_dict_producer btdp; - { - auto btlp = btdp.append_list("routers"); - for (const auto& relay : node_db->whitelist()) - { - btlp.append(relay.ToView()); - } - } - btdp.append_signature("signature", [this](ustring_view to_sign) { - std::array sig; - - if (!crypto::sign(const_cast(sig.data()), _router.identity(), to_sign)) - throw std::runtime_error{"Failed to sign fetch RouterIDs response"}; + source.from_string(btdc.require("source")); + } + catch (const std::exception& e) + { + log::info(link_cat, "Error fulfilling fetch RouterIDs request: {}", e.what()); + } - return sig; - }); - m.respond(std::move(btdp).str()); - return; - } + // if bad request, silently fail + if (source.size() != RouterID::SIZE) + return; + if (source != local) + { send_control_message( - source_rid, + source, "fetch_router_ids"s, m.body_str(), - [source_rid = std::move(source_rid), - orig_mess = std::move(m)](oxen::quic::message m) mutable { - if (not m.timed_out) - orig_mess.respond(m.body_str(), not m); - // on timeout, just silently drop (as original requester will just time out anyway) + [source_rid = std::move(source), original = std::move(m)](oxen::quic::message m) mutable { + original.respond(m.body_str(), not m); }); + return; } - catch (const std::exception& e) + + oxenc::bt_dict_producer btdp; + { - log::info(link_cat, "Error fulfilling fetch RouterIDs request: {}", e.what()); + auto btlp = btdp.append_list("routers"); + + const auto& known_rcs = node_db->get_known_rcs(); + + for (const auto& rc : known_rcs) + btlp.append_encoded(rc.view()); } + + btdp.append_signature("signature", [this](ustring_view to_sign) { + std::array sig; + + if (!crypto::sign(const_cast(sig.data()), _router.identity(), to_sign)) + throw std::runtime_error{"Failed to sign fetch RouterIDs response"}; + + return sig; + }); + + m.respond(std::move(btdp).str()); } void diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 35e9d18ec..933521544 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -4,7 +4,7 @@ namespace llarp { - namespace RCFetchMessage + namespace FetchRCMessage { inline const auto INVALID_REQUEST = messages::serialize_response({{messages::STATUS_KEY, "Invalid relay ID requested"}}); @@ -33,7 +33,7 @@ namespace llarp return std::move(btdp).str(); } - } // namespace RCFetchMessage + } // namespace FetchRCMessage namespace BootstrapFetchMessage { diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 700cdc35c..6779d2a27 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -56,6 +56,15 @@ namespace llarp fetch_counters.clear(); } + std::optional + NodeDB::get_rc_by_rid(const RouterID& rid) + { + if (auto itr = rc_lookup.find(rid); itr != rc_lookup.end()) + return itr->second; + + return std::nullopt; + } + std::optional NodeDB::get_random_rc() const { @@ -178,7 +187,8 @@ namespace llarp { if (not _router.is_service_node()) return true; - return registered_routers.count(rid); + + return known_rids.count(rid); } void @@ -358,6 +368,7 @@ namespace llarp } std::vector needed; + const auto now = time_point_now(); for (const auto& [rid, rc] : rc_lookup) @@ -370,7 +381,7 @@ namespace llarp _router.link_manager().fetch_rcs( src, - RCFetchMessage::serialize(_router.last_rc_fetch, needed), + FetchRCMessage::serialize(_router.last_rc_fetch, needed), [this, src, initial](oxen::quic::message m) mutable { if (m.timed_out) { @@ -730,6 +741,7 @@ namespace llarp replace_subset(rid_sources, specific, known_rids, RID_SOURCE_COUNT, csrng); } + // TODO: nuke all this shit void NodeDB::set_router_whitelist( const std::vector& whitelist, @@ -744,6 +756,9 @@ namespace llarp registered_routers.insert(greylist.begin(), greylist.end()); registered_routers.insert(greenlist.begin(), greenlist.end()); + for (const auto& rid : whitelist) + known_rids.insert(rid); + router_whitelist.clear(); router_whitelist.insert(whitelist.begin(), whitelist.end()); router_greylist.clear(); @@ -752,19 +767,16 @@ namespace llarp router_greenlist.insert(greenlist.begin(), greenlist.end()); log::info( - logcat, "lokinet service node list now has ", router_whitelist.size(), " active routers"); + logcat, "lokinet service node list now has ", known_rids.size(), " active router RIDs"); } std::optional NodeDB::get_random_whitelist_router() const { - const auto sz = router_whitelist.size(); - if (sz == 0) - return std::nullopt; - auto itr = router_whitelist.begin(); - if (sz > 1) - std::advance(itr, randint() % sz); - return *itr; + if (auto rc = get_random_rc()) + return rc->router_id(); + + return std::nullopt; } bool @@ -777,7 +789,7 @@ namespace llarp if (not _router.is_service_node()) return true; - return router_whitelist.count(remote) or router_greylist.count(remote); + return known_rids.count(remote) or router_greylist.count(remote); } bool diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 0cd66e74a..6470cd8c5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -130,7 +130,7 @@ namespace llarp std::map rc_lookup; - /** RouterID lists + /** RouterID lists // TODO: get rid of all these, replace with better decom/not staked sets - white: active routers - gray: fully funded, but decommissioned routers - green: registered, but not fully-staked routers @@ -187,6 +187,21 @@ namespace llarp /// in memory nodedb NodeDB(); + const std::set& + get_known_rids() const + { + return known_rids; + } + + const std::set& + get_known_rcs() const + { + return known_rcs; + } + + std::optional + get_rc_by_rid(const RouterID& rid); + bool needs_initial_fetch() const { @@ -270,7 +285,7 @@ namespace llarp bool is_path_allowed(const RouterID& remote) const { - return router_whitelist.count(remote); + return known_rids.count(remote); } // if pinned edges were specified, the remote must be in that set, else any remote @@ -293,10 +308,10 @@ namespace llarp void set_bootstrap_routers(std::unique_ptr from_router); - const std::unordered_set& + const std::set& whitelist() const { - return router_whitelist; + return known_rids; } const std::unordered_set& diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2c9b86802..d3a8513fe 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -1026,7 +1026,7 @@ namespace llarp return _link_manager.get_random_connected(result); } - const std::unordered_set& + const std::set& Router::get_whitelist() const { return _node_db->whitelist(); diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 08a8891a2..3fe14c868 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -155,6 +155,12 @@ namespace llarp std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip}; public: + RouterID + local_rid() const + { + return RouterID{pubkey()}; + } + bool needs_initial_fetch() const; @@ -284,7 +290,7 @@ namespace llarp util::StatusObject ExtractSummaryStatus() const; - const std::unordered_set& + const std::set& get_whitelist() const; void diff --git a/llarp/router_id.cpp b/llarp/router_id.cpp index d8f1e1581..a046f0aee 100644 --- a/llarp/router_id.cpp +++ b/llarp/router_id.cpp @@ -29,7 +29,7 @@ namespace llarp } bool - RouterID::FromString(std::string_view str) + RouterID::from_string(std::string_view str) { auto pos = str.find(SNODE_TLD); if (pos != str.size() - SNODE_TLD.size()) diff --git a/llarp/router_id.hpp b/llarp/router_id.hpp index 8a82f924d..fea9d4d54 100644 --- a/llarp/router_id.hpp +++ b/llarp/router_id.hpp @@ -31,7 +31,7 @@ namespace llarp ShortString() const; bool - FromString(std::string_view str); + from_string(std::string_view str); RouterID& operator=(const byte_t* ptr) @@ -49,7 +49,6 @@ namespace llarp template <> constexpr inline bool IsToStringFormattable = true; - } // namespace llarp namespace std diff --git a/llarp/rpc/lokid_rpc_client.cpp b/llarp/rpc/lokid_rpc_client.cpp index aa4e3f3ad..764b8136c 100644 --- a/llarp/rpc/lokid_rpc_client.cpp +++ b/llarp/rpc/lokid_rpc_client.cpp @@ -269,6 +269,7 @@ namespace llarp::rpc keymap = std::move(keymap), router = std::move(router)]() mutable { m_KeyMap = std::move(keymap); + router->set_router_whitelist(active, decomm, unfunded); }); } diff --git a/llarp/rpc/rpc_server.cpp b/llarp/rpc/rpc_server.cpp index b5854d940..ea657fb81 100644 --- a/llarp/rpc/rpc_server.cpp +++ b/llarp/rpc/rpc_server.cpp @@ -322,7 +322,7 @@ namespace llarp::rpc return; } - if (not routerID.FromString(lookupsnode.request.routerID)) + if (not routerID.from_string(lookupsnode.request.routerID)) { SetJSONError("Invalid remote: " + lookupsnode.request.routerID, lookupsnode.response); return; diff --git a/llarp/service/address.cpp b/llarp/service/address.cpp index 2f18907de..cd37138a4 100644 --- a/llarp/service/address.cpp +++ b/llarp/service/address.cpp @@ -79,7 +79,7 @@ namespace llarp::service { RouterID router{}; service::Address addr{}; - if (router.FromString(lokinet_addr)) + if (router.from_string(lokinet_addr)) return router; if (addr.FromString(lokinet_addr)) return addr; diff --git a/llarp/service/intro.cpp b/llarp/service/intro.cpp index 5e25e6b62..2c8bb5596 100644 --- a/llarp/service/intro.cpp +++ b/llarp/service/intro.cpp @@ -39,7 +39,7 @@ namespace llarp::service { oxenc::bt_dict_consumer btdc{std::move(buf)}; - router.FromString(btdc.require("k")); + router.from_string(btdc.require("k")); latency = std::chrono::milliseconds{btdc.require("l")}; path_id.from_string(btdc.require("p")); expiry = std::chrono::milliseconds{btdc.require("x")}; From cef2ff77823845590740a86eb101df38f634effc Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 6 Dec 2023 13:54:51 -0800 Subject: [PATCH 13/93] Local router mode - Up and running locally, no connections yet - Next: flip testnet and do the gosh dang thing --- .gitignore | 2 +- llarp/bootstrap.cpp | 1 + llarp/config/config.cpp | 53 ++++--- llarp/config/config.hpp | 4 +- llarp/config/key_manager.cpp | 122 ++++++++------- llarp/config/key_manager.hpp | 30 ++-- llarp/exit/context.cpp | 52 +++---- llarp/exit/context.hpp | 18 +-- llarp/link/contacts.cpp | 62 ++------ llarp/link/contacts.hpp | 29 +--- llarp/link/link_manager.cpp | 50 ++++-- llarp/link/link_manager.hpp | 14 +- llarp/nodedb.cpp | 11 +- llarp/nodedb.hpp | 26 +++- llarp/path/path_context.cpp | 27 +--- llarp/path/path_context.hpp | 17 +- llarp/path/pathbuilder.cpp | 2 +- llarp/router/router.cpp | 266 ++++++++++++++++++-------------- llarp/router/router.hpp | 42 +++-- llarp/router_contact.cpp | 5 +- llarp/router_contact.hpp | 2 +- llarp/router_contact_local.cpp | 5 +- llarp/router_contact_remote.cpp | 2 +- llarp/rpc/rpc_server.cpp | 4 +- llarp/service/endpoint.cpp | 6 +- llarp/service/identity.cpp | 2 +- 26 files changed, 434 insertions(+), 420 deletions(-) diff --git a/.gitignore b/.gitignore index a1d1593df..0b20f7ba5 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,7 @@ testnet_tmp vsproject/ .vs -daemon.ini +*.ini .gradle/ diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index 65d53773c..b377ec715 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -77,6 +77,7 @@ namespace llarp else { RemoteRC rc; + if (not rc.read(fpath)) { throw std::runtime_error{ diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index 77e6d8206..a777a4795 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -19,14 +19,7 @@ namespace llarp { - // constants for config file default values - constexpr int DefaultMinConnectionsForRouter = 6; - constexpr int DefaultMaxConnectionsForRouter = 60; - - constexpr int DefaultMinConnectionsForClient = 4; - constexpr int DefaultMaxConnectionsForClient = 6; - - constexpr int DefaultPublicPort = 1090; + constexpr int DEFAULT_PUBLIC_PORT = 1090; using namespace config; namespace @@ -72,38 +65,44 @@ namespace llarp net_id = std::move(arg); }); - int minConnections = - (params.is_relay ? DefaultMinConnectionsForRouter : DefaultMinConnectionsForClient); conf.define_option( "router", - "min-connections", - Default{minConnections}, + "relay-connections", + Default{CLIENT_ROUTER_CONNECTIONS}, + ClientOnly, Comment{ - "Minimum number of routers lokinet will attempt to maintain connections to.", + "Minimum number of routers lokinet client will attempt to maintain connections to.", }, [=](int arg) { - if (arg < minConnections) + if (arg < CLIENT_ROUTER_CONNECTIONS) throw std::invalid_argument{ - fmt::format("min-connections must be >= {}", minConnections)}; + fmt::format("Client relay connections must be >= {}", CLIENT_ROUTER_CONNECTIONS)}; - min_connected_routers = arg; + client_router_connections = arg; + }); + + conf.define_option( + "router", + "min-connections", + Deprecated, + Comment{ + "Minimum number of routers lokinet will attempt to maintain connections to.", + }, + [=](int) { + log::warning( + logcat, "Router min-connections is deprecated; use relay-connections for clients"); }); - int maxConnections = - (params.is_relay ? DefaultMaxConnectionsForRouter : DefaultMaxConnectionsForClient); conf.define_option( "router", "max-connections", - Default{maxConnections}, + Deprecated, Comment{ "Maximum number (hard limit) of routers lokinet will be connected to at any time.", }, - [=](int arg) { - if (arg < maxConnections) - throw std::invalid_argument{ - fmt::format("max-connections must be >= {}", maxConnections)}; - - max_connected_routers = arg; + [=](int) { + log::warning( + logcat, "Router max-connections is deprecated; use relay-connections for clients"); }); conf.define_option("router", "nickname", Deprecated); @@ -159,7 +158,7 @@ namespace llarp "router", "public-port", RelayOnly, - Default{DefaultPublicPort}, + Default{DEFAULT_PUBLIC_PORT}, Comment{ "When specifying public-ip=, this specifies the public UDP port at which this lokinet", "router is reachable. Required when public-ip is used.", @@ -1162,7 +1161,7 @@ namespace llarp conf.define_option( "lokid", "disable-testing", - Default{true}, + Default{false}, Hidden, RelayOnly, Comment{ diff --git a/llarp/config/config.hpp b/llarp/config/config.hpp index f47c92230..b294d9263 100644 --- a/llarp/config/config.hpp +++ b/llarp/config/config.hpp @@ -35,6 +35,7 @@ namespace llarp using ConfigMap = llarp::ConfigParser::ConfigMap; inline static constexpr uint16_t DEFAULT_LISTEN_PORT{1090}; + constexpr int CLIENT_ROUTER_CONNECTIONS = 4; // TODO: don't use these maps. they're sloppy and difficult to follow /// Small struct to gather all parameters needed for config generation to reduce the number of @@ -57,8 +58,7 @@ namespace llarp struct RouterConfig { - size_t min_connected_routers = 0; - size_t max_connected_routers = 0; + int client_router_connections{CLIENT_ROUTER_CONNECTIONS}; std::string net_id; diff --git a/llarp/config/key_manager.cpp b/llarp/config/key_manager.cpp index d1da613c5..070a4ce86 100644 --- a/llarp/config/key_manager.cpp +++ b/llarp/config/key_manager.cpp @@ -10,20 +10,20 @@ namespace llarp { - KeyManager::KeyManager() : m_initialized(false), m_needBackup(false) + KeyManager::KeyManager() : is_initialized(false), backup_keys(false) {} bool - KeyManager::initialize(const llarp::Config& config, bool genIfAbsent, bool isSNode) + KeyManager::initialize(const llarp::Config& config, bool gen_if_absent, bool is_snode) { - if (m_initialized) + if (is_initialized) return false; - if (not isSNode) + if (not is_snode) { - crypto::identity_keygen(identityKey); - crypto::encryption_keygen(encryptionKey); - crypto::encryption_keygen(transportKey); + crypto::identity_keygen(identity_key); + crypto::encryption_keygen(encryption_key); + crypto::encryption_keygen(transport_key); return true; } @@ -46,80 +46,87 @@ namespace llarp } }; - m_rcPath = deriveFile(our_rc_filename, config.router.rc_file); - m_idKeyPath = deriveFile(our_identity_filename, config.router.idkey_file); - m_encKeyPath = deriveFile(our_enc_key_filename, config.router.enckey_file); - m_transportKeyPath = deriveFile(our_transport_key_filename, config.router.transkey_file); + rc_path = deriveFile(our_rc_filename, config.router.rc_file); + idkey_path = deriveFile(our_identity_filename, config.router.idkey_file); + enckey_path = deriveFile(our_enc_key_filename, config.router.enckey_file); + transkey_path = deriveFile(our_transport_key_filename, config.router.transkey_file); RemoteRC rc; - bool exists = rc.read(m_rcPath); - if (not exists and not genIfAbsent) - { - LogError("Could not read RouterContact at path ", m_rcPath); - return false; - } - // we need to back up keys if our self.signed doesn't appear to have a - // valid signature - m_needBackup = (isSNode and not rc.verify()); - - // if our RC file can't be verified, assume it is out of date (e.g. uses - // older encryption) and needs to be regenerated. before doing so, backup - // files that will be overwritten - if (exists and m_needBackup) + if (auto exists = rc.read(rc_path); not exists) { - if (!genIfAbsent) + if (not gen_if_absent) { - LogError("Our RouterContact ", m_rcPath, " is invalid or out of date"); + log::error(logcat, "Could not read RC at path {}", rc_path); return false; } - else + } + else + { + if (backup_keys = (is_snode and not rc.verify()); backup_keys) { - LogWarn( - "Our RouterContact ", - m_rcPath, - " seems out of date, backing up and regenerating private keys"); + auto err = "RC (path:{}) is invalid or out of date"_format(rc_path); - if (!backupKeyFilesByMoving()) + if (not gen_if_absent) { - LogError( - "Could not mv some key files, please ensure key files" - " are backed up if needed and remove"); + log::error(logcat, err); return false; } - } - } - if (not config.router.is_relay) - { - // load identity key or create if needed - auto identityKeygen = [](llarp::SecretKey& key) { - // TODO: handle generating from service node seed - llarp::crypto::identity_keygen(key); - }; - if (not loadOrCreateKey(m_idKeyPath, identityKey, identityKeygen)) - return false; + log::warning(logcat, "{}; backing up and regenerating private keys...", err); + + if (not copy_backup_keyfiles()) + { + log::error(logcat, "Failed to copy-backup key files"); + return false; + } + } } // load encryption key - auto encryptionKeygen = [](llarp::SecretKey& key) { llarp::crypto::encryption_keygen(key); }; - if (not loadOrCreateKey(m_encKeyPath, encryptionKey, encryptionKeygen)) + auto enckey_gen = [](llarp::SecretKey& key) { llarp::crypto::encryption_keygen(key); }; + if (not keygen(enckey_path, encryption_key, enckey_gen)) + { + log::critical( + logcat, "KeyManager::keygen failed to generate encryption key line:{}", __LINE__); return false; + } // TODO: transport key (currently done in LinkLayer) - auto transportKeygen = [](llarp::SecretKey& key) { + auto transkey_gen = [](llarp::SecretKey& key) { key.Zero(); crypto::encryption_keygen(key); }; - if (not loadOrCreateKey(m_transportKeyPath, transportKey, transportKeygen)) + + if (not keygen(transkey_path, transport_key, transkey_gen)) + { + log::critical( + logcat, "KeyManager::keygen failed to generate transport key line:{}", __LINE__); return false; + } - m_initialized = true; + if (not config.router.is_relay) + { + // load identity key or create if needed + auto idkey_gen = [](llarp::SecretKey& key) { + // TODO: handle generating from service node seed + llarp::crypto::identity_keygen(key); + }; + + if (not keygen(idkey_path, identity_key, idkey_gen)) + { + log::critical( + logcat, "KeyManager::keygen failed to generate identity key line:{}", __LINE__); + return false; + } + } + + is_initialized = true; return true; } bool - KeyManager::backupFileByMoving(const fs::path& filepath) + KeyManager::copy_backup_keyfile(const fs::path& filepath) { auto findFreeBackupFilename = [](const fs::path& filepath) { for (int i = 0; i < 9; i++) @@ -136,6 +143,7 @@ namespace llarp std::error_code ec; bool exists = fs::exists(filepath, ec); + if (ec) { LogError("Could not determine status of file ", filepath, ": ", ec.message()); @@ -168,13 +176,13 @@ namespace llarp } bool - KeyManager::backupKeyFilesByMoving() const + KeyManager::copy_backup_keyfiles() const { - std::vector files = {m_rcPath, m_idKeyPath, m_encKeyPath, m_transportKeyPath}; + std::vector files = {rc_path, idkey_path, enckey_path, transkey_path}; for (auto& filepath : files) { - if (not backupFileByMoving(filepath)) + if (not copy_backup_keyfile(filepath)) return false; } @@ -182,7 +190,7 @@ namespace llarp } bool - KeyManager::loadOrCreateKey( + KeyManager::keygen( fs::path path, llarp::SecretKey& key, std::function keygen) { if (not fs::exists(path)) diff --git a/llarp/config/key_manager.hpp b/llarp/config/key_manager.hpp index 20a45d0c9..b0906f4d5 100644 --- a/llarp/config/key_manager.hpp +++ b/llarp/config/key_manager.hpp @@ -29,7 +29,7 @@ namespace llarp /// @param filepath is the name of the original file to backup. /// @return true if the file could be moved or didn't exist, false otherwise static bool - backupFileByMoving(const fs::path& filepath); + copy_backup_keyfile(const fs::path& filepath); /// Constructor KeyManager(); @@ -52,37 +52,37 @@ namespace llarp /// @param rc (out) will be modified to contian the RouterContact /// @return true on success, false otherwise bool - getRouterContact(llarp::RouterContact& rc) const; + gen_rc(llarp::RouterContact& rc) const; /// Return whether or not we need to backup keys as we load them bool - needBackup() const + needs_backup() const { - return m_needBackup; + return backup_keys; } - llarp::SecretKey identityKey; - llarp::SecretKey encryptionKey; - llarp::SecretKey transportKey; + llarp::SecretKey identity_key; + llarp::SecretKey encryption_key; + llarp::SecretKey transport_key; - fs::path m_rcPath; - fs::path m_idKeyPath; - fs::path m_encKeyPath; - fs::path m_transportKeyPath; + fs::path rc_path; + fs::path idkey_path; + fs::path enckey_path; + fs::path transkey_path; private: - std::atomic_bool m_initialized; - std::atomic_bool m_needBackup; + std::atomic_bool is_initialized; + std::atomic_bool backup_keys; /// Backup each key file (by copying, e.g. foo -> foo.bak) bool - backupKeyFilesByMoving() const; + copy_backup_keyfiles() const; /// Load the key at a given filepath or create it /// /// @param keygen is a function that will generate the key if needed static bool - loadOrCreateKey( + keygen( fs::path filepath, llarp::SecretKey& key, std::function keygen); diff --git a/llarp/exit/context.cpp b/llarp/exit/context.cpp index aa3761cc9..ec03c38e4 100644 --- a/llarp/exit/context.cpp +++ b/llarp/exit/context.cpp @@ -13,19 +13,19 @@ namespace llarp::exit Context::Tick(llarp_time_t now) { { - auto itr = m_Exits.begin(); - while (itr != m_Exits.end()) + auto itr = _exits.begin(); + while (itr != _exits.end()) { itr->second->Tick(now); ++itr; } } { - auto itr = m_Closed.begin(); - while (itr != m_Closed.end()) + auto itr = _closed.begin(); + while (itr != _closed.end()) { if ((*itr)->ShouldRemove()) - itr = m_Closed.erase(itr); + itr = _closed.erase(itr); else ++itr; } @@ -33,14 +33,14 @@ namespace llarp::exit } void - Context::Stop() + Context::stop() { - auto itr = m_Exits.begin(); - while (itr != m_Exits.end()) + auto itr = _exits.begin(); + while (itr != _exits.end()) { itr->second->Stop(); - m_Closed.emplace_back(std::move(itr->second)); - itr = m_Exits.erase(itr); + _closed.emplace_back(std::move(itr->second)); + itr = _exits.erase(itr); } } @@ -48,8 +48,8 @@ namespace llarp::exit Context::ExtractStatus() const { util::StatusObject obj{}; - auto itr = m_Exits.begin(); - while (itr != m_Exits.end()) + auto itr = _exits.begin(); + while (itr != _exits.end()) { obj[itr->first] = itr->second->ExtractStatus(); ++itr; @@ -58,10 +58,10 @@ namespace llarp::exit } void - Context::CalculateExitTraffic(TrafficStats& stats) + Context::calculate_exit_traffic(TrafficStats& stats) { - auto itr = m_Exits.begin(); - while (itr != m_Exits.end()) + auto itr = _exits.begin(); + while (itr != _exits.end()) { itr->second->CalculateTrafficStats(stats); ++itr; @@ -69,10 +69,10 @@ namespace llarp::exit } exit::Endpoint* - Context::FindEndpointForPath(const PathID_t& path) const + Context::find_endpoint_for_path(const PathID_t& path) const { - auto itr = m_Exits.begin(); - while (itr != m_Exits.end()) + auto itr = _exits.begin(); + while (itr != _exits.end()) { auto ep = itr->second->FindEndpointByPath(path); if (ep) @@ -83,10 +83,10 @@ namespace llarp::exit } bool - Context::ObtainNewExit(const PubKey& pk, const PathID_t& path, bool permitInternet) + Context::obtain_new_exit(const PubKey& pk, const PathID_t& path, bool permitInternet) { - auto itr = m_Exits.begin(); - while (itr != m_Exits.end()) + auto itr = _exits.begin(); + while (itr != _exits.end()) { if (itr->second->AllocateNewExit(pk, path, permitInternet)) return true; @@ -96,9 +96,9 @@ namespace llarp::exit } std::shared_ptr - Context::GetExitEndpoint(std::string name) const + Context::get_exit_endpoint(std::string name) const { - if (auto itr = m_Exits.find(name); itr != m_Exits.end()) + if (auto itr = _exits.find(name); itr != _exits.end()) { return itr->second; } @@ -106,10 +106,10 @@ namespace llarp::exit } void - Context::AddExitEndpoint( + Context::add_exit_endpoint( const std::string& name, const NetworkConfig& networkConfig, const DnsConfig& dnsConfig) { - if (m_Exits.find(name) != m_Exits.end()) + if (_exits.find(name) != _exits.end()) throw std::invalid_argument{fmt::format("An exit with name {} already exists", name)}; auto endpoint = std::make_unique(name, router); @@ -119,7 +119,7 @@ namespace llarp::exit if (!endpoint->Start()) throw std::runtime_error{fmt::format("Failed to start endpoint {}", name)}; - m_Exits.emplace(name, std::move(endpoint)); + _exits.emplace(name, std::move(endpoint)); } } // namespace llarp::exit diff --git a/llarp/exit/context.hpp b/llarp/exit/context.hpp index 31a9e766f..6e52da3bc 100644 --- a/llarp/exit/context.hpp +++ b/llarp/exit/context.hpp @@ -18,37 +18,37 @@ namespace llarp::exit Tick(llarp_time_t now); void - ClearAllEndpoints(); + clear_all_endpoints(); util::StatusObject ExtractStatus() const; /// send close to all exit sessions and remove all sessions void - Stop(); + stop(); void - AddExitEndpoint( + add_exit_endpoint( const std::string& name, const NetworkConfig& networkConfig, const DnsConfig& dnsConfig); bool - ObtainNewExit(const PubKey& remote, const PathID_t& path, bool permitInternet); + obtain_new_exit(const PubKey& remote, const PathID_t& path, bool permitInternet); exit::Endpoint* - FindEndpointForPath(const PathID_t& path) const; + find_endpoint_for_path(const PathID_t& path) const; /// calculate (pk, tx, rx) for all exit traffic using TrafficStats = std::unordered_map>; void - CalculateExitTraffic(TrafficStats& stats); + calculate_exit_traffic(TrafficStats& stats); std::shared_ptr - GetExitEndpoint(std::string name) const; + get_exit_endpoint(std::string name) const; private: Router* router; - std::unordered_map> m_Exits; - std::list> m_Closed; + std::unordered_map> _exits; + std::list> _closed; }; } // namespace llarp::exit diff --git a/llarp/link/contacts.cpp b/llarp/link/contacts.cpp index a31a231db..ffb78ad10 100644 --- a/llarp/link/contacts.cpp +++ b/llarp/link/contacts.cpp @@ -5,82 +5,38 @@ namespace llarp { - Contacts::Contacts(const dht::Key_t& k, Router& r) : _local_key{k}, _router{r} + Contacts::Contacts(Router& r) : _router{r}, _local_key{r.pubkey()} { timer_keepalive = std::make_shared(0); - _router.loop()->call_every(1s, timer_keepalive, [this]() { on_clean_contacts(); }); - _rc_nodes = std::make_unique>(_local_key, llarp::randint); _introset_nodes = std::make_unique>(_local_key, llarp::randint); } std::optional Contacts::get_introset_by_location(const dht::Key_t& key) const { - return _router.loop()->call_get([this, key]() -> std::optional { - auto& introsets = _introset_nodes->nodes; + std::optional enc = std::nullopt; - if (auto itr = introsets.find(key); itr != introsets.end()) - return itr->second.introset; + auto& introsets = _introset_nodes->nodes; - return std::nullopt; - }); - } - - void - Contacts::on_clean_contacts() - { - const auto now = llarp::time_now_ms(); - - if (_rc_nodes) - { - auto& nodes = _rc_nodes->nodes; - auto itr = nodes.begin(); - - while (itr != nodes.end()) - { - if (itr->second.rc.is_expired(now)) - itr = nodes.erase(itr); - else - ++itr; - } - } + if (auto itr = introsets.find(key); itr != introsets.end()) + enc = itr->second.introset; - if (_introset_nodes) - { - auto& svcs = _introset_nodes->nodes; - auto itr = svcs.begin(); - - while (itr != svcs.end()) - { - if (itr->second.introset.IsExpired(now)) - itr = svcs.erase(itr); - else - ++itr; - } - } + return enc; } util::StatusObject Contacts::ExtractStatus() const { util::StatusObject obj{ - {"nodes", _rc_nodes->ExtractStatus()}, - {"services", _introset_nodes->ExtractStatus()}, - {"local_key", _local_key.ToHex()}}; + {"services", _introset_nodes->ExtractStatus()}, {"local_key", _local_key.ToHex()}}; return obj; } void - Contacts::put_rc_node_async(const dht::RCNode& val) - { - _router.loop()->call([this, val]() { _rc_nodes->PutNode(val); }); - } - - void - Contacts::delete_rc_node_async(const dht::Key_t& val) + Contacts::put_intro(service::EncryptedIntroSet enc) { - _router.loop()->call([this, val]() { _rc_nodes->DelNode(val); }); + _introset_nodes->PutNode(std::move(enc)); } } // namespace llarp diff --git a/llarp/link/contacts.hpp b/llarp/link/contacts.hpp index 8423a346f..37056cbbc 100644 --- a/llarp/link/contacts.hpp +++ b/llarp/link/contacts.hpp @@ -15,28 +15,14 @@ namespace llarp private: // TODO: why was this a shared ptr in the original implementation? revisit this std::shared_ptr timer_keepalive; - const dht::Key_t& _local_key; Router& _router; - std::atomic transit_allowed{false}; + const dht::Key_t _local_key; - // holds router contacts - std::unique_ptr> _rc_nodes; // holds introsets for remote services std::unique_ptr> _introset_nodes; public: - Contacts(const dht::Key_t& local, Router& r); - - /// Sets the value of transit_allowed to the value of `b`. Returns false if the - /// value was already b, true otherwise - bool - set_transit_allowed(bool b) - { - return not transit_allowed.exchange(b) == b; - } - - void - on_clean_contacts(); + Contacts(Router& r); std::optional get_introset_by_location(const dht::Key_t& key) const; @@ -46,16 +32,7 @@ namespace llarp ExtractStatus() const; void - put_rc_node_async(const dht::RCNode& val); - - void - delete_rc_node_async(const dht::Key_t& val); - - dht::Bucket* - rc_nodes() const - { - return _rc_nodes.get(); - } + put_intro(service::EncryptedIntroSet enc); dht::Bucket* services() const diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 2f61820b9..6db101b36 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -183,9 +183,11 @@ namespace llarp - connection close callback - stream constructor callback - will return a BTRequestStream on the first call to get_new_stream + - bt stream construction contains a stream close callback that shuts down the connection + if the btstream closes unexpectedly */ auto ep = quic->endpoint( - _router.local_addr(), + _router.listen_addr(), [this](oxen::quic::connection_interface& ci) { return on_conn_open(ci); }, [this](oxen::quic::connection_interface& ci, uint64_t ec) { return on_conn_closed(ci, ec); @@ -198,7 +200,14 @@ namespace llarp std::optional id) -> std::shared_ptr { if (id && id == 0) { - auto s = std::make_shared(c, e); + auto s = std::make_shared( + c, e, [](oxen::quic::Stream& s, uint64_t error_code) { + log::warning( + logcat, + "BTRequestStream closed unexpectedly (ec:{}); closing connection...", + error_code); + s.conn.close_connection(error_code); + }); register_commands(s); return s; } @@ -216,6 +225,13 @@ namespace llarp , ep{startup_endpoint(), *this} {} + std::unique_ptr + LinkManager::make(Router& r) + { + std::unique_ptr p{new LinkManager(r)}; + return p; + } + bool LinkManager::send_control_message( const RouterID& remote, @@ -496,6 +512,7 @@ namespace llarp { is_stopping = false; node_db = _router.node_db(); + client_router_connections = _router.required_num_client_conns(); } void @@ -898,7 +915,7 @@ namespace llarp "Received PublishIntroMessage in which we are peer index {}.. storing introset", relay_order); - _router.contacts()->services()->PutNode(dht::ISNode{std::move(enc)}); + _router.contacts().put_intro(std::move(enc)); respond(serialize_response({{messages::STATUS_KEY, ""}})); } else @@ -936,7 +953,7 @@ namespace llarp { log::info(link_cat, "Received PublishIntroMessage for {} (TXID: {}); we are candidate {}"); - _router.contacts()->services()->PutNode(dht::ISNode{std::move(enc)}); + _router.contacts().put_intro(std::move(enc)); respond(serialize_response({{messages::STATUS_KEY, ""}})); } else @@ -1064,7 +1081,7 @@ namespace llarp } else { - if (auto maybe_intro = _router.contacts()->get_introset_by_location(addr)) + if (auto maybe_intro = _router.contacts().get_introset_by_location(addr)) respond(serialize_response({{"INTROSET", maybe_intro->bt_encode()}})); else { @@ -1102,7 +1119,7 @@ namespace llarp if (m) { service::EncryptedIntroSet enc{payload}; - _router.contacts()->services()->PutNode(std::move(enc)); + _router.contacts().put_intro(std::move(enc)); } else { @@ -1114,7 +1131,7 @@ namespace llarp void LinkManager::handle_path_build(oxen::quic::message m, const RouterID& from) { - if (!_router.path_context().AllowingTransit()) + if (!_router.path_context().is_transit_allowed()) { log::warning(link_cat, "got path build request when not permitting transit"); m.respond(serialize_response({{messages::STATUS_KEY, PathBuildMessage::NO_TRANSIT}}), true); @@ -1200,7 +1217,7 @@ namespace llarp hop->info.upstream.from_string(upstream); - if (_router.path_context().HasTransitHop(hop->info)) + if (_router.path_context().has_transit_hop(hop->info)) { log::warning(link_cat, "Invalid PathID; PathIDs must be unique"); m.respond(serialize_response({{messages::STATUS_KEY, PathBuildMessage::BAD_PATHID}}), true); @@ -1237,7 +1254,7 @@ namespace llarp { hop->terminal_hop = true; // we are terminal hop and everything is okay - _router.path_context().PutTransitHop(hop); + _router.path_context().put_transit_hop(hop); m.respond(messages::OK_RESPONSE, false); return; } @@ -1270,7 +1287,7 @@ namespace llarp link_cat, "Upstream returned successful path build response; giving hop info to Router, " "then relaying response"); - _router.path_context().PutTransitHop(hop); + _router.path_context().put_transit_hop(hop); } if (m.timed_out) log::info(link_cat, "Upstream timed out on path build; relaying timeout"); @@ -1380,7 +1397,7 @@ namespace llarp auto success = (crypto::verify(pubkey, to_usv(dict_data), sig) - and _router.exitContext().ObtainNewExit(PubKey{pubkey.data()}, rx_id, flag != 0)); + and _router.exitContext().obtain_new_exit(PubKey{pubkey.data()}, rx_id, flag != 0)); m.respond( ObtainExitMessage::sign_and_serialize_response(_router.identity(), tx_id), not success); @@ -1417,7 +1434,7 @@ namespace llarp throw; } - auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); + auto path_ptr = _router.path_context().get_path(PathID_t{to_usv(tx_id).data()}); if (crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) path_ptr->enable_exit_traffic(); @@ -1449,7 +1466,8 @@ namespace llarp auto transit_hop = _router.path_context().GetTransitHop(_router.pubkey(), PathID_t{to_usv(tx_id).data()}); - if (auto exit_ep = _router.exitContext().FindEndpointForPath(PathID_t{to_usv(path_id).data()})) + if (auto exit_ep = + _router.exitContext().find_endpoint_for_path(PathID_t{to_usv(path_id).data()})) { if (crypto::verify(exit_ep->PubKey().data(), to_usv(dict_data), sig)) { @@ -1495,7 +1513,7 @@ namespace llarp return; } - auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); + auto path_ptr = _router.path_context().get_path(PathID_t{to_usv(tx_id).data()}); if (crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) { @@ -1536,7 +1554,7 @@ namespace llarp const auto rx_id = transit_hop->info.rxID; - if (auto exit_ep = router().exitContext().FindEndpointForPath(rx_id)) + if (auto exit_ep = router().exitContext().find_endpoint_for_path(rx_id)) { if (crypto::verify(exit_ep->PubKey().data(), to_usv(dict_data), sig)) { @@ -1580,7 +1598,7 @@ namespace llarp return; } - auto path_ptr = _router.path_context().GetPath(PathID_t{to_usv(tx_id).data()}); + auto path_ptr = _router.path_context().get_path(PathID_t{to_usv(tx_id).data()}); if (path_ptr->SupportsAnyRoles(path::ePathRoleExit | path::ePathRoleSVC) and crypto::verify(_router.pubkey(), to_usv(dict_data), sig)) diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 47e148054..e038db178 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -30,6 +30,8 @@ namespace llarp using conn_open_hook = oxen::quic::connection_established_callback; using conn_closed_hook = oxen::quic::connection_closed_callback; + using stream_open_hook = oxen::quic::stream_open_callback; + using stream_closed_hook = oxen::quic::stream_close_callback; namespace link { @@ -141,7 +143,8 @@ namespace llarp struct LinkManager { public: - explicit LinkManager(Router& r); + static std::unique_ptr + make(Router& r); bool send_control_message( @@ -160,6 +163,8 @@ namespace llarp } private: + explicit LinkManager(Router& r); + bool send_control_message_impl( const RouterID& remote, @@ -310,11 +315,8 @@ namespace llarp void connect_to_random(int num_conns); - // TODO: tune these (maybe even remove max?) now that we're switching to quic - /// always maintain this many connections to other routers - size_t min_connected_routers = 4; - /// hard upperbound limit on the number of router to router connections - size_t max_connected_routers = 6; + /// always maintain this many client connections to other routers + int client_router_connections = 4; private: // DHT messages diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 6779d2a27..34adf4ced 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -766,8 +766,7 @@ namespace llarp router_greenlist.clear(); router_greenlist.insert(greenlist.begin(), greenlist.end()); - log::info( - logcat, "lokinet service node list now has ", known_rids.size(), " active router RIDs"); + log::info(logcat, "lokinet service node list now has {} active router RIDs", known_rids.size()); } std::optional @@ -843,7 +842,7 @@ namespace llarp const auto& rid = rc.router_id(); - auto [itr, b] = known_rcs.emplace(std::move(rc)); + auto [itr, b] = known_rcs.insert(std::move(rc)); rc_lookup.emplace(rid, *itr); known_rids.insert(rid); @@ -930,7 +929,7 @@ namespace llarp known_rcs.erase(rc); rc_lookup.erase(rid); - auto [itr, b] = known_rcs.emplace(std::move(rc)); + auto [itr, b] = known_rcs.insert(std::move(rc)); rc_lookup.emplace(rid, *itr); known_rids.insert(rid); @@ -939,9 +938,9 @@ namespace llarp } size_t - NodeDB::num_loaded() const + NodeDB::num_rcs() const { - return _router.loop()->call_get([this]() { return known_rcs.size(); }); + return known_rcs.size(); } bool diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 6470cd8c5..2cd2fde01 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -178,7 +178,7 @@ namespace llarp fs::path get_path_by_pubkey(RouterID pk) const; - std::unique_ptr _bootstraps; + std::unique_ptr _bootstraps{}; public: explicit NodeDB( @@ -299,10 +299,28 @@ namespace llarp return _pinned_edges; } - std::unique_ptr& + size_t + num_bootstraps() const + { + return _bootstraps ? _bootstraps->size() : 0; + } + + bool + has_bootstraps() const + { + return _bootstraps ? _bootstraps->empty() : false; + } + + const BootstrapList& + bootstrap_list() const + { + return *_bootstraps; + } + + BootstrapList& bootstrap_list() { - return _bootstraps; + return *_bootstraps; } void @@ -354,7 +372,7 @@ namespace llarp /// the number of RCs that are loaded from disk size_t - num_loaded() const; + num_rcs() const; /// do periodic tasks like flush to disk and expiration void diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp index a8e30a5a9..b8d9224f4 100644 --- a/llarp/path/path_context.cpp +++ b/llarp/path/path_context.cpp @@ -13,19 +13,19 @@ namespace llarp::path {} void - PathContext::AllowTransit() + PathContext::allow_transit() { m_AllowTransit = true; } bool - PathContext::AllowingTransit() const + PathContext::is_transit_allowed() const { return m_AllowTransit; } bool - PathContext::CheckPathLimitHitByIP(const IpAddress& ip) + PathContext::check_path_limit_hit_by_ip(const IpAddress& ip) { #ifdef TESTNET return false; @@ -39,21 +39,6 @@ namespace llarp::path #endif } - bool - PathContext::CheckPathLimitHitByIP(const std::string& ip) - { -#ifdef TESTNET - return false; -#else - IpAddress remote{ip}; - // null out the port -- we don't care about it for path limiting purposes - remote.setPort(0); - // try inserting remote address by ip into decaying hash set - // if it cannot insert it has hit a limit - return not path_limits.Insert(remote); -#endif - } - const EventLoop_ptr& PathContext::loop() { @@ -102,7 +87,7 @@ namespace llarp::path } bool - PathContext::HasTransitHop(const TransitHopInfo& info) + PathContext::has_transit_hop(const TransitHopInfo& info) { TransitHopID downstream{info.downstream, info.rxID}; if (transit_hops.count(downstream)) @@ -125,7 +110,7 @@ namespace llarp::path } Path_ptr - PathContext::GetPath(const PathID_t& path_id) + PathContext::get_path(const PathID_t& path_id) { if (auto itr = own_paths.find(path_id); itr != own_paths.end()) return itr->second; @@ -186,7 +171,7 @@ namespace llarp::path } void - PathContext::PutTransitHop(std::shared_ptr hop) + PathContext::put_transit_hop(std::shared_ptr hop) { TransitHopID downstream{hop->info.downstream, hop->info.rxID}; TransitHopID upstream{hop->info.upstream, hop->info.txID}; diff --git a/llarp/path/path_context.hpp b/llarp/path/path_context.hpp index 3f6b4485c..676831d21 100644 --- a/llarp/path/path_context.hpp +++ b/llarp/path/path_context.hpp @@ -62,28 +62,25 @@ namespace llarp::path ExpirePaths(llarp_time_t now); void - AllowTransit(); + allow_transit(); void - RejectTransit(); + reject_transit(); bool - CheckPathLimitHitByIP(const IpAddress& ip); + check_path_limit_hit_by_ip(const IpAddress& ip); bool - CheckPathLimitHitByIP(const std::string& ip); + is_transit_allowed() const; bool - AllowingTransit() const; - - bool - HasTransitHop(const TransitHopInfo& info); + has_transit_hop(const TransitHopInfo& info); void - PutTransitHop(std::shared_ptr hop); + put_transit_hop(std::shared_ptr hop); Path_ptr - GetPath(const PathID_t& path_id); + get_path(const PathID_t& path_id); bool TransitHopPreviousIsRouter(const PathID_t& path, const RouterID& r); diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index 9ba1392c8..1151f64aa 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -224,7 +224,7 @@ namespace llarp const auto& rid = rc.router_id(); #ifndef TESTNET - if (router->IsBootstrapNode(rid)) + if (router->is_bootstrap_node(rid)) return; #endif if (exclude.count(rid)) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index d3a8513fe..866e3ab03 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -45,7 +45,7 @@ namespace llarp , _disk_thread{_lmq->add_tagged_thread("disk")} , _rpc_server{nullptr} , _randomStartDelay{platform::is_simulation ? std::chrono::milliseconds{(llarp::randint() % 1250) + 2000} : 0s} - , _link_manager{*this} + // , _link_manager{*this} , _hidden_service_context{this} { _key_manager = std::make_shared(); @@ -59,9 +59,7 @@ namespace llarp } Router::~Router() - { - _contacts.reset(); - } + {} // TODO: investigate changes needed for libquic integration // still needed at all? @@ -86,11 +84,10 @@ namespace llarp return util::StatusObject{ {"running", true}, - {"numNodesKnown", _node_db->num_loaded()}, - {"contacts", _contacts->ExtractStatus()}, + {"numNodesKnown", _node_db->num_rcs()}, {"services", _hidden_service_context.ExtractStatus()}, {"exit", _exit_context.ExtractStatus()}, - {"links", _link_manager.extract_status()}}; + {"links", _link_manager->extract_status()}}; } // TODO: investigate changes needed for libquic integration @@ -102,7 +99,7 @@ namespace llarp auto services = _hidden_service_context.ExtractStatus(); - auto link_types = _link_manager.extract_status(); + auto link_types = _link_manager->extract_status(); uint64_t tx_rate = 0; uint64_t rx_rate = 0; @@ -175,7 +172,7 @@ namespace llarp {"uptime", to_json(Uptime())}, {"numPathsBuilt", pathsCount}, {"numPeersConnected", peers}, - {"numRoutersKnown", _node_db->num_loaded()}, + {"numRoutersKnown", _node_db->num_rcs()}, {"ratio", ratio}, {"txRate", tx_rate}, {"rxRate", rx_rate}, @@ -227,14 +224,14 @@ namespace llarp loop()->call([this, &peer_pubkeys]() { for (auto& pk : peer_pubkeys) - _link_manager.close_connection(pk); + _link_manager->close_connection(pk); }); } void Router::persist_connection_until(const RouterID& remote, llarp_time_t until) { - _link_manager.set_conn_persist(remote, until); + _link_manager->set_conn_persist(remote, until); } std::optional @@ -262,19 +259,19 @@ namespace llarp void Router::connect_to(const RouterID& rid) { - _link_manager.connect_to(rid); + _link_manager->connect_to(rid); } void Router::connect_to(const RemoteRC& rc) { - _link_manager.connect_to(rc); + _link_manager->connect_to(rc); } bool Router::send_data_message(const RouterID& remote, std::string payload) { - return _link_manager.send_data_message(remote, std::move(payload)); + return _link_manager->send_data_message(remote, std::move(payload)); } bool @@ -284,20 +281,20 @@ namespace llarp std::string body, std::function func) { - return _link_manager.send_control_message( + return _link_manager->send_control_message( remote, std::move(ep), std::move(body), std::move(func)); } void Router::for_each_connection(std::function func) { - return _link_manager.for_each_connection(func); + return _link_manager->for_each_connection(func); } bool Router::EnsureIdentity() { - _encryption = _key_manager->encryptionKey; + _encryption = _key_manager->encryption_key; if (is_service_node()) { @@ -340,13 +337,19 @@ namespace llarp } else { - _identity = _key_manager->identityKey; + _identity = _key_manager->identity_key; } if (_identity.IsZero()) + { + log::critical(logcat, "FUCK @ line:{}", __LINE__); return false; + } if (_encryption.IsZero()) + { + log::critical(logcat, "FUCK @ line:{}", __LINE__); return false; + } return true; } @@ -435,7 +438,7 @@ namespace llarp Router::insufficient_peers() const { constexpr int KnownPeerWarningThreshold = 5; - return node_db()->num_loaded() < KnownPeerWarningThreshold; + return node_db()->num_rcs() < KnownPeerWarningThreshold; } std::optional @@ -509,21 +512,22 @@ namespace llarp } size_t - Router::NumberOfConnectedRouters() const + Router::num_router_connections() const { - return _link_manager.get_num_connected(); + return _link_manager->get_num_connected(); } size_t - Router::NumberOfConnectedClients() const + Router::num_client_connections() const { - return _link_manager.get_num_connected_clients(); + return _link_manager->get_num_connected_clients(); } void Router::save_rc() { _node_db->put_rc(router_contact.view()); + log::info(logcat, "Saving RC file to {}", our_rc_file); queue_disk_io([&]() { router_contact.write(our_rc_file); }); } @@ -552,13 +556,14 @@ namespace llarp } // Router config - _link_manager.max_connected_routers = conf.router.max_connected_routers; - _link_manager.min_connected_routers = conf.router.min_connected_routers; + client_router_connections = conf.router.client_router_connections; - encryption_keyfile = _key_manager->m_encKeyPath; - our_rc_file = _key_manager->m_rcPath; - transport_keyfile = _key_manager->m_transportKeyPath; - identity_keyfile = _key_manager->m_idKeyPath; + encryption_keyfile = _key_manager->enckey_path; + our_rc_file = _key_manager->rc_path; + transport_keyfile = _key_manager->transkey_path; + identity_keyfile = _key_manager->idkey_path; + + std::optional _ourAddress; if (auto maybe_ip = conf.links.public_addr) _ourAddress = var::visit([](auto&& ip) { return SockAddr{ip}; }, *maybe_ip); @@ -574,11 +579,22 @@ namespace llarp else throw std::runtime_error{"public ip provided without public port"}; log::debug(logcat, "Using {} for our public address", *_ourAddress); + + _public_address = oxen::quic::Address{static_cast(*_ourAddress)}; + log::critical(logcat, "PUBLIC ADDR: {}", *_public_address); } else - log::debug(logcat, "No explicit public address given; will auto-detect during link setup"); + { + log::debug(logcat, "No explicit public address given; inferring now..."); - _local_addr = conf.links.addr; + if (auto maybe_addr = net().GetBestNetIF()) + { + _public_address = oxen::quic::Address{static_cast(*maybe_addr)}; + log::critical(logcat, "PUBLIC ADDR: {}", *_public_address); + } + } + + _listen_addr = conf.links.addr; RouterContact::BLOCK_BOGONS = conf.router.block_bogons; @@ -618,6 +634,26 @@ namespace llarp auto _bootstrap_rc_list = std::make_unique(); + auto clear_bad_rcs = [&]() mutable { + // in case someone has an old bootstrap file and is trying to use a bootstrap + // that no longer exists + for (auto it = _bootstrap_rc_list->begin(); it != _bootstrap_rc_list->end();) + { + if (it->is_obsolete_bootstrap()) + log::warning(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); + else if (not it->verify()) + log::warning(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); + else + { + ++it; + continue; + } + + // we are in one of the above error cases that we warned about: + it = _bootstrap_rc_list->erase(it); + } + }; + for (const auto& router : configRouters) { log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); @@ -629,11 +665,18 @@ namespace llarp _bootstrap_rc_list->emplace(rc); } + clear_bad_rcs(); + if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode) { auto fallbacks = llarp::load_bootstrap_fallbacks(); - if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode) + if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) + { + _bootstrap_rc_list->merge(itr->second); + } + + if (_bootstrap_rc_list->empty()) { // empty after trying fallback, if set log::error( @@ -644,35 +687,15 @@ namespace llarp throw std::runtime_error("No bootstrap nodes available."); } - } - - // in case someone has an old bootstrap file and is trying to use a bootstrap - // that no longer exists - for (auto it = _bootstrap_rc_list->begin(); it != _bootstrap_rc_list->end();) - { - if (it->is_obsolete_bootstrap()) - log::warning(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); - else if (not it->verify()) - log::warning(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); - else - { - ++it; - continue; - } - // we are in one of the above error cases that we warned about: - it = _bootstrap_rc_list->erase(it); + log::info( + logcat, "Loaded {} default fallback bootstrap routers!", _bootstrap_rc_list->size()); + clear_bad_rcs(); + node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); } - node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); - if (conf.bootstrap.seednode) - LogInfo("we are a seed node"); - else - LogInfo("Loaded ", _bootstrap_rc_list->size(), " bootstrap routers"); - - // Init components after relevant config settings loaded - _link_manager.init(); + log::critical(logcat, "We are a bootstrap seed node!"); // TODO: RC refactor here if (_is_service_node) @@ -713,14 +736,18 @@ namespace llarp } bool - Router::IsBootstrapNode(const RouterID r) const + Router::is_bootstrap_node(const RouterID r) const { - const auto& b = _node_db->bootstrap_list(); - return std::count_if( - b->begin(), - b->end(), - [r](const RemoteRC& rc) -> bool { return rc.router_id() == r; }) - > 0; + if (_node_db->has_bootstraps()) + { + const auto& b = _node_db->bootstrap_list(); + return std::count_if( + b.begin(), + b.end(), + [r](const RemoteRC& rc) -> bool { return rc.router_id() == r; }) + > 0; + } + return false; } bool @@ -737,16 +764,16 @@ namespace llarp log::info( logcat, "{} RCs loaded with {} bootstrap peers and {} router connections!", - node_db()->num_loaded(), - _node_db->bootstrap_list()->size(), - NumberOfConnectedRouters()); + _node_db->num_rcs(), + _node_db->num_bootstraps(), + num_router_connections()); if (is_service_node()) { log::info( logcat, "Local service node has {} client connections since last RC update ({} to expiry)", - NumberOfConnectedClients(), + num_client_connections(), router_contact.age(now), router_contact.time_to_expiry(now)); } @@ -766,9 +793,9 @@ namespace llarp fmt::format_to( out, " snode | known/svc/clients: {}/{}/{}", - node_db()->num_loaded(), - NumberOfConnectedRouters(), - NumberOfConnectedClients()); + node_db()->num_rcs(), + num_router_connections(), + num_client_connections()); fmt::format_to( out, " | {} active paths | block {} ", @@ -784,10 +811,7 @@ namespace llarp else { fmt::format_to( - out, - " client | known/connected: {}/{}", - node_db()->num_loaded(), - NumberOfConnectedRouters()); + out, " client | known/connected: {}/{}", node_db()->num_rcs(), num_router_connections()); if (auto ep = hidden_service_context().GetDefault()) { @@ -850,7 +874,7 @@ namespace llarp auto view = router_contact.view(); - _link_manager.gossip_rc( + _link_manager->gossip_rc( pubkey(), std::string{reinterpret_cast(view.data()), view.size()}); last_rc_gossip = now_timepoint; @@ -891,14 +915,14 @@ namespace llarp // remove RCs for nodes that are no longer allowed by network policy node_db()->RemoveIf([&](const RemoteRC& rc) -> bool { // don't purge bootstrap nodes from nodedb - if (IsBootstrapNode(rc.router_id())) + if (is_bootstrap_node(rc.router_id())) { log::trace(logcat, "Not removing {}: is bootstrap node", rc.router_id()); return false; } // if for some reason we stored an RC that isn't a valid router // purge this entry - if (not rc.is_public_router()) + if (not rc.is_public_addressable()) { log::debug(logcat, "Removing {}: not a valid router", rc.router_id()); return true; @@ -956,11 +980,11 @@ namespace llarp } */ - _link_manager.check_persisting_conns(now); + _link_manager->check_persisting_conns(now); - size_t connected = NumberOfConnectedRouters(); + size_t connected = num_router_connections(); - size_t connectToNum = _link_manager.min_connected_routers; + size_t connectToNum = _link_manager->client_router_connections; const auto& pinned_edges = _node_db->pinned_edges(); const auto pinned_count = pinned_edges.size(); @@ -989,7 +1013,7 @@ namespace llarp log::error( logcat, "We appear to be an active service node, but have only {} known peers.", - node_db()->num_loaded()); + node_db()->num_rcs()); _next_decomm_warning = now + DecommissionWarnInterval; } } @@ -1000,7 +1024,7 @@ namespace llarp { size_t dlt = connectToNum - connected; LogDebug("connecting to ", dlt, " random routers to keep alive"); - _link_manager.connect_to_random(dlt); + _link_manager->connect_to_random(dlt); } _hidden_service_context.Tick(now); @@ -1023,7 +1047,7 @@ namespace llarp bool Router::GetRandomConnectedRouter(RemoteRC& result) const { - return _link_manager.get_random_connected(result); + return _link_manager->get_random_connected(result); } const std::set& @@ -1054,16 +1078,22 @@ namespace llarp bool Router::Run() { + log::critical(logcat, "{} called", __PRETTY_FUNCTION__); + if (is_running || is_stopping) return false; - // TODO: look at _ourAddress + router_contact = LocalRC::make( + identity(), _is_service_node and _public_address ? *_public_address : _listen_addr); - router_contact = LocalRC::make(identity(), local_addr()); + _link_manager = LinkManager::make(*this); - if (is_service_node() and not router_contact.is_public_router()) + // Init components after relevant config settings loaded + _link_manager->init(); + + if (is_service_node()) { - if (not router_contact.is_public_router()) + if (not router_contact.is_public_addressable()) { log::error(logcat, "Router is configured as relay but has no reachable addresses!"); return false; @@ -1101,16 +1131,19 @@ namespace llarp log::info(logcat, "Loading NodeDB from disk..."); _node_db->load_from_disk(); - _contacts = std::make_shared(llarp::dht::Key_t(pubkey()), *this); + log::info(logcat, "Creating Introset Contacts..."); + _contacts = std::make_unique(*this); - for (const auto& rc : *_node_db->bootstrap_list()) + if (_node_db->has_bootstraps()) { - node_db()->put_rc(rc); - _contacts->rc_nodes()->PutNode(rc); - log::info(logcat, "Added bootstrap node (rid: {})", rc.router_id()); - } + for (const auto& rc : _node_db->bootstrap_list()) + { + node_db()->put_rc(rc); + log::info(logcat, "Added bootstrap node (rid: {})", rc.router_id()); + } - log::info(logcat, "Router populated NodeDB with {} routers", _node_db->num_loaded()); + log::info(logcat, "Router populated NodeDB with {} routers", _node_db->num_rcs()); + } _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); @@ -1155,7 +1188,7 @@ namespace llarp // try to make a session to this random router // this will do a dht lookup if needed - _link_manager.test_reachability( + _link_manager->test_reachability( router, [this, rid = router, previous = fails](oxen::quic::connection_interface& conn) { log::info( @@ -1224,7 +1257,7 @@ namespace llarp void Router::StopLinks() { - _link_manager.stop(); + _link_manager->stop(); } void @@ -1241,7 +1274,7 @@ namespace llarp LogWarn("stopping router hard"); llarp::sys::service_manager->stopping(); hidden_service_context().StopAll(); - _exit_context.Stop(); + _exit_context.stop(); StopLinks(); Close(); } @@ -1261,27 +1294,27 @@ namespace llarp } is_stopping.store(true); + if (auto level = log::get_level_default(); level > log::Level::info and level != log::Level::off) log::reset_level(log::Level::info); - log::info(logcat, "stopping"); + + log::info(logcat, "stopping service manager..."); llarp::sys::service_manager->stopping(); - log::debug(logcat, "stopping hidden service context"); + + log::debug(logcat, "stopping hidden service context..."); hidden_service_context().StopAll(); - llarp::sys::service_manager->stopping(); - log::debug(logcat, "stopping exit context"); - _exit_context.Stop(); - llarp::sys::service_manager->stopping(); - log::debug(logcat, "final upstream pump"); - llarp::sys::service_manager->stopping(); - log::debug(logcat, "final links pump"); + + log::debug(logcat, "stopping exit context..."); + _exit_context.stop(); + _loop->call_later(200ms, [this] { AfterStopIssued(); }); } bool Router::HasSessionTo(const RouterID& remote) const { - return _link_manager.have_connection_to(remote); + return _link_manager->have_connection_to(remote); } std::string @@ -1300,19 +1333,20 @@ namespace llarp Router::ConnectToRandomRouters(int _want) { const size_t want = _want; - auto connected = NumberOfConnectedRouters(); + auto connected = num_router_connections(); + if (connected >= want) return; - _link_manager.connect_to_random(want); + + _link_manager->connect_to_random(want); } bool Router::init_service_node() { - LogInfo("accepting transit traffic"); - paths.AllowTransit(); - _contacts->set_transit_allowed(true); - _exit_context.AddExitEndpoint("default", _config->network, _config->dns); + log::info(logcat, "Router accepting transit traffic..."); + paths.allow_transit(); + _exit_context.add_exit_endpoint("default", _config->network, _config->dns); return true; } @@ -1338,9 +1372,9 @@ namespace llarp } oxen::quic::Address - Router::local_addr() const + Router::listen_addr() const { - return _local_addr; + return _listen_addr; } void diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 3fe14c868..db73d753d 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -91,8 +91,8 @@ namespace llarp consensus::reachability_testing router_testing; - std::optional _ourAddress; - oxen::quic::Address _local_addr; + std::optional _public_address; // public addr for relays + oxen::quic::Address _listen_addr; EventLoop_ptr _loop; std::shared_ptr _vpn; @@ -122,7 +122,9 @@ namespace llarp oxenmq::address rpc_addr; Profiling _router_profiling; fs::path _profile_file; - LinkManager _link_manager{*this}; + + std::unique_ptr _link_manager; + int client_router_connections; // should we be sending padded messages every interval? bool send_padding = false; @@ -155,6 +157,12 @@ namespace llarp std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip}; public: + int + required_num_client_conns() const + { + return client_router_connections; + } + RouterID local_rid() const { @@ -176,10 +184,16 @@ namespace llarp void connect_to(const RemoteRC& rc); - Contacts* + const Contacts& contacts() const { - return _contacts.get(); + return *_contacts; + } + + Contacts& + contacts() + { + return *_contacts; } std::shared_ptr @@ -212,10 +226,16 @@ namespace llarp LinkManager& link_manager() { - return _link_manager; + return *_link_manager; + } + + const LinkManager& + link_manager() const + { + return *_link_manager; } - inline int + int outbound_udp_socket() const { return _outbound_udp_socket; @@ -282,7 +302,7 @@ namespace llarp } oxen::quic::Address - local_addr() const; + listen_addr() const; util::StatusObject ExtractStatus() const; @@ -473,7 +493,7 @@ namespace llarp std::string body, std::function func = nullptr); - bool IsBootstrapNode(RouterID) const; + bool is_bootstrap_node(RouterID) const; /// call internal router ticker void @@ -490,11 +510,11 @@ namespace llarp /// count the number of unique service nodes connected via pubkey size_t - NumberOfConnectedRouters() const; + num_router_connections() const; /// count the number of unique clients connected by pubkey size_t - NumberOfConnectedClients() const; + num_client_connections() const; bool GetRandomConnectedRouter(RemoteRC& result) const; diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index 0cd402802..585e72f99 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -102,7 +102,7 @@ namespace llarp { util::StatusObject obj{ {"lastUpdated", _timestamp.time_since_epoch().count()}, - {"publicRouter", is_public_router()}, + {"publicRouter", is_public_addressable()}, {"identity", _router_id.ToString()}, {"address", _addr.to_string()}}; @@ -221,10 +221,11 @@ namespace llarp } bool - RouterContact::is_public_router() const + RouterContact::is_public_addressable() const { if (_router_version.empty()) return false; + return _addr.is_addressable(); } diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index ce99d3822..0aee92c27 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -176,7 +176,7 @@ namespace llarp decode_key(const llarp_buffer_t& k, llarp_buffer_t* buf); bool - is_public_router() const; + is_public_addressable() const; /// does this RC expire soon? default delta is 1 minute bool diff --git a/llarp/router_contact_local.cpp b/llarp/router_contact_local.cpp index 9b82551cc..47ed6009c 100644 --- a/llarp/router_contact_local.cpp +++ b/llarp/router_contact_local.cpp @@ -22,7 +22,8 @@ namespace llarp { _router_id = llarp::seckey_to_pubkey(_secret_key); _addr = std::move(local); - _addr6.emplace(&_addr.in6()); + if (_addr.is_ipv6()) + _addr6.emplace(&_addr.in6()); resign(); } @@ -123,8 +124,6 @@ namespace llarp static_assert(llarp::LOKINET_VERSION.size() == 3); btdp.append( "v", std::string_view{reinterpret_cast(llarp::LOKINET_VERSION.data()), 3}); - - bt_sign(btdp); } void diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 8e8188052..f62849adf 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -89,7 +89,7 @@ namespace llarp } catch (const std::exception& e) { - log::error(logcat, "Failed to read or validate RC from {}: {}", fname, e.what()); + log::warning(logcat, "Failed to read or validate RC from {}: {}", fname, e.what()); return false; } diff --git a/llarp/rpc/rpc_server.cpp b/llarp/rpc/rpc_server.cpp index ea657fb81..c6873ff0f 100644 --- a/llarp/rpc/rpc_server.cpp +++ b/llarp/rpc/rpc_server.cpp @@ -76,7 +76,7 @@ namespace llarp::rpc { if (r.is_service_node()) { - return r.exitContext().GetExitEndpoint(name); + return r.exitContext().get_exit_endpoint(name); } return r.hidden_service_context().GetEndpointByName(name); @@ -329,7 +329,7 @@ namespace llarp::rpc } m_Router.loop()->call([&]() { - auto endpoint = m_Router.exitContext().GetExitEndpoint("default"); + auto endpoint = m_Router.exitContext().get_exit_endpoint("default"); if (endpoint == nullptr) { diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 6f7ea08a0..8c3264fe3 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -516,7 +516,7 @@ namespace llarp::service const auto& keyfile = _state->key_file; if (!keyfile.empty()) { - _identity.EnsureKeys(keyfile, router()->key_manager()->needBackup()); + _identity.EnsureKeys(keyfile, router()->key_manager()->needs_backup()); } else { @@ -1310,7 +1310,7 @@ namespace llarp::service // TODO: if all requests fail, call callback with failure? for (const auto& path : paths) { - path->find_intro(location, false, 0, [this, hook, got_it](std::string resp) mutable { + path->find_intro(location, false, 0, [hook, got_it, this](std::string resp) mutable { // asking many, use only first successful if (*got_it) return; @@ -1335,7 +1335,7 @@ namespace llarp::service } service::EncryptedIntroSet enc{introset}; - router()->contacts()->services()->PutNode(std::move(enc)); + router()->contacts().put_intro(std::move(enc)); // TODO: finish this /* diff --git a/llarp/service/identity.cpp b/llarp/service/identity.cpp index a79c6ae97..63ba3c5d6 100644 --- a/llarp/service/identity.cpp +++ b/llarp/service/identity.cpp @@ -97,7 +97,7 @@ namespace llarp::service if (exists and needBackup) { - KeyManager::backupFileByMoving(fname); + KeyManager::copy_backup_keyfile(fname); exists = false; } From aaf284b39fe86bf0702b4e4afe948ca909c8a7c6 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 7 Dec 2023 10:05:33 -0800 Subject: [PATCH 14/93] libquic vbump - now pointing to jason/void-listen --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 3ced484e8..a2d89aa79 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 3ced484e8cc543b90c5fc554ccc0ea2e54ec8d37 +Subproject commit a2d89aa79dd06cbd7ee864da285cf4d8ac1b09b9 From d00257b9f0aa44de9c2ad2db2a1f61ca12c78cc3 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 7 Dec 2023 14:39:39 -0800 Subject: [PATCH 15/93] address parsing - straightened out setting of public addr and public port in config vs listen ("bind") addr - fixed small bug in router contact writing and saving --- daemon/lokinet.cpp | 6 +-- llarp/config/config.cpp | 93 ++++++++++++++++----------------- llarp/config/config.hpp | 22 ++++---- llarp/link/link_manager.hpp | 10 +++- llarp/nodedb.hpp | 2 +- llarp/router/router.cpp | 79 ++++++++++++++-------------- llarp/router/router.hpp | 2 +- llarp/router_contact.hpp | 4 +- llarp/router_contact_local.cpp | 2 +- llarp/router_contact_remote.cpp | 4 +- 10 files changed, 117 insertions(+), 107 deletions(-) diff --git a/daemon/lokinet.cpp b/daemon/lokinet.cpp index 89b3d155a..bf15d6394 100644 --- a/daemon/lokinet.cpp +++ b/daemon/lokinet.cpp @@ -593,7 +593,7 @@ namespace exit_code.set_value(1); return; } - catch (std::exception& ex) + catch (const std::exception& ex) { llarp::LogError(fmt::format("failed to start up lokinet: {}", ex.what())); exit_code.set_value(1); @@ -604,9 +604,9 @@ namespace auto result = ctx->Run(opts); exit_code.set_value(result); } - catch (std::exception& e) + catch (const std::exception& e) { - llarp::LogError("Fatal: caught exception while running: ", e.what()); + llarp::LogError("Fatal: caught exception while running: {}", e.what()); exit_code.set_exception(std::current_exception()); } catch (...) diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index a777a4795..5b92f0fe9 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -19,8 +19,6 @@ namespace llarp { - constexpr int DEFAULT_PUBLIC_PORT = 1090; - using namespace config; namespace { @@ -134,19 +132,7 @@ namespace llarp "this setting specifies the public IP at which this router is reachable. When", "provided the public-port option must also be specified.", }, - [this, net = params.Net_ptr()](std::string arg) { - if (arg.empty()) - return; - nuint32_t addr{}; - if (not addr.FromString(arg)) - throw std::invalid_argument{fmt::format("{} is not a valid IPv4 address", arg)}; - - if (net->IsBogonIP(addr)) - throw std::invalid_argument{ - fmt::format("{} is not a publicly routable ip address", addr)}; - - public_ip = addr; - }); + [this](std::string arg) { public_ip = std::move(arg); }); conf.define_option("router", "public-address", Hidden, [](std::string) { throw std::invalid_argument{ @@ -154,19 +140,18 @@ namespace llarp "[router]:public-port instead"}; }); - conf.define_option( + conf.define_option( "router", "public-port", RelayOnly, - Default{DEFAULT_PUBLIC_PORT}, Comment{ "When specifying public-ip=, this specifies the public UDP port at which this lokinet", "router is reachable. Required when public-ip is used.", }, - [this](int arg) { + [this](uint16_t arg) { if (arg <= 0 || arg > std::numeric_limits::max()) throw std::invalid_argument("public-port must be >= 0 and <= 65536"); - public_port = ToNet(huint16_t{static_cast(arg)}); + public_port = arg; }); conf.define_option( @@ -912,27 +897,38 @@ namespace llarp conf.define_option( "bind", "public-ip", + Hidden, RelayOnly, Comment{ "The IP address to advertise to the network instead of the incoming= or auto-detected", "IP. This is typically required only when incoming= is used to listen on an internal", "private range IP address that received traffic forwarded from the public IP.", }, - [this](std::string_view arg) { - SockAddr pubaddr{arg}; - public_addr = pubaddr.getIP(); + [this](std::string arg) { + public_addr = std::move(arg); + log::warning( + logcat, + "Using deprecated option; pass this value to [Router]:public-ip instead PLEASE"); }); conf.define_option( "bind", "public-port", + Hidden, RelayOnly, Comment{ "The port to advertise to the network instead of the incoming= (or default) port.", "This is typically required only when incoming= is used to listen on an internal", "private range IP address/port that received traffic forwarded from the public IP.", }, - [this](uint16_t arg) { public_port = net::port_t::from_host(arg); }); + [this](uint16_t arg) { + if (arg <= 0 || arg > std::numeric_limits::max()) + throw std::invalid_argument("public-port must be >= 0 and <= 65536"); + public_port = arg; + log::warning( + logcat, + "Using deprecated option; pass this value to [Router]:public-port instead PLEASE"); + }); auto parse_addr_for_link = [net_ptr](const std::string& arg) { std::optional maybe = std::nullopt; @@ -1004,11 +1000,13 @@ namespace llarp }, [this, parse_addr_for_link](const std::string& arg) { if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) - addr = *a; + { + listen_addr = *a; + using_user_value = true; + using_new_api = true; + } else throw std::invalid_argument{"Could not parse listen address!"}; - - using_new_api = true; }); conf.define_option( @@ -1022,21 +1020,17 @@ namespace llarp throw std::runtime_error{"USE THE NEW API -- SPECIFY LOCAL ADDRESS UNDER [LISTEN]"}; if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) - addr = *a; + { + log::warning( + logcat, + "Loaded address from deprecated [inbound] options; update your config to use " + "[bind]:listen instead PLEASE"); + listen_addr = *a; + using_user_value = true; + } }); - conf.define_option( - "bind", - "outbound", - MultiValue, - Hidden, - [this, parse_addr_for_link](const std::string& arg) { - if (using_new_api) - throw std::runtime_error{"USE THE NEW API -- SPECIFY LOCAL ADDRESS UNDER [LISTEN]"}; - - if (auto a = parse_addr_for_link(arg); a and a->is_addressable()) - addr = *a; - }); + conf.define_option("bind", "outbound", MultiValue, Deprecated, Hidden); conf.add_undeclared_handler( "bind", [this](std::string_view, std::string_view key, std::string_view val) { @@ -1055,7 +1049,11 @@ namespace llarp // special case: wildcard for outbound if (key == "*") { - addr = oxen::quic::Address{port}; + log::warning( + logcat, + "Wildcat address referencing port {} is referencing deprecated outbound config " + "options; use [bind]:listen instead", + port); return; } @@ -1074,16 +1072,17 @@ namespace llarp e.what())}; } - if (temp.is_addressable()) + if (not temp.is_addressable()) { - addr = std::move(temp); - return; + throw std::runtime_error{fmt::format( + "Invalid address: {}; stop using this deprecated handler, update your config to " + "use " + "[bind]:listen instead PLEASE", + temp)}; } - throw std::runtime_error{fmt::format( - "Invalid address: {}; stop using this deprecated handler, update your config to use " - "[bind]:listen instead PLEASE", - temp)}; + listen_addr = std::move(temp); + using_user_value = true; }); } diff --git a/llarp/config/config.hpp b/llarp/config/config.hpp index b294d9263..b0a9231a9 100644 --- a/llarp/config/config.hpp +++ b/llarp/config/config.hpp @@ -34,8 +34,9 @@ namespace llarp using SectionValues = llarp::ConfigParser::SectionValues; using ConfigMap = llarp::ConfigParser::ConfigMap; - inline static constexpr uint16_t DEFAULT_LISTEN_PORT{1090}; - constexpr int CLIENT_ROUTER_CONNECTIONS = 4; + inline const std::string QUAD_ZERO{"0.0.0.0"}; + inline constexpr uint16_t DEFAULT_LISTEN_PORT{1090}; + inline constexpr int CLIENT_ROUTER_CONNECTIONS = 4; // TODO: don't use these maps. they're sloppy and difficult to follow /// Small struct to gather all parameters needed for config generation to reduce the number of @@ -77,10 +78,9 @@ namespace llarp std::string transkey_file; bool is_relay = false; - /// deprecated - std::optional public_ip; - /// deprecated - std::optional public_port; + + std::optional public_ip; + std::optional public_port; void define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params); @@ -170,10 +170,14 @@ namespace llarp struct LinksConfig { - std::optional public_addr; - std::optional public_port; + // DEPRECATED -- use [Router]:public_addr + std::optional public_addr; + // DEPRECATED -- use [Router]:public_port + std::optional public_port; + + std::optional listen_addr; - oxen::quic::Address addr{""s, DEFAULT_LISTEN_PORT}; + bool using_user_value = false; bool using_new_api = false; void diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index e038db178..990e54cf9 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -417,8 +417,14 @@ namespace llarp connid_map.emplace(conn_interface->scid(), rc.router_id()); auto [itr, b] = conns.emplace(rc.router_id(), nullptr); - auto control_stream = - conn_interface->template get_new_stream(); + auto control_stream = conn_interface->template get_new_stream( + [](oxen::quic::Stream& s, uint64_t error_code) { + log::warning( + logcat, + "BTRequestStream closed unexpectedly (ec:{}); closing connection...", + error_code); + s.conn.close_connection(error_code); + }); itr->second = std::make_shared(conn_interface, control_stream, rc); return true; diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 2cd2fde01..15997655c 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -165,7 +165,7 @@ namespace llarp std::atomic fetch_failures{0}, bootstrap_failures{0}; std::atomic _using_bootstrap_fallback{false}, _needs_rebootstrap{false}, - _needs_initial_fetch{false}; + _needs_initial_fetch{true}; bool want_rc(const RouterID& rid) const; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 866e3ab03..5be99f6dc 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -563,38 +563,39 @@ namespace llarp transport_keyfile = _key_manager->transkey_path; identity_keyfile = _key_manager->idkey_path; - std::optional _ourAddress; + std::optional paddr = (conf.router.public_ip) ? conf.router.public_ip + : (conf.links.public_addr) ? conf.links.public_addr + : std::nullopt; + std::optional pport = (conf.router.public_port) ? conf.router.public_port + : (conf.links.public_port) ? conf.links.public_port + : std::nullopt; - if (auto maybe_ip = conf.links.public_addr) - _ourAddress = var::visit([](auto&& ip) { return SockAddr{ip}; }, *maybe_ip); - else if (auto maybe_ip = conf.router.public_ip) - _ourAddress = var::visit([](auto&& ip) { return SockAddr{ip}; }, *maybe_ip); + if (pport.has_value() and not paddr.has_value()) + throw std::runtime_error{"If public-port is specified, public-addr must be as well!"}; - if (_ourAddress) + if (conf.links.listen_addr) { - if (auto maybe_port = conf.links.public_port) - _ourAddress->setPort(*maybe_port); - else if (auto maybe_port = conf.router.public_port) - _ourAddress->setPort(*maybe_port); - else - throw std::runtime_error{"public ip provided without public port"}; - log::debug(logcat, "Using {} for our public address", *_ourAddress); - - _public_address = oxen::quic::Address{static_cast(*_ourAddress)}; - log::critical(logcat, "PUBLIC ADDR: {}", *_public_address); + _listen_address = *conf.links.listen_addr; } else { - log::debug(logcat, "No explicit public address given; inferring now..."); + if (paddr or pport) + throw std::runtime_error{"Must specify [bind]:listen in config with public ip/addr!"}; if (auto maybe_addr = net().GetBestNetIF()) { - _public_address = oxen::quic::Address{static_cast(*maybe_addr)}; - log::critical(logcat, "PUBLIC ADDR: {}", *_public_address); + _listen_address = oxen::quic::Address{static_cast(*maybe_addr)}; + _listen_address.set_port(DEFAULT_LISTEN_PORT); } + else + throw std::runtime_error{"Could not find net interface on current platform!"}; } - _listen_addr = conf.links.addr; + _public_address = (not paddr and not pport) + ? _listen_address + : oxen::quic::Address{*paddr, pport ? *pport : DEFAULT_LISTEN_PORT}; + + log::critical(logcat, "listen_addr:{} \t public_addr:{}", _listen_address, _public_address); RouterContact::BLOCK_BOGONS = conf.router.block_bogons; @@ -886,29 +887,27 @@ namespace llarp next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; } } + + if (needs_initial_fetch()) + { + node_db()->fetch_initial(); + } + else if (needs_rebootstrap() and next_bootstrap_attempt > now_timepoint) + { + node_db()->fallback_to_bootstrap(); + } else { - if (needs_initial_fetch()) + // (client-only) periodically fetch updated RCs + if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) { - node_db()->fetch_initial(); + node_db()->fetch_rcs(); } - else if (needs_rebootstrap() and next_bootstrap_attempt > now_timepoint) - { - node_db()->fallback_to_bootstrap(); - } - else - { - // (client-only) periodically fetch updated RCs - if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) - { - node_db()->fetch_rcs(); - } - // (client-only) periodically fetch updated RouterID list - if (now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) - { - node_db()->fetch_rids(); - } + // (client-only) periodically fetch updated RouterID list + if (now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) + { + node_db()->fetch_rids(); } } @@ -1084,7 +1083,7 @@ namespace llarp return false; router_contact = LocalRC::make( - identity(), _is_service_node and _public_address ? *_public_address : _listen_addr); + identity(), _is_service_node and _public_address ? *_public_address : _listen_address); _link_manager = LinkManager::make(*this); @@ -1374,7 +1373,7 @@ namespace llarp oxen::quic::Address Router::listen_addr() const { - return _listen_addr; + return _listen_address; } void diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index db73d753d..bee0ef337 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -92,7 +92,7 @@ namespace llarp consensus::reachability_testing router_testing; std::optional _public_address; // public addr for relays - oxen::quic::Address _listen_addr; + oxen::quic::Address _listen_address; EventLoop_ptr _loop; std::shared_ptr _vpn; diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 0aee92c27..19e63b2e1 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -312,7 +312,9 @@ namespace llarp public: RemoteRC() = default; RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} - {} + { + _payload = {reinterpret_cast(data.data()), data.size()}; + } RemoteRC(ustring_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} { _payload = data; diff --git a/llarp/router_contact_local.cpp b/llarp/router_contact_local.cpp index 47ed6009c..b1b0268d4 100644 --- a/llarp/router_contact_local.cpp +++ b/llarp/router_contact_local.cpp @@ -79,7 +79,7 @@ namespace llarp return sig; }); - _payload = btdp.view(); + _payload = ustring{btdp.view()}; } void diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index f62849adf..16f99cf6f 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -75,11 +75,11 @@ namespace llarp RemoteRC::read(const fs::path& fname) { ustring buf; - buf.reserve(MAX_RC_SIZE); + buf.resize(MAX_RC_SIZE); try { - util::file_to_buffer(fname, buf.data(), MAX_RC_SIZE); + util::file_to_buffer(fname, buf.data(), buf.size()); oxenc::bt_dict_consumer btdc{buf}; bt_load(btdc); From 8af38d3d44c825d8d2dc3ef4e1e3d505e0baa920 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 7 Dec 2023 16:33:06 -0800 Subject: [PATCH 16/93] TELL ME WHEN IT WORKS --- llarp/link/link_manager.cpp | 10 ++++++++-- llarp/nodedb.cpp | 1 - llarp/router/router.cpp | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 6db101b36..d48d28578 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -356,10 +356,10 @@ namespace llarp std::move(on_close)); rv) { - log::info(quic_cat, "Connection to {} successfully established!", remote_addr); + log::info(quic_cat, "Begun establishing connection to {}", remote_addr); return; } - log::warning(quic_cat, "Connection to {} successfully established!", remote_addr); + log::warning(quic_cat, "Failed to begin establishing connection to {}", remote_addr); } // TODO: should we add routes here now that Router::SessionOpen is gone? @@ -370,6 +370,12 @@ namespace llarp const auto& scid = conn_interface.scid(); const auto& rid = ep.connid_map[scid]; + log::critical( + logcat, + "SERVICE NODE (RID:{}) ESTABLISHED CONNECTION TO RID:{}", + _router.local_rid(), + rid); + // check to see if this connection was established while we were attempting to queue // messages to the remote if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end()) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 34adf4ced..60ab2f3cb 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -368,7 +368,6 @@ namespace llarp } std::vector needed; - const auto now = time_point_now(); for (const auto& [rid, rc] : rc_lookup) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 5be99f6dc..e9c6e5d94 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -526,7 +526,7 @@ namespace llarp void Router::save_rc() { - _node_db->put_rc(router_contact.view()); + // _node_db->put_rc(router_contact.view()); log::info(logcat, "Saving RC file to {}", our_rc_file); queue_disk_io([&]() { router_contact.write(our_rc_file); }); } From bc2cb46d9c0a43be9425b64a9cc0133bc950dabd Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 7 Dec 2023 20:52:21 -0400 Subject: [PATCH 17/93] testnet: don't give oxend and error when testing is disabled --- llarp/router/router.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index e9c6e5d94..0bae8c652 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -446,7 +446,7 @@ namespace llarp { // If we're in the white or gray list then we *should* be establishing connections to other // routers, so if we have almost no peers then something is almost certainly wrong. - if (appears_funded() and insufficient_peers()) + if (appears_funded() and insufficient_peers() and not _testing_disabled) return "too few peer connections; lokinet is not adequately connected to the network"; return std::nullopt; } From 9084d39e5aec353071a09a36bbeafb7ef888932c Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 7 Dec 2023 17:07:32 -0800 Subject: [PATCH 18/93] Squashed misc testnet fixes --- external/oxen-libquic | 2 +- llarp/bootstrap.cpp | 2 + llarp/bootstrap.hpp | 20 ++++++---- llarp/link/link_manager.cpp | 75 ++++++++++++++++++++++++++----------- llarp/link/link_manager.hpp | 5 ++- llarp/nodedb.cpp | 22 ++++++++--- llarp/nodedb.hpp | 10 ++--- llarp/router/router.cpp | 10 ++++- 8 files changed, 102 insertions(+), 44 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index a2d89aa79..92fa0e987 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit a2d89aa79dd06cbd7ee864da285cf4d8ac1b09b9 +Subproject commit 92fa0e987e0513dfcd4efc683cb4dbff21de9622 diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index b377ec715..23b7c034e 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -85,5 +85,7 @@ namespace llarp } insert(rc); } + + _curr = begin(); } } // namespace llarp diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index f74faa3cb..472406861 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -12,8 +12,13 @@ namespace llarp { struct BootstrapList final : public std::set { - size_t index; - std::set::iterator current; + std::set::iterator _curr; + + const RemoteRC& + current() + { + return *_curr; + } bool bt_decode(std::string_view buf); @@ -32,12 +37,12 @@ namespace llarp const RemoteRC& next() { - ++current; + ++_curr; - if (current == this->end()) - current = this->begin(); + if (_curr == this->end()) + _curr = this->begin(); - return *current; + return *_curr; } bool @@ -46,7 +51,8 @@ namespace llarp void randomize() { - current = std::next(begin(), std::uniform_int_distribution{0, size() - 1}(csrng)); + if (size() > 1) + _curr = std::next(begin(), std::uniform_int_distribution{0, size() - 1}(csrng)); } void diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index d48d28578..b70ce4f80 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -193,26 +193,43 @@ namespace llarp return on_conn_closed(ci, ec); }, [this](oxen::quic::dgram_interface& di, bstring dgram) { recv_data_message(di, dgram); }); - ep->listen( - tls_creds, - [&](oxen::quic::Connection& c, - oxen::quic::Endpoint& e, - std::optional id) -> std::shared_ptr { - if (id && id == 0) - { - auto s = std::make_shared( - c, e, [](oxen::quic::Stream& s, uint64_t error_code) { - log::warning( - logcat, - "BTRequestStream closed unexpectedly (ec:{}); closing connection...", - error_code); - s.conn.close_connection(error_code); - }); - register_commands(s); - return s; - } - return std::make_shared(c, e); - }); + tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) { + bool result = false; + RouterID other{key.data()}; + if (auto itr = rids_pending_verification.find(other); itr != rids_pending_verification.end()) + { + rids_pending_verification.erase(itr); + result = true; + } + if (_router.node_db()->has_rc(other)) + result = true; + + log::critical(logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Un", other); + return result; + }); + if (_router.is_service_node()) + { + ep->listen( + tls_creds, + [&](oxen::quic::Connection& c, + oxen::quic::Endpoint& e, + std::optional id) -> std::shared_ptr { + if (id && id == 0) + { + auto s = std::make_shared( + c, e, [](oxen::quic::Stream& s, uint64_t error_code) { + log::warning( + logcat, + "BTRequestStream closed unexpectedly (ec:{}); closing connection...", + error_code); + s.conn.close_connection(error_code); + }); + register_commands(s); + return s; + } + return std::make_shared(c, e); + }); + } return ep; } @@ -346,6 +363,9 @@ namespace llarp } const auto& remote_addr = rc.addr(); + const auto& rid = rc.router_id(); + + rids_pending_verification.insert(rid); // TODO: confirm remote end is using the expected pubkey (RouterID). // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) @@ -415,6 +435,10 @@ namespace llarp { const auto& rid = c_itr->second; + if (auto maybe = rids_pending_verification.find(rid); + maybe != rids_pending_verification.end()) + rids_pending_verification.erase(maybe); + // in case this didn't clear earlier, do it now if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) pending_conn_msg_queue.erase(p_itr); @@ -582,9 +606,16 @@ namespace llarp void LinkManager::fetch_bootstrap_rcs( - const RouterID& source, std::string payload, std::function func) + const RemoteRC& source, std::string payload, std::function func) { - send_control_message(source, "bfetch_rcs", std::move(payload), std::move(func)); + _router.loop()->call([this, source, payload, f = std::move(func)]() { + auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f); + + auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue()); + itr->second.push_back(std::move(pending)); + + connect_to(source); + }); } void diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 990e54cf9..943521c96 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -181,6 +181,9 @@ namespace llarp // holds any messages we attempt to send while connections are establishing std::unordered_map pending_conn_msg_queue; + // when establishing a connection, the rid of the remote is placed here to be cross- + // checked by the tls verification callback + std::set rids_pending_verification; util::DecayingHashSet clients{path::DEFAULT_LIFETIME}; @@ -252,7 +255,7 @@ namespace llarp void fetch_bootstrap_rcs( - const RouterID& source, + const RemoteRC& source, std::string payload, std::function func); diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 60ab2f3cb..1c4eea3fe 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -196,7 +196,8 @@ namespace llarp { // TODO: if this needs to be called more than once (ex: drastic failures), then // change this assert to a bootstraps.clear() call - assert(_bootstraps->empty()); + if (_bootstraps) + assert(_bootstraps->empty()); _bootstraps = std::move(from_router); _bootstraps->randomize(); @@ -343,8 +344,9 @@ namespace llarp void NodeDB::fetch_initial() { - if (known_rids.empty()) + if (known_rcs.empty()) { + log::critical(logcat, "No RC's held locally... BOOTSTRAP TIME"); fallback_to_bootstrap(); } else @@ -605,6 +607,14 @@ namespace llarp { _router.last_rc_fetch = llarp::time_point_now(); + if (_router.is_service_node()) + { + _needs_rebootstrap = false; + fail_sources.clear(); + fetch_failures = 0; + return; + } + if (initial) fetch_rids(initial); } @@ -619,7 +629,10 @@ namespace llarp _needs_rebootstrap = false; if (initial) + { _needs_initial_fetch = false; + _initial_completed = true; + } } void @@ -656,7 +669,7 @@ namespace llarp _needs_rebootstrap = false; _router.link_manager().fetch_bootstrap_rcs( - fetch_source, + _bootstraps->current(), BootstrapFetchMessage::serialize(BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { if (not m) @@ -755,9 +768,6 @@ namespace llarp registered_routers.insert(greylist.begin(), greylist.end()); registered_routers.insert(greenlist.begin(), greenlist.end()); - for (const auto& rid : whitelist) - known_rids.insert(rid); - router_whitelist.clear(); router_whitelist.insert(whitelist.begin(), whitelist.end()); router_greylist.clear(); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 15997655c..91541ec76 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -135,9 +135,9 @@ namespace llarp - gray: fully funded, but decommissioned routers - green: registered, but not fully-staked routers */ - std::unordered_set router_whitelist; - std::unordered_set router_greylist; - std::unordered_set router_greenlist; + std::set router_whitelist{}; + std::set router_greylist{}; + std::set router_greenlist{}; // All registered relays (service nodes) std::set registered_routers; @@ -165,7 +165,7 @@ namespace llarp std::atomic fetch_failures{0}, bootstrap_failures{0}; std::atomic _using_bootstrap_fallback{false}, _needs_rebootstrap{false}, - _needs_initial_fetch{true}; + _needs_initial_fetch{true}, _initial_completed{false}; bool want_rc(const RouterID& rid) const; @@ -332,7 +332,7 @@ namespace llarp return known_rids; } - const std::unordered_set& + const std::set& greylist() const { return router_greylist; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 0bae8c652..16a277750 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -670,6 +670,8 @@ namespace llarp if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode) { + log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node"); + auto fallbacks = llarp::load_bootstrap_fallbacks(); if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) @@ -692,9 +694,12 @@ namespace llarp log::info( logcat, "Loaded {} default fallback bootstrap routers!", _bootstrap_rc_list->size()); clear_bad_rcs(); - node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); } + log::critical(logcat, "We have {} bootstrap routers!", _bootstrap_rc_list->size()); + + node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); + if (conf.bootstrap.seednode) log::critical(logcat, "We are a bootstrap seed node!"); @@ -890,7 +895,8 @@ namespace llarp if (needs_initial_fetch()) { - node_db()->fetch_initial(); + if (not _config->bootstrap.seednode) + node_db()->fetch_initial(); } else if (needs_rebootstrap() and next_bootstrap_attempt > now_timepoint) { From 30d58911fabc51ccb8830a1213078c5d2c2b36c3 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 7 Dec 2023 22:45:10 -0400 Subject: [PATCH 19/93] Update logging in daemon.cpp Some of these were wrong (trying to use formats that didn't work); this updates them all to new-style logging. --- daemon/lokinet.cpp | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/daemon/lokinet.cpp b/daemon/lokinet.cpp index bf15d6394..97324b356 100644 --- a/daemon/lokinet.cpp +++ b/daemon/lokinet.cpp @@ -125,7 +125,7 @@ namespace if (!GetModuleFileName(nullptr, szPath.data(), MAX_PATH)) { - llarp::LogError("Cannot install service ", GetLastError()); + llarp::log::error(logcat, "Cannot install service {}", GetLastError()); return; } @@ -137,7 +137,7 @@ namespace if (nullptr == schSCManager) { - llarp::LogError("OpenSCManager failed ", GetLastError()); + llarp::log::error(logcat, "OpenSCManager failed {}", GetLastError()); return; } @@ -159,12 +159,12 @@ namespace if (schService == nullptr) { - llarp::LogError("CreateService failed ", GetLastError()); + llarp::log::error(logcat, "CreateService failed {}", GetLastError()); CloseServiceHandle(schSCManager); return; } else - llarp::LogInfo("Service installed successfully"); + llarp::log::info(logcat, "Service installed successfully"); CloseServiceHandle(schService); CloseServiceHandle(schSCManager); @@ -189,7 +189,7 @@ namespace if (nullptr == schSCManager) { - llarp::LogError("OpenSCManager failed ", GetLastError()); + llarp::log::error(logcat, "OpenSCManager failed {}", GetLastError()); return; } @@ -201,7 +201,7 @@ namespace if (schService == nullptr) { - llarp::LogError("OpenService failed ", GetLastError()); + llarp::log::error(logcat, "OpenService failed {}", GetLastError()); CloseServiceHandle(schSCManager); return; } @@ -214,10 +214,10 @@ namespace SERVICE_CONFIG_DESCRIPTION, // change: description &sd)) // new description { - llarp::LogError("ChangeServiceConfig2 failed"); + llarp::log::error(logcat, "ChangeServiceConfig2 failed"); } else - llarp::LogInfo("Service description updated successfully."); + llarp::log::info(log_cat, "Service description updated successfully."); CloseServiceHandle(schService); CloseServiceHandle(schSCManager); @@ -237,7 +237,7 @@ namespace if (nullptr == schSCManager) { - llarp::LogError("OpenSCManager failed ", GetLastError()); + llarp::log::error(logcat, "OpenSCManager failed {}", GetLastError()); return; } @@ -249,7 +249,7 @@ namespace if (schService == nullptr) { - llarp::LogError("OpenService failed ", GetLastError()); + llarp::log::error(logcat, "OpenService failed {}", GetLastError()); CloseServiceHandle(schSCManager); return; } @@ -257,10 +257,10 @@ namespace // Delete the service. if (!DeleteService(schService)) { - llarp::LogError("DeleteService failed ", GetLastError()); + llarp::log::error(logcat, "DeleteService failed {}", GetLastError()); } else - llarp::LogInfo("Service deleted successfully\n"); + llarp::log::info(logcat, "Service deleted successfully"); CloseServiceHandle(schService); CloseServiceHandle(schSCManager); @@ -337,7 +337,7 @@ namespace if (svc->handle == nullptr) { - llarp::LogError("failed to register daemon control handler"); + llarp::log::error(logcat, "failed to register daemon control handler"); return; } @@ -454,7 +454,7 @@ namespace } catch (std::exception& ex) { - llarp::LogError("cannot generate config at ", *configFile, ": ", ex.what()); + llarp::log::error(logcat, "cannot generate config at {}: {}", *configFile, ex.what()); return 1; } } @@ -464,13 +464,13 @@ namespace { if (!fs::exists(*configFile)) { - llarp::LogError("Config file not found ", *configFile); + llarp::log::error(logcat, "Config file not found {}", *configFile); return 1; } } catch (std::exception& ex) { - llarp::LogError("cannot check if ", *configFile, " exists: ", ex.what()); + llarp::log::error(logcat, "cannot check if ", *configFile, " exists: ", ex.what()); return 1; } } @@ -487,7 +487,7 @@ namespace } catch (std::exception& ex) { - llarp::LogError("cannot ensure config: ", ex.what()); + llarp::log::error(logcat, "cannot ensure config: {}", ex.what()); return 1; } configFile = llarp::GetDefaultConfigPath(); @@ -548,14 +548,14 @@ namespace static void run_main_context(std::optional confFile, const llarp::RuntimeOptions opts) { - llarp::LogInfo(fmt::format( - "starting up {} {}", llarp::LOKINET_VERSION_FULL, llarp::LOKINET_RELEASE_MOTTO)); + llarp::log::info( + logcat, "starting up {} {}", llarp::LOKINET_VERSION_FULL, llarp::LOKINET_RELEASE_MOTTO); try { std::shared_ptr conf; if (confFile) { - llarp::LogInfo("Using config file: ", *confFile); + llarp::log::info(logcat, "Using config file: {}", *confFile); conf = std::make_shared(confFile->parent_path()); } else @@ -564,7 +564,7 @@ namespace } if (not conf->load(confFile, opts.isSNode)) { - llarp::LogError("failed to parse configuration"); + llarp::log::error(logcat, "failed to parse configuration"); exit_code.set_value(1); return; } @@ -589,13 +589,13 @@ namespace } catch (llarp::util::bind_socket_error& ex) { - llarp::LogError(fmt::format("{}, is lokinet already running? 🤔", ex.what())); + llarp::log::error(logcat, "{}; is lokinet already running?", ex.what()); exit_code.set_value(1); return; } catch (const std::exception& ex) { - llarp::LogError(fmt::format("failed to start up lokinet: {}", ex.what())); + llarp::log::error(logcat, "failed to start up lokinet: {}", ex.what()); exit_code.set_value(1); return; } @@ -606,12 +606,12 @@ namespace } catch (const std::exception& e) { - llarp::LogError("Fatal: caught exception while running: {}", e.what()); + llarp::log::error(logcat, "Fatal: caught exception while running: {}", e.what()); exit_code.set_exception(std::current_exception()); } catch (...) { - llarp::LogError("Fatal: caught non-standard exception while running"); + llarp::log::error(logcat, "Fatal: caught non-standard exception while running"); exit_code.set_exception(std::current_exception()); } } From b3dc23e1bf60f89aae04b6404dc755d7e81f854d Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Thu, 7 Dec 2023 22:49:26 -0400 Subject: [PATCH 20/93] Remove release motto This doesn't really have a purpose. --- CMakeLists.txt | 2 -- daemon/lokinet.cpp | 3 +-- llarp/constants/version.cpp.in | 1 - llarp/constants/version.hpp | 1 - llarp/context.cpp | 3 +-- 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c3b407d8..cc7bb6a91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,8 +41,6 @@ if(APPLE) set(LOKINET_APPLE_BUILD 5) endif() -set(LOKINET_RELEASE_MOTTO "Anonymous, decentralized, IP-based overlay network" CACHE STRING "Release motto") - list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") set(DEFAULT_WITH_BOOTSTRAP ON) diff --git a/daemon/lokinet.cpp b/daemon/lokinet.cpp index 97324b356..19c673780 100644 --- a/daemon/lokinet.cpp +++ b/daemon/lokinet.cpp @@ -548,8 +548,7 @@ namespace static void run_main_context(std::optional confFile, const llarp::RuntimeOptions opts) { - llarp::log::info( - logcat, "starting up {} {}", llarp::LOKINET_VERSION_FULL, llarp::LOKINET_RELEASE_MOTTO); + llarp::log::info(logcat, "starting up {}", llarp::LOKINET_VERSION_FULL); try { std::shared_ptr conf; diff --git a/llarp/constants/version.cpp.in b/llarp/constants/version.cpp.in index 432f81e8e..3c57aa0ef 100644 --- a/llarp/constants/version.cpp.in +++ b/llarp/constants/version.cpp.in @@ -8,7 +8,6 @@ namespace llarp const char* const LOKINET_VERSION_TAG = "@VERSIONTAG@"; const char* const LOKINET_VERSION_FULL = "lokinet-@lokinet_VERSION_MAJOR@.@lokinet_VERSION_MINOR@.@lokinet_VERSION_PATCH@-@LOKINET_VERSION_TAG@"; - const char* const LOKINET_RELEASE_MOTTO = "@RELEASE_MOTTO@"; const char* const LOKINET_DEFAULT_NETID = "lokinet"; const char* const LOKINET_TESTNET_NETID = "testnet"; // clang-format on diff --git a/llarp/constants/version.hpp b/llarp/constants/version.hpp index 75b764cd0..31c2ebee2 100644 --- a/llarp/constants/version.hpp +++ b/llarp/constants/version.hpp @@ -10,7 +10,6 @@ namespace llarp extern const char* const LOKINET_VERSION_TAG; extern const char* const LOKINET_VERSION_FULL; - extern const char* const LOKINET_RELEASE_MOTTO; extern const char* const LOKINET_DEFAULT_NETID; extern const char* const LOKINET_TESTNET_NETID; } // namespace llarp diff --git a/llarp/context.cpp b/llarp/context.cpp index 378a5d9d6..cb0ca52d1 100644 --- a/llarp/context.cpp +++ b/llarp/context.cpp @@ -57,8 +57,7 @@ namespace llarp throw std::runtime_error("Cannot call Setup() on context without a Config"); if (opts.showBanner) - llarp::LogInfo( - fmt::format("{} {}", llarp::LOKINET_VERSION_FULL, llarp::LOKINET_RELEASE_MOTTO)); + llarp::LogInfo(fmt::format("{}", llarp::LOKINET_VERSION_FULL)); if (!loop) { From b82b4c7fe610d48f17534a95a5264c546f96168f Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 8 Dec 2023 15:47:28 -0400 Subject: [PATCH 21/93] Update libquic --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 92fa0e987..2d38e6a75 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 92fa0e987e0513dfcd4efc683cb4dbff21de9622 +Subproject commit 2d38e6a75819ddc047b46ecace182396115971d7 From 0e6a2941bd3749bd6106bdeaa49ac83b18683e0a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 8 Dec 2023 17:23:41 -0400 Subject: [PATCH 22/93] Bump libquic --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 2d38e6a75..f903a8f6d 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 2d38e6a75819ddc047b46ecace182396115971d7 +Subproject commit f903a8f6d08398c591c9fd12e93a3c35c5fa65c7 From daeff3ee7efa7d36b545439540cb22223ff1d7cb Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 8 Dec 2023 19:02:14 -0400 Subject: [PATCH 23/93] Remove duplicate connection closing method `deregister_peer` does the exact same thing as `close_connection` so just remove it. Also removes an unnecessary loop dispatch call (because we *have* to be in the logic thread already to be able to touch the variables we are touching before the dispatch). --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 52 ++++++------------------------------- llarp/link/link_manager.hpp | 6 ----- llarp/router/router.cpp | 2 +- 4 files changed, 10 insertions(+), 52 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index f903a8f6d..fdc8b2a51 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit f903a8f6d08398c591c9fd12e93a3c35c5fa65c7 +Subproject commit fdc8b2a514c3da0012adfcb6a0d08809c1bc5e06 diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index b70ce4f80..2b94d844f 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -51,27 +51,6 @@ namespace llarp return false; } - bool - Endpoint::deregister_peer(RouterID _rid) - { - if (auto itr = conns.find(_rid); itr != conns.end()) - { - auto& c = itr->second; - auto& _scid = c->conn->scid(); - - link_manager._router.loop()->call([this, scid = _scid, rid = _rid]() { - endpoint->close_connection(scid); - - conns.erase(rid); - connid_map.erase(scid); - }); - - return true; - } - - return false; - } - size_t Endpoint::num_connected(bool clients_only) const { @@ -111,18 +90,15 @@ namespace llarp void Endpoint::close_connection(RouterID _rid) { - if (auto itr = conns.find(_rid); itr != conns.end()) - { - auto& c = itr->second; - auto& _scid = c->conn->scid(); - - link_manager._router.loop()->call([this, scid = _scid, rid = _rid]() { - endpoint->close_connection(scid); + assert(link_manager._router.loop()->inEventLoop()); + auto itr = conns.find(_rid); + if (itr != conns.end()) + return; - conns.erase(rid); - connid_map.erase(scid); - }); - } + auto& conn = *itr->second->conn; + conn.close_connection(); + connid_map.erase(conn.scid()); + conns.erase(itr); } } // namespace link @@ -464,18 +440,6 @@ namespace llarp return ep.have_conn(remote, true); } - void - LinkManager::deregister_peer(RouterID remote) - { - if (auto rv = ep.deregister_peer(remote); rv) - { - persisting_conns.erase(remote); - log::info(logcat, "Peer {} successfully de-registered", remote); - } - else - log::warning(logcat, "Peer {} not found for de-registration!", remote); - } - void LinkManager::stop() { diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 943521c96..6a1b3cba8 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -61,9 +61,6 @@ namespace llarp bool have_conn(const RouterID& remote, bool client_only) const; - bool - deregister_peer(RouterID remote); - size_t num_connected(bool clients_only) const; @@ -268,9 +265,6 @@ namespace llarp bool have_client_connection_to(const RouterID& remote) const; - void - deregister_peer(RouterID remote); - void test_reachability(const RouterID& rid, conn_open_hook, conn_closed_hook); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 16a277750..9d941e065 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -981,7 +981,7 @@ namespace llarp // mark peers as de-registered for (auto& peer : close_peers) - _link_manager.deregister_peer(peer); + _link_manager.close_connection(peer); } */ From 412ce441391df9502230efaa204f29bb27ae410e Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 8 Dec 2023 21:49:58 -0400 Subject: [PATCH 24/93] Update libquic --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 4 ++-- llarp/router/router.cpp | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index fdc8b2a51..c9271e87e 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit fdc8b2a514c3da0012adfcb6a0d08809c1bc5e06 +Subproject commit c9271e87eaa2d6bcbdf4e49936fcce2b7bc9e693 diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 2b94d844f..f764fc394 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -192,7 +192,7 @@ namespace llarp std::optional id) -> std::shared_ptr { if (id && id == 0) { - auto s = std::make_shared( + auto s = e.make_shared( c, e, [](oxen::quic::Stream& s, uint64_t error_code) { log::warning( logcat, @@ -203,7 +203,7 @@ namespace llarp register_commands(s); return s; } - return std::make_shared(c, e); + return nullptr; }); } return ep; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 9d941e065..56035e6cc 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -595,8 +595,6 @@ namespace llarp ? _listen_address : oxen::quic::Address{*paddr, pport ? *pport : DEFAULT_LISTEN_PORT}; - log::critical(logcat, "listen_addr:{} \t public_addr:{}", _listen_address, _public_address); - RouterContact::BLOCK_BOGONS = conf.router.block_bogons; auto& networkConfig = conf.network; From f2feea74ee7ed9b27b3c6e657b4a8f4728fc6c79 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 03:25:41 -0800 Subject: [PATCH 25/93] Crit logging - TODO: discuss authentication for nodes connecting to bootstrap seed - crit log num connected/RC's, conn open/closed, etc --- llarp/link/link_manager.cpp | 16 +++++++++++++--- llarp/link/link_manager.hpp | 3 +++ llarp/router/router.cpp | 17 +++++++++++------ llarp/router/router.hpp | 7 +++++++ 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index f764fc394..74a7d0ff0 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -172,15 +172,25 @@ namespace llarp tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) { bool result = false; RouterID other{key.data()}; + if (auto itr = rids_pending_verification.find(other); itr != rids_pending_verification.end()) { rids_pending_verification.erase(itr); result = true; } + if (_router.node_db()->has_rc(other)) result = true; - log::critical(logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Un", other); + // TODO: discuss pubkey verification for bootstraps connecting to seed node + if (_router.is_bootstrap_seed()) + { + log::warning(logcat, "Allowing connection -- we are bootstrap seed"); + result = true; + } + + log::critical( + logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Uns", other); return result; }); if (_router.is_service_node()) @@ -405,7 +415,7 @@ namespace llarp _router.loop()->call([this, &conn_interface = ci, error_code = ec]() { const auto& scid = conn_interface.scid(); - log::debug(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); + log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end()) { @@ -423,7 +433,7 @@ namespace llarp ep.connid_map.erase(c_itr); - log::debug(quic_cat, "Quic connection CID:{} purged successfully", scid); + log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); } }); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 6a1b3cba8..55bd7d686 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -414,6 +414,8 @@ namespace llarp connid_map.emplace(conn_interface->scid(), rc.router_id()); auto [itr, b] = conns.emplace(rc.router_id(), nullptr); + log::critical(logcat, "Establishing connection to {}...", rc.router_id()); + auto control_stream = conn_interface->template get_new_stream( [](oxen::quic::Stream& s, uint64_t error_code) { log::warning( @@ -422,6 +424,7 @@ namespace llarp error_code); s.conn.close_connection(error_code); }); + itr->second = std::make_shared(conn_interface, control_stream, rc); return true; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 56035e6cc..d21c8edef 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -666,7 +666,12 @@ namespace llarp clear_bad_rcs(); - if (_bootstrap_rc_list->empty() and not conf.bootstrap.seednode) + _bootstrap_seed = conf.bootstrap.seednode; + + if (_bootstrap_seed) + log::critical(logcat, "We are a bootstrap seed node!"); + + if (_bootstrap_rc_list->empty() and not _bootstrap_seed) { log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node"); @@ -698,9 +703,6 @@ namespace llarp node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); - if (conf.bootstrap.seednode) - log::critical(logcat, "We are a bootstrap seed node!"); - // TODO: RC refactor here if (_is_service_node) init_inbounds(); @@ -765,7 +767,8 @@ namespace llarp Router::report_stats() { const auto now = llarp::time_now_ms(); - log::info( + + log::critical( logcat, "{} RCs loaded with {} bootstrap peers and {} router connections!", _node_db->num_rcs(), @@ -774,15 +777,17 @@ namespace llarp if (is_service_node()) { - log::info( + log::critical( logcat, "Local service node has {} client connections since last RC update ({} to expiry)", num_client_connections(), router_contact.age(now), router_contact.time_to_expiry(now)); } + if (_last_stats_report > 0s) log::info(logcat, "Last reported stats time {}", now - _last_stats_report); + _last_stats_report = now; } diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index bee0ef337..86eb5de13 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -88,6 +88,7 @@ namespace llarp bool _testnet = false; bool _testing_disabled = false; + bool _bootstrap_seed = false; consensus::reachability_testing router_testing; @@ -157,6 +158,12 @@ namespace llarp std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip}; public: + bool + is_bootstrap_seed() const + { + return _bootstrap_seed; + } + int required_num_client_conns() const { From 63644d3d84b60c462119db95e85a743cbb6abf3b Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 07:08:02 -0800 Subject: [PATCH 26/93] Bootstrap seed - nodes now send their RC to the bootstrap seed on making a request - allows the bootstrap seed to distribute RCs --- external/oxen-libquic | 2 +- llarp/link/connection.cpp | 2 +- llarp/link/connection.hpp | 2 +- llarp/link/link_manager.cpp | 33 ++++++++++++++++++++++++++++++--- llarp/link/link_manager.hpp | 8 +++++++- llarp/messages/fetch.hpp | 4 +++- llarp/nodedb.cpp | 3 ++- llarp/router_contact.hpp | 5 +++++ llarp/router_contact_local.cpp | 6 ++++++ llarp/router_id.hpp | 3 +++ 10 files changed, 59 insertions(+), 9 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index c9271e87e..55d81cbbc 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit c9271e87eaa2d6bcbdf4e49936fcce2b7bc9e693 +Subproject commit 55d81cbbc7bc5ea3667e87389d8143702731a32a diff --git a/llarp/link/connection.cpp b/llarp/link/connection.cpp index dace3d299..eb16b9174 100644 --- a/llarp/link/connection.cpp +++ b/llarp/link/connection.cpp @@ -3,7 +3,7 @@ namespace llarp::link { Connection::Connection( - std::shared_ptr& c, + const std::shared_ptr& c, std::shared_ptr& s, const RemoteRC& rc) : conn{c}, control_stream{s}, remote_rc{std::move(rc)} diff --git a/llarp/link/connection.hpp b/llarp/link/connection.hpp index 46aef9328..b71a6e9f9 100644 --- a/llarp/link/connection.hpp +++ b/llarp/link/connection.hpp @@ -18,7 +18,7 @@ namespace llarp::link bool remote_is_relay{true}; Connection( - std::shared_ptr& c, + const std::shared_ptr& c, std::shared_ptr& s, const RemoteRC& rc); }; diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 74a7d0ff0..323ee132e 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -175,6 +175,7 @@ namespace llarp if (auto itr = rids_pending_verification.find(other); itr != rids_pending_verification.end()) { + verified_rids[other] = itr->second; rids_pending_verification.erase(itr); result = true; } @@ -351,7 +352,7 @@ namespace llarp const auto& remote_addr = rc.addr(); const auto& rid = rc.router_id(); - rids_pending_verification.insert(rid); + rids_pending_verification[rid] = rc; // TODO: confirm remote end is using the expected pubkey (RouterID). // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) @@ -368,13 +369,34 @@ namespace llarp log::warning(quic_cat, "Failed to begin establishing connection to {}", remote_addr); } + void + LinkManager::on_inbound_conn(oxen::quic::connection_interface& ci) + { + const auto& scid = ci.scid(); + RouterID rid{ci.remote_key()}; + + const auto& rc = verified_rids[rid]; + ep.connid_map.emplace(scid, rid); + auto [itr, b] = ep.conns.emplace(rid, nullptr); + + auto control_stream = ci.get_new_stream(); + itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); + log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); + } + // TODO: should we add routes here now that Router::SessionOpen is gone? void LinkManager::on_conn_open(oxen::quic::connection_interface& ci) { _router.loop()->call([this, &conn_interface = ci]() { - const auto& scid = conn_interface.scid(); - const auto& rid = ep.connid_map[scid]; + const auto rid = RouterID{conn_interface.remote_key()}; + const auto& remote = conn_interface.remote(); + + if (conn_interface.is_inbound()) + { + log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote); + on_inbound_conn(conn_interface); + } log::critical( logcat, @@ -599,12 +621,15 @@ namespace llarp assert(_router.is_service_node()); const auto& rcs = node_db->get_rcs(); + RemoteRC remote; size_t quantity; try { oxenc::bt_dict_consumer btdc{m.body()}; quantity = btdc.require("quantity"); + btdc.required("local"); + remote = RemoteRC{btdc.consume_dict_consumer()}; } catch (const std::exception& e) { @@ -613,6 +638,8 @@ namespace llarp return; } + node_db->put_rc(remote); + auto rc_size = rcs.size(); auto now = llarp::time_now_ms(); size_t i = 0; diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 55bd7d686..b97a6c7a9 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -180,7 +180,10 @@ namespace llarp std::unordered_map pending_conn_msg_queue; // when establishing a connection, the rid of the remote is placed here to be cross- // checked by the tls verification callback - std::set rids_pending_verification; + std::map rids_pending_verification; + // in the interim of verifying an inbound connection and the creation of its link::Connection + // object, we store the rid and rc here + std::map verified_rids; util::DecayingHashSet clients{path::DEFAULT_LIFETIME}; @@ -203,6 +206,9 @@ namespace llarp void recv_control_message(oxen::quic::message msg); + void + on_inbound_conn(oxen::quic::connection_interface& ci); + void on_conn_open(oxen::quic::connection_interface& ci); diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 933521544..839a5d458 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -37,10 +37,12 @@ namespace llarp namespace BootstrapFetchMessage { + // the LocalRC is converted to a RemoteRC type to send to the bootstrap seed inline static std::string - serialize(size_t quantity) + serialize(const RemoteRC& local_rc, size_t quantity) { oxenc::bt_dict_producer btdp; + btdp.append_encoded("local", local_rc.view()); btdp.append("quantity", quantity); return std::move(btdp).str(); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 1c4eea3fe..000f51734 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -670,7 +670,8 @@ namespace llarp _router.link_manager().fetch_bootstrap_rcs( _bootstraps->current(), - BootstrapFetchMessage::serialize(BOOTSTRAP_SOURCE_COUNT), + BootstrapFetchMessage::serialize( + _router.router_contact.to_remote(), BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { if (not m) { diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 19e63b2e1..3723301cb 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -207,6 +207,8 @@ namespace llarp bt_load(oxenc::bt_dict_consumer& data); }; + struct RemoteRC; + /// Extension of RouterContact used to store a local "RC," and inserts a RouterContact by /// re-parsing and sending it out. This sub-class contains a pubkey and all the other attributes /// required for signing and serialization @@ -236,6 +238,9 @@ namespace llarp explicit LocalRC(std::string payload, const SecretKey sk); ~LocalRC() = default; + RemoteRC + to_remote(); + void resign(); diff --git a/llarp/router_contact_local.cpp b/llarp/router_contact_local.cpp index b1b0268d4..da692787f 100644 --- a/llarp/router_contact_local.cpp +++ b/llarp/router_contact_local.cpp @@ -27,6 +27,12 @@ namespace llarp resign(); } + RemoteRC + LocalRC::to_remote() + { + return RemoteRC{view()}; + } + LocalRC::LocalRC(std::string payload, const SecretKey sk) : _secret_key{std::move(sk)} { _router_id = llarp::seckey_to_pubkey(_secret_key); diff --git a/llarp/router_id.hpp b/llarp/router_id.hpp index fea9d4d54..6a25ff9d4 100644 --- a/llarp/router_id.hpp +++ b/llarp/router_id.hpp @@ -21,6 +21,9 @@ namespace llarp RouterID(const Data& data) : PubKey(data) {} + RouterID(ustring_view data) : PubKey(data.data()) + {} + util::StatusObject ExtractStatus() const; From 5f430a392bad6affda2f93ef5e84c5b656f4e2e5 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 07:46:11 -0800 Subject: [PATCH 27/93] kick --- llarp/link/link_manager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 323ee132e..e5856e7eb 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -214,7 +214,8 @@ namespace llarp register_commands(s); return s; } - return nullptr; + + return e.make_shared(c, e); }); } return ep; @@ -379,7 +380,7 @@ namespace llarp ep.connid_map.emplace(scid, rid); auto [itr, b] = ep.conns.emplace(rid, nullptr); - auto control_stream = ci.get_new_stream(); + auto control_stream = ci.template get_new_stream(); itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); } @@ -605,6 +606,7 @@ namespace llarp const RemoteRC& source, std::string payload, std::function func) { _router.loop()->call([this, source, payload, f = std::move(func)]() { + log::critical(logcat, "Queuing bootstrap fetch request"); auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f); auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue()); From 238cc3d0dac28b072fc508b79a4d1622dae71e8f Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 08:16:13 -0800 Subject: [PATCH 28/93] kick --- llarp/link/link_manager.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index e5856e7eb..5a6f06b25 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -135,6 +135,11 @@ namespace llarp [this, msg = std::move(m)]() mutable { handle_gossip_rc(std::move(msg)); }); }); + s->register_command("bfetc_rcs"s, [this, rid](oxen::quic::message m) { + _router.loop()->call( + [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); + }); + for (auto& method : direct_requests) { s->register_command( From 22edd632841ef4bee526f744a38dc8b301bde6f0 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 08:35:20 -0800 Subject: [PATCH 29/93] libquic vbump --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 55d81cbbc..69e577ffd 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 55d81cbbc7bc5ea3667e87389d8143702731a32a +Subproject commit 69e577ffd788e85091eaf9ecde9b3ec2e00727ea From 0bf3a3323a1d0b0bc376c0f32b38ed183f9c8de5 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 08:36:37 -0800 Subject: [PATCH 30/93] kick --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 69e577ffd..ad8904deb 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 69e577ffd788e85091eaf9ecde9b3ec2e00727ea +Subproject commit ad8904debc55f9495667e8571ee5bb963156d54e diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 5a6f06b25..6a6386743 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -135,7 +135,7 @@ namespace llarp [this, msg = std::move(m)]() mutable { handle_gossip_rc(std::move(msg)); }); }); - s->register_command("bfetc_rcs"s, [this, rid](oxen::quic::message m) { + s->register_command("bfetch_rcs"s, [this, rid](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); }); @@ -385,7 +385,7 @@ namespace llarp ep.connid_map.emplace(scid, rid); auto [itr, b] = ep.conns.emplace(rid, nullptr); - auto control_stream = ci.template get_new_stream(); + auto control_stream = ci.get_stream(0); itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); } From efe5fd3b5b598970edb3faca16891a3ac54c4d52 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 09:17:29 -0800 Subject: [PATCH 31/93] try queueing --- llarp/link/link_manager.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 6a6386743..39df9bfeb 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -385,7 +385,12 @@ namespace llarp ep.connid_map.emplace(scid, rid); auto [itr, b] = ep.conns.emplace(rid, nullptr); - auto control_stream = ci.get_stream(0); + auto control_stream = ci.queue_stream([](oxen::quic::Stream& s, + uint64_t error_code) { + log::warning( + logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); + s.conn.close_connection(error_code); + }); itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); } From e0c1069790fab4f67e5ba73542a29984e632a08d Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 09:29:56 -0800 Subject: [PATCH 32/93] kiiiiiick --- llarp/link/link_manager.cpp | 15 ++++++++++++++- llarp/router_contact_local.cpp | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 39df9bfeb..b04cfad6b 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -419,6 +419,7 @@ namespace llarp // messages to the remote if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end()) { + log::critical(logcat, "Clearing pending queue for RID:{}", rid); auto& que = itr->second; while (not que.empty()) @@ -428,6 +429,7 @@ namespace llarp if (m.is_control) { auto& msg = reinterpret_cast(m); + log::critical(logcat, "Dispatching {} request!", msg.endpoint); ep.conns[rid]->control_stream->command(msg.endpoint, msg.body, msg.func); } else @@ -438,7 +440,9 @@ namespace llarp que.pop_front(); } + return; } + log::warning(logcat, "No pending queue to clear for RID:{}", rid); }); }; @@ -616,6 +620,14 @@ namespace llarp const RemoteRC& source, std::string payload, std::function func) { _router.loop()->call([this, source, payload, f = std::move(func)]() { + + if (auto conn = ep.get_conn(source); conn) + { + log::critical(logcat, "Dispatched bootstrap fetch request!"); + conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f)); + return; + } + log::critical(logcat, "Queuing bootstrap fetch request"); auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f); @@ -631,6 +643,7 @@ namespace llarp { // this handler should not be registered for clients assert(_router.is_service_node()); + log::critical(logcat, "Handling fetch bootstrap fetch request..."); const auto& rcs = node_db->get_rcs(); RemoteRC remote; @@ -639,9 +652,9 @@ namespace llarp try { oxenc::bt_dict_consumer btdc{m.body()}; - quantity = btdc.require("quantity"); btdc.required("local"); remote = RemoteRC{btdc.consume_dict_consumer()}; + quantity = btdc.require("quantity"); } catch (const std::exception& e) { diff --git a/llarp/router_contact_local.cpp b/llarp/router_contact_local.cpp index da692787f..937c98908 100644 --- a/llarp/router_contact_local.cpp +++ b/llarp/router_contact_local.cpp @@ -30,6 +30,7 @@ namespace llarp RemoteRC LocalRC::to_remote() { + resign(); return RemoteRC{view()}; } From f725c07f2d2b94c2e28571ce955d62776150860d Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 10:17:46 -0800 Subject: [PATCH 33/93] dont bomb with bootstrap reqs --- llarp/link/link_manager.cpp | 5 +++-- llarp/nodedb.cpp | 21 +++++++++++---------- llarp/nodedb.hpp | 2 +- llarp/router/router.cpp | 4 +++- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index b04cfad6b..8ec85bf32 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -391,8 +391,10 @@ namespace llarp logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); s.conn.close_connection(error_code); }); + itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); + node_db->put_rc(rc); } // TODO: should we add routes here now that Router::SessionOpen is gone? @@ -620,14 +622,13 @@ namespace llarp const RemoteRC& source, std::string payload, std::function func) { _router.loop()->call([this, source, payload, f = std::move(func)]() { - if (auto conn = ep.get_conn(source); conn) { log::critical(logcat, "Dispatched bootstrap fetch request!"); conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f)); return; } - + log::critical(logcat, "Queuing bootstrap fetch request"); auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f); diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 000f51734..6e6f2beb5 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -512,7 +512,7 @@ namespace llarp { if (error) { - auto& fail_count = (_using_bootstrap_fallback) ? bootstrap_failures : fetch_failures; + auto& fail_count = (_using_bootstrap_fallback) ? bootstrap_attempts : fetch_failures; auto& THRESHOLD = (_using_bootstrap_fallback) ? MAX_BOOTSTRAP_FETCH_ATTEMPTS : MAX_FETCH_ATTEMPTS; @@ -638,7 +638,7 @@ namespace llarp void NodeDB::fallback_to_bootstrap() { - auto at_max_failures = bootstrap_failures >= MAX_BOOTSTRAP_FETCH_ATTEMPTS; + auto at_max_failures = bootstrap_attempts >= MAX_BOOTSTRAP_FETCH_ATTEMPTS; // base case: we have failed to query all bootstraps, or we received a sample of // the network, but the sample was unusable or unreachable. We will also enter this @@ -646,7 +646,7 @@ namespace llarp // checking not using_bootstrap_fallback) if (at_max_failures || not _using_bootstrap_fallback) { - bootstrap_failures = 0; + bootstrap_attempts = 0; // Fail case: if we have returned to the front of the bootstrap list, we're in a // bad spot; we are unable to do anything @@ -667,6 +667,7 @@ namespace llarp // By passing the last conditional, we ensure this is set to true _using_bootstrap_fallback = true; _needs_rebootstrap = false; + ++bootstrap_attempts; _router.link_manager().fetch_bootstrap_rcs( _bootstraps->current(), @@ -675,12 +676,12 @@ namespace llarp [this](oxen::quic::message m) mutable { if (not m) { - ++bootstrap_failures; + // ++bootstrap_attempts; log::warning( logcat, "BootstrapRC fetch request to {} failed (error {}/{})", fetch_source, - bootstrap_failures, + bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS); fallback_to_bootstrap(); return; @@ -704,12 +705,12 @@ namespace llarp } catch (const std::exception& e) { - ++bootstrap_failures; + // ++bootstrap_attempts; log::warning( logcat, "Failed to parse BootstrapRC fetch response from {} (error {}/{}): {}", fetch_source, - bootstrap_failures, + bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS, e.what()); fallback_to_bootstrap(); @@ -719,7 +720,7 @@ namespace llarp // We set this to the max allowable value because if this result is bad, we won't // try this bootstrap again. If this result is undersized, we roll right into the // next call to fallback_to_bootstrap() and hit the base case, rotating sources - bootstrap_failures = MAX_BOOTSTRAP_FETCH_ATTEMPTS; + // bootstrap_attempts = MAX_BOOTSTRAP_FETCH_ATTEMPTS; if (rids.size() == BOOTSTRAP_SOURCE_COUNT) { @@ -728,13 +729,13 @@ namespace llarp } else { - ++bootstrap_failures; + // ++bootstrap_attempts; log::warning( logcat, "BootstrapRC fetch response from {} returned insufficient number of RC's (error " "{}/{})", fetch_source, - bootstrap_failures, + bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS); fallback_to_bootstrap(); } diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 91541ec76..f8e99372a 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -162,7 +162,7 @@ namespace llarp - bootstrap_failures: tracks errors fetching both RC's from bootstrasps and RID requests they mediate. This is a different counter as we only bootstrap in problematic cases */ - std::atomic fetch_failures{0}, bootstrap_failures{0}; + std::atomic fetch_failures{0}, bootstrap_attempts{0}; std::atomic _using_bootstrap_fallback{false}, _needs_rebootstrap{false}, _needs_initial_fetch{true}, _initial_completed{false}; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index d21c8edef..a700aaf9c 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -894,6 +894,8 @@ namespace llarp next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; } + + report_stats(); } if (needs_initial_fetch()) @@ -901,7 +903,7 @@ namespace llarp if (not _config->bootstrap.seednode) node_db()->fetch_initial(); } - else if (needs_rebootstrap() and next_bootstrap_attempt > now_timepoint) + else if (needs_rebootstrap() and now_timepoint > next_bootstrap_attempt) { node_db()->fallback_to_bootstrap(); } From 4c1f6112a7adf9ce85280a36ef9052904263d0f0 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 14:46:29 -0400 Subject: [PATCH 34/93] REVERT ME: debug incoming dict data --- llarp/link/link_manager.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 8ec85bf32..38369a332 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -654,7 +654,9 @@ namespace llarp { oxenc::bt_dict_consumer btdc{m.body()}; btdc.required("local"); - remote = RemoteRC{btdc.consume_dict_consumer()}; + auto rc_dict = btdc.consume_dict_data(); + log::critical(logcat, "incoming dict data: {}", oxenc::to_hex(rc_dict)); + remote = RemoteRC{oxenc::bt_dict_consumer{rc_dict}}; quantity = btdc.require("quantity"); } catch (const std::exception& e) From 1a7f47f5cd1173f8bf26cf39baec6e6229e63256 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 15:03:52 -0400 Subject: [PATCH 35/93] Fix version encoding We are reinterpret_cast'ing the version to a string to send it as raw bytes, but it was sending \x00\x00\x09 instead of \x00\x09\x0a because the version constant was actually a uint16_t array. This just makes the version constant a uint8_t array instead so that it works (and I am not at all worried about any version component getting larger than 255). --- llarp/constants/version.cpp.in | 2 +- llarp/constants/version.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llarp/constants/version.cpp.in b/llarp/constants/version.cpp.in index 3c57aa0ef..e2e095147 100644 --- a/llarp/constants/version.cpp.in +++ b/llarp/constants/version.cpp.in @@ -4,7 +4,7 @@ namespace llarp { // clang-format off - const std::array LOKINET_VERSION{{@lokinet_VERSION_MAJOR@, @lokinet_VERSION_MINOR@, @lokinet_VERSION_PATCH@}}; + const std::array LOKINET_VERSION{{@lokinet_VERSION_MAJOR@, @lokinet_VERSION_MINOR@, @lokinet_VERSION_PATCH@}}; const char* const LOKINET_VERSION_TAG = "@VERSIONTAG@"; const char* const LOKINET_VERSION_FULL = "lokinet-@lokinet_VERSION_MAJOR@.@lokinet_VERSION_MINOR@.@lokinet_VERSION_PATCH@-@LOKINET_VERSION_TAG@"; diff --git a/llarp/constants/version.hpp b/llarp/constants/version.hpp index 31c2ebee2..bd83bdde6 100644 --- a/llarp/constants/version.hpp +++ b/llarp/constants/version.hpp @@ -6,7 +6,7 @@ namespace llarp { // Given a full lokinet version of: lokinet-1.2.3-abc these are: - extern const std::array LOKINET_VERSION; + extern const std::array LOKINET_VERSION; extern const char* const LOKINET_VERSION_TAG; extern const char* const LOKINET_VERSION_FULL; From f5d959c65afce72909dbcb4656ee83eb422074b0 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 11:09:59 -0800 Subject: [PATCH 36/93] D --- llarp/router_contact_local.cpp | 4 ++-- llarp/router_contact_remote.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/llarp/router_contact_local.cpp b/llarp/router_contact_local.cpp index 937c98908..8c9d8b39b 100644 --- a/llarp/router_contact_local.cpp +++ b/llarp/router_contact_local.cpp @@ -55,13 +55,13 @@ namespace llarp if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) { - auto err = "Unable to verify expired RemoteRC!"; + auto err = "Unable to verify expired LocalRC!"; log::info(logcat, err); throw std::runtime_error{err}; } if (not crypto::verify(router_id(), msg, sig)) - throw std::runtime_error{"Failed to verify RemoteRC"}; + throw std::runtime_error{"Failed to verify LocalRC"}; }); } catch (const std::exception& e) diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 16f99cf6f..d47e41933 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -29,13 +29,13 @@ namespace llarp if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) { - auto err = "Unable to verify expired RemoteRC!"; + auto err = "Unable to verify RemoteRC address!"; log::info(logcat, err); throw std::runtime_error{err}; } if (not crypto::verify(router_id(), msg, sig)) - throw std::runtime_error{"Failed to verify RemoteRC"}; + throw std::runtime_error{"Failed to verify RemoteRC signature"}; }); } catch (const std::exception& e) @@ -61,13 +61,13 @@ namespace llarp if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) { - auto err = "Unable to verify expired RemoteRC!"; + auto err = "Unable to verify expired RemoteRC address!"; log::info(logcat, err); throw std::runtime_error{err}; } if (not crypto::verify(router_id(), msg, sig)) - throw std::runtime_error{"Failed to verify RemoteRC"}; + throw std::runtime_error{"Failed to verify RemoteRC signature"}; }); } From 5e2c18ff1955b7b7f9b51837f01157ca1a93eb41 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 15:13:49 -0400 Subject: [PATCH 37/93] RemoteRC: `explicit` single-argument constructors --- llarp/router_contact.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 3723301cb..466d00719 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -316,11 +316,11 @@ namespace llarp public: RemoteRC() = default; - RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} + explicit RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} { _payload = {reinterpret_cast(data.data()), data.size()}; } - RemoteRC(ustring_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} + explicit RemoteRC(ustring_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} { _payload = data; } From daa3a6fd9473045a30ab2162b99782e1127f11c5 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 11:16:03 -0800 Subject: [PATCH 38/93] shoot --- llarp/link/link_manager.cpp | 8 ++++---- llarp/router_contact.cpp | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 38369a332..6bb9cf94a 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -656,7 +656,7 @@ namespace llarp btdc.required("local"); auto rc_dict = btdc.consume_dict_data(); log::critical(logcat, "incoming dict data: {}", oxenc::to_hex(rc_dict)); - remote = RemoteRC{oxenc::bt_dict_consumer{rc_dict}}; + remote = RemoteRC{rc_dict}; quantity = btdc.require("quantity"); } catch (const std::exception& e) @@ -808,10 +808,10 @@ namespace llarp { auto btlp = btdp.append_list("routers"); - const auto& known_rcs = node_db->get_known_rcs(); + const auto& known_rids = node_db->get_known_rids(); - for (const auto& rc : known_rcs) - btlp.append_encoded(rc.view()); + for (const auto& rid : known_rids) + btlp.append(rid.ToView()); } btdp.append_signature("signature", [this](ustring_view to_sign) { diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index 585e72f99..7f1280f7a 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -61,12 +61,14 @@ namespace llarp "Invalid RC netid: expected {}, got {}; this is an RC for a different network!"_format( ACTIVE_NETID, netid)}; - auto pk = data.require("p"); + _router_id.from_string(data.require("p")); - if (pk.size() != RouterID::SIZE) - throw std::runtime_error{"Invalid RC: router id has invalid size {}"_format(pk.size())}; + // auto pk = data.require("p"); - std::memcpy(_router_id.data(), pk.data(), RouterID::SIZE); + // if (pk.size() != RouterID::SIZE) + // throw std::runtime_error{"Invalid RC: router id has invalid size {}"_format(pk.size())}; + + // std::memcpy(_router_id.data(), pk.data(), RouterID::SIZE); _timestamp = rc_time{std::chrono::seconds{data.require("t")}}; From 636de93b1a082cf3bd3230bd01366f8bdf84eacc Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 15:17:35 -0400 Subject: [PATCH 39/93] Revert me: debug print the msg/sig/signer --- llarp/router_contact_remote.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index d47e41933..1f402a4aa 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -66,6 +66,9 @@ namespace llarp throw std::runtime_error{err}; } + log::error(log::Cat("FIXME"), "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", + oxenc::to_hex(msg), oxenc::to_hex(sig), oxenc::to_hex(router_id().ToHex())); + if (not crypto::verify(router_id(), msg, sig)) throw std::runtime_error{"Failed to verify RemoteRC signature"}; }); From febcd44ea1f61375d0058ebc7397f6ce71575656 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 15:41:16 -0400 Subject: [PATCH 40/93] Fix pubkey parsing to be read as bytes from_string was trying to parse it as base32z.snode Also leave FIXMEs behind for the badly named methods (both in RouterID itself and in ancestor classes). --- llarp/crypto/types.hpp | 2 ++ llarp/router_contact.cpp | 6 +++++- llarp/router_id.hpp | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/llarp/crypto/types.hpp b/llarp/crypto/types.hpp index e62c0476a..0092adcfb 100644 --- a/llarp/crypto/types.hpp +++ b/llarp/crypto/types.hpp @@ -32,9 +32,11 @@ namespace llarp std::string ToString() const; + // FIXME: this is deceptively named: it should be "from_hex" since that's what it does. bool FromString(const std::string& str); + // FIXME: this is deceptively named: it should be "from_hex" since that's what it does. static PubKey from_string(const std::string& s); diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index 7f1280f7a..11aa3341f 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -61,7 +61,11 @@ namespace llarp "Invalid RC netid: expected {}, got {}; this is an RC for a different network!"_format( ACTIVE_NETID, netid)}; - _router_id.from_string(data.require("p")); + auto pubkey = data.require("p"); + if (pubkey.size() != 32) + throw std::runtime_error{ + "Invalid RC pubkey: expected 32 bytes, got {}"_format(pubkey.size())}; + std::memcpy(_router_id.data(), pubkey.data(), 32); // auto pk = data.require("p"); diff --git a/llarp/router_id.hpp b/llarp/router_id.hpp index 6a25ff9d4..d587f8365 100644 --- a/llarp/router_id.hpp +++ b/llarp/router_id.hpp @@ -33,6 +33,9 @@ namespace llarp std::string ShortString() const; + // FIXME: this is deceptively named: it parses something base32z formatted with .snode on the + // end, so should probably be called "from_snode_address" or "from_base32z" or something that + // doesn't sound exactly like the other (different) from_strings of its base classes. bool from_string(std::string_view str); From 3509693c49c2384785ff1f0315589c854c109f7f Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 15:42:36 -0400 Subject: [PATCH 41/93] Revert me: fix double-hex in debug output --- llarp/router_contact_remote.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 1f402a4aa..598aeead6 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -67,7 +67,7 @@ namespace llarp } log::error(log::Cat("FIXME"), "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", - oxenc::to_hex(msg), oxenc::to_hex(sig), oxenc::to_hex(router_id().ToHex())); + oxenc::to_hex(msg), oxenc::to_hex(sig), router_id().ToHex()); if (not crypto::verify(router_id(), msg, sig)) throw std::runtime_error{"Failed to verify RemoteRC signature"}; From 3c521c492347b12738c2cac2fd95ffcd1ece6e68 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 11:48:14 -0800 Subject: [PATCH 42/93] log --- llarp/router_contact_remote.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 598aeead6..527cfb333 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -34,6 +34,9 @@ namespace llarp throw std::runtime_error{err}; } + log::error(log::Cat("FIXME"), "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", + oxenc::to_hex(msg), oxenc::to_hex(sig), router_id().ToHex()); + if (not crypto::verify(router_id(), msg, sig)) throw std::runtime_error{"Failed to verify RemoteRC signature"}; }); From f812d5471b1cd02a25b4500cdc285183c8343cb5 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 12:06:16 -0800 Subject: [PATCH 43/93] send localrc, receive as remoterc --- llarp/bootstrap.cpp | 2 +- llarp/messages/fetch.hpp | 6 ++++-- llarp/nodedb.cpp | 6 +++--- llarp/router_contact.hpp | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index 23b7c034e..3100eada6 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -14,7 +14,7 @@ namespace llarp oxenc::bt_list_consumer btlc{buf}; while (not btlc.is_finished()) - emplace(btlc.consume_dict_consumer()); + emplace(btlc.consume_dict_data()); } catch (...) { diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 839a5d458..859603759 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -1,6 +1,7 @@ #pragma once #include "common.hpp" +#include namespace llarp { @@ -39,10 +40,11 @@ namespace llarp { // the LocalRC is converted to a RemoteRC type to send to the bootstrap seed inline static std::string - serialize(const RemoteRC& local_rc, size_t quantity) + serialize(const LocalRC& local_rc, size_t quantity) { oxenc::bt_dict_producer btdp; - btdp.append_encoded("local", local_rc.view()); + btdp.append_encoded("local", oxen::quic::to_sv(local_rc.view())); + log::critical(logcat, "Serializing localRC: {}", buffer_printer{local_rc.view()}); btdp.append("quantity", quantity); return std::move(btdp).str(); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 6e6f2beb5..17c42924d 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -408,7 +408,7 @@ namespace llarp std::set rcs; while (not btlc.is_finished()) - rcs.emplace(btlc.consume_dict_consumer()); + rcs.emplace(btlc.consume_dict_data()); // if process_fetched_rcs returns false, then the trust model rejected the fetched RC's fetch_rcs_result(initial, not ingest_fetched_rcs(std::move(rcs), timestamp)); @@ -672,7 +672,7 @@ namespace llarp _router.link_manager().fetch_bootstrap_rcs( _bootstraps->current(), BootstrapFetchMessage::serialize( - _router.router_contact.to_remote(), BOOTSTRAP_SOURCE_COUNT), + _router.router_contact, BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { if (not m) { @@ -698,7 +698,7 @@ namespace llarp while (not btlc.is_finished()) { - auto rc = RemoteRC{btlc.consume_dict_consumer()}; + auto rc = RemoteRC{btlc.consume_dict_data()}; rids.emplace(rc.router_id()); } } diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 466d00719..ea7fff249 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -314,6 +314,7 @@ namespace llarp void bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired = false) const; + explicit RemoteRC(oxenc::bt_dict_consumer btdc); public: RemoteRC() = default; explicit RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} @@ -324,7 +325,6 @@ namespace llarp { _payload = data; } - explicit RemoteRC(oxenc::bt_dict_consumer btdc); ~RemoteRC() = default; std::string_view From 88b9d9f9762e29bb40d1921923781baec3718baa Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 12:20:02 -0800 Subject: [PATCH 44/93] print but better --- llarp/messages/fetch.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 859603759..1d4453810 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -44,7 +44,7 @@ namespace llarp { oxenc::bt_dict_producer btdp; btdp.append_encoded("local", oxen::quic::to_sv(local_rc.view())); - log::critical(logcat, "Serializing localRC: {}", buffer_printer{local_rc.view()}); + log::critical(logcat, "Serializing localRC: {}", oxenc::to_hex(local_rc.view())); btdp.append("quantity", quantity); return std::move(btdp).str(); } From a6da88cbbef0850e90332cbb961382281565c30a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 15:56:08 -0400 Subject: [PATCH 45/93] DRY duplicated code in RCRemote --- llarp/router_contact_remote.cpp | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 527cfb333..1e9005455 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -17,29 +17,7 @@ namespace llarp { bt_load(btdc); - btdc.require_signature("~", [this](ustring_view msg, ustring_view sig) { - if (sig.size() != 64) - throw std::runtime_error{"Invalid signature: not 64 bytes"}; - - if (is_expired(time_now_ms())) - throw std::runtime_error{"Unable to verify expired RemoteRC!"}; - - // TODO: revisit if this is needed; detail from previous implementation - const auto* net = net::Platform::Default_ptr(); - - if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) - { - auto err = "Unable to verify RemoteRC address!"; - log::info(logcat, err); - throw std::runtime_error{err}; - } - - log::error(log::Cat("FIXME"), "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", - oxenc::to_hex(msg), oxenc::to_hex(sig), router_id().ToHex()); - - if (not crypto::verify(router_id(), msg, sig)) - throw std::runtime_error{"Failed to verify RemoteRC signature"}; - }); + bt_verify(btdc, /*reject_expired=*/true); } catch (const std::exception& e) { From 6c58f07a410ce61b4cb3e42f16c474adb9736470 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 16:35:11 -0400 Subject: [PATCH 46/93] Bump libquic --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index ad8904deb..400e1cad8 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit ad8904debc55f9495667e8571ee5bb963156d54e +Subproject commit 400e1cad8fd4509e925ab285363326b6e4f99f06 From cece742cf3438ff5f11950f46a0735dccec1553d Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 11 Dec 2023 17:20:49 -0400 Subject: [PATCH 47/93] Bump libquic for message move/copy fixes --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 400e1cad8..fdc92554a 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 400e1cad8fd4509e925ab285363326b6e4f99f06 +Subproject commit fdc92554a40414fdcdea1fa1bd84b2c827d21b1c diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 6bb9cf94a..72cea824c 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -254,7 +254,7 @@ namespace llarp if (func) { func = [this, f = std::move(func)](oxen::quic::message m) mutable { - _router.loop()->call([func = std::move(f), msg = std::move(m)]() mutable { func(msg); }); + _router.loop()->call([func = std::move(f), msg = std::move(m)]() mutable { func(std::move(msg)); }); }; } From 7544436f6afd1149eafc893027c87ac08e4e4bc0 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 11 Dec 2023 13:45:35 -0800 Subject: [PATCH 48/93] who cares --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 14 +++++++++++--- llarp/messages/fetch.hpp | 1 + llarp/nodedb.cpp | 3 +-- llarp/nodedb.hpp | 2 +- llarp/router_contact.cpp | 4 ++-- llarp/router_contact.hpp | 1 + llarp/router_contact_remote.cpp | 8 ++++++-- 8 files changed, 24 insertions(+), 11 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index fdc92554a..ad8904deb 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit fdc92554a40414fdcdea1fa1bd84b2c827d21b1c +Subproject commit ad8904debc55f9495667e8571ee5bb963156d54e diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 72cea824c..f1dff5b44 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -646,7 +646,6 @@ namespace llarp assert(_router.is_service_node()); log::critical(logcat, "Handling fetch bootstrap fetch request..."); - const auto& rcs = node_db->get_rcs(); RemoteRC remote; size_t quantity; @@ -668,7 +667,16 @@ namespace llarp node_db->put_rc(remote); - auto rc_size = rcs.size(); + auto& bootstraps = node_db->bootstrap_list(); + auto count = bootstraps.size(); + + if (count == 0) + { + log::error(logcat, "No bootstraps locally to send!"); + m.respond(messages::ERROR_RESPONSE, true); + return; + } + auto now = llarp::time_now_ms(); size_t i = 0; @@ -679,7 +687,7 @@ namespace llarp while (i < quantity) { - auto& next_rc = *std::next(rcs.begin(), csrng() % rc_size); + auto& next_rc = bootstraps.next(); if (next_rc.is_expired(now)) continue; diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 1d4453810..8981b6dde 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -1,6 +1,7 @@ #pragma once #include "common.hpp" + #include namespace llarp diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 17c42924d..b67069e47 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -671,8 +671,7 @@ namespace llarp _router.link_manager().fetch_bootstrap_rcs( _bootstraps->current(), - BootstrapFetchMessage::serialize( - _router.router_contact, BOOTSTRAP_SOURCE_COUNT), + BootstrapFetchMessage::serialize(_router.router_contact, BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { if (not m) { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index f8e99372a..eb595fec9 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -44,7 +44,7 @@ namespace llarp inline constexpr double GOOD_RID_FETCH_THRESHOLD{}; /* Bootstrap Constants */ // the number of rc's we query the bootstrap for - inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{50}; + inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{10}; // the maximum number of fetch requests we make across all bootstraps inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{5}; // if all bootstraps fail, router will trigger re-bootstrapping after this cooldown diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index 11aa3341f..dbb57d2da 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -63,8 +63,8 @@ namespace llarp auto pubkey = data.require("p"); if (pubkey.size() != 32) - throw std::runtime_error{ - "Invalid RC pubkey: expected 32 bytes, got {}"_format(pubkey.size())}; + throw std::runtime_error{ + "Invalid RC pubkey: expected 32 bytes, got {}"_format(pubkey.size())}; std::memcpy(_router_id.data(), pubkey.data(), 32); // auto pk = data.require("p"); diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index ea7fff249..d8dfd3ae3 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -315,6 +315,7 @@ namespace llarp bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired = false) const; explicit RemoteRC(oxenc::bt_dict_consumer btdc); + public: RemoteRC() = default; explicit RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 1e9005455..857d2f4a6 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -47,8 +47,12 @@ namespace llarp throw std::runtime_error{err}; } - log::error(log::Cat("FIXME"), "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", - oxenc::to_hex(msg), oxenc::to_hex(sig), router_id().ToHex()); + log::error( + log::Cat("FIXME"), + "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", + oxenc::to_hex(msg), + oxenc::to_hex(sig), + router_id().ToHex()); if (not crypto::verify(router_id(), msg, sig)) throw std::runtime_error{"Failed to verify RemoteRC signature"}; From 94f307283fb1fa5ba5035a8254f6ee0e2462ebb5 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 05:43:28 -0800 Subject: [PATCH 49/93] libquic vbump --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index ad8904deb..3ace46701 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit ad8904debc55f9495667e8571ee5bb963156d54e +Subproject commit 3ace46701449c4c01f74aae2e0be3b4164768911 From b69f7545999939406d5f26f2d3b3a64c45c5ead9 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 07:45:38 -0800 Subject: [PATCH 50/93] come on already work already --- llarp/bootstrap.cpp | 4 +-- llarp/bootstrap.hpp | 6 ++-- llarp/link/link_manager.cpp | 53 +++++++++++++++++------------- llarp/link/link_manager.hpp | 12 ++----- llarp/nodedb.cpp | 40 ++++++++++++++++------- llarp/nodedb.hpp | 36 +++++++++++++++++---- llarp/router/router.cpp | 46 +++++++++++--------------- llarp/router_contact.cpp | 57 +++++++++++++++++---------------- llarp/router_contact_remote.cpp | 1 - 9 files changed, 142 insertions(+), 113 deletions(-) diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index 3100eada6..b5f35c605 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -26,7 +26,7 @@ namespace llarp } bool - BootstrapList::contains(const RouterID& rid) + BootstrapList::contains(const RouterID& rid) const { for (const auto& it : *this) { @@ -38,7 +38,7 @@ namespace llarp } bool - BootstrapList::contains(const RemoteRC& rc) + BootstrapList::contains(const RemoteRC& rc) const { return count(rc); } diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index 472406861..b2cad4205 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -12,7 +12,7 @@ namespace llarp { struct BootstrapList final : public std::set { - std::set::iterator _curr; + std::set::iterator _curr = begin(); const RemoteRC& current() @@ -30,7 +30,7 @@ namespace llarp read_from_file(const fs::path& fpath); bool - contains(const RouterID& rid); + contains(const RouterID& rid) const; // returns a reference to the next index and a boolean that equals true if // this is the front of the set @@ -46,7 +46,7 @@ namespace llarp } bool - contains(const RemoteRC& rc); + contains(const RemoteRC& rc) const; void randomize() diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index f1dff5b44..71137e381 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -178,12 +178,13 @@ namespace llarp bool result = false; RouterID other{key.data()}; - if (auto itr = rids_pending_verification.find(other); itr != rids_pending_verification.end()) - { - verified_rids[other] = itr->second; - rids_pending_verification.erase(itr); - result = true; - } + // if (auto itr = rids_pending_verification.find(other); itr != + // rids_pending_verification.end()) + // { + // verified_rids[other] = itr->second; + // rids_pending_verification.erase(itr); + // result = true; + // } if (_router.node_db()->has_rc(other)) result = true; @@ -254,7 +255,8 @@ namespace llarp if (func) { func = [this, f = std::move(func)](oxen::quic::message m) mutable { - _router.loop()->call([func = std::move(f), msg = std::move(m)]() mutable { func(std::move(msg)); }); + _router.loop()->call( + [func = std::move(f), msg = std::move(m)]() mutable { func(std::move(msg)); }); }; } @@ -358,12 +360,12 @@ namespace llarp const auto& remote_addr = rc.addr(); const auto& rid = rc.router_id(); - rids_pending_verification[rid] = rc; + // rids_pending_verification[rid] = rc; // TODO: confirm remote end is using the expected pubkey (RouterID). // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) if (auto rv = ep.establish_connection( - oxen::quic::RemoteAddress{rc.router_id().ToView(), remote_addr}, + oxen::quic::RemoteAddress{rid.ToView(), remote_addr}, rc, std::move(on_open), std::move(on_close)); @@ -391,6 +393,7 @@ namespace llarp logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); s.conn.close_connection(error_code); }); + register_commands(control_stream); itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); @@ -460,9 +463,10 @@ namespace llarp { const auto& rid = c_itr->second; - if (auto maybe = rids_pending_verification.find(rid); - maybe != rids_pending_verification.end()) - rids_pending_verification.erase(maybe); + // if (auto maybe = rids_pending_verification.find(rid); + // maybe != rids_pending_verification.end()) + // rids_pending_verification.erase(maybe); + // in case this didn't clear earlier, do it now if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) pending_conn_msg_queue.erase(p_itr); @@ -665,14 +669,14 @@ namespace llarp return; } - node_db->put_rc(remote); + auto is_seed = _router.is_bootstrap_seed(); - auto& bootstraps = node_db->bootstrap_list(); - auto count = bootstraps.size(); + auto& src = is_seed ? node_db->bootstrap_seeds() : node_db->get_known_rcs(); + auto count = src.size(); if (count == 0) { - log::error(logcat, "No bootstraps locally to send!"); + log::error(logcat, "No {} locally to send!", is_seed ? "bootstrap seeds" : "known RCs"); m.respond(messages::ERROR_RESPONSE, true); return; } @@ -685,18 +689,21 @@ namespace llarp { auto sublist = btdp.append_list("rcs"); - while (i < quantity) + for (const auto& rc : src) { - auto& next_rc = bootstraps.next(); + if (not rc.is_expired(now)) + sublist.append_encoded(rc.view()); - if (next_rc.is_expired(now)) - continue; - - sublist.append_encoded(next_rc.view()); - ++i; + if (++i >= quantity) + break; } } + if (is_seed) + node_db->bootstrap_seeds().insert(remote); + else + node_db->put_rc(remote); + m.respond(std::move(btdp).str()); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index b97a6c7a9..e8b9c9431 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -181,7 +181,7 @@ namespace llarp // when establishing a connection, the rid of the remote is placed here to be cross- // checked by the tls verification callback std::map rids_pending_verification; - // in the interim of verifying an inbound connection and the creation of its link::Connection + // in the interim of verifying an outbound connection and the creation of its link::Connection // object, we store the rid and rc here std::map verified_rids; @@ -354,18 +354,9 @@ namespace llarp {"find_name"sv, &LinkManager::handle_find_name}, {"publish_intro"sv, &LinkManager::handle_publish_intro}, {"find_intro"sv, &LinkManager::handle_find_intro}}; - /* - {"path_confirm", &LinkManager::handle_path_confirm}, - {"path_latency", &LinkManager::handle_path_latency}, - {"update_exit", &LinkManager::handle_update_exit}, - {"obtain_exit", &LinkManager::handle_obtain_exit}, - {"close_exit", &LinkManager::handle_close_exit}, - {"convo_intro", &LinkManager::handle_convo_intro}}; - */ // these requests are direct, i.e. not over a path; // the rest are relay->relay - // TODO: new RC fetch endpoint (which will be both client->relay and relay->relay) std::unordered_map< std::string_view, void (LinkManager::*)(std::string_view body, std::function respond)> @@ -431,6 +422,7 @@ namespace llarp s.conn.close_connection(error_code); }); + link_manager.register_commands(control_stream); itr->second = std::make_shared(conn_interface, control_stream, rc); return true; diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index b67069e47..674af63d3 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -192,15 +192,10 @@ namespace llarp } void - NodeDB::set_bootstrap_routers(std::unique_ptr from_router) + NodeDB::set_bootstrap_routers(BootstrapList from_router) { - // TODO: if this needs to be called more than once (ex: drastic failures), then - // change this assert to a bootstraps.clear() call - if (_bootstraps) - assert(_bootstraps->empty()); - - _bootstraps = std::move(from_router); - _bootstraps->randomize(); + _bootstraps.merge(from_router); + _bootstraps.randomize(); } bool @@ -610,6 +605,8 @@ namespace llarp if (_router.is_service_node()) { _needs_rebootstrap = false; + _needs_initial_fetch = false; + _using_bootstrap_fallback = false; fail_sources.clear(); fetch_failures = 0; return; @@ -627,6 +624,7 @@ namespace llarp _router.last_rid_fetch = llarp::time_point_now(); fetch_counters.clear(); _needs_rebootstrap = false; + _using_bootstrap_fallback = false; if (initial) { @@ -638,6 +636,7 @@ namespace llarp void NodeDB::fallback_to_bootstrap() { + log::critical(logcat, "{} called", __PRETTY_FUNCTION__); auto at_max_failures = bootstrap_attempts >= MAX_BOOTSTRAP_FETCH_ATTEMPTS; // base case: we have failed to query all bootstraps, or we received a sample of @@ -660,7 +659,7 @@ namespace llarp return; } - auto rc = _bootstraps->next(); + auto rc = (_using_bootstrap_fallback) ? _bootstraps.next() : _bootstraps.current(); fetch_source = rc.router_id(); } @@ -669,8 +668,10 @@ namespace llarp _needs_rebootstrap = false; ++bootstrap_attempts; + log::critical(logcat, "Dispatching BootstrapRC fetch request to {}", _bootstraps.current().view()); + _router.link_manager().fetch_bootstrap_rcs( - _bootstraps->current(), + _bootstraps.current(), BootstrapFetchMessage::serialize(_router.router_contact, BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { if (not m) @@ -723,7 +724,7 @@ namespace llarp if (rids.size() == BOOTSTRAP_SOURCE_COUNT) { - known_rids.swap(rids); + known_rids.merge(rids); fetch_initial(); } else @@ -745,6 +746,7 @@ namespace llarp NodeDB::bootstrap_cooldown() { _needs_rebootstrap = true; + _using_bootstrap_fallback = false; _router.next_bootstrap_attempt = llarp::time_point_now() + BOOTSTRAP_COOLDOWN; } @@ -792,7 +794,7 @@ namespace llarp NodeDB::is_connection_allowed(const RouterID& remote) const { if (_pinned_edges.size() && _pinned_edges.count(remote) == 0 - && not _bootstraps->contains(remote)) + && not _bootstraps.contains(remote)) return false; if (not _router.is_service_node()) @@ -810,6 +812,20 @@ namespace llarp return true; } + void + NodeDB::store_bootstraps() + { + if (_bootstraps.empty()) + return; + + for (const auto& rc : _bootstraps) + { + put_rc(rc); + } + + log::critical(logcat, "NodeDB populated with {} routers", num_rcs()); + } + void NodeDB::load_from_disk() { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index eb595fec9..db1b98ef5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -121,6 +121,9 @@ namespace llarp the unknown_rids container - rc_lookup: holds all the same rc's as known_rcs, but can be used to look them up by their rid + - bootstrap_seeds: if we are the seed node, we insert the rc's of bootstrap fetch requests + senders into this container to "introduce" them to each other + - _bootstraps: the standard container for bootstrap RemoteRCs */ std::set known_rids; std::set> unconfirmed_rids; @@ -130,6 +133,9 @@ namespace llarp std::map rc_lookup; + std::set _bootstrap_seeds; + BootstrapList _bootstraps{}; + /** RouterID lists // TODO: get rid of all these, replace with better decom/not staked sets - white: active routers - gray: fully funded, but decommissioned routers @@ -178,8 +184,6 @@ namespace llarp fs::path get_path_by_pubkey(RouterID pk) const; - std::unique_ptr _bootstraps{}; - public: explicit NodeDB( fs::path rootdir, std::function)> diskCaller, Router* r); @@ -214,6 +218,9 @@ namespace llarp return _needs_rebootstrap; } + void + ingest_bootstrap_seed(); + bool ingest_fetched_rcs(std::set rcs, rc_time timestamp); @@ -299,32 +306,47 @@ namespace llarp return _pinned_edges; } + void + store_bootstraps(); + size_t num_bootstraps() const { - return _bootstraps ? _bootstraps->size() : 0; + return _bootstraps.size(); } bool has_bootstraps() const { - return _bootstraps ? _bootstraps->empty() : false; + return _bootstraps.empty(); } const BootstrapList& bootstrap_list() const { - return *_bootstraps; + return _bootstraps; } BootstrapList& bootstrap_list() { - return *_bootstraps; + return _bootstraps; + } + + const std::set& + bootstrap_seeds() const + { + return _bootstrap_seeds; + } + + std::set& + bootstrap_seeds() + { + return _bootstrap_seeds; } void - set_bootstrap_routers(std::unique_ptr from_router); + set_bootstrap_routers(BootstrapList from_router); const std::set& whitelist() const diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index a700aaf9c..426e8397a 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -631,17 +631,18 @@ namespace llarp configRouters.push_back(defaultBootstrapFile); } - auto _bootstrap_rc_list = std::make_unique(); + BootstrapList _bootstrap_rc_list; auto clear_bad_rcs = [&]() mutable { + log::critical(logcat, "Clearing bad RCs..."); // in case someone has an old bootstrap file and is trying to use a bootstrap // that no longer exists - for (auto it = _bootstrap_rc_list->begin(); it != _bootstrap_rc_list->end();) + for (auto it = _bootstrap_rc_list.begin(); it != _bootstrap_rc_list.end();) { if (it->is_obsolete_bootstrap()) - log::warning(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); + log::critical(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); else if (not it->verify()) - log::warning(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); + log::critical(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); else { ++it; @@ -649,29 +650,27 @@ namespace llarp } // we are in one of the above error cases that we warned about: - it = _bootstrap_rc_list->erase(it); + it = _bootstrap_rc_list.erase(it); } }; for (const auto& router : configRouters) { log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); - _bootstrap_rc_list->read_from_file(router); + _bootstrap_rc_list.read_from_file(router); } for (const auto& rc : conf.bootstrap.routers) { - _bootstrap_rc_list->emplace(rc); + _bootstrap_rc_list.emplace(rc); } - clear_bad_rcs(); - _bootstrap_seed = conf.bootstrap.seednode; if (_bootstrap_seed) log::critical(logcat, "We are a bootstrap seed node!"); - if (_bootstrap_rc_list->empty() and not _bootstrap_seed) + if (_bootstrap_rc_list.empty() and not _bootstrap_seed) { log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node"); @@ -679,10 +678,10 @@ namespace llarp if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) { - _bootstrap_rc_list->merge(itr->second); + _bootstrap_rc_list.merge(itr->second); } - if (_bootstrap_rc_list->empty()) + if (_bootstrap_rc_list.empty()) { // empty after trying fallback, if set log::error( @@ -694,12 +693,12 @@ namespace llarp throw std::runtime_error("No bootstrap nodes available."); } - log::info( - logcat, "Loaded {} default fallback bootstrap routers!", _bootstrap_rc_list->size()); - clear_bad_rcs(); + log::critical( + logcat, "Loaded {} default fallback bootstrap routers!", _bootstrap_rc_list.size()); } - log::critical(logcat, "We have {} bootstrap routers!", _bootstrap_rc_list->size()); + clear_bad_rcs(); + log::critical(logcat, "We have {} bootstrap routers!", _bootstrap_rc_list.size()); node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); @@ -1140,21 +1139,11 @@ namespace llarp log::info(logcat, "Loading NodeDB from disk..."); _node_db->load_from_disk(); + _node_db->store_bootstraps(); log::info(logcat, "Creating Introset Contacts..."); _contacts = std::make_unique(*this); - if (_node_db->has_bootstraps()) - { - for (const auto& rc : _node_db->bootstrap_list()) - { - node_db()->put_rc(rc); - log::info(logcat, "Added bootstrap node (rid: {})", rc.router_id()); - } - - log::info(logcat, "Router populated NodeDB with {} routers", _node_db->num_rcs()); - } - _loop->call_every(ROUTER_TICK_INTERVAL, weak_from_this(), [this] { Tick(); }); _route_poker->start(); @@ -1163,7 +1152,7 @@ namespace llarp _started_at = now(); - if (is_service_node()) + if (is_service_node() and not _testing_disabled) { // do service node testing if we are in service node whitelist mode _loop->call_every(consensus::REACHABILITY_TESTING_TIMER_INTERVAL, weak_from_this(), [this] { @@ -1225,6 +1214,7 @@ namespace llarp } }); } + llarp::sys::service_manager->ready(); return is_running; } diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index dbb57d2da..b660c7990 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -17,45 +17,55 @@ namespace llarp if (int rc_ver = data.require(""); rc_ver != RC_VERSION) throw std::runtime_error{"Invalid RC: do not know how to parse v{} RCs"_format(rc_ver)}; - auto ipv4_port = data.require("4"); + auto addr_key = data.key(); + bool addr_set = false; - if (ipv4_port.size() != 6) - throw std::runtime_error{ - "Invalid RC address: expected 6-byte IPv4 IP/port, got {}"_format(ipv4_port.size())}; - - sockaddr_in s4; - s4.sin_family = AF_INET; + if (addr_key == "4") + { + auto ipv4_port = data.consume(); - std::memcpy(&s4.sin_addr.s_addr, ipv4_port.data(), 4); - std::memcpy(&s4.sin_port, ipv4_port.data() + 4, 2); + if (ipv4_port.size() != 6) + throw std::runtime_error{ + "Invalid RC address: expected 6-byte IPv4 IP/port, got {}"_format(ipv4_port.size())}; - _addr = oxen::quic::Address{&s4}; + sockaddr_in s4; + s4.sin_family = AF_INET; - if (!_addr.is_public()) - throw std::runtime_error{"Invalid RC: IPv4 address is not a publicly routable IP"}; + std::memcpy(&s4.sin_addr.s_addr, ipv4_port.data(), 4); + std::memcpy(&s4.sin_port, ipv4_port.data() + 4, 2); - if (auto ipv6_port = data.maybe("6")) + _addr = oxen::quic::Address{&s4}; + // advance in case ipv4 and ipv6 are included + addr_key = data.key(); + addr_set = true; + } + if (addr_key == "6") { - if (ipv6_port->size() != 18) + auto ipv6_port = data.consume(); + + if (ipv6_port.size() != 18) throw std::runtime_error{ - "Invalid RC address: expected 18-byte IPv6 IP/port, got {}"_format(ipv6_port->size())}; + "Invalid RC address: expected 18-byte IPv6 IP/port, got {}"_format(ipv6_port.size())}; sockaddr_in6 s6{}; s6.sin6_family = AF_INET6; - std::memcpy(&s6.sin6_addr.s6_addr, ipv6_port->data(), 16); - std::memcpy(&s6.sin6_port, ipv6_port->data() + 16, 2); + std::memcpy(&s6.sin6_addr.s6_addr, ipv6_port.data(), 16); + std::memcpy(&s6.sin6_port, ipv6_port.data() + 16, 2); _addr6.emplace(&s6); + if (!_addr6->is_public()) throw std::runtime_error{"Invalid RC: IPv6 address is not a publicly routable IP"}; } else - { _addr6.reset(); - } + + if (not addr_set) + throw std::runtime_error{"Invalid RC: could not discern ipv4 vs ipv6 in payload"}; auto netid = data.maybe("i").value_or(llarp::LOKINET_DEFAULT_NETID); + if (netid != ACTIVE_NETID) throw std::runtime_error{ "Invalid RC netid: expected {}, got {}; this is an RC for a different network!"_format( @@ -67,14 +77,7 @@ namespace llarp "Invalid RC pubkey: expected 32 bytes, got {}"_format(pubkey.size())}; std::memcpy(_router_id.data(), pubkey.data(), 32); - // auto pk = data.require("p"); - - // if (pk.size() != RouterID::SIZE) - // throw std::runtime_error{"Invalid RC: router id has invalid size {}"_format(pk.size())}; - - // std::memcpy(_router_id.data(), pk.data(), RouterID::SIZE); - - _timestamp = rc_time{std::chrono::seconds{data.require("t")}}; + _timestamp = rc_time{std::chrono::seconds{data.require("t")}}; auto ver = data.require("v"); diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 857d2f4a6..ef405662b 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -16,7 +16,6 @@ namespace llarp try { bt_load(btdc); - bt_verify(btdc, /*reject_expired=*/true); } catch (const std::exception& e) From 6cb2f57abd6b867c3c972631c5cebaa8c1a8ccaa Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 09:01:02 -0800 Subject: [PATCH 51/93] whatever --- llarp/bootstrap.hpp | 3 +++ llarp/link/link_manager.cpp | 7 +++--- llarp/link/link_manager.hpp | 4 ++++ llarp/net/posix.cpp | 2 +- llarp/nodedb.cpp | 19 ++++++++------- llarp/nodedb.hpp | 2 +- llarp/router/router.cpp | 39 +++++++++++++++++------------- llarp/router/router.hpp | 2 +- llarp/router_contact.cpp | 47 +++++++++++++++---------------------- 9 files changed, 66 insertions(+), 59 deletions(-) diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index b2cad4205..7d037ff55 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -37,6 +37,9 @@ namespace llarp const RemoteRC& next() { + if (size() < 2) + return *_curr; + ++_curr; if (_curr == this->end()) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 71137e381..a49a310b6 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -352,6 +352,7 @@ namespace llarp { if (auto conn = ep.get_conn(rc.router_id()); conn) { + log::error(logcat, "We should not be here!"); // TODO: should implement some connection failed logic, but not the same logic that // would be executed for another failure case return; @@ -628,13 +629,13 @@ namespace llarp _router.loop()->call([this, source, payload, f = std::move(func)]() { if (auto conn = ep.get_conn(source); conn) { - log::critical(logcat, "Dispatched bootstrap fetch request!"); conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f)); + log::critical(logcat, "Dispatched bootstrap fetch request!"); return; } - log::critical(logcat, "Queuing bootstrap fetch request"); - auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, f); + log::critical(logcat, "Queuing bootstrap fetch request to {}", source.router_id()); + auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, std::move(f)); auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue()); itr->second.push_back(std::move(pending)); diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index e8b9c9431..64ef45b11 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -404,6 +404,10 @@ namespace llarp { try { + std::this_thread::sleep_for(5s); + oxen::log::flush(); + log::critical(logcat, "Establishing connection to {}", remote); + auto conn_interface = endpoint->connect(remote, link_manager.tls_creds, std::forward(opts)...); diff --git a/llarp/net/posix.cpp b/llarp/net/posix.cpp index d48fbb39e..3715e1571 100644 --- a/llarp/net/posix.cpp +++ b/llarp/net/posix.cpp @@ -55,7 +55,7 @@ namespace llarp::net { std::optional found; - iter_all([this, &found, af](auto i) { + iter_all([this, &found, af](ifaddrs* i) { if (found) return; if (i and i->ifa_addr and i->ifa_addr->sa_family == af) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 674af63d3..c2eb9f559 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -192,7 +192,7 @@ namespace llarp } void - NodeDB::set_bootstrap_routers(BootstrapList from_router) + NodeDB::set_bootstrap_routers(BootstrapList& from_router) { _bootstraps.merge(from_router); _bootstraps.randomize(); @@ -339,9 +339,9 @@ namespace llarp void NodeDB::fetch_initial() { - if (known_rcs.empty()) + if (known_rids.empty()) { - log::critical(logcat, "No RC's held locally... BOOTSTRAP TIME"); + log::critical(logcat, "No RouterID's held locally... BOOTSTRAP TIME"); fallback_to_bootstrap(); } else @@ -658,20 +658,21 @@ namespace llarp bootstrap_cooldown(); return; } - - auto rc = (_using_bootstrap_fallback) ? _bootstraps.next() : _bootstraps.current(); - fetch_source = rc.router_id(); } + auto& rc = (_using_bootstrap_fallback) ? _bootstraps.next() : _bootstraps.current(); + fetch_source = rc.router_id(); + // By passing the last conditional, we ensure this is set to true _using_bootstrap_fallback = true; _needs_rebootstrap = false; ++bootstrap_attempts; - log::critical(logcat, "Dispatching BootstrapRC fetch request to {}", _bootstraps.current().view()); + log::critical( + logcat, "Dispatching BootstrapRC fetch request to {}", _bootstraps.current().view()); _router.link_manager().fetch_bootstrap_rcs( - _bootstraps.current(), + rc, BootstrapFetchMessage::serialize(_router.router_contact, BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { if (not m) @@ -823,7 +824,7 @@ namespace llarp put_rc(rc); } - log::critical(logcat, "NodeDB populated with {} routers", num_rcs()); + log::critical(logcat, "NodeDB stored {} bootstrap routers", _bootstraps.size()); } void diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index db1b98ef5..6644c4b05 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -346,7 +346,7 @@ namespace llarp } void - set_bootstrap_routers(BootstrapList from_router); + set_bootstrap_routers(BootstrapList& from_router); const std::set& whitelist() const diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 426e8397a..c1ea59519 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -631,13 +631,14 @@ namespace llarp configRouters.push_back(defaultBootstrapFile); } - BootstrapList _bootstrap_rc_list; + // BootstrapList _bootstrap_rc_list; + auto& node_bstrap = _node_db->bootstrap_list(); auto clear_bad_rcs = [&]() mutable { log::critical(logcat, "Clearing bad RCs..."); // in case someone has an old bootstrap file and is trying to use a bootstrap // that no longer exists - for (auto it = _bootstrap_rc_list.begin(); it != _bootstrap_rc_list.end();) + for (auto it = node_bstrap.begin(); it != node_bstrap.end();) { if (it->is_obsolete_bootstrap()) log::critical(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); @@ -650,19 +651,22 @@ namespace llarp } // we are in one of the above error cases that we warned about: - it = _bootstrap_rc_list.erase(it); + it = node_bstrap.erase(it); } }; + for (const auto& router : configRouters) { log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); - _bootstrap_rc_list.read_from_file(router); + node_bstrap.read_from_file(router); + // _bootstrap_rc_list.read_from_file(router); } for (const auto& rc : conf.bootstrap.routers) { - _bootstrap_rc_list.emplace(rc); + // _bootstrap_rc_list.emplace(rc); + node_bstrap.emplace(rc); } _bootstrap_seed = conf.bootstrap.seednode; @@ -670,7 +674,7 @@ namespace llarp if (_bootstrap_seed) log::critical(logcat, "We are a bootstrap seed node!"); - if (_bootstrap_rc_list.empty() and not _bootstrap_seed) + if (node_bstrap.empty() and not _bootstrap_seed) { log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node"); @@ -678,10 +682,11 @@ namespace llarp if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) { - _bootstrap_rc_list.merge(itr->second); + // _bootstrap_rc_list.merge(itr->second); + node_bstrap.merge(itr->second); } - if (_bootstrap_rc_list.empty()) + if (node_bstrap.empty()) { // empty after trying fallback, if set log::error( @@ -694,19 +699,19 @@ namespace llarp } log::critical( - logcat, "Loaded {} default fallback bootstrap routers!", _bootstrap_rc_list.size()); + logcat, "Loaded {} default fallback bootstrap routers!", node_bstrap.size()); } clear_bad_rcs(); - log::critical(logcat, "We have {} bootstrap routers!", _bootstrap_rc_list.size()); + log::critical(logcat, "We have {} bootstrap routers!", node_bstrap.size()); - node_db()->set_bootstrap_routers(std::move(_bootstrap_rc_list)); + // node_db()->set_bootstrap_routers(_bootstrap_rc_list); // TODO: RC refactor here - if (_is_service_node) - init_inbounds(); - else - init_outbounds(); + // if (_is_service_node) + // init_inbounds(); + // else + // init_outbounds(); // profiling _profile_file = conf.router.data_dir / "profiles.dat"; @@ -1117,7 +1122,7 @@ namespace llarp } log::info(logcat, "Router initialized as service node!"); - const RouterID us = pubkey(); + // relays do not use profiling router_profiling().Disable(); } @@ -1141,6 +1146,8 @@ namespace llarp _node_db->load_from_disk(); _node_db->store_bootstraps(); + oxen::log::flush(); + log::info(logcat, "Creating Introset Contacts..."); _contacts = std::make_unique(*this); diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 86eb5de13..87affae61 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -105,7 +105,7 @@ namespace llarp std::shared_ptr _node_db; llarp_time_t _started_at; const oxenmq::TaggedThreadID _disk_thread; - oxen::quic::Network _net; + // oxen::quic::Network _net; // DISCUSS: we don't use this anywhere..? llarp_time_t _last_stats_report = 0s; llarp_time_t _next_decomm_warning = time_now_ms() + 15s; diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index b660c7990..d75c5625e 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -17,52 +17,43 @@ namespace llarp if (int rc_ver = data.require(""); rc_ver != RC_VERSION) throw std::runtime_error{"Invalid RC: do not know how to parse v{} RCs"_format(rc_ver)}; - auto addr_key = data.key(); - bool addr_set = false; + auto ipv4_port = data.require("4"); - if (addr_key == "4") - { - auto ipv4_port = data.consume(); + if (ipv4_port.size() != 6) + throw std::runtime_error{ + "Invalid RC address: expected 6-byte IPv4 IP/port, got {}"_format(ipv4_port.size())}; - if (ipv4_port.size() != 6) - throw std::runtime_error{ - "Invalid RC address: expected 6-byte IPv4 IP/port, got {}"_format(ipv4_port.size())}; + sockaddr_in s4; + s4.sin_family = AF_INET; - sockaddr_in s4; - s4.sin_family = AF_INET; + std::memcpy(&s4.sin_addr.s_addr, ipv4_port.data(), 4); + std::memcpy(&s4.sin_port, ipv4_port.data() + 4, 2); - std::memcpy(&s4.sin_addr.s_addr, ipv4_port.data(), 4); - std::memcpy(&s4.sin_port, ipv4_port.data() + 4, 2); + _addr = oxen::quic::Address{&s4}; - _addr = oxen::quic::Address{&s4}; - // advance in case ipv4 and ipv6 are included - addr_key = data.key(); - addr_set = true; - } - if (addr_key == "6") - { - auto ipv6_port = data.consume(); + if (!_addr.is_public()) + throw std::runtime_error{"Invalid RC: IPv4 address is not a publicly routable IP"}; - if (ipv6_port.size() != 18) + if (auto ipv6_port = data.maybe("6")) + { + if (ipv6_port->size() != 18) throw std::runtime_error{ - "Invalid RC address: expected 18-byte IPv6 IP/port, got {}"_format(ipv6_port.size())}; + "Invalid RC address: expected 18-byte IPv6 IP/port, got {}"_format(ipv6_port->size())}; sockaddr_in6 s6{}; s6.sin6_family = AF_INET6; - std::memcpy(&s6.sin6_addr.s6_addr, ipv6_port.data(), 16); - std::memcpy(&s6.sin6_port, ipv6_port.data() + 16, 2); + std::memcpy(&s6.sin6_addr.s6_addr, ipv6_port->data(), 16); + std::memcpy(&s6.sin6_port, ipv6_port->data() + 16, 2); _addr6.emplace(&s6); - if (!_addr6->is_public()) throw std::runtime_error{"Invalid RC: IPv6 address is not a publicly routable IP"}; } else + { _addr6.reset(); - - if (not addr_set) - throw std::runtime_error{"Invalid RC: could not discern ipv4 vs ipv6 in payload"}; + } auto netid = data.maybe("i").value_or(llarp::LOKINET_DEFAULT_NETID); From 4c7f9d080f0c9c7d5879ed1a3de4933ac1013989 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 12 Dec 2023 13:38:31 -0400 Subject: [PATCH 52/93] Replace GetBestNetIF with quic::Address version It is now called get_best_public_address, and takes (bool, port) argument to return an optional quic::Address to make life easier: the caller now can just give the default port to set, and we keep the C sockaddr* more constrained. --- llarp/config/config.cpp | 10 +++------- llarp/net/net.hpp | 8 ++++++-- llarp/net/posix.cpp | 18 +++++++++++------- llarp/net/win32.cpp | 4 ++-- llarp/router/router.cpp | 7 ++----- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index 5b92f0fe9..9b094eae9 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -961,14 +961,10 @@ namespace llarp throw std::invalid_argument{fmt::format("{} is a loopback address", arg)}; } if (not maybe) - { // infer public address - if (auto maybe_ifname = net_ptr->GetBestNetIF()) - maybe = oxen::quic::Address{*maybe_ifname}; - } - - if (maybe && maybe->port() == 0) - maybe = oxen::quic::Address{maybe->host(), DEFAULT_LISTEN_PORT}; + maybe = net_ptr->get_best_public_address(true, DEFAULT_LISTEN_PORT); + else if (maybe && maybe->port() == 0) + maybe->set_port(DEFAULT_LISTEN_PORT); return maybe; }; diff --git a/llarp/net/net.hpp b/llarp/net/net.hpp index c821c2879..0074b4594 100644 --- a/llarp/net/net.hpp +++ b/llarp/net/net.hpp @@ -10,6 +10,8 @@ #include #include +#include + #include // for itoa #include #include @@ -121,8 +123,10 @@ namespace llarp return var::visit([](auto&& ip) { return not ip.n; }, ip); } - virtual std::optional - GetBestNetIF(int af = AF_INET) const = 0; + // Attempts to guess a good default public network address from the system's public IP + // addresses; the returned Address (if set) will have its port set to the given value. + virtual std::optional + get_best_public_address(bool ipv4, uint16_t port) const = 0; virtual std::optional FindFreeRange() const = 0; diff --git a/llarp/net/posix.cpp b/llarp/net/posix.cpp index 3715e1571..f96088786 100644 --- a/llarp/net/posix.cpp +++ b/llarp/net/posix.cpp @@ -10,6 +10,8 @@ #include #endif +#include + #include namespace llarp::net @@ -50,19 +52,21 @@ namespace llarp::net return ifname; } - std::optional - GetBestNetIF(int af) const override + std::optional + get_best_public_address(bool ipv4, uint16_t port) const override { - std::optional found; + std::optional found; - iter_all([this, &found, af](ifaddrs* i) { + iter_all([&found, ipv4, port](ifaddrs* i) { if (found) return; - if (i and i->ifa_addr and i->ifa_addr->sa_family == af) + if (i and i->ifa_addr and i->ifa_addr->sa_family == (ipv4 ? AF_INET : AF_INET6)) { - if (not IsBogon(*i->ifa_addr)) + oxen::quic::Address a{i->ifa_addr}; + if (a.is_public_ip()) { - found = i->ifa_addr; + a.set_port(port); + found = std::move(a); } } }); diff --git a/llarp/net/win32.cpp b/llarp/net/win32.cpp index 88b011b20..37af7ad26 100644 --- a/llarp/net/win32.cpp +++ b/llarp/net/win32.cpp @@ -129,8 +129,8 @@ namespace llarp::net return "lokitun0"; } - std::optional - GetBestNetIF(int) const override + std::optional + get_best_public_address(bool, uint16_t) const override { // TODO: implement me ? return std::nullopt; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index c1ea59519..30604f423 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -582,11 +582,8 @@ namespace llarp if (paddr or pport) throw std::runtime_error{"Must specify [bind]:listen in config with public ip/addr!"}; - if (auto maybe_addr = net().GetBestNetIF()) - { - _listen_address = oxen::quic::Address{static_cast(*maybe_addr)}; - _listen_address.set_port(DEFAULT_LISTEN_PORT); - } + if (auto maybe_addr = net().get_best_public_address(true, DEFAULT_LISTEN_PORT)) + _listen_address = std::move(*maybe_addr); else throw std::runtime_error{"Could not find net interface on current platform!"}; } From b0d6d0cc2ba75406427dcdc4187ba6d921ef4a50 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 12 Dec 2023 14:05:39 -0400 Subject: [PATCH 53/93] Bump libquic for register_command fix --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 3ace46701..921a85852 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 3ace46701449c4c01f74aae2e0be3b4164768911 +Subproject commit 921a85852a12510d374e1042d983e98f40f07689 From eb971265c78dbaa2523c6c54dd54538b0ef40859 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 10:19:32 -0800 Subject: [PATCH 54/93] wrap dat --- llarp/link/link_manager.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index a49a310b6..dd25393dd 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -626,7 +626,17 @@ namespace llarp LinkManager::fetch_bootstrap_rcs( const RemoteRC& source, std::string payload, std::function func) { - _router.loop()->call([this, source, payload, f = std::move(func)]() { + _router.loop()->call([this, source, payload, f = std::move(func)]() mutable { + + if (f) + { + f = [this, func = std::move(f)](oxen::quic::message m) mutable { + _router.loop()->call([f = std::move(func), msg = std::move(m)]() mutable { + f(std::move(msg)); + }); + }; + } + if (auto conn = ep.get_conn(source); conn) { conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f)); From 63c9bd6e6311743ff79bdfa10b3a9d239fa14b4c Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 10:24:31 -0800 Subject: [PATCH 55/93] update seeds before returning no --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 921a85852..3ace46701 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 921a85852a12510d374e1042d983e98f40f07689 +Subproject commit 3ace46701449c4c01f74aae2e0be3b4164768911 diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index dd25393dd..0c8f15387 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -684,6 +684,11 @@ namespace llarp auto& src = is_seed ? node_db->bootstrap_seeds() : node_db->get_known_rcs(); auto count = src.size(); + + if (is_seed) + node_db->bootstrap_seeds().insert(remote); + else + node_db->put_rc(remote); if (count == 0) { @@ -710,10 +715,6 @@ namespace llarp } } - if (is_seed) - node_db->bootstrap_seeds().insert(remote); - else - node_db->put_rc(remote); m.respond(std::move(btdp).str()); } From 2f1917040c3c96f685284f11854aa5da49e0fd3a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Tue, 12 Dec 2023 14:30:40 -0400 Subject: [PATCH 56/93] Fix std::set move semantics This is, apparently, the only way to move an element out of a std::set. --- llarp/nodedb.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index c2eb9f559..75cb8247c 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -254,8 +254,8 @@ namespace llarp return false; } - for (auto& rc : rcs) - put_rc_if_newer(std::move(rc), timestamp); + while (!rcs.empty()) + put_rc_if_newer(std::move(rcs.extract(rcs.begin()).value()), timestamp); return true; } From 49dbdf1062e51440d62cba3d625bf33b1fc41412 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 10:44:55 -0800 Subject: [PATCH 57/93] doofus --- llarp/link/link_manager.cpp | 11 ++++++----- llarp/link/link_manager.hpp | 2 -- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 0c8f15387..49583a2c7 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -118,24 +118,24 @@ namespace llarp LinkManager::register_commands(std::shared_ptr& s) { assert(ep.connid_map.count(s->conn_id())); - const RouterID& rid = ep.connid_map[s->conn_id()]; + const RouterID& router_id = ep.connid_map[s->conn_id()]; - s->register_command("path_build"s, [this, rid](oxen::quic::message m) { + s->register_command("path_build"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( [this, &rid, msg = std::move(m)]() mutable { handle_path_build(std::move(msg), rid); }); }); - s->register_command("path_control"s, [this, rid](oxen::quic::message m) { + s->register_command("path_control"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( [this, &rid, msg = std::move(m)]() mutable { handle_path_control(std::move(msg), rid); }); }); - s->register_command("gossip_rc"s, [this, rid](oxen::quic::message m) { + s->register_command("gossip_rc"s, [this](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_gossip_rc(std::move(msg)); }); }); - s->register_command("bfetch_rcs"s, [this, rid](oxen::quic::message m) { + s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); }); @@ -394,6 +394,7 @@ namespace llarp logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); s.conn.close_connection(error_code); }); + log::critical(logcat, "Opened BTStream ID:{}", control_stream->stream_id()); register_commands(control_stream); itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 64ef45b11..4cef45722 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -404,8 +404,6 @@ namespace llarp { try { - std::this_thread::sleep_for(5s); - oxen::log::flush(); log::critical(logcat, "Establishing connection to {}", remote); auto conn_interface = From ea614ed141b56889d76e924d6d792cd1f661b613 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 12 Dec 2023 11:46:45 -0800 Subject: [PATCH 58/93] better! --- llarp/link/connection.cpp | 5 +-- llarp/link/connection.hpp | 5 +-- llarp/link/link_manager.cpp | 75 ++++++++++++++++++++---------------- llarp/link/link_manager.hpp | 2 +- llarp/nodedb.cpp | 6 ++- llarp/nodedb.hpp | 7 ++++ llarp/path/pathbuilder.cpp | 5 +-- llarp/router/route_poker.cpp | 4 +- llarp/router/router.cpp | 6 +-- 9 files changed, 65 insertions(+), 50 deletions(-) diff --git a/llarp/link/connection.cpp b/llarp/link/connection.cpp index eb16b9174..9b8760746 100644 --- a/llarp/link/connection.cpp +++ b/llarp/link/connection.cpp @@ -4,9 +4,8 @@ namespace llarp::link { Connection::Connection( const std::shared_ptr& c, - std::shared_ptr& s, - const RemoteRC& rc) - : conn{c}, control_stream{s}, remote_rc{std::move(rc)} + std::shared_ptr& s) + : conn{c}, control_stream{s}/* , remote_rc{std::move(rc)} */ {} } // namespace llarp::link diff --git a/llarp/link/connection.hpp b/llarp/link/connection.hpp index b71a6e9f9..1c5000657 100644 --- a/llarp/link/connection.hpp +++ b/llarp/link/connection.hpp @@ -11,7 +11,7 @@ namespace llarp::link { std::shared_ptr conn; std::shared_ptr control_stream; - RemoteRC remote_rc; + // std::optional remote_rc; // one side of a connection will be responsible for some things, e.g. heartbeat bool inbound{false}; @@ -19,8 +19,7 @@ namespace llarp::link Connection( const std::shared_ptr& c, - std::shared_ptr& s, - const RemoteRC& rc); + std::shared_ptr& s); }; } // namespace llarp::link diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 49583a2c7..9787d774f 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -72,8 +72,16 @@ namespace llarp { auto itr = conns.begin(); std::advance(itr, randint() % size); - router = itr->second->remote_rc; - return true; + + RouterID rid{itr->second->conn->remote_key()}; + + if (auto maybe = link_manager.node_db->get_rc(rid)) + { + router = *maybe; + return true; + } + + return false; } log::warning(quic_cat, "Error: failed to fetch random connection"); @@ -143,7 +151,7 @@ namespace llarp for (auto& method : direct_requests) { s->register_command( - std::string{method.first}, [this, func = method.second](oxen::quic::message m) { + std::string{method.first}, [this, func = std::move(method.second)](oxen::quic::message m) { _router.loop()->call([this, msg = std::move(m), func = std::move(func)]() mutable { auto body = msg.body_str(); auto respond = [m = std::move(msg)](std::string response) mutable { @@ -178,24 +186,23 @@ namespace llarp bool result = false; RouterID other{key.data()}; - // if (auto itr = rids_pending_verification.find(other); itr != - // rids_pending_verification.end()) - // { - // verified_rids[other] = itr->second; - // rids_pending_verification.erase(itr); - // result = true; - // } - - if (_router.node_db()->has_rc(other)) - result = true; - - // TODO: discuss pubkey verification for bootstraps connecting to seed node if (_router.is_bootstrap_seed()) { - log::warning(logcat, "Allowing connection -- we are bootstrap seed"); - result = true; + if (node_db->whitelist().count(other)) + { + auto [it, b] = node_db->seeds().emplace(other); + result &= b; + } + log::critical( + logcat, + "Bootstrap seed node was {} to confirm fetch requester is white-listed; saving RID", + result ? "able" : "unable"); + return result; } + if (node_db->has_rc(other)) + result = true; + log::critical( logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Uns", other); return result; @@ -383,8 +390,6 @@ namespace llarp { const auto& scid = ci.scid(); RouterID rid{ci.remote_key()}; - - const auto& rc = verified_rids[rid]; ep.connid_map.emplace(scid, rid); auto [itr, b] = ep.conns.emplace(rid, nullptr); @@ -395,11 +400,9 @@ namespace llarp s.conn.close_connection(error_code); }); log::critical(logcat, "Opened BTStream ID:{}", control_stream->stream_id()); - register_commands(control_stream); - itr->second = std::make_shared(ci.shared_from_this(), control_stream, rc); - log::critical(logcat, "Successfully configured inbound connection fom {}; storing RC...", rid); - node_db->put_rc(rc); + itr->second = std::make_shared(ci.shared_from_this(), control_stream); + log::critical(logcat, "Successfully configured inbound connection fom {}...", rid); } // TODO: should we add routes here now that Router::SessionOpen is gone? @@ -628,13 +631,11 @@ namespace llarp const RemoteRC& source, std::string payload, std::function func) { _router.loop()->call([this, source, payload, f = std::move(func)]() mutable { - if (f) { f = [this, func = std::move(f)](oxen::quic::message m) mutable { - _router.loop()->call([f = std::move(func), msg = std::move(m)]() mutable { - f(std::move(msg)); - }); + _router.loop()->call( + [f = std::move(func), msg = std::move(m)]() mutable { f(std::move(msg)); }); }; } @@ -682,14 +683,23 @@ namespace llarp } auto is_seed = _router.is_bootstrap_seed(); + auto& rid = remote.router_id(); + + // TODO: if we are not the seed, how do we check the requester + if (is_seed) + { + // we already insert the + auto& seeds = node_db->seeds(); + + if (auto itr = seeds.find(rid); itr != seeds.end()) + { + log::critical(logcat, "Bootstrap seed confirmed RID:{} is white-listed seeds; approving fetch request and saving RC!", rid); + node_db->put_rc(remote); + } + } auto& src = is_seed ? node_db->bootstrap_seeds() : node_db->get_known_rcs(); auto count = src.size(); - - if (is_seed) - node_db->bootstrap_seeds().insert(remote); - else - node_db->put_rc(remote); if (count == 0) { @@ -716,7 +726,6 @@ namespace llarp } } - m.respond(std::move(btdp).str()); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 4cef45722..69a3107da 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -425,7 +425,7 @@ namespace llarp }); link_manager.register_commands(control_stream); - itr->second = std::make_shared(conn_interface, control_stream, rc); + itr->second = std::make_shared(conn_interface, control_stream); return true; } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 75cb8247c..36d58693f 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -779,12 +779,16 @@ namespace llarp router_greenlist.clear(); router_greenlist.insert(greenlist.begin(), greenlist.end()); - log::info(logcat, "lokinet service node list now has {} active router RIDs", known_rids.size()); + log::info( + logcat, + "lokinet service node whitelist now has {} active router RIDs", + router_whitelist.size()); } std::optional NodeDB::get_random_whitelist_router() const { + // TODO: this should be checking whitelist not known_rcs if (auto rc = get_random_rc()) return rc->router_id(); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 6644c4b05..08baa110f 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -134,6 +134,7 @@ namespace llarp std::map rc_lookup; std::set _bootstrap_seeds; + std::set _seeds; BootstrapList _bootstraps{}; /** RouterID lists // TODO: get rid of all these, replace with better decom/not staked sets @@ -191,6 +192,12 @@ namespace llarp /// in memory nodedb NodeDB(); + std::set& + seeds() + { + return _seeds; + } + const std::set& get_known_rids() const { diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index 1151f64aa..7c1e94f1b 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -220,8 +220,7 @@ namespace llarp { std::optional found = std::nullopt; router->for_each_connection([&](link::Connection& conn) { - const auto& rc = conn.remote_rc; - const auto& rid = rc.router_id(); + RouterID rid{conn.conn->remote_key()}; #ifndef TESTNET if (router->is_bootstrap_node(rid)) @@ -236,7 +235,7 @@ namespace llarp if (router->router_profiling().IsBadForPath(rid)) return; - found = rc; + found = router->node_db()->get_rc(rid); }); return found; } diff --git a/llarp/router/route_poker.cpp b/llarp/router/route_poker.cpp index b2a9a1ed5..d578ff7d4 100644 --- a/llarp/router/route_poker.cpp +++ b/llarp/router/route_poker.cpp @@ -219,7 +219,7 @@ namespace llarp // explicit route pokes for first hops router.for_each_connection( - [this](link::Connection conn) { add_route(conn.remote_rc.addr()); }); + [this](link::Connection conn) { add_route(conn.conn->remote()); }); add_route(router.link_manager().local()); // add default route @@ -238,7 +238,7 @@ namespace llarp { // unpoke routes for first hops router.for_each_connection( - [this](link::Connection conn) { delete_route(conn.remote_rc.addr()); }); + [this](link::Connection conn) { delete_route(conn.conn->remote()); }); if (is_enabled() and is_up) { vpn::AbstractRouteManager& route = router.vpn_platform()->RouteManager(); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 30604f423..f35bdbfbf 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -219,7 +219,7 @@ namespace llarp std::unordered_set peer_pubkeys; for_each_connection([&peer_pubkeys](link::Connection& conn) { - peer_pubkeys.emplace(conn.remote_rc.router_id()); + peer_pubkeys.emplace(conn.conn->remote_key()); }); loop()->call([this, &peer_pubkeys]() { @@ -652,7 +652,6 @@ namespace llarp } }; - for (const auto& router : configRouters) { log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); @@ -695,8 +694,7 @@ namespace llarp throw std::runtime_error("No bootstrap nodes available."); } - log::critical( - logcat, "Loaded {} default fallback bootstrap routers!", node_bstrap.size()); + log::critical(logcat, "Loaded {} default fallback bootstrap routers!", node_bstrap.size()); } clear_bad_rcs(); From fbc71847ef3cecaf388c47bdc046a3e80988e55e Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 05:09:23 -0800 Subject: [PATCH 59/93] libquic vbump --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 3ace46701..921a85852 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 3ace46701449c4c01f74aae2e0be3b4164768911 +Subproject commit 921a85852a12510d374e1042d983e98f40f07689 From d016951d2ff15db16e35f579b9cfb38942247ab3 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 05:49:59 -0800 Subject: [PATCH 60/93] Fixed pending message queue weirdness --- llarp/link/connection.cpp | 2 +- llarp/link/link_manager.cpp | 32 ++++++++++++++++++-------------- llarp/link/link_manager.hpp | 25 ++++++++----------------- llarp/nodedb.cpp | 5 +++-- llarp/router/router.cpp | 5 ++--- 5 files changed, 32 insertions(+), 37 deletions(-) diff --git a/llarp/link/connection.cpp b/llarp/link/connection.cpp index 9b8760746..d5fa0abf9 100644 --- a/llarp/link/connection.cpp +++ b/llarp/link/connection.cpp @@ -5,7 +5,7 @@ namespace llarp::link Connection::Connection( const std::shared_ptr& c, std::shared_ptr& s) - : conn{c}, control_stream{s}/* , remote_rc{std::move(rc)} */ + : conn{c}, control_stream{s} /* , remote_rc{std::move(rc)} */ {} } // namespace llarp::link diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 9787d774f..4f2bc3f91 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -74,7 +74,7 @@ namespace llarp std::advance(itr, randint() % size); RouterID rid{itr->second->conn->remote_key()}; - + if (auto maybe = link_manager.node_db->get_rc(rid)) { router = *maybe; @@ -151,7 +151,8 @@ namespace llarp for (auto& method : direct_requests) { s->register_command( - std::string{method.first}, [this, func = std::move(method.second)](oxen::quic::message m) { + std::string{method.first}, + [this, func = std::move(method.second)](oxen::quic::message m) { _router.loop()->call([this, msg = std::move(m), func = std::move(func)]() mutable { auto body = msg.body_str(); auto respond = [m = std::move(msg)](std::string response) mutable { @@ -291,7 +292,7 @@ namespace llarp endpoint = std::move(endpoint), body = std::move(body), f = std::move(func)]() { - auto pending = PendingControlMessage(std::move(body), std::move(endpoint), f); + auto pending = PendingMessage(std::move(body), std::move(endpoint), std::move(f)); auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); itr->second.push_back(std::move(pending)); @@ -315,7 +316,7 @@ namespace llarp } _router.loop()->call([this, body = std::move(body), remote]() { - auto pending = PendingDataMessage(body); + auto pending = PendingMessage(std::move(body)); auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); itr->second.push_back(std::move(pending)); @@ -434,17 +435,16 @@ namespace llarp while (not que.empty()) { - auto& m = que.front(); + auto& msg = que.front(); - if (m.is_control) + if (msg.is_control) { - auto& msg = reinterpret_cast(m); - log::critical(logcat, "Dispatching {} request!", msg.endpoint); - ep.conns[rid]->control_stream->command(msg.endpoint, msg.body, msg.func); + log::critical(logcat, "Dispatching {} request!", *msg.endpoint); + ep.conns[rid]->control_stream->command( + std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); } else { - auto& msg = reinterpret_cast(m); conn_interface.send_datagram(std::move(msg.body)); } @@ -647,7 +647,7 @@ namespace llarp } log::critical(logcat, "Queuing bootstrap fetch request to {}", source.router_id()); - auto pending = PendingControlMessage(std::move(payload), "bfetch_rcs"s, std::move(f)); + auto pending = PendingMessage(std::move(payload), "bfetch_rcs"s, std::move(f)); auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue()); itr->second.push_back(std::move(pending)); @@ -688,12 +688,16 @@ namespace llarp // TODO: if we are not the seed, how do we check the requester if (is_seed) { - // we already insert the + // we already insert the auto& seeds = node_db->seeds(); - + if (auto itr = seeds.find(rid); itr != seeds.end()) { - log::critical(logcat, "Bootstrap seed confirmed RID:{} is white-listed seeds; approving fetch request and saving RC!", rid); + log::critical( + logcat, + "Bootstrap seed confirmed RID:{} is white-listed seeds; approving fetch request and " + "saving RC!", + rid); node_db->put_rc(remote); } } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 69a3107da..d44b389a1 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -109,27 +109,18 @@ namespace llarp struct PendingMessage { std::string body; - RouterID rid; - bool is_control{false}; + std::optional endpoint = std::nullopt; + std::function func = nullptr; - PendingMessage(std::string b, bool control = false) : body{std::move(b)}, is_control{control} - {} - }; + RouterID rid; + bool is_control = false; - struct PendingDataMessage : PendingMessage - { - PendingDataMessage(std::string b) : PendingMessage(b) + PendingMessage(std::string b) : body{std::move(b)} {} - }; - - struct PendingControlMessage : PendingMessage - { - std::string endpoint; - std::function func; - PendingControlMessage( - std::string b, std::string e, std::function f = nullptr) - : PendingMessage(b, true), endpoint{std::move(e)}, func{std::move(f)} + PendingMessage( + std::string b, std::string ep, std::function f = nullptr) + : body{std::move(b)}, endpoint{std::move(ep)}, func{std::move(f)}, is_control{true} {} }; diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 36d58693f..9c460b624 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -668,13 +668,14 @@ namespace llarp _needs_rebootstrap = false; ++bootstrap_attempts; - log::critical( - logcat, "Dispatching BootstrapRC fetch request to {}", _bootstraps.current().view()); + log::critical(logcat, "Dispatching BootstrapRC fetch request to {}", fetch_source); _router.link_manager().fetch_bootstrap_rcs( rc, BootstrapFetchMessage::serialize(_router.router_contact, BOOTSTRAP_SOURCE_COUNT), [this](oxen::quic::message m) mutable { + log::critical(logcat, "Received response to BootstrapRC fetch request..."); + if (not m) { // ++bootstrap_attempts; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index f35bdbfbf..a4c3fc654 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -218,9 +218,8 @@ namespace llarp std::unordered_set peer_pubkeys; - for_each_connection([&peer_pubkeys](link::Connection& conn) { - peer_pubkeys.emplace(conn.conn->remote_key()); - }); + for_each_connection( + [&peer_pubkeys](link::Connection& conn) { peer_pubkeys.emplace(conn.conn->remote_key()); }); loop()->call([this, &peer_pubkeys]() { for (auto& pk : peer_pubkeys) From fd21eb3a009d4d0aa1ded1a6f13d30db50bba453 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 09:41:56 -0800 Subject: [PATCH 61/93] Big fix! - pending conns container stops them from being counted towards active conns in the interim - un-abstracted pendingmessages vs pendingdatamessages vs pendingcontrolmessages (gross) - fixed bootstrap fetching and storage! --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 361 +++++++++++++++++++-------------- llarp/link/link_manager.hpp | 18 +- llarp/nodedb.cpp | 62 +++--- llarp/nodedb.hpp | 3 + llarp/path/path.cpp | 2 +- llarp/path/pathbuilder.cpp | 4 +- llarp/router/router.cpp | 5 +- llarp/rpc/lokid_rpc_client.cpp | 4 +- 9 files changed, 266 insertions(+), 195 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 921a85852..357548ba0 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 921a85852a12510d374e1042d983e98f40f07689 +Subproject commit 357548ba0b6f3d99126148c27091afebac77aae9 diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 4f2bc3f91..ee47863b8 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -24,7 +24,7 @@ namespace llarp std::shared_ptr Endpoint::get_conn(const RemoteRC& rc) const { - if (auto itr = conns.find(rc.router_id()); itr != conns.end()) + if (auto itr = active_conns.find(rc.router_id()); itr != active_conns.end()) return itr->second; return nullptr; @@ -33,7 +33,7 @@ namespace llarp std::shared_ptr Endpoint::get_conn(const RouterID& rid) const { - if (auto itr = conns.find(rid); itr != conns.end()) + if (auto itr = active_conns.find(rid); itr != active_conns.end()) return itr->second; return nullptr; @@ -42,7 +42,7 @@ namespace llarp bool Endpoint::have_conn(const RouterID& remote, bool client_only) const { - if (auto itr = conns.find(remote); itr != conns.end()) + if (auto itr = active_conns.find(remote); itr != active_conns.end()) { if (not(itr->second->remote_is_relay and client_only)) return true; @@ -56,7 +56,7 @@ namespace llarp { size_t count = 0; - for (const auto& c : conns) + for (const auto& c : active_conns) { if (not(c.second->remote_is_relay and clients_only)) count += 1; @@ -68,9 +68,9 @@ namespace llarp bool Endpoint::get_random_connection(RemoteRC& router) const { - if (const auto size = conns.size(); size) + if (const auto size = active_conns.size(); size) { - auto itr = conns.begin(); + auto itr = active_conns.begin(); std::advance(itr, randint() % size); RouterID rid{itr->second->conn->remote_key()}; @@ -91,7 +91,7 @@ namespace llarp void Endpoint::for_each_connection(std::function func) { - for (const auto& [rid, conn] : conns) + for (const auto& [rid, conn] : active_conns) func(*conn); } @@ -99,14 +99,14 @@ namespace llarp Endpoint::close_connection(RouterID _rid) { assert(link_manager._router.loop()->inEventLoop()); - auto itr = conns.find(_rid); - if (itr != conns.end()) + auto itr = active_conns.find(_rid); + if (itr != active_conns.end()) return; auto& conn = *itr->second->conn; conn.close_connection(); connid_map.erase(conn.scid()); - conns.erase(itr); + active_conns.erase(itr); } } // namespace link @@ -125,8 +125,22 @@ namespace llarp void LinkManager::register_commands(std::shared_ptr& s) { - assert(ep.connid_map.count(s->conn_id())); - const RouterID& router_id = ep.connid_map[s->conn_id()]; + log::critical(logcat, "{} called", __PRETTY_FUNCTION__); + const RouterID& router_id {s->conn.remote_key()}; + + log::critical(logcat, "Registering commands (RID:{})", router_id); + + s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { + _router.loop()->call( + [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); + }); + + // s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { + // _router.loop()->call( + // [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); + // }); + + log::critical(logcat, "Registered `bfetch_rcs` (RID:{})", router_id); s->register_command("path_build"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( @@ -143,11 +157,6 @@ namespace llarp [this, msg = std::move(m)]() mutable { handle_gossip_rc(std::move(msg)); }); }); - s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { - _router.loop()->call( - [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); - }); - for (auto& method : direct_requests) { s->register_command( @@ -162,6 +171,24 @@ namespace llarp }); }); } + + log::critical(logcat, "Registered all commands! (RID:{})", router_id); + } + + LinkManager::LinkManager(Router& r) + : _router{r} + , quic{std::make_unique()} + , tls_creds{oxen::quic::GNUTLSCreds::make_from_ed_keys( + {reinterpret_cast(_router.identity().data()), size_t{32}}, + {reinterpret_cast(_router.identity().toPublic().data()), size_t{32}})} + , ep{startup_endpoint(), *this} + {} + + std::unique_ptr + LinkManager::make(Router& r) + { + std::unique_ptr p{new LinkManager(r)}; + return p; } std::shared_ptr @@ -184,25 +211,26 @@ namespace llarp }, [this](oxen::quic::dgram_interface& di, bstring dgram) { recv_data_message(di, dgram); }); tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) { - bool result = false; + bool result = true; RouterID other{key.data()}; if (_router.is_bootstrap_seed()) { - if (node_db->whitelist().count(other)) + // FIXME: remove "|| true", this is just for local testing! + if (node_db->whitelist().count(other) || true) { + log::critical(logcat, "Saving bootstrap seed requester..."); auto [it, b] = node_db->seeds().emplace(other); - result &= b; + result |= b; } log::critical( logcat, - "Bootstrap seed node was {} to confirm fetch requester is white-listed; saving RID", - result ? "able" : "unable"); + "Bootstrap seed node was {} to confirm fetch requester is white-listed; {}successfully saved RID", + result ? "able" : "unable", result ? "" : "un"); return result; } - if (node_db->has_rc(other)) - result = true; + result = node_db->has_rc(other); log::critical( logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Uns", other); @@ -215,8 +243,10 @@ namespace llarp [&](oxen::quic::Connection& c, oxen::quic::Endpoint& e, std::optional id) -> std::shared_ptr { - if (id && id == 0) + + if (id && *id == 0) { + log::critical(logcat, "Stream constructor constructing BTStream (ID:{})", id); auto s = e.make_shared( c, e, [](oxen::quic::Stream& s, uint64_t error_code) { log::warning( @@ -229,28 +259,133 @@ namespace llarp return s; } + log::critical(logcat, "Stream constructor constructing Stream (ID:{})!", id); + return e.make_shared(c, e); }); } return ep; } - LinkManager::LinkManager(Router& r) - : _router{r} - , quic{std::make_unique()} - , tls_creds{oxen::quic::GNUTLSCreds::make_from_ed_keys( - {reinterpret_cast(_router.identity().data()), size_t{32}}, - {reinterpret_cast(_router.identity().toPublic().data()), size_t{32}})} - , ep{startup_endpoint(), *this} - {} + void + LinkManager::on_inbound_conn(oxen::quic::connection_interface& ci) + { + const auto& scid = ci.scid(); + RouterID rid{ci.remote_key()}; + ep.connid_map.emplace(scid, rid); + auto [itr, b] = ep.active_conns.emplace(rid, nullptr); - std::unique_ptr - LinkManager::make(Router& r) + log::critical(logcat, "Queueing BTStream to be opened..."); + + auto control_stream = ci.queue_stream([](oxen::quic::Stream& s, + uint64_t error_code) { + log::warning( + logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); + s.conn.close_connection(error_code); + }); + + log::critical(logcat, "Queued BTStream to be opened ID:{}", control_stream->stream_id()); + assert(control_stream->stream_id() == 0); + // register_commands(control_stream); + + itr->second = std::make_shared(ci.shared_from_this(), control_stream); + log::critical(logcat, "Successfully configured inbound connection fom {}...", rid); + } + + // TODO: should we add routes here now that Router::SessionOpen is gone? + void + LinkManager::on_conn_open(oxen::quic::connection_interface& ci) { - std::unique_ptr p{new LinkManager(r)}; - return p; + _router.loop()->call([this, &conn_interface = ci]() { + const auto rid = RouterID{conn_interface.remote_key()}; + const auto& remote = conn_interface.remote(); + const auto& scid = conn_interface.scid(); + + if (conn_interface.is_inbound()) + { + log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote); + on_inbound_conn(conn_interface); + } + else + { + if (auto itr = ep.pending_conns.find(rid); itr != ep.pending_conns.end()) + { + ep.connid_map.emplace(scid, rid); + auto [it, b] = ep.active_conns.emplace(rid, nullptr); + it->second = std::move(itr->second); + log::critical(logcat, "Connection to RID:{} moved from pending to active conns!", rid); + } + else + throw std::runtime_error{"Could not find newly established connection in pending conns!"}; + } + + log::critical( + logcat, + "SERVICE NODE (RID:{}) ESTABLISHED CONNECTION TO RID:{}", + _router.local_rid(), + rid); + + // check to see if this connection was established while we were attempting to queue + // messages to the remote + if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end()) + { + log::critical(logcat, "Clearing pending queue for RID:{}", rid); + auto& que = itr->second; + + while (not que.empty()) + { + auto& msg = que.front(); + + if (msg.is_control) + { + log::critical(logcat, "Dispatching {} request!", *msg.endpoint); + ep.active_conns[rid]->control_stream->command( + std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); + } + else + { + conn_interface.send_datagram(std::move(msg.body)); + } + + que.pop_front(); + } + return; + } + log::warning(logcat, "No pending queue to clear for RID:{}", rid); + }); + }; + + void + LinkManager::on_conn_closed(oxen::quic::connection_interface& ci, uint64_t ec) + { + _router.loop()->call([this, &conn_interface = ci, error_code = ec]() { + const auto& scid = conn_interface.scid(); + + log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); + + if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end()) + { + const auto& rid = c_itr->second; + + // if (auto maybe = rids_pending_verification.find(rid); + // maybe != rids_pending_verification.end()) + // rids_pending_verification.erase(maybe); + + // in case this didn't clear earlier, do it now + if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) + pending_conn_msg_queue.erase(p_itr); + + if (auto m_itr = ep.active_conns.find(rid); m_itr != ep.active_conns.end()) + ep.active_conns.erase(m_itr); + + ep.connid_map.erase(c_itr); + + log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); + } + }); } + bool LinkManager::send_control_message( const RouterID& remote, @@ -293,11 +428,23 @@ namespace llarp body = std::move(body), f = std::move(func)]() { auto pending = PendingMessage(std::move(body), std::move(endpoint), std::move(f)); + + if (auto it1 = ep.pending_conns.find(remote); it1 != ep.pending_conns.end()) + { + if (auto it2 = pending_conn_msg_queue.find(remote); it2 != pending_conn_msg_queue.end()) + { + it2->second.push_back(std::move(pending)); + log::critical(logcat, "Connection (RID:{}) is pending; message appended to send queue!", remote); + } + } + else + { + log::critical(logcat, "Connection (RID:{}) not found in pending conns; creating send queue!", remote); + auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); + itr->second.push_back(std::move(pending)); + connect_to(remote); + } - auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); - itr->second.push_back(std::move(pending)); - - connect_to(remote); }); return false; @@ -386,106 +533,6 @@ namespace llarp log::warning(quic_cat, "Failed to begin establishing connection to {}", remote_addr); } - void - LinkManager::on_inbound_conn(oxen::quic::connection_interface& ci) - { - const auto& scid = ci.scid(); - RouterID rid{ci.remote_key()}; - ep.connid_map.emplace(scid, rid); - auto [itr, b] = ep.conns.emplace(rid, nullptr); - - auto control_stream = ci.queue_stream([](oxen::quic::Stream& s, - uint64_t error_code) { - log::warning( - logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); - s.conn.close_connection(error_code); - }); - log::critical(logcat, "Opened BTStream ID:{}", control_stream->stream_id()); - - itr->second = std::make_shared(ci.shared_from_this(), control_stream); - log::critical(logcat, "Successfully configured inbound connection fom {}...", rid); - } - - // TODO: should we add routes here now that Router::SessionOpen is gone? - void - LinkManager::on_conn_open(oxen::quic::connection_interface& ci) - { - _router.loop()->call([this, &conn_interface = ci]() { - const auto rid = RouterID{conn_interface.remote_key()}; - const auto& remote = conn_interface.remote(); - - if (conn_interface.is_inbound()) - { - log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote); - on_inbound_conn(conn_interface); - } - - log::critical( - logcat, - "SERVICE NODE (RID:{}) ESTABLISHED CONNECTION TO RID:{}", - _router.local_rid(), - rid); - - // check to see if this connection was established while we were attempting to queue - // messages to the remote - if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end()) - { - log::critical(logcat, "Clearing pending queue for RID:{}", rid); - auto& que = itr->second; - - while (not que.empty()) - { - auto& msg = que.front(); - - if (msg.is_control) - { - log::critical(logcat, "Dispatching {} request!", *msg.endpoint); - ep.conns[rid]->control_stream->command( - std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); - } - else - { - conn_interface.send_datagram(std::move(msg.body)); - } - - que.pop_front(); - } - return; - } - log::warning(logcat, "No pending queue to clear for RID:{}", rid); - }); - }; - - void - LinkManager::on_conn_closed(oxen::quic::connection_interface& ci, uint64_t ec) - { - _router.loop()->call([this, &conn_interface = ci, error_code = ec]() { - const auto& scid = conn_interface.scid(); - - log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); - - if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end()) - { - const auto& rid = c_itr->second; - - // if (auto maybe = rids_pending_verification.find(rid); - // maybe != rids_pending_verification.end()) - // rids_pending_verification.erase(maybe); - - // in case this didn't clear earlier, do it now - if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) - pending_conn_msg_queue.erase(p_itr); - - if (auto m_itr = ep.conns.find(rid); m_itr != ep.conns.end()) - ep.conns.erase(m_itr); - - ep.connid_map.erase(c_itr); - - log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); - } - }); - } - bool LinkManager::have_connection_to(const RouterID& remote, bool client_only) const { @@ -598,7 +645,7 @@ namespace llarp void LinkManager::gossip_rc(const RouterID& rc_rid, std::string serialized_rc) { - for (auto& [rid, conn] : ep.conns) + for (auto& [rid, conn] : ep.active_conns) { // don't send back to the owner... if (rid == rc_rid) @@ -607,7 +654,9 @@ namespace llarp if (not conn->remote_is_relay) continue; - send_control_message(rid, "gossip_rc", serialized_rc); + send_control_message(rid, "gossip_rc", serialized_rc, [](oxen::quic::message) mutable { + log::critical(logcat, "PLACEHOLDER FOR GOSSIP RC RESPONSE HANDLER"); + }); } } @@ -702,12 +751,12 @@ namespace llarp } } - auto& src = is_seed ? node_db->bootstrap_seeds() : node_db->get_known_rcs(); + auto& src = node_db->get_known_rcs(); auto count = src.size(); if (count == 0) { - log::error(logcat, "No {} locally to send!", is_seed ? "bootstrap seeds" : "known RCs"); + log::error(logcat, "No known RCs locally to send!"); m.respond(messages::ERROR_RESPONSE, true); return; } @@ -899,9 +948,9 @@ namespace llarp void LinkManager::handle_find_name_response(oxen::quic::message m) { - if (m.timed_out) + if (not m) { - log::info(link_cat, "FindNameMessage timed out!"); + log::info(link_cat, "FindNameMessage failed!"); return; } @@ -1035,7 +1084,7 @@ namespace llarp "publish_intro", PublishIntroMessage::serialize(introset, relay_order, is_relayed), [respond = std::move(respond)](oxen::quic::message m) { - if (m.timed_out) + if (not m) return; // drop if timed out; requester will have timed out as well respond(m.body_str()); }); @@ -1073,7 +1122,7 @@ namespace llarp void LinkManager::handle_publish_intro_response(oxen::quic::message m) { - if (m.timed_out) + if (not m) { log::info(link_cat, "PublishIntroMessage timed out!"); return; @@ -1176,7 +1225,7 @@ namespace llarp link_cat, "Relayed FindIntroMessage returned successful response; transmitting to initial " "requester"); - else if (relay_response.timed_out) + else if (not relay_response) log::critical( link_cat, "Relayed FindIntroMessage timed out! Notifying initial requester"); else @@ -1203,7 +1252,7 @@ namespace llarp void LinkManager::handle_find_intro_response(oxen::quic::message m) { - if (m.timed_out) + if (not m) { log::info(link_cat, "FindIntroMessage timed out!"); return; @@ -1396,7 +1445,7 @@ namespace llarp "then relaying response"); _router.path_context().put_transit_hop(hop); } - if (m.timed_out) + if (not m) log::info(link_cat, "Upstream timed out on path build; relaying timeout"); else log::info(link_cat, "Upstream returned path build failure; relaying response"); @@ -1513,7 +1562,7 @@ namespace llarp void LinkManager::handle_obtain_exit_response(oxen::quic::message m) { - if (m.timed_out) + if (not m) { log::info(link_cat, "ObtainExitMessage timed out!"); return; @@ -1591,7 +1640,7 @@ namespace llarp void LinkManager::handle_update_exit_response(oxen::quic::message m) { - if (m.timed_out) + if (not m) { log::info(link_cat, "UpdateExitMessage timed out!"); return; @@ -1676,7 +1725,7 @@ namespace llarp void LinkManager::handle_close_exit_response(oxen::quic::message m) { - if (m.timed_out) + if (not m) { log::info(link_cat, "CloseExitMessage timed out!"); return; @@ -1828,7 +1877,7 @@ namespace llarp void LinkManager::handle_convo_intro(oxen::quic::message m) { - if (m.timed_out) + if (not m) { log::info(link_cat, "Path control message timed out!"); return; diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index d44b389a1..bc1b13ddd 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -48,9 +48,12 @@ namespace llarp // for outgoing packets, we route via RouterID; map RouterID->Connection // for incoming packets, we get a ConnectionID; map ConnectionID->RouterID - std::unordered_map> conns; + std::unordered_map> active_conns; std::unordered_map connid_map; + // for pending connections, cleared in LinkManager::on_conn_open + std::unordered_map> pending_conns; + // TODO: see which of these is actually useful and delete the other std::shared_ptr get_conn(const RemoteRC&) const; @@ -395,16 +398,18 @@ namespace llarp { try { - log::critical(logcat, "Establishing connection to {}", remote); + const auto& rid = rc.router_id(); + log::critical(logcat, "Establishing connection to RID:{}", rid); auto conn_interface = endpoint->connect(remote, link_manager.tls_creds, std::forward(opts)...); - // emplace immediately for connection open callback to find scid - connid_map.emplace(conn_interface->scid(), rc.router_id()); - auto [itr, b] = conns.emplace(rc.router_id(), nullptr); + // add to pending conns + auto [itr, b] = pending_conns.emplace(rid, nullptr); - log::critical(logcat, "Establishing connection to {}...", rc.router_id()); + // emplace immediately for connection open callback to find scid + // connid_map.emplace(conn_interface->scid(), rc.router_id()); + // auto [itr, b] = conns.emplace(rc.router_id(), nullptr); auto control_stream = conn_interface->template get_new_stream( [](oxen::quic::Stream& s, uint64_t error_code) { @@ -418,6 +423,7 @@ namespace llarp link_manager.register_commands(control_stream); itr->second = std::make_shared(conn_interface, control_stream); + log::critical(logcat, "Connection to RID:{} added to pending connections...", rid); return true; } catch (...) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 9c460b624..bc2d02246 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -379,16 +379,16 @@ namespace llarp src, FetchRCMessage::serialize(_router.last_rc_fetch, needed), [this, src, initial](oxen::quic::message m) mutable { - if (m.timed_out) + if (not m) { - log::info(logcat, "RC fetch to {} timed out", src); + log::info(logcat, "RC fetch to {} failed!", src); fetch_rcs_result(initial, true); return; } try { oxenc::bt_dict_consumer btdc{m.body()}; - + // TODO: fix this shit after removing ::timed_out from message type if (not m) { auto reason = btdc.require(messages::STATUS_KEY); @@ -689,7 +689,8 @@ namespace llarp return; } - std::set rids; + // std::set rids; + size_t num = 0; try { @@ -701,7 +702,8 @@ namespace llarp while (not btlc.is_finished()) { auto rc = RemoteRC{btlc.consume_dict_data()}; - rids.emplace(rc.router_id()); + put_rc(rc); + ++num; } } } @@ -724,23 +726,30 @@ namespace llarp // next call to fallback_to_bootstrap() and hit the base case, rotating sources // bootstrap_attempts = MAX_BOOTSTRAP_FETCH_ATTEMPTS; - if (rids.size() == BOOTSTRAP_SOURCE_COUNT) - { - known_rids.merge(rids); - fetch_initial(); - } - else - { - // ++bootstrap_attempts; - log::warning( - logcat, - "BootstrapRC fetch response from {} returned insufficient number of RC's (error " - "{}/{})", - fetch_source, - bootstrap_attempts, - MAX_BOOTSTRAP_FETCH_ATTEMPTS); - fallback_to_bootstrap(); - } + // const auto& num = rids.size(); + + log::critical(logcat, "BootstrapRC fetch response from {} returned {}/{} needed RCs", fetch_source, num, BOOTSTRAP_SOURCE_COUNT); + // known_rids.merge(rids); + fetch_initial(); + + // FIXME: when moving to testnet, uncomment this + // if (rids.size() == BOOTSTRAP_SOURCE_COUNT) + // { + // known_rids.merge(rids); + // fetch_initial(); + // } + // else + // { + // // ++bootstrap_attempts; + // log::warning( + // logcat, + // "BootstrapRC fetch response from {} returned insufficient number of RC's (error " + // "{}/{})", + // fetch_source, + // bootstrap_attempts, + // MAX_BOOTSTRAP_FETCH_ATTEMPTS); + // fallback_to_bootstrap(); + // } }); } @@ -955,9 +964,6 @@ namespace llarp { const auto& rid = rc.router_id(); - if (not want_rc(rid)) - return false; - known_rcs.erase(rc); rc_lookup.erase(rid); @@ -975,6 +981,12 @@ namespace llarp return known_rcs.size(); } + size_t + NodeDB::num_rids() const + { + return known_rids.size(); + } + bool NodeDB::put_rc_if_newer(RemoteRC rc, rc_time now) { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 08baa110f..c9bbc5ec2 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -403,6 +403,9 @@ namespace llarp size_t num_rcs() const; + size_t + num_rids() const; + /// do periodic tasks like flush to disk and expiration void Tick(llarp_time_t now); diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp index 73de6465f..955716089 100644 --- a/llarp/path/path.cpp +++ b/llarp/path/path.cpp @@ -118,7 +118,7 @@ namespace llarp::path if ((not self) or (not response_cb)) return; - if (m.timed_out) + if (not m) { response_cb(messages::TIMEOUT_RESPONSE); return; diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index 7c1e94f1b..d0b07bf4c 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -513,9 +513,9 @@ namespace llarp path->EnterState(path::ePathEstablished); return; } - if (m.timed_out) + if (not m) { - log::warning(path_cat, "Path build timed out"); + log::warning(path_cat, "Path build request failed!"); } else { diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index a4c3fc654..04941c36c 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -768,8 +768,9 @@ namespace llarp log::critical( logcat, - "{} RCs loaded with {} bootstrap peers and {} router connections!", + "{} RCs loaded with {} RIDs, {} bootstrap peers, and {} router connections!", _node_db->num_rcs(), + _node_db->num_rids(), _node_db->num_bootstraps(), num_router_connections()); @@ -1138,7 +1139,7 @@ namespace llarp log::info(logcat, "Loading NodeDB from disk..."); _node_db->load_from_disk(); - _node_db->store_bootstraps(); + // _node_db->store_bootstraps(); oxen::log::flush(); diff --git a/llarp/rpc/lokid_rpc_client.cpp b/llarp/rpc/lokid_rpc_client.cpp index 764b8136c..6f41b8c4e 100644 --- a/llarp/rpc/lokid_rpc_client.cpp +++ b/llarp/rpc/lokid_rpc_client.cpp @@ -97,7 +97,7 @@ namespace llarp::rpc return; // bail } - LogDebug("new block at height ", m_BlockHeight); + log::trace(logcat, "new block at height {}", m_BlockHeight); // don't upadate on block notification if an update is pending if (not m_UpdatingList) UpdateServiceNodeList(); @@ -138,7 +138,7 @@ namespace llarp::rpc throw std::runtime_error{"get_service_nodes did not return 'OK' status"}; if (auto it = json.find("unchanged"); it != json.end() and it->is_boolean() and it->get()) - LogDebug("service node list unchanged"); + log::trace(logcat, "service node list unchanged"); else { self->HandleNewServiceNodeList(json.at("service_node_states")); From d6e5aca572cdb6ac0039397560370ac0f280eb69 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 10:12:36 -0800 Subject: [PATCH 62/93] libquic vbump --- external/oxen-libquic | 2 +- llarp/link/connection.hpp | 1 - llarp/link/link_manager.cpp | 7 ------- llarp/link/link_manager.hpp | 4 ---- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 357548ba0..0e431b912 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 357548ba0b6f3d99126148c27091afebac77aae9 +Subproject commit 0e431b912eb4bf76a9219861afc06cd8ceafa781 diff --git a/llarp/link/connection.hpp b/llarp/link/connection.hpp index 1c5000657..fb194b714 100644 --- a/llarp/link/connection.hpp +++ b/llarp/link/connection.hpp @@ -11,7 +11,6 @@ namespace llarp::link { std::shared_ptr conn; std::shared_ptr control_stream; - // std::optional remote_rc; // one side of a connection will be responsible for some things, e.g. heartbeat bool inbound{false}; diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index ee47863b8..99edc3240 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -135,13 +135,6 @@ namespace llarp [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); }); - // s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { - // _router.loop()->call( - // [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); - // }); - - log::critical(logcat, "Registered `bfetch_rcs` (RID:{})", router_id); - s->register_command("path_build"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( [this, &rid, msg = std::move(m)]() mutable { handle_path_build(std::move(msg), rid); }); diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index bc1b13ddd..bc4e2b0d9 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -407,10 +407,6 @@ namespace llarp // add to pending conns auto [itr, b] = pending_conns.emplace(rid, nullptr); - // emplace immediately for connection open callback to find scid - // connid_map.emplace(conn_interface->scid(), rc.router_id()); - // auto [itr, b] = conns.emplace(rc.router_id(), nullptr); - auto control_stream = conn_interface->template get_new_stream( [](oxen::quic::Stream& s, uint64_t error_code) { log::warning( From 08c2c26c2974112f9a654c504919423a891b0832 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 11:13:22 -0800 Subject: [PATCH 63/93] bootstrap tweaking --- llarp/link/link_manager.cpp | 21 +++++++++++---------- llarp/nodedb.cpp | 16 +++++++++++++--- llarp/nodedb.hpp | 1 + llarp/router/router.hpp | 6 ++++++ 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 99edc3240..bd7d93c6c 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -126,7 +126,7 @@ namespace llarp LinkManager::register_commands(std::shared_ptr& s) { log::critical(logcat, "{} called", __PRETTY_FUNCTION__); - const RouterID& router_id {s->conn.remote_key()}; + const RouterID& router_id{s->conn.remote_key()}; log::critical(logcat, "Registering commands (RID:{})", router_id); @@ -209,7 +209,7 @@ namespace llarp if (_router.is_bootstrap_seed()) { - // FIXME: remove "|| true", this is just for local testing! + // FIXME: remove "|| true", this is just for local testing! if (node_db->whitelist().count(other) || true) { log::critical(logcat, "Saving bootstrap seed requester..."); @@ -218,8 +218,10 @@ namespace llarp } log::critical( logcat, - "Bootstrap seed node was {} to confirm fetch requester is white-listed; {}successfully saved RID", - result ? "able" : "unable", result ? "" : "un"); + "Bootstrap seed node was {} to confirm fetch requester is white-listed; {}successfully " + "saved RID", + result ? "able" : "unable", + result ? "" : "un"); return result; } @@ -236,7 +238,6 @@ namespace llarp [&](oxen::quic::Connection& c, oxen::quic::Endpoint& e, std::optional id) -> std::shared_ptr { - if (id && *id == 0) { log::critical(logcat, "Stream constructor constructing BTStream (ID:{})", id); @@ -378,7 +379,6 @@ namespace llarp }); } - bool LinkManager::send_control_message( const RouterID& remote, @@ -421,23 +421,24 @@ namespace llarp body = std::move(body), f = std::move(func)]() { auto pending = PendingMessage(std::move(body), std::move(endpoint), std::move(f)); - + if (auto it1 = ep.pending_conns.find(remote); it1 != ep.pending_conns.end()) { if (auto it2 = pending_conn_msg_queue.find(remote); it2 != pending_conn_msg_queue.end()) { it2->second.push_back(std::move(pending)); - log::critical(logcat, "Connection (RID:{}) is pending; message appended to send queue!", remote); + log::critical( + logcat, "Connection (RID:{}) is pending; message appended to send queue!", remote); } } else { - log::critical(logcat, "Connection (RID:{}) not found in pending conns; creating send queue!", remote); + log::critical( + logcat, "Connection (RID:{}) not found in pending conns; creating send queue!", remote); auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); itr->second.push_back(std::move(pending)); connect_to(remote); } - }); return false; diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index bc2d02246..7f773641b 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -339,9 +339,11 @@ namespace llarp void NodeDB::fetch_initial() { - if (known_rids.empty()) + auto sz = num_rcs(); + + if (num_rcs() < MIN_ACTIVE_RCS) { - log::critical(logcat, "No RouterID's held locally... BOOTSTRAP TIME"); + log::critical(logcat, "{}/{} RCs held locally... BOOTSTRAP TIME", sz, MIN_ACTIVE_RCS); fallback_to_bootstrap(); } else @@ -728,7 +730,12 @@ namespace llarp // const auto& num = rids.size(); - log::critical(logcat, "BootstrapRC fetch response from {} returned {}/{} needed RCs", fetch_source, num, BOOTSTRAP_SOURCE_COUNT); + log::critical( + logcat, + "BootstrapRC fetch response from {} returned {}/{} needed RCs", + fetch_source, + num, + BOOTSTRAP_SOURCE_COUNT); // known_rids.merge(rids); fetch_initial(); @@ -964,6 +971,9 @@ namespace llarp { const auto& rid = rc.router_id(); + if (rid == _router.local_rid()) + return false; + known_rcs.erase(rc); rc_lookup.erase(rid); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index c9bbc5ec2..8ba55cbaa 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -23,6 +23,7 @@ namespace llarp struct Router; /* RC Fetch Constants */ + inline constexpr size_t MIN_ACTIVE_RCS{6}; // max number of attempts we make in non-bootstrap fetch requests inline constexpr int MAX_FETCH_ATTEMPTS{10}; // the total number of returned rcs that are held locally should be at least this diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 87affae61..da3375ab9 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -158,6 +158,12 @@ namespace llarp std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip}; public: + bool + testnet() const + { + return _testnet; + } + bool is_bootstrap_seed() const { From 2c3763b61c760e934c15395007dfcb8d3e8889d6 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 11:17:33 -0800 Subject: [PATCH 64/93] libquic vbump --- external/oxen-libquic | 2 +- llarp/router/router.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 0e431b912..0060e9b9f 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 0e431b912eb4bf76a9219861afc06cd8ceafa781 +Subproject commit 0060e9b9fa1d8fc75ad882d1764e032f65d19cdc diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 04941c36c..f38fd09b4 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -1139,7 +1139,7 @@ namespace llarp log::info(logcat, "Loading NodeDB from disk..."); _node_db->load_from_disk(); - // _node_db->store_bootstraps(); + _node_db->store_bootstraps(); oxen::log::flush(); From 575494c3da2bbb95f71e5e4870151a5ed9f15f84 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 13 Dec 2023 13:08:05 -0800 Subject: [PATCH 65/93] actually have rid in command registration --- llarp/link/link_manager.cpp | 14 +++++++------- llarp/link/link_manager.hpp | 4 ++-- llarp/router/router.cpp | 5 +++-- llarp/router/router.hpp | 3 +-- llarp/router_contact_remote.cpp | 7 ------- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index bd7d93c6c..d4c4f02f5 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -123,12 +123,9 @@ namespace llarp } void - LinkManager::register_commands(std::shared_ptr& s) + LinkManager::register_commands(std::shared_ptr& s, const RouterID& router_id) { log::critical(logcat, "{} called", __PRETTY_FUNCTION__); - const RouterID& router_id{s->conn.remote_key()}; - - log::critical(logcat, "Registering commands (RID:{})", router_id); s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { _router.loop()->call( @@ -249,7 +246,7 @@ namespace llarp error_code); s.conn.close_connection(error_code); }); - register_commands(s); + // register_commands(s); return s; } @@ -280,7 +277,7 @@ namespace llarp log::critical(logcat, "Queued BTStream to be opened ID:{}", control_stream->stream_id()); assert(control_stream->stream_id() == 0); - // register_commands(control_stream); + register_commands(control_stream, rid); itr->second = std::make_shared(ci.shared_from_this(), control_stream); log::critical(logcat, "Successfully configured inbound connection fom {}...", rid); @@ -369,6 +366,9 @@ namespace llarp if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) pending_conn_msg_queue.erase(p_itr); + if (auto c_itr = ep.pending_conns.find(rid); c_itr != ep.pending_conns.end()) + ep.pending_conns.erase(c_itr); + if (auto m_itr = ep.active_conns.find(rid); m_itr != ep.active_conns.end()) ep.active_conns.erase(m_itr); @@ -714,7 +714,7 @@ namespace llarp oxenc::bt_dict_consumer btdc{m.body()}; btdc.required("local"); auto rc_dict = btdc.consume_dict_data(); - log::critical(logcat, "incoming dict data: {}", oxenc::to_hex(rc_dict)); + // log::critical(logcat, "incoming dict data: {}", oxenc::to_hex(rc_dict)); remote = RemoteRC{rc_dict}; quantity = btdc.require("quantity"); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index bc4e2b0d9..4c67505a9 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -213,7 +213,7 @@ namespace llarp startup_endpoint(); void - register_commands(std::shared_ptr& s); + register_commands(std::shared_ptr& s, const RouterID& rid); public: const link::Endpoint& @@ -416,7 +416,7 @@ namespace llarp s.conn.close_connection(error_code); }); - link_manager.register_commands(control_stream); + link_manager.register_commands(control_stream, rid); itr->second = std::make_shared(conn_interface, control_stream); log::critical(logcat, "Connection to RID:{} added to pending connections...", rid); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index f38fd09b4..20ba41b3e 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -776,7 +776,7 @@ namespace llarp if (is_service_node()) { - log::critical( + log::info( logcat, "Local service node has {} client connections since last RC update ({} to expiry)", num_client_connections(), @@ -911,11 +911,12 @@ namespace llarp // (client-only) periodically fetch updated RCs if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) { + log::critical(logcat, "Time to fetch RCs!"); node_db()->fetch_rcs(); } // (client-only) periodically fetch updated RouterID list - if (now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) + if (not is_snode and now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) { node_db()->fetch_rids(); } diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index da3375ab9..b80ffd5c1 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -48,7 +48,7 @@ namespace llarp static constexpr size_t INTROSET_STORAGE_REDUNDANCY = (INTROSET_RELAY_REDUNDANCY * INTROSET_REQS_PER_RELAY); - static const std::chrono::seconds RC_UPDATE_INTERVAL = 5min; + static const std::chrono::seconds RC_UPDATE_INTERVAL = 4min; static const std::chrono::seconds ROUTERID_UPDATE_INTERVAL = 1h; struct Contacts; @@ -105,7 +105,6 @@ namespace llarp std::shared_ptr _node_db; llarp_time_t _started_at; const oxenmq::TaggedThreadID _disk_thread; - // oxen::quic::Network _net; // DISCUSS: we don't use this anywhere..? llarp_time_t _last_stats_report = 0s; llarp_time_t _next_decomm_warning = time_now_ms() + 15s; diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index ef405662b..387ebf89c 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -46,13 +46,6 @@ namespace llarp throw std::runtime_error{err}; } - log::error( - log::Cat("FIXME"), - "ABOUT TO VERIFY THIS: {}, WITH SIG {}, SIGNED BY {}", - oxenc::to_hex(msg), - oxenc::to_hex(sig), - router_id().ToHex()); - if (not crypto::verify(router_id(), msg, sig)) throw std::runtime_error{"Failed to verify RemoteRC signature"}; }); From 13305f703f7300bf8a87afc9305c61c845e8931a Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 14 Dec 2023 06:13:14 -0800 Subject: [PATCH 66/93] libquic vbump --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 0060e9b9f..89d85ffea 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 0060e9b9fa1d8fc75ad882d1764e032f65d19cdc +Subproject commit 89d85ffeabbc773a1ea20d05b7f9c2a13c5e8370 From ef1897c25aafa293b0a2e812ad0643921b9fdfe3 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 05:52:31 -0800 Subject: [PATCH 67/93] Added connection keepalive - implemented ngtcp2 ping to keep connections alive - fixed weird lambda captures - fetch logic - lets see what happens --- llarp/config/key_manager.hpp | 6 +-- llarp/constants/time.hpp | 3 +- llarp/context.cpp | 2 +- llarp/link/link_manager.cpp | 99 +++++++++++++++++++++++------------- llarp/link/link_manager.hpp | 17 ++++++- llarp/net/sock_addr.cpp | 4 +- llarp/nodedb.cpp | 51 ++++++++++++++----- llarp/nodedb.hpp | 6 ++- llarp/router/router.cpp | 93 ++++++++++++++++----------------- llarp/router/router.hpp | 25 ++++++--- llarp/router_contact.hpp | 2 - 11 files changed, 189 insertions(+), 119 deletions(-) diff --git a/llarp/config/key_manager.hpp b/llarp/config/key_manager.hpp index b0906f4d5..9573fa128 100644 --- a/llarp/config/key_manager.hpp +++ b/llarp/config/key_manager.hpp @@ -34,10 +34,8 @@ namespace llarp /// Constructor KeyManager(); - /// Initializes keys using the provided config, loading from disk - /// - /// NOTE: Must be called prior to obtaining any keys. - /// NOTE: blocks on I/O + /// Initializes keys using the provided config, loading from disk. Must be called + /// prior to obtaining any keys and blocks on I/O /// /// @param config should be a prepared config object /// @param genIfAbsent determines whether or not we will create files if they diff --git a/llarp/constants/time.hpp b/llarp/constants/time.hpp index 69866e2c8..79072418e 100644 --- a/llarp/constants/time.hpp +++ b/llarp/constants/time.hpp @@ -5,6 +5,5 @@ namespace llarp { using namespace std::literals; - /// how big of a time skip before we reset network state - constexpr auto TimeskipDetectedDuration = 1min; + } // namespace llarp diff --git a/llarp/context.cpp b/llarp/context.cpp index cb0ca52d1..2624fb6cd 100644 --- a/llarp/context.cpp +++ b/llarp/context.cpp @@ -182,7 +182,7 @@ namespace llarp { if (router) { - llarp::log::debug(logcat, "Handling SIGINT"); + llarp::log::error(logcat, "Handling SIGINT"); /// async stop router on sigint router->Stop(); } diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index d4c4f02f5..76fec975c 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -27,6 +27,9 @@ namespace llarp if (auto itr = active_conns.find(rc.router_id()); itr != active_conns.end()) return itr->second; + // if (auto itr = pending_conns.find(rc.router_id()); itr != pending_conns.end()) + // return itr->second; + return nullptr; } @@ -36,6 +39,9 @@ namespace llarp if (auto itr = active_conns.find(rid); itr != active_conns.end()) return itr->second; + // if (auto itr = pending_conns.find(rid); itr != pending_conns.end()) + // return itr->second; + return nullptr; } @@ -48,6 +54,12 @@ namespace llarp return true; } + if (auto itr = pending_conns.find(remote); itr != pending_conns.end()) + { + if (not(itr->second->remote_is_relay and client_only)) + return true; + } + return false; } @@ -99,14 +111,21 @@ namespace llarp Endpoint::close_connection(RouterID _rid) { assert(link_manager._router.loop()->inEventLoop()); - auto itr = active_conns.find(_rid); - if (itr != active_conns.end()) - return; - auto& conn = *itr->second->conn; - conn.close_connection(); - connid_map.erase(conn.scid()); - active_conns.erase(itr); + // deletion from pending_conns, pending_conn_msg_queue, active_conns, etc is taken care + // of by LinkManager::on_conn_closed + if (auto itr = active_conns.find(_rid); itr != active_conns.end()) + { + auto& conn = *itr->second->conn; + conn.close_connection(); + } + else if (auto itr = pending_conns.find(_rid); itr != pending_conns.end()) + { + auto& conn = *itr->second->conn; + conn.close_connection(); + } + else + return; } } // namespace link @@ -123,7 +142,8 @@ namespace llarp } void - LinkManager::register_commands(std::shared_ptr& s, const RouterID& router_id) + LinkManager::register_commands( + std::shared_ptr& s, const RouterID& router_id) { log::critical(logcat, "{} called", __PRETTY_FUNCTION__); @@ -206,8 +226,7 @@ namespace llarp if (_router.is_bootstrap_seed()) { - // FIXME: remove "|| true", this is just for local testing! - if (node_db->whitelist().count(other) || true) + if (node_db->whitelist().count(other)) { log::critical(logcat, "Saving bootstrap seed requester..."); auto [it, b] = node_db->seeds().emplace(other); @@ -232,6 +251,7 @@ namespace llarp { ep->listen( tls_creds, + ROUTER_KEEP_ALIVE, [&](oxen::quic::Connection& c, oxen::quic::Endpoint& e, std::optional id) -> std::shared_ptr { @@ -304,6 +324,8 @@ namespace llarp ep.connid_map.emplace(scid, rid); auto [it, b] = ep.active_conns.emplace(rid, nullptr); it->second = std::move(itr->second); + ep.pending_conns.erase(itr); + log::critical(logcat, "Connection to RID:{} moved from pending to active conns!", rid); } else @@ -340,43 +362,46 @@ namespace llarp que.pop_front(); } - return; } - log::warning(logcat, "No pending queue to clear for RID:{}", rid); + + log::warning(logcat, "Pending queue empty for RID:{}", rid); }); }; void LinkManager::on_conn_closed(oxen::quic::connection_interface& ci, uint64_t ec) { - _router.loop()->call([this, &conn_interface = ci, error_code = ec]() { - const auto& scid = conn_interface.scid(); - - log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); - - if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end()) - { - const auto& rid = c_itr->second; + _router.loop()->call( + [this, scid = ci.scid(), _rid = RouterID{ci.remote_key()}, error_code = ec]() { + log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); - // if (auto maybe = rids_pending_verification.find(rid); - // maybe != rids_pending_verification.end()) - // rids_pending_verification.erase(maybe); + // a pending connection would not be in the connid_map + if (auto v_itr = ep.pending_conns.find(_rid); v_itr != ep.pending_conns.end()) + { + ep.pending_conns.erase(v_itr); - // in case this didn't clear earlier, do it now - if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) - pending_conn_msg_queue.erase(p_itr); + // in case this didn't clear earlier, do it now + if (auto p_itr = pending_conn_msg_queue.find(_rid); + p_itr != pending_conn_msg_queue.end()) + pending_conn_msg_queue.erase(p_itr); - if (auto c_itr = ep.pending_conns.find(rid); c_itr != ep.pending_conns.end()) - ep.pending_conns.erase(c_itr); + log::critical(quic_cat, "Pending quic connection CID:{} purged successfully", scid); + } + else if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end()) + { + const auto& rid = c_itr->second; + assert(_rid == rid); // this should hold true - if (auto m_itr = ep.active_conns.find(rid); m_itr != ep.active_conns.end()) - ep.active_conns.erase(m_itr); + if (auto m_itr = ep.active_conns.find(rid); m_itr != ep.active_conns.end()) + ep.active_conns.erase(m_itr); - ep.connid_map.erase(c_itr); + ep.connid_map.erase(c_itr); - log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); - } - }); + log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); + } + else + log::critical(quic_cat, "Nothing to purge for quic connection CID:{}", scid); + }); } bool @@ -585,6 +610,12 @@ namespace llarp return ep.get_random_connection(router); } + bool + LinkManager::is_service_node() const + { + return _router.is_service_node(); + } + // TODO: this? perhaps no longer necessary in the same way? void LinkManager::check_persisting_conns(llarp_time_t) diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 4c67505a9..5c1ae69cd 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -33,6 +33,11 @@ namespace llarp using stream_open_hook = oxen::quic::stream_open_callback; using stream_closed_hook = oxen::quic::stream_close_callback; + using keep_alive = oxen::quic::opt::keep_alive; + + inline const keep_alive ROUTER_KEEP_ALIVE{10s}; + inline const keep_alive CLIENT_KEEP_ALIVE{0s}; + namespace link { struct Connection; @@ -292,6 +297,9 @@ namespace llarp bool get_random_connected(RemoteRC& router) const; + bool + is_service_node() const; + void check_persisting_conns(llarp_time_t now); @@ -401,8 +409,13 @@ namespace llarp const auto& rid = rc.router_id(); log::critical(logcat, "Establishing connection to RID:{}", rid); - auto conn_interface = - endpoint->connect(remote, link_manager.tls_creds, std::forward(opts)...); + bool is_snode = link_manager.is_service_node(); + + auto conn_interface = endpoint->connect( + remote, + link_manager.tls_creds, + is_snode ? ROUTER_KEEP_ALIVE : CLIENT_KEEP_ALIVE, + std::forward(opts)...); // add to pending conns auto [itr, b] = pending_conns.emplace(rid, nullptr); diff --git a/llarp/net/sock_addr.cpp b/llarp/net/sock_addr.cpp index a4a897ee3..e3630295e 100644 --- a/llarp/net/sock_addr.cpp +++ b/llarp/net/sock_addr.cpp @@ -232,8 +232,8 @@ namespace llarp return; } - // NOTE: this potentially involves multiple memory allocations, - // reimplement without split() if it is performance bottleneck + // TOFIX: This potentially involves multiple memory allocations, + // reimplement without split() if it is performance bottleneck auto splits = split(str, ":"); // TODO: having ":port" at the end makes this ambiguous with IPv6 diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 7f773641b..f393fa17e 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -337,15 +337,20 @@ namespace llarp } void - NodeDB::fetch_initial() + NodeDB::fetch_initial(bool is_snode) { auto sz = num_rcs(); - if (num_rcs() < MIN_ACTIVE_RCS) + if (sz < MIN_ACTIVE_RCS) { log::critical(logcat, "{}/{} RCs held locally... BOOTSTRAP TIME", sz, MIN_ACTIVE_RCS); fallback_to_bootstrap(); } + else if (is_snode) + { + // service nodes who have sufficient local RC's can bypass initial fetching + _needs_initial_fetch = false; + } else { // Set fetch source as random selection of known active client routers @@ -675,7 +680,7 @@ namespace llarp _router.link_manager().fetch_bootstrap_rcs( rc, BootstrapFetchMessage::serialize(_router.router_contact, BOOTSTRAP_SOURCE_COUNT), - [this](oxen::quic::message m) mutable { + [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); if (not m) @@ -736,8 +741,19 @@ namespace llarp fetch_source, num, BOOTSTRAP_SOURCE_COUNT); - // known_rids.merge(rids); - fetch_initial(); + + if (not is_snode) + { + log::critical( + logcat, + "Client completed processing BootstrapRC fetch; proceeding to initial fetch"); + fetch_initial(); + } + else + { + log::critical(logcat, "Service node completed processing BootstrapRC fetch!"); + post_snode_bootstrap(); + } // FIXME: when moving to testnet, uncomment this // if (rids.size() == BOOTSTRAP_SOURCE_COUNT) @@ -760,6 +776,14 @@ namespace llarp }); } + void + NodeDB::post_snode_bootstrap() + { + _needs_rebootstrap = false; + _using_bootstrap_fallback = false; + _needs_initial_fetch = false; + } + void NodeDB::bootstrap_cooldown() { @@ -805,22 +829,21 @@ namespace llarp std::optional NodeDB::get_random_whitelist_router() const { - // TODO: this should be checking whitelist not known_rcs - if (auto rc = get_random_rc()) - return rc->router_id(); + std::optional rand = std::nullopt; - return std::nullopt; + std::sample(router_whitelist.begin(), router_whitelist.end(), &*rand, 1, csrng); + return rand; } bool NodeDB::is_connection_allowed(const RouterID& remote) const { - if (_pinned_edges.size() && _pinned_edges.count(remote) == 0 - && not _bootstraps.contains(remote)) - return false; - if (not _router.is_service_node()) - return true; + { + if (_pinned_edges.size() && _pinned_edges.count(remote) == 0 + && not _bootstraps.contains(remote)) + return false; + } return known_rids.count(remote) or router_greylist.count(remote); } diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 8ba55cbaa..09880ca17 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -22,6 +22,8 @@ namespace llarp { struct Router; + // TESTNET: the following constants have been shortened for testing purposes + /* RC Fetch Constants */ inline constexpr size_t MIN_ACTIVE_RCS{6}; // max number of attempts we make in non-bootstrap fetch requests @@ -242,7 +244,7 @@ namespace llarp process_fetched_rids(); void - fetch_initial(); + fetch_initial(bool is_snode = false); // RouterContact fetching void @@ -264,6 +266,8 @@ namespace llarp void fallback_to_bootstrap(); void + post_snode_bootstrap(); + void bootstrap_cooldown(); // Populate rid_sources with random sample from known_rids. A set of rids is passed diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 20ba41b3e..205f9f5f9 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -58,12 +58,8 @@ namespace llarp loop_wakeup = _loop->make_waker([this]() { PumpLL(); }); } - Router::~Router() - {} - // TODO: investigate changes needed for libquic integration // still needed at all? - // TODO: No. The answer is No. // TONUKE: EVERYTHING ABOUT THIS void @@ -237,14 +233,10 @@ namespace llarp Router::GetRandomGoodRouter() { if (is_service_node()) - { return node_db()->get_random_whitelist_router(); - } if (auto maybe = node_db()->get_random_rc()) - { return maybe->router_id(); - } return std::nullopt; } @@ -742,23 +734,13 @@ namespace llarp bool Router::is_bootstrap_node(const RouterID r) const { - if (_node_db->has_bootstraps()) - { - const auto& b = _node_db->bootstrap_list(); - return std::count_if( - b.begin(), - b.end(), - [r](const RemoteRC& rc) -> bool { return rc.router_id() == r; }) - > 0; - } - return false; + return _node_db->has_bootstraps() ? _node_db->bootstrap_list().contains(r) : false; } bool Router::should_report_stats(llarp_time_t now) const { - static constexpr auto ReportStatsInterval = 1h; - return now - _last_stats_report > ReportStatsInterval; + return now - _last_stats_report > REPORT_STATS_INTERVAL; } void @@ -766,23 +748,29 @@ namespace llarp { const auto now = llarp::time_now_ms(); - log::critical( - logcat, - "{} RCs loaded with {} RIDs, {} bootstrap peers, and {} router connections!", - _node_db->num_rcs(), - _node_db->num_rids(), - _node_db->num_bootstraps(), - num_router_connections()); - if (is_service_node()) { - log::info( + log::critical( logcat, - "Local service node has {} client connections since last RC update ({} to expiry)", + "Local Service Node has {} RCs, {} RIDs, {} bootstrap peers, {} router " + "connections, and {} client connections since last RC update ({} to expiry)", + _node_db->num_rcs(), + _node_db->num_rids(), + _node_db->num_bootstraps(), + num_router_connections(), num_client_connections(), - router_contact.age(now), router_contact.time_to_expiry(now)); } + else + { + log::critical( + logcat, + "{} RCs loaded with {} RIDs, {} bootstrap peers, and {} router connections!", + _node_db->num_rcs(), + _node_db->num_rids(), + _node_db->num_bootstraps(), + num_router_connections()); + } if (_last_stats_report > 0s) log::info(logcat, "Last reported stats time {}", now - _last_stats_report); @@ -844,12 +832,18 @@ namespace llarp { if (is_stopping) return; - // LogDebug("tick router"); + + const bool is_snode = is_service_node(); + const bool is_decommed = appears_decommed(); + const auto now = llarp::time_now_ms(); - if (const auto delta = now - _last_tick; _last_tick != 0s and delta > TimeskipDetectedDuration) + auto now_timepoint = std::chrono::system_clock::time_point(now); + + if (const auto delta = now - _last_tick; + _last_tick != 0s and delta > NETWORK_RESET_SKIP_INTERVAL) { // we detected a time skip into the futre, thaw the network - LogWarn("Timeskip of ", ToString(delta), " detected. Resetting network state"); + log::warning(logcat, "Timeskip of {} detected, resetting network state!", delta.count()); Thaw(); } @@ -864,18 +858,13 @@ namespace llarp report_stats(); } - const bool is_snode = is_service_node(); - const bool is_decommed = appears_decommed(); - // (relay-only) if we have fetched the relay list from oxend and // we are registered and funded, we want to gossip our RC periodically - auto now_timepoint = std::chrono::system_clock::time_point(now); - if (is_snode) { - if (appears_funded() and now_timepoint > next_rc_gossip) + if (now_timepoint > next_rc_gossip) { - log::info(logcat, "regenerating and gossiping RC"); + log::critical(logcat, "Regenerating and gossiping RC..."); router_contact.resign(); save_rc(); @@ -887,11 +876,15 @@ namespace llarp last_rc_gossip = now_timepoint; - // 1min to 5min before "stale time" is next gossip time + // TESTNET: 1 to 2 minutes before testnet gossip interval auto random_delta = std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; + // 1min to 5min before "stale time" is next gossip time + // auto random_delta = + // std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; - next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; + next_rc_gossip = now_timepoint + TESTNET_GOSSIP_INTERVAL - random_delta; + // next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; } report_stats(); @@ -900,13 +893,13 @@ namespace llarp if (needs_initial_fetch()) { if (not _config->bootstrap.seednode) - node_db()->fetch_initial(); + node_db()->fetch_initial(is_service_node()); } else if (needs_rebootstrap() and now_timepoint > next_bootstrap_attempt) { node_db()->fallback_to_bootstrap(); } - else + else if (not is_snode) { // (client-only) periodically fetch updated RCs if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) @@ -916,8 +909,9 @@ namespace llarp } // (client-only) periodically fetch updated RouterID list - if (not is_snode and now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) + if (now_timepoint - last_rid_fetch > ROUTERID_UPDATE_INTERVAL) { + log::critical(logcat, "Time to fetch RIDs!"); node_db()->fetch_rids(); } } @@ -1005,7 +999,6 @@ namespace llarp if (is_snode and now >= _next_decomm_warning) { - constexpr auto DecommissionWarnInterval = 5min; if (auto registered = appears_registered(), funded = appears_funded(); not(registered and funded and not is_decommed)) { @@ -1016,7 +1009,7 @@ namespace llarp not registered ? "deregistered" : is_decommed ? "decommissioned" : "not fully staked"); - _next_decomm_warning = now + DecommissionWarnInterval; + _next_decomm_warning = now + DECOMM_WARNING_INTERVAL; } else if (insufficient_peers()) { @@ -1024,7 +1017,7 @@ namespace llarp logcat, "We appear to be an active service node, but have only {} known peers.", node_db()->num_rcs()); - _next_decomm_warning = now + DecommissionWarnInterval; + _next_decomm_warning = now + DECOMM_WARNING_INTERVAL; } } @@ -1033,7 +1026,7 @@ namespace llarp if (connected < connectToNum and (appears_funded() or not is_snode)) { size_t dlt = connectToNum - connected; - LogDebug("connecting to ", dlt, " random routers to keep alive"); + log::debug(logcat, "Connecting to {} random routers to keep alive", dlt); _link_manager->connect_to_random(dlt); } diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index b80ffd5c1..792ec5dee 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -40,16 +40,27 @@ namespace llarp { /// number of routers to publish to - static constexpr size_t INTROSET_RELAY_REDUNDANCY = 2; + inline constexpr size_t INTROSET_RELAY_REDUNDANCY{2}; /// number of dht locations handled per relay - static constexpr size_t INTROSET_REQS_PER_RELAY = 2; + inline constexpr size_t INTROSET_REQS_PER_RELAY{2}; - static constexpr size_t INTROSET_STORAGE_REDUNDANCY = - (INTROSET_RELAY_REDUNDANCY * INTROSET_REQS_PER_RELAY); + inline constexpr size_t INTROSET_STORAGE_REDUNDANCY{ + (INTROSET_RELAY_REDUNDANCY * INTROSET_REQS_PER_RELAY)}; - static const std::chrono::seconds RC_UPDATE_INTERVAL = 4min; - static const std::chrono::seconds ROUTERID_UPDATE_INTERVAL = 1h; + // TESTNET: these constants are shortened for testing purposes + inline constexpr std::chrono::milliseconds TESTNET_GOSSIP_INTERVAL{4min}; + inline constexpr std::chrono::milliseconds RC_UPDATE_INTERVAL{4min}; + + inline constexpr std::chrono::milliseconds ROUTERID_UPDATE_INTERVAL{1h}; + + // DISCUSS: ask tom and jason about this + // how big of a time skip before we reset network state + inline constexpr std::chrono::milliseconds NETWORK_RESET_SKIP_INTERVAL{1min}; + + inline constexpr std::chrono::milliseconds REPORT_STATS_INTERVAL{1h}; + + inline constexpr std::chrono::milliseconds DECOMM_WARNING_INTERVAL{5min}; struct Contacts; @@ -59,7 +70,7 @@ namespace llarp explicit Router(EventLoop_ptr loop, std::shared_ptr vpnPlatform); - ~Router(); + ~Router() = default; private: std::shared_ptr _route_poker; diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index d8dfd3ae3..47bf2e633 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -53,8 +53,6 @@ namespace llarp static inline constexpr size_t MAX_RC_SIZE = 1024; - /// Timespans for RCs: - /// How long (from its signing time) before an RC is considered "stale". Relays republish /// their RCs slightly more frequently than this so that ideally this won't happen. static constexpr auto STALE_AGE = 6h; From 42aa92ab9504d26108b62422876d737cb1d4efdb Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 08:43:58 -0800 Subject: [PATCH 68/93] gossip fetch and response handling implemented --- llarp/link/link_manager.cpp | 50 ++++++++++++++++++++++++++----------- llarp/link/link_manager.hpp | 2 +- llarp/messages/fetch.hpp | 22 ++++++++++++++++ llarp/nodedb.cpp | 20 ++++++++++++--- llarp/nodedb.hpp | 8 +++++- llarp/router/router.cpp | 6 ++--- 6 files changed, 86 insertions(+), 22 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 76fec975c..42810637c 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -668,20 +668,28 @@ namespace llarp } void - LinkManager::gossip_rc(const RouterID& rc_rid, std::string serialized_rc) + LinkManager::gossip_rc( + const RouterID& gossip_src, const RouterID& last_sender, std::string serialized_rc) { for (auto& [rid, conn] : ep.active_conns) { - // don't send back to the owner... - if (rid == rc_rid) + // don't send back to the gossip source or the last sender + if (rid == gossip_src or rid == last_sender) continue; + // don't gossip RCs to clients if (not conn->remote_is_relay) continue; - send_control_message(rid, "gossip_rc", serialized_rc, [](oxen::quic::message) mutable { - log::critical(logcat, "PLACEHOLDER FOR GOSSIP RC RESPONSE HANDLER"); - }); + log::critical(logcat, "Dispatching gossip_rc to {}", rid); + + send_control_message( + rid, + "gossip_rc"s, + GossipRCMessage::serialize(gossip_src, last_sender, serialized_rc), + [](oxen::quic::message) mutable { + log::critical(logcat, "PLACEHOLDER FOR GOSSIP RC RESPONSE HANDLER"); + }); } } @@ -689,15 +697,31 @@ namespace llarp LinkManager::handle_gossip_rc(oxen::quic::message m) { // RemoteRC constructor wraps deserialization in a try/catch - RemoteRC rc{m.body()}; + RemoteRC rc; + RouterID src, sender; + + try + { + oxenc::bt_dict_consumer btdc{m.body()}; + + btdc.required("rc"); + rc = RemoteRC{btdc.consume_dict_data()}; + src.from_string(btdc.require("sender")); + sender.from_string(btdc.require("src")); + } + catch (const std::exception& e) + { + log::info(link_cat, "Exception handling GossipRC request: {}", e.what()); + return; + } - if (node_db->put_rc_if_newer(rc)) + if (node_db->verify_store_gossip_rc(rc)) { - log::info(link_cat, "Received updated RC, forwarding to relay peers."); - gossip_rc(rc.router_id(), m.body_str()); + log::critical(link_cat, "Received updated RC, forwarding to relay peers."); + gossip_rc(src, _router.local_rid(), std::string{rc.view()}); } else - log::debug(link_cat, "Received known or old RC, not storing or forwarding."); + log::critical(link_cat, "Received known or old RC, not storing or forwarding."); } void @@ -744,9 +768,7 @@ namespace llarp { oxenc::bt_dict_consumer btdc{m.body()}; btdc.required("local"); - auto rc_dict = btdc.consume_dict_data(); - // log::critical(logcat, "incoming dict data: {}", oxenc::to_hex(rc_dict)); - remote = RemoteRC{rc_dict}; + remote = RemoteRC{btdc.consume_dict_data()}; quantity = btdc.require("quantity"); } catch (const std::exception& e) diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 5c1ae69cd..2e85e83d4 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -234,7 +234,7 @@ namespace llarp } void - gossip_rc(const RouterID& rc_rid, std::string serialized_rc); + gossip_rc(const RouterID& gossip_src, const RouterID& last_sender, std::string serialized_rc); void handle_gossip_rc(oxen::quic::message m); diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 8981b6dde..af804f161 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -6,6 +6,28 @@ namespace llarp { + namespace GossipRCMessage + { + inline static std::string + serialize(const RouterID& gossip_src, const RouterID& last_sender, std::string rc) + { + oxenc::bt_dict_producer btdp; + + try + { + btdp.append_encoded("rc", rc); + btdp.append("sender", last_sender.ToView()); + btdp.append("src", gossip_src.ToView()); + } + catch (...) + { + log::error(link_cat, "Error: GossipRCMessage failed to bt encode contents"); + } + + return std::move(btdp).str(); + } + } // namespace GossipRCMessage + namespace FetchRCMessage { inline const auto INVALID_REQUEST = diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index f393fa17e..7bfb62aae 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -945,7 +945,13 @@ namespace llarp } bool - NodeDB::has_rc(RouterID pk) const + NodeDB::has_rc(const RemoteRC& rc) const + { + return known_rcs.count(rc); + } + + bool + NodeDB::has_rc(const RouterID& pk) const { return rc_lookup.count(pk); } @@ -1020,11 +1026,19 @@ namespace llarp return known_rids.size(); } + bool + NodeDB::verify_store_gossip_rc(const RemoteRC& rc) + { + if (not router_whitelist.count(rc.router_id())) + return put_rc_if_newer(rc); + + return false; + } + bool NodeDB::put_rc_if_newer(RemoteRC rc, rc_time now) { - if (auto itr = rc_lookup.find(rc.router_id()); - itr == rc_lookup.end() or itr->second.other_is_newer(rc)) + if (not has_rc(rc)) return put_rc(std::move(rc), now); return false; diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 09880ca17..90f45f73c 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -425,7 +425,10 @@ namespace llarp /// return true if we have an rc by its ident pubkey bool - has_rc(RouterID pk) const; + has_rc(const RouterID& pk) const; + + bool + has_rc(const RemoteRC& rc) const; /// maybe get an rc by its ident pubkey std::optional @@ -584,6 +587,9 @@ namespace llarp /// returns true if the rc was inserted bool put_rc_if_newer(RemoteRC rc, rc_time now = time_point_now()); + + bool + verify_store_gossip_rc(const RemoteRC& rc); }; } // namespace llarp diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 205f9f5f9..68426852e 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -869,10 +869,10 @@ namespace llarp router_contact.resign(); save_rc(); - auto view = router_contact.view(); - _link_manager->gossip_rc( - pubkey(), std::string{reinterpret_cast(view.data()), view.size()}); + router_contact.router_id(), + router_contact.router_id(), + std::string{oxen::quic::to_sv(router_contact.view())}); last_rc_gossip = now_timepoint; From c71e76751150b3707f99ea95c864c78a30094e10 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 09:45:00 -0800 Subject: [PATCH 69/93] full mesh proto implementation --- llarp/link/link_manager.cpp | 40 +++++++++---------- llarp/link/link_manager.hpp | 9 +++-- llarp/nodedb.cpp | 27 +++++++++---- llarp/nodedb.hpp | 9 +++-- llarp/router/router.cpp | 80 ++++++++++++++++++++++--------------- llarp/router/router.hpp | 3 +- 6 files changed, 100 insertions(+), 68 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 42810637c..6c6041a13 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -46,23 +46,27 @@ namespace llarp } bool - Endpoint::have_conn(const RouterID& remote, bool client_only) const + Endpoint::have_client_conn(const RouterID& remote) const { if (auto itr = active_conns.find(remote); itr != active_conns.end()) { - if (not(itr->second->remote_is_relay and client_only)) - return true; + return not itr->second->remote_is_relay; } if (auto itr = pending_conns.find(remote); itr != pending_conns.end()) { - if (not(itr->second->remote_is_relay and client_only)) - return true; + return not itr->second->remote_is_relay; } return false; } + bool + Endpoint::have_conn(const RouterID& remote) const + { + return active_conns.count(remote) or pending_conns.count(remote); + } + size_t Endpoint::num_connected(bool clients_only) const { @@ -535,8 +539,6 @@ namespace llarp const auto& remote_addr = rc.addr(); const auto& rid = rc.router_id(); - // rids_pending_verification[rid] = rc; - // TODO: confirm remote end is using the expected pubkey (RouterID). // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) if (auto rv = ep.establish_connection( @@ -553,15 +555,15 @@ namespace llarp } bool - LinkManager::have_connection_to(const RouterID& remote, bool client_only) const + LinkManager::have_connection_to(const RouterID& remote) const { - return ep.have_conn(remote, client_only); + return ep.have_conn(remote); } bool LinkManager::have_client_connection_to(const RouterID& remote) const { - return ep.have_conn(remote, true); + return ep.have_client_conn(remote); } void @@ -636,20 +638,17 @@ namespace llarp { is_stopping = false; node_db = _router.node_db(); - client_router_connections = _router.required_num_client_conns(); } void - LinkManager::connect_to_random(int num_conns) + LinkManager::connect_to_random(int num_conns, bool client_only) { - std::set exclude; - auto remainder = num_conns; - - auto filter = [exclude](const RemoteRC& rc) -> bool { - return exclude.count(rc.router_id()) == 0; + auto filter = [this, client_only](const RemoteRC& rc) -> bool { + return client_only ? not ep.have_client_conn(rc.router_id()) + : not ep.have_conn(rc.router_id()); }; - if (auto maybe = node_db->get_n_random_rcs_conditional(remainder, filter)) + if (auto maybe = node_db->get_n_random_rcs_conditional(num_conns, filter)) { std::vector& rcs = *maybe; @@ -696,6 +695,8 @@ namespace llarp void LinkManager::handle_gossip_rc(oxen::quic::message m) { + log::critical(logcat, "Handling GossipRC request..."); + // RemoteRC constructor wraps deserialization in a try/catch RemoteRC rc; RouterID src, sender; @@ -864,7 +865,6 @@ namespace llarp } const auto& rcs = node_db->get_rcs(); - const auto now = time_point_now(); oxenc::bt_dict_producer btdp; const auto& last_time = node_db->get_last_rc_update_times(); @@ -895,8 +895,6 @@ namespace llarp } } - btdp.append("time", now.time_since_epoch().count()); - m.respond(std::move(btdp).str()); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 2e85e83d4..88adc56e6 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -67,7 +67,10 @@ namespace llarp get_conn(const RouterID&) const; bool - have_conn(const RouterID& remote, bool client_only) const; + have_client_conn(const RouterID& remote) const; + + bool + have_conn(const RouterID& remote) const; size_t num_connected(bool clients_only) const; @@ -265,7 +268,7 @@ namespace llarp handle_fetch_bootstrap_rcs(oxen::quic::message m); bool - have_connection_to(const RouterID& remote, bool client_only = false) const; + have_connection_to(const RouterID& remote) const; bool have_client_connection_to(const RouterID& remote) const; @@ -318,7 +321,7 @@ namespace llarp // check if we already have a connection to any of the random set, as making // that thread safe would be slow...I think. void - connect_to_random(int num_conns); + connect_to_random(int num_conns, bool client_only = false); /// always maintain this many client connections to other routers int client_router_connections = 4; diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 7bfb62aae..7b09f26b4 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -245,7 +245,7 @@ namespace llarp } bool - NodeDB::ingest_fetched_rcs(std::set rcs, rc_time timestamp) + NodeDB::ingest_fetched_rcs(std::set rcs) { // if we are not bootstrapping, we should check the rc's against the ones we currently hold if (not _using_bootstrap_fallback) @@ -255,7 +255,7 @@ namespace llarp } while (!rcs.empty()) - put_rc_if_newer(std::move(rcs.extract(rcs.begin()).value()), timestamp); + put_rc_if_newer(std::move(rcs.extract(rcs.begin()).value())); return true; } @@ -405,7 +405,6 @@ namespace llarp } auto btlc = btdc.require("rcs"sv); - auto timestamp = rc_time{std::chrono::seconds{btdc.require("time"sv)}}; std::set rcs; @@ -413,7 +412,7 @@ namespace llarp rcs.emplace(btlc.consume_dict_data()); // if process_fetched_rcs returns false, then the trust model rejected the fetched RC's - fetch_rcs_result(initial, not ingest_fetched_rcs(std::move(rcs), timestamp)); + fetch_rcs_result(initial, not ingest_fetched_rcs(std::move(rcs))); } catch (const std::exception& e) { @@ -957,7 +956,16 @@ namespace llarp } std::optional - NodeDB::get_rc(RouterID pk) const + NodeDB::get_rc(const RemoteRC& pk) const + { + if (auto itr = known_rcs.find(pk); itr != known_rcs.end()) + return *itr; + + return std::nullopt; + } + + std::optional + NodeDB::get_rc(const RouterID& pk) const { if (auto itr = rc_lookup.find(pk); itr != rc_lookup.end()) return itr->second; @@ -1036,10 +1044,13 @@ namespace llarp } bool - NodeDB::put_rc_if_newer(RemoteRC rc, rc_time now) + NodeDB::put_rc_if_newer(RemoteRC rc) { - if (not has_rc(rc)) - return put_rc(std::move(rc), now); + if (auto maybe = get_rc(rc)) + { + if (maybe->other_is_newer(rc)) + return put_rc(rc); + } return false; } diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 90f45f73c..340f03922 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -232,7 +232,7 @@ namespace llarp ingest_bootstrap_seed(); bool - ingest_fetched_rcs(std::set rcs, rc_time timestamp); + ingest_fetched_rcs(std::set rcs); bool process_fetched_rcs(std::set& rcs); @@ -432,7 +432,10 @@ namespace llarp /// maybe get an rc by its ident pubkey std::optional - get_rc(RouterID pk) const; + get_rc(const RouterID& pk) const; + + std::optional + get_rc(const RemoteRC& pk) const; std::optional get_random_rc() const; @@ -586,7 +589,7 @@ namespace llarp /// put this rc into the cache if it is not there or is newer than the one there already /// returns true if the rc was inserted bool - put_rc_if_newer(RemoteRC rc, rc_time now = time_point_now()); + put_rc_if_newer(RemoteRC rc); bool verify_store_gossip_rc(const RemoteRC& rc); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 68426852e..061ff5efc 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -986,48 +986,64 @@ namespace llarp _link_manager->check_persisting_conns(now); - size_t connected = num_router_connections(); + auto num_conns = num_router_connections(); - size_t connectToNum = _link_manager->client_router_connections; - const auto& pinned_edges = _node_db->pinned_edges(); - const auto pinned_count = pinned_edges.size(); + const auto& num_rcs = node_db()->num_rcs(); - if (pinned_count > 0 && connectToNum > pinned_count) - { - connectToNum = pinned_count; - } - - if (is_snode and now >= _next_decomm_warning) + if (is_snode) { - if (auto registered = appears_registered(), funded = appears_funded(); - not(registered and funded and not is_decommed)) + if (now >= _next_decomm_warning) { - // complain about being deregistered/decommed/unfunded - log::error( - logcat, - "We are running as a service node but we seem to be {}", - not registered ? "deregistered" - : is_decommed ? "decommissioned" - : "not fully staked"); - _next_decomm_warning = now + DECOMM_WARNING_INTERVAL; + if (auto registered = appears_registered(), funded = appears_funded(); + not(registered and funded and not is_decommed)) + { + // complain about being deregistered/decommed/unfunded + log::error( + logcat, + "We are running as a service node but we seem to be {}", + not registered ? "deregistered" + : is_decommed ? "decommissioned" + : "not fully staked"); + _next_decomm_warning = now + DECOMM_WARNING_INTERVAL; + } + else if (insufficient_peers()) + { + log::error( + logcat, + "We appear to be an active service node, but have only {} known peers.", + node_db()->num_rcs()); + _next_decomm_warning = now + DECOMM_WARNING_INTERVAL; + } } - else if (insufficient_peers()) + + if (num_conns < num_rcs) { - log::error( + log::critical( logcat, - "We appear to be an active service node, but have only {} known peers.", - node_db()->num_rcs()); - _next_decomm_warning = now + DECOMM_WARNING_INTERVAL; + "Service Node connecting to {} random routers to achieve full mesh", + FULL_MESH_ITERATION); + _link_manager->connect_to_random(FULL_MESH_ITERATION); } + else + log::critical(logcat, "SERVICE NODE IS FULLY MESHED"); } - - // if we need more sessions to routers and we are not a service node kicked from the network or - // we are a client we shall connect out to others - if (connected < connectToNum and (appears_funded() or not is_snode)) + else { - size_t dlt = connectToNum - connected; - log::debug(logcat, "Connecting to {} random routers to keep alive", dlt); - _link_manager->connect_to_random(dlt); + size_t min_client_conns = _link_manager->client_router_connections; + const auto& pinned_edges = _node_db->pinned_edges(); + const auto pinned_count = pinned_edges.size(); + + if (pinned_count > 0 && min_client_conns > pinned_count) + min_client_conns = pinned_count; + + // if we need more sessions to routers and we are not a service node kicked from the network + // or we are a client we shall connect out to others + if (num_conns < min_client_conns) + { + size_t needed = min_client_conns - num_conns; + log::critical(logcat, "Client connecting to {} random routers to keep alive", needed); + _link_manager->connect_to_random(needed); + } } _hidden_service_context.Tick(now); diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 792ec5dee..bd3b33330 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -51,7 +51,8 @@ namespace llarp // TESTNET: these constants are shortened for testing purposes inline constexpr std::chrono::milliseconds TESTNET_GOSSIP_INTERVAL{4min}; inline constexpr std::chrono::milliseconds RC_UPDATE_INTERVAL{4min}; - + // as we advance towards full mesh, we try to connect to this number per tick + inline constexpr int FULL_MESH_ITERATION{1}; inline constexpr std::chrono::milliseconds ROUTERID_UPDATE_INTERVAL{1h}; // DISCUSS: ask tom and jason about this From 5c13a0e7b492ac72893fefcf89b9cf55ded964db Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 15 Dec 2023 14:30:30 -0400 Subject: [PATCH 70/93] Revert me: print out the failed bootstrap response --- llarp/nodedb.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 7b09f26b4..455d6ca8c 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -723,6 +723,7 @@ namespace llarp bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS, e.what()); + log::critical(logcat, "DEBUG FIXME THIS IS WHAT I GOT: {}", oxenc::to_hex(m.body())); fallback_to_bootstrap(); return; } From 49794295b1b1e0349295a0192887d582e3c12d66 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 10:37:42 -0800 Subject: [PATCH 71/93] booyakasha --- llarp/link/link_manager.cpp | 2 +- llarp/router_contact_remote.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 6c6041a13..1e9c4a950 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -238,7 +238,7 @@ namespace llarp } log::critical( logcat, - "Bootstrap seed node was {} to confirm fetch requester is white-listed; {}successfully " + "Bootstrap seed node was {} to confirm remote is white-listed; {}successfully " "saved RID", result ? "able" : "unable", result ? "" : "un"); diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 387ebf89c..98d70a4d6 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -59,7 +59,8 @@ namespace llarp try { - util::file_to_buffer(fname, buf.data(), buf.size()); + auto nread = util::file_to_buffer(fname, buf.data(), buf.size()); + buf.resize(nread); oxenc::bt_dict_consumer btdc{buf}; bt_load(btdc); From b63733381fa9dd4e154e97f33c1e82422afbdc1e Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 12:45:45 -0800 Subject: [PATCH 72/93] libquic/oxenc vbumps --- external/oxen-encoding | 2 +- external/oxen-libquic | 2 +- llarp/router/router.cpp | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/external/oxen-encoding b/external/oxen-encoding index f6172d58d..24fbdb794 160000 --- a/external/oxen-encoding +++ b/external/oxen-encoding @@ -1 +1 @@ -Subproject commit f6172d58d3358473a4c98d96270058a32e166d5f +Subproject commit 24fbdb794ef26bf5324d1b56d48d1da8de8a140c diff --git a/external/oxen-libquic b/external/oxen-libquic index 89d85ffea..78b46d4f4 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 89d85ffeabbc773a1ea20d05b7f9c2a13c5e8370 +Subproject commit 78b46d4f4df45747b70918386672228ead2c7ed3 diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 061ff5efc..a7a17c79f 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -866,8 +866,12 @@ namespace llarp { log::critical(logcat, "Regenerating and gossiping RC..."); +<<<<<<< Updated upstream router_contact.resign(); save_rc(); +======= + auto view = router_contact.view(); +>>>>>>> Stashed changes _link_manager->gossip_rc( router_contact.router_id(), From 87ae0686d06d3012d2180966d1b52c5a67c8f8a3 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 13:47:14 -0800 Subject: [PATCH 73/93] rc parsing - new btdc method used to ensure no junk at the end of our bt data - DRYed out the RC code - check inbound bootstraps against all registered routers, not just whitelist - libquic vbump --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 11 ++- llarp/nodedb.cpp | 14 ++- llarp/nodedb.hpp | 12 ++- llarp/router/router.cpp | 10 +-- llarp/router/router.hpp | 4 +- llarp/router_contact.cpp | 146 +++++++++----------------------- llarp/router_contact.hpp | 13 +-- llarp/router_contact_local.cpp | 37 -------- llarp/router_contact_remote.cpp | 36 +------- 10 files changed, 72 insertions(+), 213 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 78b46d4f4..b35cab3a3 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 78b46d4f4df45747b70918386672228ead2c7ed3 +Subproject commit b35cab3a310cf358580ef9f418553f71436766ab diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 1e9c4a950..ac99d38c0 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -230,15 +230,16 @@ namespace llarp if (_router.is_bootstrap_seed()) { - if (node_db->whitelist().count(other)) + if (node_db->registered_routers().count(other)) { log::critical(logcat, "Saving bootstrap seed requester..."); - auto [it, b] = node_db->seeds().emplace(other); - result |= b; + auto [it, b] = node_db->seeds().insert(other); + result &= b; } + log::critical( logcat, - "Bootstrap seed node was {} to confirm remote is white-listed; {}successfully " + "Bootstrap seed node was {} to confirm remote is registered; {}successfully " "saved RID", result ? "able" : "unable", result ? "" : "un"); @@ -323,6 +324,8 @@ namespace llarp } else { + log::critical(logcat, "Searching for RID:{} in pending conns...", rid); + if (auto itr = ep.pending_conns.find(rid); itr != ep.pending_conns.end()) { ep.connid_map.emplace(scid, rid); diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 455d6ca8c..6244d104a 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -808,10 +808,10 @@ namespace llarp if (whitelist.empty()) return; - registered_routers.clear(); - registered_routers.insert(whitelist.begin(), whitelist.end()); - registered_routers.insert(greylist.begin(), greylist.end()); - registered_routers.insert(greenlist.begin(), greenlist.end()); + _registered_routers.clear(); + _registered_routers.insert(whitelist.begin(), whitelist.end()); + _registered_routers.insert(greylist.begin(), greylist.end()); + _registered_routers.insert(greenlist.begin(), greenlist.end()); router_whitelist.clear(); router_whitelist.insert(whitelist.begin(), whitelist.end()); @@ -820,10 +820,8 @@ namespace llarp router_greenlist.clear(); router_greenlist.insert(greenlist.begin(), greenlist.end()); - log::info( - logcat, - "lokinet service node whitelist now has {} active router RIDs", - router_whitelist.size()); + log::critical( + logcat, "Service node whitelist now has {} active router RIDs", router_whitelist.size()); } std::optional diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 340f03922..5f4588edf 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -150,7 +150,7 @@ namespace llarp std::set router_greenlist{}; // All registered relays (service nodes) - std::set registered_routers; + std::set _registered_routers; // timing (note: Router holds the variables for last rc and rid request times) std::unordered_map last_rc_update_times; // if populated from a config file, lists specific exclusively used as path first-hops @@ -372,10 +372,16 @@ namespace llarp return router_greylist; } + std::set& + registered_routers() + { + return _registered_routers; + } + const std::set& - get_registered_routers() const + registered_routers() const { - return registered_routers; + return _registered_routers; } const std::set& diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index a7a17c79f..a530c6712 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -476,7 +476,7 @@ namespace llarp Router::appears_registered() const { return _is_service_node and have_snode_whitelist() - and node_db()->get_registered_routers().count(pubkey()); + and node_db()->registered_routers().count(pubkey()); } bool @@ -866,17 +866,11 @@ namespace llarp { log::critical(logcat, "Regenerating and gossiping RC..."); -<<<<<<< Updated upstream router_contact.resign(); save_rc(); -======= - auto view = router_contact.view(); ->>>>>>> Stashed changes _link_manager->gossip_rc( - router_contact.router_id(), - router_contact.router_id(), - std::string{oxen::quic::to_sv(router_contact.view())}); + local_rid(), local_rid(), std::string{oxen::quic::to_sv(router_contact.view())}); last_rc_gossip = now_timepoint; diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index bd3b33330..39d0c8693 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -187,10 +187,10 @@ namespace llarp return client_router_connections; } - RouterID + const RouterID& local_rid() const { - return RouterID{pubkey()}; + return router_contact.router_id(); } bool diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index d75c5625e..ebd87e701 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -12,12 +12,42 @@ namespace llarp { void - RouterContact::bt_load(oxenc::bt_dict_consumer& data) + RouterContact::bt_verify(oxenc::bt_dict_consumer& btdc, bool reject_expired) const { - if (int rc_ver = data.require(""); rc_ver != RC_VERSION) + btdc.require_signature("~", [this, reject_expired](ustring_view msg, ustring_view sig) { + if (sig.size() != 64) + throw std::runtime_error{"Invalid signature: not 64 bytes"}; + + if (reject_expired and is_expired(time_now_ms())) + throw std::runtime_error{"Rejecting expired RemoteRC!"}; + + // TODO: revisit if this is needed; detail from previous implementation + const auto* net = net::Platform::Default_ptr(); + + if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) + { + auto err = "Unable to verify expired RemoteRC address!"; + log::info(logcat, err); + throw std::runtime_error{err}; + } + + if (not crypto::verify(router_id(), msg, sig)) + throw std::runtime_error{"Failed to verify RemoteRC signature"}; + }); + + if (not btdc.is_finished()) + throw std::runtime_error{"RouterContact has some fucked up shit at the end"}; + + btdc.finish(); + } + + void + RouterContact::bt_load(oxenc::bt_dict_consumer& btdc) + { + if (int rc_ver = btdc.require(""); rc_ver != RC_VERSION) throw std::runtime_error{"Invalid RC: do not know how to parse v{} RCs"_format(rc_ver)}; - auto ipv4_port = data.require("4"); + auto ipv4_port = btdc.require("4"); if (ipv4_port.size() != 6) throw std::runtime_error{ @@ -34,7 +64,7 @@ namespace llarp if (!_addr.is_public()) throw std::runtime_error{"Invalid RC: IPv4 address is not a publicly routable IP"}; - if (auto ipv6_port = data.maybe("6")) + if (auto ipv6_port = btdc.maybe("6")) { if (ipv6_port->size() != 18) throw std::runtime_error{ @@ -55,22 +85,22 @@ namespace llarp _addr6.reset(); } - auto netid = data.maybe("i").value_or(llarp::LOKINET_DEFAULT_NETID); + auto netid = btdc.maybe("i").value_or(llarp::LOKINET_DEFAULT_NETID); if (netid != ACTIVE_NETID) throw std::runtime_error{ "Invalid RC netid: expected {}, got {}; this is an RC for a different network!"_format( ACTIVE_NETID, netid)}; - auto pubkey = data.require("p"); + auto pubkey = btdc.require("p"); if (pubkey.size() != 32) throw std::runtime_error{ "Invalid RC pubkey: expected 32 bytes, got {}"_format(pubkey.size())}; std::memcpy(_router_id.data(), pubkey.data(), 32); - _timestamp = rc_time{std::chrono::seconds{data.require("t")}}; + _timestamp = rc_time{std::chrono::seconds{btdc.require("t")}}; - auto ver = data.require("v"); + auto ver = btdc.require("v"); if (ver.size() != 3) throw std::runtime_error{ @@ -120,106 +150,6 @@ namespace llarp return obj; } - bool - RouterContact::BDecode(llarp_buffer_t* buf) - { - // TODO: unfuck all of this - - (void)buf; - - // clear(); - - // if (*buf->cur == 'd') // old format - // { - // return DecodeVersion_0(buf); - // } - // else if (*buf->cur != 'l') // if not dict, should be new format and start with list - // { - // return false; - // } - - // try - // { - // std::string_view buf_view(reinterpret_cast(buf->cur), buf->size_left()); - // oxenc::bt_list_consumer btlist(buf_view); - - // uint64_t outer_version = btlist.consume_integer(); - - // if (outer_version == 1) - // { - // bool decode_result = DecodeVersion_1(btlist); - - // // advance the llarp_buffer_t since lokimq serialization is unaware of it. - // // FIXME: this is broken (current_buffer got dropped), but the whole thing is getting - // // replaced. - // // buf->cur += btlist. - // // current_buffer().data() - buf_view.data() + 1; - - // return decode_result; - // } - // else - // { - // log::warning(logcat, "Received RouterContact with unkown version ({})", outer_version); - // return false; - // } - // } - // catch (const std::exception& e) - // { - // log::debug(logcat, "RouterContact::BDecode failed: {}", e.what()); - // } - - return false; - } - - bool - RouterContact::decode_key(const llarp_buffer_t& key, llarp_buffer_t* buf) - { - bool read = false; - (void)key; - - // TOFIX: fuck everything about llarp_buffer_t - - // if (!BEncodeMaybeReadDictEntry("a", addr, read, key, buf)) - // return false; - - // if (!BEncodeMaybeReadDictEntry("i", netID, read, key, buf)) - // return false; - - // if (!BEncodeMaybeReadDictEntry("k", _router_id, read, key, buf)) - // return false; - - // if (key.startswith("r")) - // { - // RouterVersion r; - // if (not r.BDecode(buf)) - // return false; - // routerVersion = r; - // return true; - // } - - // if (not BEncodeMaybeReadDictList("s", srvRecords, read, key, buf)) - // return false; - - // if (!BEncodeMaybeReadDictEntry("p", enckey, read, key, buf)) - // return false; - - // if (!BEncodeMaybeReadDictInt("u", _timestamp, read, key, buf)) - // return false; - - // if (!BEncodeMaybeReadDictInt("v", version, read, key, buf)) - // return false; - - // if (key.startswith("x") and serializeExit) - // { - // return bencode_discard(buf); - // } - - // if (!BEncodeMaybeReadDictEntry("z", signature, read, key, buf)) - // return false; - - return read or bencode_discard(buf); - } - bool RouterContact::is_public_addressable() const { diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index 47bf2e633..c37652fe5 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -167,12 +167,6 @@ namespace llarp clear() {} - bool - BDecode(llarp_buffer_t* buf); - - bool - decode_key(const llarp_buffer_t& k, llarp_buffer_t* buf); - bool is_public_addressable() const; @@ -201,6 +195,9 @@ namespace llarp bool is_obsolete_bootstrap() const; + void + bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired = false) const; + void bt_load(oxenc::bt_dict_consumer& data); }; @@ -233,7 +230,6 @@ namespace llarp public: LocalRC() = default; - explicit LocalRC(std::string payload, const SecretKey sk); ~LocalRC() = default; RemoteRC @@ -309,9 +305,6 @@ namespace llarp struct RemoteRC final : public RouterContact { private: - void - bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired = false) const; - explicit RemoteRC(oxenc::bt_dict_consumer btdc); public: diff --git a/llarp/router_contact_local.cpp b/llarp/router_contact_local.cpp index 8c9d8b39b..82520daa4 100644 --- a/llarp/router_contact_local.cpp +++ b/llarp/router_contact_local.cpp @@ -34,43 +34,6 @@ namespace llarp return RemoteRC{view()}; } - LocalRC::LocalRC(std::string payload, const SecretKey sk) : _secret_key{std::move(sk)} - { - _router_id = llarp::seckey_to_pubkey(_secret_key); - - try - { - oxenc::bt_dict_consumer btdc{payload}; - bt_load(btdc); - - btdc.require_signature("~", [this](ustring_view msg, ustring_view sig) { - if (sig.size() != 64) - throw std::runtime_error{"Invalid signature: not 64 bytes"}; - - if (is_expired(time_now_ms())) - throw std::runtime_error{"Unable to verify expired RemoteRC!"}; - - // TODO: revisit if this is needed; detail from previous implementation - const auto* net = net::Platform::Default_ptr(); - - if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) - { - auto err = "Unable to verify expired LocalRC!"; - log::info(logcat, err); - throw std::runtime_error{err}; - } - - if (not crypto::verify(router_id(), msg, sig)) - throw std::runtime_error{"Failed to verify LocalRC"}; - }); - } - catch (const std::exception& e) - { - log::warning(logcat, "Failed to parse LocalRC: {}", e.what()); - throw; - } - } - void LocalRC::bt_sign(oxenc::bt_dict_producer& btdp) { diff --git a/llarp/router_contact_remote.cpp b/llarp/router_contact_remote.cpp index 98d70a4d6..141c6fc84 100644 --- a/llarp/router_contact_remote.cpp +++ b/llarp/router_contact_remote.cpp @@ -26,47 +26,19 @@ namespace llarp } } - void - RemoteRC::bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired) const - { - data.require_signature("~", [this, reject_expired](ustring_view msg, ustring_view sig) { - if (sig.size() != 64) - throw std::runtime_error{"Invalid signature: not 64 bytes"}; - - if (reject_expired and is_expired(time_now_ms())) - throw std::runtime_error{"Rejecting expired RemoteRC!"}; - - // TODO: revisit if this is needed; detail from previous implementation - const auto* net = net::Platform::Default_ptr(); - - if (net->IsBogon(addr().in4()) and BLOCK_BOGONS) - { - auto err = "Unable to verify expired RemoteRC address!"; - log::info(logcat, err); - throw std::runtime_error{err}; - } - - if (not crypto::verify(router_id(), msg, sig)) - throw std::runtime_error{"Failed to verify RemoteRC signature"}; - }); - } - bool RemoteRC::read(const fs::path& fname) { - ustring buf; - buf.resize(MAX_RC_SIZE); + _payload.resize(MAX_RC_SIZE); try { - auto nread = util::file_to_buffer(fname, buf.data(), buf.size()); - buf.resize(nread); + auto nread = util::file_to_buffer(fname, _payload.data(), _payload.size()); + _payload.resize(nread); - oxenc::bt_dict_consumer btdc{buf}; + oxenc::bt_dict_consumer btdc{_payload}; bt_load(btdc); bt_verify(btdc); - - _payload = buf; } catch (const std::exception& e) { From 2a090b6e42244d66165d88f2e836c29d915d024e Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 14:52:54 -0800 Subject: [PATCH 74/93] btreq destructor - try closing the connection via link::Endpoint like we probably should be doing.. - testing out dropping stream constructor, godspeed --- llarp/link/link_manager.cpp | 69 ++++++++++++++++--------------------- llarp/link/link_manager.hpp | 4 +-- llarp/nodedb.cpp | 4 +-- llarp/nodedb.hpp | 27 +++------------ 4 files changed, 39 insertions(+), 65 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index ac99d38c0..655cefa50 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -225,38 +225,24 @@ namespace llarp }, [this](oxen::quic::dgram_interface& di, bstring dgram) { recv_data_message(di, dgram); }); tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) { - bool result = true; RouterID other{key.data()}; - if (_router.is_bootstrap_seed()) - { - if (node_db->registered_routers().count(other)) - { - log::critical(logcat, "Saving bootstrap seed requester..."); - auto [it, b] = node_db->seeds().insert(other); - result &= b; - } - - log::critical( - logcat, - "Bootstrap seed node was {} to confirm remote is registered; {}successfully " - "saved RID", - result ? "able" : "unable", - result ? "" : "un"); - return result; - } - - result = node_db->has_rc(other); + bool result = node_db->registered_routers().count(other); log::critical( - logcat, "{}uccessfully verified connection to {}!", result ? "S" : "Uns", other); + logcat, + "{} node was {} to confirm remote (RID:{}) is registered; allowing connection!", + router().is_bootstrap_seed() ? "Bootstrap seed node" : "Service node", + other, + result ? "able" : "unable"); + return result; }); if (_router.is_service_node()) { ep->listen( tls_creds, - ROUTER_KEEP_ALIVE, + ROUTER_KEEP_ALIVE/* , [&](oxen::quic::Connection& c, oxen::quic::Endpoint& e, std::optional id) -> std::shared_ptr { @@ -278,7 +264,7 @@ namespace llarp log::critical(logcat, "Stream constructor constructing Stream (ID:{})!", id); return e.make_shared(c, e); - }); + } */); } return ep; } @@ -293,11 +279,11 @@ namespace llarp log::critical(logcat, "Queueing BTStream to be opened..."); - auto control_stream = ci.queue_stream([](oxen::quic::Stream& s, + auto control_stream = ci.queue_stream([this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { log::warning( logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); - s.conn.close_connection(error_code); + ep.close_connection(rid); }); log::critical(logcat, "Queued BTStream to be opened ID:{}", control_stream->stream_id()); @@ -789,13 +775,13 @@ namespace llarp if (is_seed) { // we already insert the - auto& seeds = node_db->seeds(); + auto& registered = node_db->registered_routers(); - if (auto itr = seeds.find(rid); itr != seeds.end()) + if (auto itr = registered.find(rid); itr != registered.end()) { log::critical( logcat, - "Bootstrap seed confirmed RID:{} is white-listed seeds; approving fetch request and " + "Bootstrap seed confirmed RID:{} is registered; approving fetch request and " "saving RC!", rid); node_db->put_rc(remote); @@ -805,12 +791,9 @@ namespace llarp auto& src = node_db->get_known_rcs(); auto count = src.size(); - if (count == 0) - { - log::error(logcat, "No known RCs locally to send!"); - m.respond(messages::ERROR_RESPONSE, true); - return; - } + // if quantity is 0, then the service node requesting this wants all the RC's; otherwise, + // send the amount requested in the message + quantity = quantity == 0 ? count : quantity; auto now = llarp::time_now_ms(); size_t i = 0; @@ -820,13 +803,18 @@ namespace llarp { auto sublist = btdp.append_list("rcs"); - for (const auto& rc : src) + if (count == 0) + log::error(logcat, "No known RCs locally to send!"); + else { - if (not rc.is_expired(now)) - sublist.append_encoded(rc.view()); + for (const auto& rc : src) + { + if (not rc.is_expired(now)) + sublist.append_encoded(rc.view()); - if (++i >= quantity) - break; + if (++i >= quantity) + break; + } } } @@ -837,6 +825,9 @@ namespace llarp LinkManager::fetch_rcs( const RouterID& source, std::string payload, std::function func) { + // this handler should not be registered for service nodes + assert(not _router.is_service_node()); + send_control_message(source, "fetch_rcs", std::move(payload), std::move(func)); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 88adc56e6..54d06eb1e 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -424,12 +424,12 @@ namespace llarp auto [itr, b] = pending_conns.emplace(rid, nullptr); auto control_stream = conn_interface->template get_new_stream( - [](oxen::quic::Stream& s, uint64_t error_code) { + [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { log::warning( logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); - s.conn.close_connection(error_code); + close_connection(rid); }); link_manager.register_commands(control_stream, rid); diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 6244d104a..4a7eb7fee 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -678,7 +678,7 @@ namespace llarp _router.link_manager().fetch_bootstrap_rcs( rc, - BootstrapFetchMessage::serialize(_router.router_contact, BOOTSTRAP_SOURCE_COUNT), + BootstrapFetchMessage::serialize(_router.router_contact, CLIENT_BOOTSTRAP_SOURCE_COUNT), [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); @@ -740,7 +740,7 @@ namespace llarp "BootstrapRC fetch response from {} returned {}/{} needed RCs", fetch_source, num, - BOOTSTRAP_SOURCE_COUNT); + CLIENT_BOOTSTRAP_SOURCE_COUNT); if (not is_snode) { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 5f4588edf..3f6b73e31 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -45,9 +45,12 @@ namespace llarp inline constexpr size_t MIN_GOOD_RID_FETCH_TOTAL{}; // the ratio of accepted:rejected rids must be above this ratio inline constexpr double GOOD_RID_FETCH_THRESHOLD{}; + /* Bootstrap Constants */ - // the number of rc's we query the bootstrap for - inline constexpr size_t BOOTSTRAP_SOURCE_COUNT{10}; + // the number of rc's we query the bootstrap for; service nodes pass 0, which means + // gimme all dat RCs + inline constexpr size_t SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT{0}; + inline constexpr size_t CLIENT_BOOTSTRAP_SOURCE_COUNT{10}; // the maximum number of fetch requests we make across all bootstraps inline constexpr int MAX_BOOTSTRAP_FETCH_ATTEMPTS{5}; // if all bootstraps fail, router will trigger re-bootstrapping after this cooldown @@ -136,8 +139,6 @@ namespace llarp std::map rc_lookup; - std::set _bootstrap_seeds; - std::set _seeds; BootstrapList _bootstraps{}; /** RouterID lists // TODO: get rid of all these, replace with better decom/not staked sets @@ -195,12 +196,6 @@ namespace llarp /// in memory nodedb NodeDB(); - std::set& - seeds() - { - return _seeds; - } - const std::set& get_known_rids() const { @@ -345,18 +340,6 @@ namespace llarp return _bootstraps; } - const std::set& - bootstrap_seeds() const - { - return _bootstrap_seeds; - } - - std::set& - bootstrap_seeds() - { - return _bootstrap_seeds; - } - void set_bootstrap_routers(BootstrapList& from_router); From f41bcd00c69dcc21e368179067cd3d9e55f0c036 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 14:58:20 -0800 Subject: [PATCH 75/93] loop call --- llarp/link/link_manager.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 655cefa50..706019c7e 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -114,24 +114,24 @@ namespace llarp void Endpoint::close_connection(RouterID _rid) { - assert(link_manager._router.loop()->inEventLoop()); - - // deletion from pending_conns, pending_conn_msg_queue, active_conns, etc is taken care - // of by LinkManager::on_conn_closed - if (auto itr = active_conns.find(_rid); itr != active_conns.end()) - { - auto& conn = *itr->second->conn; - conn.close_connection(); - } - else if (auto itr = pending_conns.find(_rid); itr != pending_conns.end()) - { - auto& conn = *itr->second->conn; - conn.close_connection(); - } - else - return; + // assert(link_manager._router.loop()->inEventLoop()); + link_manager._router.loop()->call([this, rid = _rid](){ + // deletion from pending_conns, pending_conn_msg_queue, active_conns, etc is taken care + // of by LinkManager::on_conn_closed + if (auto itr = active_conns.find(rid); itr != active_conns.end()) + { + auto& conn = *itr->second->conn; + conn.close_connection(); + } + else if (auto itr = pending_conns.find(rid); itr != pending_conns.end()) + { + auto& conn = *itr->second->conn; + conn.close_connection(); + } + else + return; + }); } - } // namespace link using messages::serialize_response; From 7417c59286182ad36ba778ede77dbca676e99326 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 15 Dec 2023 19:11:56 -0400 Subject: [PATCH 76/93] Get rid of IterDir It's a gross implementation, and even if it wasn't, using it takes more code than not using it. --- llarp/config/config.cpp | 30 ++++++++-------- llarp/link/link_manager.cpp | 3 -- llarp/nodedb.cpp | 70 +++++++------------------------------ llarp/nodedb.hpp | 2 +- llarp/util/file.hpp | 31 ++-------------- 5 files changed, 30 insertions(+), 106 deletions(-) diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index 9b094eae9..4dbc60fc7 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -1377,22 +1378,19 @@ namespace llarp const auto overridesDir = GetOverridesDir(data_dir); if (fs::exists(overridesDir)) { - util::IterDir(overridesDir, [&](const fs::path& overrideFile) { - if (overrideFile.extension() == ".ini") - { - ConfigParser parser; - if (not parser.load_file(overrideFile)) - throw std::runtime_error{"cannot load '" + overrideFile.u8string() + "'"}; - - parser.iter_all_sections([&](std::string_view section, const SectionValues& values) { - for (const auto& pair : values) - { - conf.add_config_value(section, pair.first, pair.second); - } - }); - } - return true; - }); + for (const auto& f : fs::directory_iterator{overridesDir}) + { + if (not f.is_regular_file() or f.path().extension() != ".ini") + continue; + ConfigParser parser; + if (not parser.load_file(f.path())) + throw std::runtime_error{"cannot load '" + f.path().u8string() + "'"}; + + parser.iter_all_sections([&](std::string_view section, const SectionValues& values) { + for (const auto& [k, v] : values) + conf.add_config_value(section, k, v); + }); + } } } diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 706019c7e..759225118 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -248,7 +248,6 @@ namespace llarp std::optional id) -> std::shared_ptr { if (id && *id == 0) { - log::critical(logcat, "Stream constructor constructing BTStream (ID:{})", id); auto s = e.make_shared( c, e, [](oxen::quic::Stream& s, uint64_t error_code) { log::warning( @@ -261,8 +260,6 @@ namespace llarp return s; } - log::critical(logcat, "Stream constructor constructing Stream (ID:{})!", id); - return e.make_shared(c, e); } */); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 4a7eb7fee..d2ddc81fb 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -10,7 +10,6 @@ #include #include -static const char skiplist_subdirs[] = "0123456789abcdef"; static const std::string RC_FILE_EXT = ".signed"; namespace llarp @@ -31,19 +30,6 @@ namespace llarp if (not fs::is_directory(nodedbDir)) throw std::runtime_error{fmt::format("nodedb {} is not a directory", nodedbDir)}; - - for (const char& ch : skiplist_subdirs) - { - // this seems to be a problem on all targets - // perhaps cpp17::fs is just as screwed-up - // attempting to create a folder with no name - // what does this mean...? - if (!ch) - continue; - - fs::path sub = nodedbDir / std::string(&ch, 1); - fs::create_directory(sub); - } } NodeDB::NodeDB(fs::path root, std::function)> diskCaller, Router* r) @@ -169,17 +155,9 @@ namespace llarp } fs::path - NodeDB::get_path_by_pubkey(RouterID pubkey) const + NodeDB::get_path_by_pubkey(const RouterID& pubkey) const { - std::string hexString = oxenc::to_hex(pubkey.begin(), pubkey.end()); - std::string skiplistDir; - - const llarp::RouterID r{pubkey}; - std::string fname = r.ToString(); - - skiplistDir += hexString[0]; - fname += RC_FILE_EXT; - return _root / skiplistDir / fname; + return _root / (pubkey.ToString() + RC_FILE_EXT); } bool @@ -875,48 +853,26 @@ namespace llarp if (_root.empty()) return; - std::set purge; + std::vector purge; const auto now = time_now_ms(); - for (const char& ch : skiplist_subdirs) + for (const auto& f : fs::directory_iterator{_root}) { - if (!ch) + if (not f.is_regular_file() or f.path().extension() != RC_FILE_EXT) continue; - std::string p; - p += ch; - fs::path sub = _root / p; - - llarp::util::IterDir(sub, [&](const fs::path& f) -> bool { - // skip files that are not suffixed with .signed - if (not(fs::is_regular_file(f) and f.extension() == RC_FILE_EXT)) - return true; - - RemoteRC rc{}; - - if (not rc.read(f)) - { - // try loading it, purge it if it is junk - purge.emplace(f); - return true; - } - - if (rc.is_expired(now)) - { - // rc expired dont load it and purge it later - purge.emplace(f); - return true; - } + RemoteRC rc{}; - const auto& rid = rc.router_id(); + if (not rc.read(f) or rc.is_expired(now)) + // try loading it, purge it if it is junk or expired + purge.push_back(f); - auto [itr, b] = known_rcs.insert(std::move(rc)); - rc_lookup.emplace(rid, *itr); - known_rids.insert(rid); + const auto& rid = rc.router_id(); - return true; - }); + auto [itr, b] = known_rcs.insert(std::move(rc)); + rc_lookup.emplace(rid, *itr); + known_rids.insert(rid); } if (not purge.empty()) diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 3f6b73e31..455fd7eb2 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -187,7 +187,7 @@ namespace llarp /// get filename of an RC file given its public ident key fs::path - get_path_by_pubkey(RouterID pk) const; + get_path_by_pubkey(const RouterID& pk) const; public: explicit NodeDB( diff --git a/llarp/util/file.hpp b/llarp/util/file.hpp index 39c140673..8629a9d2c 100644 --- a/llarp/util/file.hpp +++ b/llarp/util/file.hpp @@ -25,7 +25,7 @@ namespace llarp::util template < typename Char, std::enable_if_t, int> = 1> - inline size_t + size_t file_to_buffer(const fs::path& filename, Char* buffer, size_t buffer_size) { return file_to_buffer(filename, reinterpret_cast(buffer), buffer_size); @@ -38,7 +38,7 @@ namespace llarp::util /// Same as above, but works via char-like buffer template = 0> - inline void + void buffer_to_file(const fs::path& filename, const Char* buffer, size_t buffer_size) { return buffer_to_file( @@ -73,31 +73,4 @@ namespace llarp::util return std::make_optional(pathname, mode); } - template - static void - IterDir(const fs::path& path, PathVisitor visit) - { - DIR* d = opendir(path.string().c_str()); - if (d == nullptr) - return; - struct dirent* ent = nullptr; - std::set entries; - do - { - ent = readdir(d); - if (not ent) - break; - if (ent->d_name[0] == '.') - continue; - entries.emplace(path / fs::path{ent->d_name}); - } while (ent); - closedir(d); - - for (const auto& p : entries) - { - if (not visit(p)) - return; - } - } - } // namespace llarp::util From 47be3cefe10adf1c196c090668d6c87877c7e33b Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 15:12:59 -0800 Subject: [PATCH 77/93] lets see which gets rejected --- llarp/link/link_manager.cpp | 44 ++++++++++++------------------------- llarp/nodedb.cpp | 2 +- llarp/nodedb.hpp | 2 +- 3 files changed, 16 insertions(+), 32 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 759225118..5297e34b0 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -115,7 +115,7 @@ namespace llarp Endpoint::close_connection(RouterID _rid) { // assert(link_manager._router.loop()->inEventLoop()); - link_manager._router.loop()->call([this, rid = _rid](){ + link_manager._router.loop()->call([this, rid = _rid]() { // deletion from pending_conns, pending_conn_msg_queue, active_conns, etc is taken care // of by LinkManager::on_conn_closed if (auto itr = active_conns.find(rid); itr != active_conns.end()) @@ -233,35 +233,14 @@ namespace llarp logcat, "{} node was {} to confirm remote (RID:{}) is registered; allowing connection!", router().is_bootstrap_seed() ? "Bootstrap seed node" : "Service node", - other, - result ? "able" : "unable"); + result ? "able" : "unable", + other); return result; }); if (_router.is_service_node()) { - ep->listen( - tls_creds, - ROUTER_KEEP_ALIVE/* , - [&](oxen::quic::Connection& c, - oxen::quic::Endpoint& e, - std::optional id) -> std::shared_ptr { - if (id && *id == 0) - { - auto s = e.make_shared( - c, e, [](oxen::quic::Stream& s, uint64_t error_code) { - log::warning( - logcat, - "BTRequestStream closed unexpectedly (ec:{}); closing connection...", - error_code); - s.conn.close_connection(error_code); - }); - // register_commands(s); - return s; - } - - return e.make_shared(c, e); - } */); + ep->listen(tls_creds, ROUTER_KEEP_ALIVE); } return ep; } @@ -276,8 +255,9 @@ namespace llarp log::critical(logcat, "Queueing BTStream to be opened..."); - auto control_stream = ci.queue_stream([this, rid = rid](oxen::quic::Stream&, - uint64_t error_code) { + auto control_stream = ci.queue_stream([this, rid = rid]( + oxen::quic::Stream&, + uint64_t error_code) { log::warning( logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); ep.close_connection(rid); @@ -630,8 +610,12 @@ namespace llarp LinkManager::connect_to_random(int num_conns, bool client_only) { auto filter = [this, client_only](const RemoteRC& rc) -> bool { - return client_only ? not ep.have_client_conn(rc.router_id()) - : not ep.have_conn(rc.router_id()); + auto res = + client_only ? not ep.have_client_conn(rc.router_id()) : not ep.have_conn(rc.router_id()); + + log::critical(logcat, "RID:{} {}", rc.router_id(), res ? "ACCEPTED" : "REJECTED"); + + return res; }; if (auto maybe = node_db->get_n_random_rcs_conditional(num_conns, filter)) @@ -824,7 +808,7 @@ namespace llarp { // this handler should not be registered for service nodes assert(not _router.is_service_node()); - + send_control_message(source, "fetch_rcs", std::move(payload), std::move(func)); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index d2ddc81fb..39d9e8c05 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -992,7 +992,7 @@ namespace llarp bool NodeDB::verify_store_gossip_rc(const RemoteRC& rc) { - if (not router_whitelist.count(rc.router_id())) + if (not registered_routers().count(rc.router_id())) return put_rc_if_newer(rc); return false; diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 455fd7eb2..3bff7be1f 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -45,7 +45,7 @@ namespace llarp inline constexpr size_t MIN_GOOD_RID_FETCH_TOTAL{}; // the ratio of accepted:rejected rids must be above this ratio inline constexpr double GOOD_RID_FETCH_THRESHOLD{}; - + /* Bootstrap Constants */ // the number of rc's we query the bootstrap for; service nodes pass 0, which means // gimme all dat RCs From 6fdfb4cef61573f38b3543f5fc06c869ac0b955b Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Fri, 15 Dec 2023 19:50:32 -0400 Subject: [PATCH 78/93] Add `exact` argument to get_n_random_rcs If given and true then return nullopt if we don't find the requested number; otherwise return them even if there aren't as many as we requested. --- llarp/nodedb.cpp | 27 ++++++++++++++++----------- llarp/nodedb.hpp | 7 +++++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 39d9e8c05..91e8d4ed2 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -61,14 +61,16 @@ namespace llarp } std::optional> - NodeDB::get_n_random_rcs(size_t n) const + NodeDB::get_n_random_rcs(size_t n, bool exact) const { - std::vector rand{}; + auto rand = std::make_optional>(); + rand->reserve(n); - std::sample(known_rcs.begin(), known_rcs.end(), std::back_inserter(rand), n, csrng); - return rand.empty() ? std::nullopt : std::make_optional(rand); + std::sample(known_rcs.begin(), known_rcs.end(), std::back_inserter(*rand), n, csrng); + if (rand->size() < (exact ? n : 1)) + rand.reset(); + return rand; } - std::optional NodeDB::get_random_rc_conditional(std::function hook) const { @@ -99,10 +101,11 @@ namespace llarp } std::optional> - NodeDB::get_n_random_rcs_conditional(size_t n, std::function hook) const + NodeDB::get_n_random_rcs_conditional( + size_t n, std::function hook, bool exact) const { - std::vector selected; - selected.reserve(n); + auto selected = std::make_optional>(); + selected->reserve(n); size_t i = 0; @@ -115,17 +118,19 @@ namespace llarp // load the first n RC's that pass the condition into selected if (++i <= n) { - selected.push_back(rc); + selected->push_back(rc); continue; } // replace selections with decreasing probability per iteration size_t x = csrng() % (i + 1); if (x < n) - selected[x] = rc; + (*selected)[x] = rc; } - return selected.size() == n ? std::make_optional(selected) : std::nullopt; + if (selected->size() < (exact ? n : 1)) + selected.reset(); + return selected; } void diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 3bff7be1f..57a835915 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -429,8 +429,11 @@ namespace llarp std::optional get_random_rc() const; + // Get `n` random RCs from all RCs we know about. If `exact` is true then we require n matches + // (and otherwise return nullopt); otherwise we return whatever we found, or nullopt if we find + // nothing at all. std::optional> - get_n_random_rcs(size_t n) const; + get_n_random_rcs(size_t n, bool exact = false) const; /** The following random conditional functions utilize a simple implementation of reservoir sampling to return either 1 or n random RC's using only one pass through the set of RC's. @@ -447,7 +450,7 @@ namespace llarp get_random_rc_conditional(std::function hook) const; std::optional> - get_n_random_rcs_conditional(size_t n, std::function hook) const; + get_n_random_rcs_conditional(size_t n, std::function hook, bool exact = false) const; // Updates `current` to not contain any of the elements of `replace` and resamples (up to // `target_size`) from population to refill it. From 9e31300d0f0fedb3d024cc457dd63b3ecca9184e Mon Sep 17 00:00:00 2001 From: dr7ana Date: Fri, 15 Dec 2023 16:04:09 -0800 Subject: [PATCH 79/93] gossip storage, logs --- llarp/link/connection.cpp | 2 +- llarp/link/link_manager.cpp | 38 +++++++++++++++++++++++++++++++------ llarp/link/link_manager.hpp | 6 ++++++ llarp/nodedb.cpp | 17 +++++------------ llarp/nodedb.hpp | 6 ++---- llarp/router/router.cpp | 10 +++++++--- llarp/router/router.hpp | 2 +- 7 files changed, 54 insertions(+), 27 deletions(-) diff --git a/llarp/link/connection.cpp b/llarp/link/connection.cpp index d5fa0abf9..ca994e99c 100644 --- a/llarp/link/connection.cpp +++ b/llarp/link/connection.cpp @@ -5,7 +5,7 @@ namespace llarp::link Connection::Connection( const std::shared_ptr& c, std::shared_ptr& s) - : conn{c}, control_stream{s} /* , remote_rc{std::move(rc)} */ + : conn{c}, control_stream{s}, inbound{conn->is_inbound()} {} } // namespace llarp::link diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 5297e34b0..5343b1341 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -67,6 +67,22 @@ namespace llarp return active_conns.count(remote) or pending_conns.count(remote); } + std::pair + Endpoint::num_in_out() const + { + size_t in{0}, out{0}; + + for (const auto& c : active_conns) + { + if (c.second->inbound) + ++in; + else + ++out; + } + + return {in, out}; + } + size_t Endpoint::num_connected(bool clients_only) const { @@ -494,16 +510,17 @@ namespace llarp void LinkManager::connect_to(const RemoteRC& rc, conn_open_hook on_open, conn_closed_hook on_close) { - if (auto conn = ep.get_conn(rc.router_id()); conn) + const auto& rid = rc.router_id(); + + if (ep.have_conn(rid)) { - log::error(logcat, "We should not be here!"); + log::warning(logcat, "We already have a connection to {}!", rid); // TODO: should implement some connection failed logic, but not the same logic that // would be executed for another failure case return; } const auto& remote_addr = rc.addr(); - const auto& rid = rc.router_id(); // TODO: confirm remote end is using the expected pubkey (RouterID). // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) @@ -560,6 +577,12 @@ namespace llarp } } + std::pair + LinkManager::num_in_out() const + { + return ep.num_in_out(); + } + size_t LinkManager::get_num_connected(bool clients_only) const { @@ -613,7 +636,7 @@ namespace llarp auto res = client_only ? not ep.have_client_conn(rc.router_id()) : not ep.have_conn(rc.router_id()); - log::critical(logcat, "RID:{} {}", rc.router_id(), res ? "ACCEPTED" : "REJECTED"); + log::debug(logcat, "RID:{} {}", rc.router_id(), res ? "ACCEPTED" : "REJECTED"); return res; }; @@ -640,6 +663,8 @@ namespace llarp LinkManager::gossip_rc( const RouterID& gossip_src, const RouterID& last_sender, std::string serialized_rc) { + int count = 0; + for (auto& [rid, conn] : ep.active_conns) { // don't send back to the gossip source or the last sender @@ -650,8 +675,6 @@ namespace llarp if (not conn->remote_is_relay) continue; - log::critical(logcat, "Dispatching gossip_rc to {}", rid); - send_control_message( rid, "gossip_rc"s, @@ -659,7 +682,10 @@ namespace llarp [](oxen::quic::message) mutable { log::critical(logcat, "PLACEHOLDER FOR GOSSIP RC RESPONSE HANDLER"); }); + ++count; } + + log::critical(logcat, "Dispatched {} GossipRC requests!", count); } void diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 54d06eb1e..a24190956 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -72,6 +72,9 @@ namespace llarp bool have_conn(const RouterID& remote) const; + std::pair + num_in_out() const; + size_t num_connected(bool clients_only) const; @@ -291,6 +294,9 @@ namespace llarp void set_conn_persist(const RouterID& remote, llarp_time_t until); + std::pair + num_in_out() const; + size_t get_num_connected(bool clients_only = false) const; diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 91e8d4ed2..b459b91a9 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -915,15 +915,6 @@ namespace llarp return rc_lookup.count(pk); } - std::optional - NodeDB::get_rc(const RemoteRC& pk) const - { - if (auto itr = known_rcs.find(pk); itr != known_rcs.end()) - return *itr; - - return std::nullopt; - } - std::optional NodeDB::get_rc(const RouterID& pk) const { @@ -997,7 +988,7 @@ namespace llarp bool NodeDB::verify_store_gossip_rc(const RemoteRC& rc) { - if (not registered_routers().count(rc.router_id())) + if (registered_routers().count(rc.router_id())) return put_rc_if_newer(rc); return false; @@ -1006,13 +997,15 @@ namespace llarp bool NodeDB::put_rc_if_newer(RemoteRC rc) { - if (auto maybe = get_rc(rc)) + if (auto maybe = get_rc(rc.router_id())) { if (maybe->other_is_newer(rc)) return put_rc(rc); + + return false; } - return false; + return put_rc(rc); } void diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 57a835915..7b8a3b4ea 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -423,9 +423,6 @@ namespace llarp std::optional get_rc(const RouterID& pk) const; - std::optional - get_rc(const RemoteRC& pk) const; - std::optional get_random_rc() const; @@ -450,7 +447,8 @@ namespace llarp get_random_rc_conditional(std::function hook) const; std::optional> - get_n_random_rcs_conditional(size_t n, std::function hook, bool exact = false) const; + get_n_random_rcs_conditional( + size_t n, std::function hook, bool exact = false) const; // Updates `current` to not contain any of the elements of `replace` and resamples (up to // `target_size`) from population to refill it. diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index a530c6712..2d7fe5e66 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -750,14 +750,18 @@ namespace llarp if (is_service_node()) { + auto [in, out] = _link_manager->num_in_out(); + log::critical( logcat, - "Local Service Node has {} RCs, {} RIDs, {} bootstrap peers, {} router " + "Local Service Node has {} RCs, {} RIDs, {} bootstrap peers, {}:{} (inbound:outbound) " + "router " "connections, and {} client connections since last RC update ({} to expiry)", _node_db->num_rcs(), _node_db->num_rids(), _node_db->num_bootstraps(), - num_router_connections(), + in, + out, num_client_connections(), router_contact.time_to_expiry(now)); } @@ -874,7 +878,7 @@ namespace llarp last_rc_gossip = now_timepoint; - // TESTNET: 1 to 2 minutes before testnet gossip interval + // TESTNET: 1 to 4 minutes before testnet gossip interval auto random_delta = std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; // 1min to 5min before "stale time" is next gossip time diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 39d0c8693..9434d2ad4 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -49,7 +49,7 @@ namespace llarp (INTROSET_RELAY_REDUNDANCY * INTROSET_REQS_PER_RELAY)}; // TESTNET: these constants are shortened for testing purposes - inline constexpr std::chrono::milliseconds TESTNET_GOSSIP_INTERVAL{4min}; + inline constexpr std::chrono::milliseconds TESTNET_GOSSIP_INTERVAL{10min}; inline constexpr std::chrono::milliseconds RC_UPDATE_INTERVAL{4min}; // as we advance towards full mesh, we try to connect to this number per tick inline constexpr int FULL_MESH_ITERATION{1}; From ea3c3e30e0a5acbbfa0323a31d2c07c42b3825e7 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 18 Dec 2023 11:07:40 -0800 Subject: [PATCH 80/93] libquic vbump (stream redux, reauth updates, alpns->ustring) --- external/oxen-libquic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index b35cab3a3..9fe1aebbd 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit b35cab3a310cf358580ef9f418553f71436766ab +Subproject commit 9fe1aebbd8604c58c9b58098cfd07a233f80fabb From 3451a30d0eaf65c0e6c44a975bafba33afc6a1cc Mon Sep 17 00:00:00 2001 From: dr7ana Date: Mon, 18 Dec 2023 14:15:45 -0800 Subject: [PATCH 81/93] ALPN verification - laying the groundwork for functional client->service node connections. this requires ALPNs verification as a secondary method of identification to the remote key - refactored btreq stream creation to use improved stream creation logic in libquic --- llarp/exit/session.cpp | 4 +- llarp/link/link_manager.cpp | 153 ++++++++++++++++++----------- llarp/link/link_manager.hpp | 34 +++++-- llarp/messages/fetch.hpp | 5 +- llarp/nodedb.cpp | 39 ++++---- llarp/nodedb.hpp | 10 +- llarp/path/path.cpp | 52 +++++----- llarp/path/path_context.hpp | 2 +- llarp/path/pathbuilder.cpp | 32 +++--- llarp/path/pathset.cpp | 6 +- llarp/path/pathset.hpp | 12 +-- llarp/router/router.cpp | 3 +- llarp/service/endpoint.cpp | 4 +- llarp/service/outbound_context.cpp | 4 +- llarp/util/buffer.hpp | 5 + 15 files changed, 213 insertions(+), 152 deletions(-) diff --git a/llarp/exit/session.cpp b/llarp/exit/session.cpp index 82dd4d44b..5f1c6192e 100644 --- a/llarp/exit/session.cpp +++ b/llarp/exit/session.cpp @@ -214,7 +214,7 @@ namespace llarp::exit BaseSession::HandleTrafficDrop(llarp::path::Path_ptr p, const PathID_t& path, uint64_t s) { llarp::LogError("dropped traffic on exit ", exit_router, " S=", s, " P=", path); - p->EnterState(path::ePathIgnore, router->now()); + p->EnterState(path::IGNORE, router->now()); return true; } @@ -238,7 +238,7 @@ namespace llarp::exit { if (BuildCooldownHit(now)) return false; - if (IsReady() and NumInStatus(path::ePathBuilding) < numDesiredPaths) + if (IsReady() and NumInStatus(path::BUILDING) < numDesiredPaths) return path::Builder::UrgentBuild(now); return false; } diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 5343b1341..6aa905650 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -21,10 +21,16 @@ namespace llarp { namespace link { + Endpoint::Endpoint(std::shared_ptr ep, LinkManager& lm) + : endpoint{std::move(ep)} + , link_manager{lm} + , _is_service_node{link_manager.is_service_node()} + {} + std::shared_ptr Endpoint::get_conn(const RemoteRC& rc) const { - if (auto itr = active_conns.find(rc.router_id()); itr != active_conns.end()) + if (auto itr = service_conns.find(rc.router_id()); itr != service_conns.end()) return itr->second; // if (auto itr = pending_conns.find(rc.router_id()); itr != pending_conns.end()) @@ -36,7 +42,7 @@ namespace llarp std::shared_ptr Endpoint::get_conn(const RouterID& rid) const { - if (auto itr = active_conns.find(rid); itr != active_conns.end()) + if (auto itr = service_conns.find(rid); itr != service_conns.end()) return itr->second; // if (auto itr = pending_conns.find(rid); itr != pending_conns.end()) @@ -48,7 +54,7 @@ namespace llarp bool Endpoint::have_client_conn(const RouterID& remote) const { - if (auto itr = active_conns.find(remote); itr != active_conns.end()) + if (auto itr = service_conns.find(remote); itr != service_conns.end()) { return not itr->second->remote_is_relay; } @@ -64,7 +70,7 @@ namespace llarp bool Endpoint::have_conn(const RouterID& remote) const { - return active_conns.count(remote) or pending_conns.count(remote); + return service_conns.count(remote) or pending_conns.count(remote); } std::pair @@ -72,7 +78,7 @@ namespace llarp { size_t in{0}, out{0}; - for (const auto& c : active_conns) + for (const auto& c : service_conns) { if (c.second->inbound) ++in; @@ -88,7 +94,7 @@ namespace llarp { size_t count = 0; - for (const auto& c : active_conns) + for (const auto& c : service_conns) { if (not(c.second->remote_is_relay and clients_only)) count += 1; @@ -100,9 +106,9 @@ namespace llarp bool Endpoint::get_random_connection(RemoteRC& router) const { - if (const auto size = active_conns.size(); size) + if (const auto size = service_conns.size(); size) { - auto itr = active_conns.begin(); + auto itr = service_conns.begin(); std::advance(itr, randint() % size); RouterID rid{itr->second->conn->remote_key()}; @@ -123,7 +129,7 @@ namespace llarp void Endpoint::for_each_connection(std::function func) { - for (const auto& [rid, conn] : active_conns) + for (const auto& [rid, conn] : service_conns) func(*conn); } @@ -134,7 +140,7 @@ namespace llarp link_manager._router.loop()->call([this, rid = _rid]() { // deletion from pending_conns, pending_conn_msg_queue, active_conns, etc is taken care // of by LinkManager::on_conn_closed - if (auto itr = active_conns.find(rid); itr != active_conns.end()) + if (auto itr = service_conns.find(rid); itr != service_conns.end()) { auto& conn = *itr->second->conn; conn.close_connection(); @@ -195,7 +201,7 @@ namespace llarp _router.loop()->call([this, msg = std::move(m), func = std::move(func)]() mutable { auto body = msg.body_str(); auto respond = [m = std::move(msg)](std::string response) mutable { - m.respond(std::move(response), not m); + m.respond(std::move(response), m.is_error()); }; std::invoke(func, this, body, std::move(respond)); }); @@ -207,6 +213,7 @@ namespace llarp LinkManager::LinkManager(Router& r) : _router{r} + , _is_service_node{_router.is_service_node()} , quic{std::make_unique()} , tls_creds{oxen::quic::GNUTLSCreds::make_from_ed_keys( {reinterpret_cast(_router.identity().data()), size_t{32}}, @@ -239,20 +246,44 @@ namespace llarp [this](oxen::quic::connection_interface& ci, uint64_t ec) { return on_conn_closed(ci, ec); }, - [this](oxen::quic::dgram_interface& di, bstring dgram) { recv_data_message(di, dgram); }); - tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view&) { + [this](oxen::quic::dgram_interface& di, bstring dgram) { recv_data_message(di, dgram); }, + is_service_node() ? alpns::SERVICE_INBOUND : alpns::CLIENT_INBOUND, + is_service_node() ? alpns::SERVICE_OUTBOUND : alpns::CLIENT_OUTBOUND); + + tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view& alpn) { + auto is_snode = is_service_node(); + RouterID other{key.data()}; - bool result = node_db->registered_routers().count(other); + auto us = router().is_bootstrap_seed() ? "Bootstrap seed node"s : "Service node"s; - log::critical( - logcat, - "{} node was {} to confirm remote (RID:{}) is registered; allowing connection!", - router().is_bootstrap_seed() ? "Bootstrap seed node" : "Service node", - result ? "able" : "unable", - other); + if (is_snode) + { + if (alpn == alpns::C_ALPNS) + { + log::critical(logcat, "{} node accepting client connection (remote ID:{})!", us, other); + return true; + } + if (alpn == alpns::SN_ALPNS) + { + // verify as service node! + bool result = node_db->registered_routers().count(other); + + log::critical( + logcat, + "{} node was {} to confirm remote (RID:{}) is registered; allowing connection!", + us, + result ? "able" : "unable", + other); + + return result; + } - return result; + log::critical(logcat, "{} node received unknown ALPN; rejecting connection!", us); + return false; + } + + throw std::runtime_error{"Clients should not be validating inbound connections!"}; }); if (_router.is_service_node()) { @@ -266,18 +297,19 @@ namespace llarp { const auto& scid = ci.scid(); RouterID rid{ci.remote_key()}; - ep.connid_map.emplace(scid, rid); - auto [itr, b] = ep.active_conns.emplace(rid, nullptr); + ep.service_connid_map.emplace(scid, rid); + auto [itr, b] = ep.service_conns.emplace(rid, nullptr); log::critical(logcat, "Queueing BTStream to be opened..."); - auto control_stream = ci.queue_stream([this, rid = rid]( - oxen::quic::Stream&, - uint64_t error_code) { - log::warning( - logcat, "BTRequestStream closed unexpectedly (ec:{}); closing connection...", error_code); - ep.close_connection(rid); - }); + auto control_stream = ci.queue_incoming_stream( + [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { + log::warning( + logcat, + "BTRequestStream closed unexpectedly (ec:{}); closing connection...", + error_code); + ep.close_connection(rid); + }); log::critical(logcat, "Queued BTStream to be opened ID:{}", control_stream->stream_id()); assert(control_stream->stream_id() == 0); @@ -307,8 +339,8 @@ namespace llarp if (auto itr = ep.pending_conns.find(rid); itr != ep.pending_conns.end()) { - ep.connid_map.emplace(scid, rid); - auto [it, b] = ep.active_conns.emplace(rid, nullptr); + ep.service_connid_map.emplace(scid, rid); + auto [it, b] = ep.service_conns.emplace(rid, nullptr); it->second = std::move(itr->second); ep.pending_conns.erase(itr); @@ -338,7 +370,7 @@ namespace llarp if (msg.is_control) { log::critical(logcat, "Dispatching {} request!", *msg.endpoint); - ep.active_conns[rid]->control_stream->command( + ep.service_conns[rid]->control_stream->command( std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); } else @@ -373,15 +405,16 @@ namespace llarp log::critical(quic_cat, "Pending quic connection CID:{} purged successfully", scid); } - else if (const auto& c_itr = ep.connid_map.find(scid); c_itr != ep.connid_map.end()) + else if (const auto& c_itr = ep.service_connid_map.find(scid); + c_itr != ep.service_connid_map.end()) { const auto& rid = c_itr->second; assert(_rid == rid); // this should hold true - if (auto m_itr = ep.active_conns.find(rid); m_itr != ep.active_conns.end()) - ep.active_conns.erase(m_itr); + if (auto m_itr = ep.service_conns.find(rid); m_itr != ep.service_conns.end()) + ep.service_conns.erase(m_itr); - ep.connid_map.erase(c_itr); + ep.service_connid_map.erase(c_itr); log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); } @@ -604,7 +637,7 @@ namespace llarp bool LinkManager::is_service_node() const { - return _router.is_service_node(); + return _is_service_node; } // TODO: this? perhaps no longer necessary in the same way? @@ -660,12 +693,12 @@ namespace llarp } void - LinkManager::gossip_rc( - const RouterID& gossip_src, const RouterID& last_sender, std::string serialized_rc) + LinkManager::gossip_rc(const RouterID& last_sender, const RemoteRC& rc) { int count = 0; + const auto& gossip_src = rc.router_id(); - for (auto& [rid, conn] : ep.active_conns) + for (auto& [rid, conn] : ep.service_conns) { // don't send back to the gossip source or the last sender if (rid == gossip_src or rid == last_sender) @@ -678,7 +711,7 @@ namespace llarp send_control_message( rid, "gossip_rc"s, - GossipRCMessage::serialize(gossip_src, last_sender, serialized_rc), + GossipRCMessage::serialize(last_sender, rc), [](oxen::quic::message) mutable { log::critical(logcat, "PLACEHOLDER FOR GOSSIP RC RESPONSE HANDLER"); }); @@ -704,7 +737,6 @@ namespace llarp btdc.required("rc"); rc = RemoteRC{btdc.consume_dict_data()}; src.from_string(btdc.require("sender")); - sender.from_string(btdc.require("src")); } catch (const std::exception& e) { @@ -715,7 +747,7 @@ namespace llarp if (node_db->verify_store_gossip_rc(rc)) { log::critical(link_cat, "Received updated RC, forwarding to relay peers."); - gossip_rc(src, _router.local_rid(), std::string{rc.view()}); + gossip_rc(_router.local_rid(), rc); } else log::critical(link_cat, "Received known or old RC, not storing or forwarding."); @@ -934,7 +966,7 @@ namespace llarp "fetch_router_ids"s, m.body_str(), [source_rid = std::move(source), original = std::move(m)](oxen::quic::message m) mutable { - original.respond(m.body_str(), not m); + original.respond(m.body_str(), m.is_error()); }); return; } @@ -994,9 +1026,9 @@ namespace llarp void LinkManager::handle_find_name_response(oxen::quic::message m) { - if (not m) + if (m.timed_out) { - log::info(link_cat, "FindNameMessage failed!"); + log::info(link_cat, "FindNameMessage request timed out!"); return; } @@ -1130,7 +1162,7 @@ namespace llarp "publish_intro", PublishIntroMessage::serialize(introset, relay_order, is_relayed), [respond = std::move(respond)](oxen::quic::message m) { - if (not m) + if (m.timed_out) return; // drop if timed out; requester will have timed out as well respond(m.body_str()); }); @@ -1165,10 +1197,13 @@ namespace llarp addr); } + // DISCUSS: I feel like ::handle_publish_intro_response should be the callback that handles the + // response to a relayed publish_intro (above line 1131-ish) + void LinkManager::handle_publish_intro_response(oxen::quic::message m) { - if (not m) + if (m.timed_out) { log::info(link_cat, "PublishIntroMessage timed out!"); return; @@ -1271,7 +1306,7 @@ namespace llarp link_cat, "Relayed FindIntroMessage returned successful response; transmitting to initial " "requester"); - else if (not relay_response) + else if (relay_response.timed_out) log::critical( link_cat, "Relayed FindIntroMessage timed out! Notifying initial requester"); else @@ -1298,7 +1333,7 @@ namespace llarp void LinkManager::handle_find_intro_response(oxen::quic::message m) { - if (not m) + if (m.timed_out) { log::info(link_cat, "FindIntroMessage timed out!"); return; @@ -1491,12 +1526,12 @@ namespace llarp "then relaying response"); _router.path_context().put_transit_hop(hop); } - if (not m) + if (m.timed_out) log::info(link_cat, "Upstream timed out on path build; relaying timeout"); else log::info(link_cat, "Upstream returned path build failure; relaying response"); - m.respond(m.body_str(), not m); + m.respond(m.body_str(), m.is_error()); }); } catch (const std::exception& e) @@ -1608,12 +1643,12 @@ namespace llarp void LinkManager::handle_obtain_exit_response(oxen::quic::message m) { - if (not m) + if (m.timed_out) { log::info(link_cat, "ObtainExitMessage timed out!"); return; } - if (m.is_error) + if (m.is_error()) { // TODO: what to do here } @@ -1686,12 +1721,12 @@ namespace llarp void LinkManager::handle_update_exit_response(oxen::quic::message m) { - if (not m) + if (m.timed_out) { log::info(link_cat, "UpdateExitMessage timed out!"); return; } - if (m.is_error) + if (m.is_error()) { // TODO: what to do here } @@ -1771,12 +1806,12 @@ namespace llarp void LinkManager::handle_close_exit_response(oxen::quic::message m) { - if (not m) + if (m.timed_out) { log::info(link_cat, "CloseExitMessage timed out!"); return; } - if (m.is_error) + if (m.is_error()) { // TODO: what to do here } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index a24190956..30232ca3f 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -34,27 +34,42 @@ namespace llarp using stream_closed_hook = oxen::quic::stream_close_callback; using keep_alive = oxen::quic::opt::keep_alive; + using inbound_alpns = oxen::quic::opt::inbound_alpns; + using outbound_alpns = oxen::quic::opt::outbound_alpns; inline const keep_alive ROUTER_KEEP_ALIVE{10s}; inline const keep_alive CLIENT_KEEP_ALIVE{0s}; + namespace alpns + { + inline const auto SN_ALPNS = "SERVICE_NODE"_us; + inline const auto C_ALPNS = "CLIENT"_us; + + inline const inbound_alpns SERVICE_INBOUND{{SN_ALPNS, C_ALPNS}}; + inline const outbound_alpns SERVICE_OUTBOUND{{SN_ALPNS}}; + + inline const inbound_alpns CLIENT_INBOUND{}; + inline const outbound_alpns CLIENT_OUTBOUND{{C_ALPNS}}; + } // namespace alpns + namespace link { struct Connection; struct Endpoint { - Endpoint(std::shared_ptr ep, LinkManager& lm) - : endpoint{std::move(ep)}, link_manager{lm} - {} + Endpoint(std::shared_ptr ep, LinkManager& lm); std::shared_ptr endpoint; LinkManager& link_manager; // for outgoing packets, we route via RouterID; map RouterID->Connection // for incoming packets, we get a ConnectionID; map ConnectionID->RouterID - std::unordered_map> active_conns; - std::unordered_map connid_map; + std::unordered_map> service_conns; + std::unordered_map service_connid_map; + + std::unordered_map> client_conns; + std::unordered_map client_connid_map; // for pending connections, cleared in LinkManager::on_conn_open std::unordered_map> pending_conns; @@ -93,6 +108,7 @@ namespace llarp close_connection(RouterID rid); private: + const bool _is_service_node; }; } // namespace link @@ -198,6 +214,8 @@ namespace llarp Router& _router; + const bool _is_service_node; + // FIXME: Lokinet currently expects to be able to kill all network functionality before // finishing other shutdown things, including destroying this class, and that is all in // Network's destructor, so we need to be able to destroy it before this class. @@ -228,7 +246,7 @@ namespace llarp public: const link::Endpoint& - endpoint() + endpoint() const { return ep; } @@ -240,7 +258,7 @@ namespace llarp } void - gossip_rc(const RouterID& gossip_src, const RouterID& last_sender, std::string serialized_rc); + gossip_rc(const RouterID& last_sender, const RemoteRC& rc); void handle_gossip_rc(oxen::quic::message m); @@ -429,7 +447,7 @@ namespace llarp // add to pending conns auto [itr, b] = pending_conns.emplace(rid, nullptr); - auto control_stream = conn_interface->template get_new_stream( + auto control_stream = conn_interface->template open_stream( [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { log::warning( logcat, diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index af804f161..95a6018aa 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -9,15 +9,14 @@ namespace llarp namespace GossipRCMessage { inline static std::string - serialize(const RouterID& gossip_src, const RouterID& last_sender, std::string rc) + serialize(const RouterID& last_sender, const RemoteRC& rc) { oxenc::bt_dict_producer btdp; try { - btdp.append_encoded("rc", rc); + btdp.append_encoded("rc", rc.view()); btdp.append("sender", last_sender.ToView()); - btdp.append("src", gossip_src.ToView()); } catch (...) { diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index b459b91a9..d15be5b1c 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -369,21 +369,21 @@ namespace llarp src, FetchRCMessage::serialize(_router.last_rc_fetch, needed), [this, src, initial](oxen::quic::message m) mutable { - if (not m) + if (m.timed_out) { - log::info(logcat, "RC fetch to {} failed!", src); - fetch_rcs_result(initial, true); + log::info(logcat, "RC fetch to {} timed out!", src); + fetch_rcs_result(initial, m.timed_out); return; } try { oxenc::bt_dict_consumer btdc{m.body()}; - // TODO: fix this shit after removing ::timed_out from message type - if (not m) + // TODO: can this just combine with the above failure case...? + if (m.is_error()) { auto reason = btdc.require(messages::STATUS_KEY); log::info(logcat, "RC fetch to {} returned error: {}", src, reason); - fetch_rcs_result(initial, true); + fetch_rcs_result(initial, m.is_error()); return; } @@ -438,9 +438,11 @@ namespace llarp src, FetchRIDMessage::serialize(target), [this, src, target, initial](oxen::quic::message m) mutable { - if (not m) + if (m.is_error()) { - log::info(link_cat, "RID fetch from {} via {} timed out", src, target); + auto err = "RID fetch from {} via {} {}"_format( + src, target, m.timed_out ? "timed out" : "failed"); + log::info(link_cat, err); ingest_rid_fetch_responses(target); fetch_rids_result(initial); return; @@ -796,15 +798,18 @@ namespace llarp _registered_routers.insert(greylist.begin(), greylist.end()); _registered_routers.insert(greenlist.begin(), greenlist.end()); - router_whitelist.clear(); - router_whitelist.insert(whitelist.begin(), whitelist.end()); - router_greylist.clear(); - router_greylist.insert(greylist.begin(), greylist.end()); - router_greenlist.clear(); - router_greenlist.insert(greenlist.begin(), greenlist.end()); + _router_whitelist.clear(); + _router_whitelist.insert(whitelist.begin(), whitelist.end()); + _router_greylist.clear(); + _router_greylist.insert(greylist.begin(), greylist.end()); + _router_greenlist.clear(); + _router_greenlist.insert(greenlist.begin(), greenlist.end()); log::critical( - logcat, "Service node whitelist now has {} active router RIDs", router_whitelist.size()); + logcat, + "Oxend provided {}:{} (whitelist:registered)", + _router_whitelist.size(), + _registered_routers.size()); } std::optional @@ -812,7 +817,7 @@ namespace llarp { std::optional rand = std::nullopt; - std::sample(router_whitelist.begin(), router_whitelist.end(), &*rand, 1, csrng); + std::sample(_router_whitelist.begin(), _router_whitelist.end(), &*rand, 1, csrng); return rand; } @@ -826,7 +831,7 @@ namespace llarp return false; } - return known_rids.count(remote) or router_greylist.count(remote); + return known_rids.count(remote) or _router_greylist.count(remote); } bool diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 7b8a3b4ea..9ff0e73a3 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -146,9 +146,9 @@ namespace llarp - gray: fully funded, but decommissioned routers - green: registered, but not fully-staked routers */ - std::set router_whitelist{}; - std::set router_greylist{}; - std::set router_greenlist{}; + std::set _router_whitelist{}; + std::set _router_greylist{}; + std::set _router_greenlist{}; // All registered relays (service nodes) std::set _registered_routers; @@ -346,13 +346,13 @@ namespace llarp const std::set& whitelist() const { - return known_rids; + return _router_whitelist; } const std::set& greylist() const { - return router_greylist; + return _router_greylist; } std::set& diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp index 955716089..8fd894d86 100644 --- a/llarp/path/path.cpp +++ b/llarp/path/path.cpp @@ -44,7 +44,7 @@ namespace llarp::path intro.router = hops[hsz - 1].rc.router_id(); intro.path_id = hops[hsz - 1].txID; if (auto parent = m_PathSet.lock()) - EnterState(ePathBuilding, parent->Now()); + EnterState(BUILDING, parent->Now()); } bool @@ -118,7 +118,7 @@ namespace llarp::path if ((not self) or (not response_cb)) return; - if (not m) + if (m.timed_out) { response_cb(messages::TIMEOUT_RESPONSE); return; @@ -186,7 +186,7 @@ namespace llarp::path { if (Expired(llarp::time_now_ms())) return false; - return intro.latency > 0s && _status == ePathEstablished; + return intro.latency > 0s && _status == ESTABLISHED; } bool @@ -227,12 +227,12 @@ namespace llarp::path if (now == 0s) now = router.now(); - if (st == ePathFailed) + if (st == FAILED) { _status = st; return; } - if (st == ePathExpired && _status == ePathBuilding) + if (st == EXPIRED && _status == BUILDING) { _status = st; if (auto parent = m_PathSet.lock()) @@ -240,16 +240,16 @@ namespace llarp::path parent->HandlePathBuildTimeout(shared_from_this()); } } - else if (st == ePathBuilding) + else if (st == BUILDING) { LogInfo("path ", name(), " is building"); buildStarted = now; } - else if (st == ePathEstablished && _status == ePathBuilding) + else if (st == ESTABLISHED && _status == BUILDING) { LogInfo("path ", name(), " is built, took ", ToString(now - buildStarted)); } - else if (st == ePathTimeout && _status == ePathEstablished) + else if (st == TIMEOUT && _status == ESTABLISHED) { LogInfo("path ", name(), " died"); _status = st; @@ -258,11 +258,11 @@ namespace llarp::path parent->HandlePathDied(shared_from_this()); } } - else if (st == ePathEstablished && _status == ePathTimeout) + else if (st == ESTABLISHED && _status == TIMEOUT) { LogInfo("path ", name(), " reanimated"); } - else if (st == ePathIgnore) + else if (st == IGNORE) { LogInfo("path ", name(), " ignored"); } @@ -309,22 +309,22 @@ namespace llarp::path switch (_status) { - case ePathBuilding: + case BUILDING: obj["status"] = "building"; break; - case ePathEstablished: + case ESTABLISHED: obj["status"] = "established"; break; - case ePathTimeout: + case TIMEOUT: obj["status"] = "timeout"; break; - case ePathExpired: + case EXPIRED: obj["status"] = "expired"; break; - case ePathFailed: + case FAILED: obj["status"] = "failed"; break; - case ePathIgnore: + case IGNORE: obj["status"] = "ignored"; break; default: @@ -385,7 +385,7 @@ namespace llarp::path m_RXRate = 0; m_TXRate = 0; - if (_status == ePathBuilding) + if (_status == BUILDING) { if (buildStarted == 0s) return; @@ -396,13 +396,13 @@ namespace llarp::path { LogWarn(name(), " waited for ", ToString(dlt), " and no path was built"); r->router_profiling().MarkPathFail(this); - EnterState(ePathExpired, now); + EnterState(EXPIRED, now); return; } } } // check to see if this path is dead - if (_status == ePathEstablished) + if (_status == ESTABLISHED) { auto dlt = now - m_LastLatencyTestTime; if (dlt > path::LATENCY_INTERVAL && m_LastLatencyTestID == 0) @@ -420,13 +420,13 @@ namespace llarp::path { LogWarn(name(), " waited for ", ToString(dlt), " and path looks dead"); r->router_profiling().MarkPathFail(this); - EnterState(ePathTimeout, now); + EnterState(TIMEOUT, now); } } - if (_status == ePathIgnore and now - m_LastRecvMessage >= path::ALIVE_TIMEOUT) + if (_status == IGNORE and now - m_LastRecvMessage >= path::ALIVE_TIMEOUT) { // clean up this path as we dont use it anymore - EnterState(ePathExpired, now); + EnterState(EXPIRED, now); } } @@ -436,15 +436,15 @@ namespace llarp::path bool Path::Expired(llarp_time_t now) const { - if (_status == ePathFailed) + if (_status == FAILED) return true; - if (_status == ePathBuilding) + if (_status == BUILDING) return false; - if (_status == ePathTimeout) + if (_status == TIMEOUT) { return now >= m_LastRecvMessage + PathReanimationTimeout; } - if (_status == ePathEstablished or _status == ePathIgnore) + if (_status == ESTABLISHED or _status == IGNORE) { return now >= ExpireTime(); } diff --git a/llarp/path/path_context.hpp b/llarp/path/path_context.hpp index 676831d21..87524951e 100644 --- a/llarp/path/path_context.hpp +++ b/llarp/path/path_context.hpp @@ -122,7 +122,7 @@ namespace llarp::path /// current number of paths we created in status uint64_t - CurrentOwnedPaths(path::PathStatus status = path::PathStatus::ePathEstablished); + CurrentOwnedPaths(path::PathStatus status = path::PathStatus::ESTABLISHED); Router* router() const diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index d0b07bf4c..6eb1cc3b7 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -261,7 +261,7 @@ namespace llarp const auto now = Now(); for (auto& item : m_Paths) { - item.second->EnterState(ePathIgnore, now); + item.second->EnterState(IGNORE, now); } return true; } @@ -275,7 +275,7 @@ namespace llarp bool Builder::ShouldRemove() const { - return IsStopped() and NumInStatus(ePathEstablished) == 0; + return IsStopped() and NumInStatus(ESTABLISHED) == 0; } bool @@ -505,40 +505,40 @@ namespace llarp // be worth doing sooner rather than later. Leaving some TODOs below where fail // and success live. auto response_cb = [path](oxen::quic::message m) { + if (m) + { + // TODO: inform success (what this means needs revisiting, badly) + path->EnterState(path::ESTABLISHED); + return; + } + try { - if (m) + // TODO: inform failure (what this means needs revisiting, badly) + if (m.timed_out) { - // TODO: inform success (what this means needs revisiting, badly) - path->EnterState(path::ePathEstablished); - return; - } - if (not m) - { - log::warning(path_cat, "Path build request failed!"); + log::warning(path_cat, "Path build request timed out!"); + path->EnterState(path::TIMEOUT); } else { oxenc::bt_dict_consumer d{m.body()}; auto status = d.require(messages::STATUS_KEY); log::warning(path_cat, "Path build returned failure status: {}", status); + path->EnterState(path::FAILED); } } catch (const std::exception& e) { - log::warning(path_cat, "Failed parsing path build response."); + log::warning(path_cat, "Exception caught parsing path build response: {}", e.what()); } - - // TODO: inform failure (what this means needs revisiting, badly) - path->EnterState(path::ePathFailed); }; if (not router->send_control_message( path->upstream(), "path_build", std::move(frames).str(), std::move(response_cb))) { log::warning(path_cat, "Error sending path_build control message"); - // TODO: inform failure (what this means needs revisiting, badly) - path->EnterState(path::ePathFailed, router->now()); + path->EnterState(path::FAILED, router->now()); } } diff --git a/llarp/path/pathset.cpp b/llarp/path/pathset.cpp index 1b1012448..fc24acb51 100644 --- a/llarp/path/pathset.cpp +++ b/llarp/path/pathset.cpp @@ -13,10 +13,10 @@ namespace llarp::path PathSet::ShouldBuildMore(llarp_time_t now) const { (void)now; - const auto building = NumInStatus(ePathBuilding); + const auto building = NumInStatus(BUILDING); if (building >= numDesiredPaths) return false; - const auto established = NumInStatus(ePathEstablished); + const auto established = NumInStatus(ESTABLISHED); return established < numDesiredPaths; } @@ -255,7 +255,7 @@ namespace llarp::path auto itr = m_Paths.begin(); while (itr != m_Paths.end()) { - if (itr->second->Status() == ePathEstablished && itr->second->SupportsAnyRoles(roles)) + if (itr->second->Status() == ESTABLISHED && itr->second->SupportsAnyRoles(roles)) ++count; ++itr; } diff --git a/llarp/path/pathset.hpp b/llarp/path/pathset.hpp index c228823ac..a93608f43 100644 --- a/llarp/path/pathset.hpp +++ b/llarp/path/pathset.hpp @@ -47,12 +47,12 @@ namespace llarp /// status of a path enum PathStatus { - ePathBuilding, - ePathEstablished, - ePathTimeout, - ePathFailed, - ePathIgnore, - ePathExpired + BUILDING, + ESTABLISHED, + TIMEOUT, + FAILED, + IGNORE, + EXPIRED }; /// Stats about all our path builds diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2d7fe5e66..c3441594a 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -873,8 +873,7 @@ namespace llarp router_contact.resign(); save_rc(); - _link_manager->gossip_rc( - local_rid(), local_rid(), std::string{oxen::quic::to_sv(router_contact.view())}); + _link_manager->gossip_rc(local_rid(), router_contact.to_remote()); last_rc_gossip = now_timepoint; diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 8c3264fe3..a9eebe4c9 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -299,7 +299,7 @@ namespace llarp::service // expire convotags EndpointUtil::ExpireConvoSessions(now, Sessions()); - if (NumInStatus(path::ePathEstablished) > 1) + if (NumInStatus(path::ESTABLISHED) > 1) { for (const auto& item : _startup_ons_mappings) { @@ -1539,7 +1539,7 @@ namespace llarp::service if (BuildCooldownHit(now)) return false; const auto requiredPaths = std::max(numDesiredPaths, path::MIN_INTRO_PATHS); - if (NumInStatus(path::ePathBuilding) >= requiredPaths) + if (NumInStatus(path::BUILDING) >= requiredPaths) return false; return NumPathsExistingAt(now + (path::DEFAULT_LIFETIME - path::INTRO_PATH_SPREAD)) < requiredPaths; diff --git a/llarp/service/outbound_context.cpp b/llarp/service/outbound_context.cpp index fe0218416..c025168b5 100644 --- a/llarp/service/outbound_context.cpp +++ b/llarp/service/outbound_context.cpp @@ -143,7 +143,7 @@ namespace llarp::service { // ignore new path if we are marked dead LogInfo(Name(), " marked bad, ignoring new path"); - p->EnterState(path::ePathIgnore, Now()); + p->EnterState(path::IGNORE, Now()); } else if (p->Endpoint() == next_intro.router) { @@ -375,7 +375,7 @@ namespace llarp::service if (marked_bad or path::Builder::BuildCooldownHit(now)) return false; - if (NumInStatus(path::ePathBuilding) >= std::max(numDesiredPaths / size_t{2}, size_t{1})) + if (NumInStatus(path::BUILDING) >= std::max(numDesiredPaths / size_t{2}, size_t{1})) return false; size_t numValidPaths = 0; diff --git a/llarp/util/buffer.hpp b/llarp/util/buffer.hpp index 2cfa8f3a6..ca688f127 100644 --- a/llarp/util/buffer.hpp +++ b/llarp/util/buffer.hpp @@ -24,6 +24,11 @@ namespace llarp using bstring = std::basic_string; using bstring_view = std::basic_string_view; + inline ustring operator""_us(const char* str, size_t len) noexcept + { + return {reinterpret_cast(str), len}; + } + // Helper function to switch between string_view and ustring_view inline ustring_view to_usv(std::string_view v) From 4437d0b3739a44ccb5a19adf7cf6c21e31de369a Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 19 Dec 2023 03:31:26 -0800 Subject: [PATCH 82/93] re-abstraction for client connections - pending_conns removed in favor of direct creation of link::Connection objects in link::Endpoint::{service,client}_conn containers - conn lookup maps removed, they were pointless --- llarp/link/connection.cpp | 7 +- llarp/link/connection.hpp | 13 +- llarp/link/link_manager.cpp | 332 ++++++++++++++++++------------------ llarp/link/link_manager.hpp | 104 ++++------- llarp/nodedb.cpp | 13 +- llarp/nodedb.hpp | 4 +- llarp/router/router.cpp | 57 ++----- llarp/router/router.hpp | 6 - 8 files changed, 240 insertions(+), 296 deletions(-) diff --git a/llarp/link/connection.cpp b/llarp/link/connection.cpp index ca994e99c..27e3935a2 100644 --- a/llarp/link/connection.cpp +++ b/llarp/link/connection.cpp @@ -3,9 +3,10 @@ namespace llarp::link { Connection::Connection( - const std::shared_ptr& c, - std::shared_ptr& s) - : conn{c}, control_stream{s}, inbound{conn->is_inbound()} + std::shared_ptr c, + std::shared_ptr s, + bool is_relay) + : conn{std::move(c)}, control_stream{std::move(s)}, remote_is_relay{is_relay} {} } // namespace llarp::link diff --git a/llarp/link/connection.hpp b/llarp/link/connection.hpp index fb194b714..097303432 100644 --- a/llarp/link/connection.hpp +++ b/llarp/link/connection.hpp @@ -12,13 +12,18 @@ namespace llarp::link std::shared_ptr conn; std::shared_ptr control_stream; - // one side of a connection will be responsible for some things, e.g. heartbeat - bool inbound{false}; bool remote_is_relay{true}; + bool + is_inbound() const + { + return conn->is_inbound(); + } + Connection( - const std::shared_ptr& c, - std::shared_ptr& s); + std::shared_ptr c, + std::shared_ptr s, + bool is_relay = true); }; } // namespace llarp::link diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 6aa905650..82ff615b0 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -28,14 +28,11 @@ namespace llarp {} std::shared_ptr - Endpoint::get_conn(const RemoteRC& rc) const + Endpoint::get_service_conn(const RouterID& rid) const { - if (auto itr = service_conns.find(rc.router_id()); itr != service_conns.end()) + if (auto itr = service_conns.find(rid); itr != service_conns.end()) return itr->second; - // if (auto itr = pending_conns.find(rc.router_id()); itr != pending_conns.end()) - // return itr->second; - return nullptr; } @@ -45,32 +42,31 @@ namespace llarp if (auto itr = service_conns.find(rid); itr != service_conns.end()) return itr->second; - // if (auto itr = pending_conns.find(rid); itr != pending_conns.end()) - // return itr->second; + if (_is_service_node) + { + if (auto itr = client_conns.find(rid); itr != client_conns.end()) + return itr->second; + } return nullptr; } bool - Endpoint::have_client_conn(const RouterID& remote) const + Endpoint::have_conn(const RouterID& remote) const { - if (auto itr = service_conns.find(remote); itr != service_conns.end()) - { - return not itr->second->remote_is_relay; - } - - if (auto itr = pending_conns.find(remote); itr != pending_conns.end()) - { - return not itr->second->remote_is_relay; - } + return have_service_conn(remote) or have_client_conn(remote); + } - return false; + bool + Endpoint::have_client_conn(const RouterID& remote) const + { + return client_conns.count(remote); } bool - Endpoint::have_conn(const RouterID& remote) const + Endpoint::have_service_conn(const RouterID& remote) const { - return service_conns.count(remote) or pending_conns.count(remote); + return service_conns.count(remote); } std::pair @@ -80,7 +76,15 @@ namespace llarp for (const auto& c : service_conns) { - if (c.second->inbound) + if (c.second->is_inbound()) + ++in; + else + ++out; + } + + for (const auto& c : client_conns) + { + if (c.second->is_inbound()) ++in; else ++out; @@ -92,38 +96,7 @@ namespace llarp size_t Endpoint::num_connected(bool clients_only) const { - size_t count = 0; - - for (const auto& c : service_conns) - { - if (not(c.second->remote_is_relay and clients_only)) - count += 1; - } - - return count; - } - - bool - Endpoint::get_random_connection(RemoteRC& router) const - { - if (const auto size = service_conns.size(); size) - { - auto itr = service_conns.begin(); - std::advance(itr, randint() % size); - - RouterID rid{itr->second->conn->remote_key()}; - - if (auto maybe = link_manager.node_db->get_rc(rid)) - { - router = *maybe; - return true; - } - - return false; - } - - log::warning(quic_cat, "Error: failed to fetch random connection"); - return false; + return clients_only ? client_conns.size() : client_conns.size() + service_conns.size(); } void @@ -131,27 +104,35 @@ namespace llarp { for (const auto& [rid, conn] : service_conns) func(*conn); + + if (_is_service_node) + { + for (const auto& [rid, conn] : client_conns) + func(*conn); + } } void Endpoint::close_connection(RouterID _rid) { - // assert(link_manager._router.loop()->inEventLoop()); link_manager._router.loop()->call([this, rid = _rid]() { - // deletion from pending_conns, pending_conn_msg_queue, active_conns, etc is taken care - // of by LinkManager::on_conn_closed if (auto itr = service_conns.find(rid); itr != service_conns.end()) { + log::critical(logcat, "Closing connection to relay RID:{}", rid); auto& conn = *itr->second->conn; conn.close_connection(); } - else if (auto itr = pending_conns.find(rid); itr != pending_conns.end()) + else if (_is_service_node) { - auto& conn = *itr->second->conn; - conn.close_connection(); + if (auto itr = client_conns.find(rid); itr != client_conns.end()) + { + log::critical(logcat, "Closing connection to client RID:{}", rid); + auto& conn = *itr->second->conn; + conn.close_connection(); + } } else - return; + log::critical(logcat, "Could not find connection to RID:{} to close!", rid); }); } } // namespace link @@ -169,9 +150,9 @@ namespace llarp void LinkManager::register_commands( - std::shared_ptr& s, const RouterID& router_id) + std::shared_ptr& s, const RouterID& router_id, bool) { - log::critical(logcat, "{} called", __PRETTY_FUNCTION__); + log::debug(logcat, "{} called", __PRETTY_FUNCTION__); s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { _router.loop()->call( @@ -240,7 +221,7 @@ namespace llarp - bt stream construction contains a stream close callback that shuts down the connection if the btstream closes unexpectedly */ - auto ep = quic->endpoint( + auto e = quic->endpoint( _router.listen_addr(), [this](oxen::quic::connection_interface& ci) { return on_conn_open(ci); }, [this](oxen::quic::connection_interface& ci, uint64_t ec) { @@ -250,18 +231,20 @@ namespace llarp is_service_node() ? alpns::SERVICE_INBOUND : alpns::CLIENT_INBOUND, is_service_node() ? alpns::SERVICE_OUTBOUND : alpns::CLIENT_OUTBOUND); + // While only service nodes accept inbound connections, clients must have this key verify + // callback set. It will reject any attempted inbound connection to a lokinet client prior to + // handshake completion tls_creds->set_key_verify_callback([this](const ustring_view& key, const ustring_view& alpn) { - auto is_snode = is_service_node(); - RouterID other{key.data()}; - auto us = router().is_bootstrap_seed() ? "Bootstrap seed node"s : "Service node"s; + auto is_snode = is_service_node(); if (is_snode) { if (alpn == alpns::C_ALPNS) { log::critical(logcat, "{} node accepting client connection (remote ID:{})!", us, other); + ep.client_conns.emplace(other, nullptr); return true; } if (alpn == alpns::SN_ALPNS) @@ -271,10 +254,14 @@ namespace llarp log::critical( logcat, - "{} node was {} to confirm remote (RID:{}) is registered; allowing connection!", + "{} node was {} to confirm remote (RID:{}) is registered; {} connection!", us, result ? "able" : "unable", - other); + other, + result ? "allowing" : "rejecting"); + + if (result) + ep.service_conns.emplace(other, nullptr); return result; } @@ -283,81 +270,76 @@ namespace llarp return false; } + // TESTNET: change this to an error message later; just because someone tries to erroneously + // connect to a local lokinet client doesn't mean we should kill the program? throw std::runtime_error{"Clients should not be validating inbound connections!"}; }); if (_router.is_service_node()) { - ep->listen(tls_creds, ROUTER_KEEP_ALIVE); + e->listen(tls_creds, ROUTER_KEEP_ALIVE); } - return ep; + return e; } - void - LinkManager::on_inbound_conn(oxen::quic::connection_interface& ci) + std::shared_ptr + LinkManager::make_control(oxen::quic::connection_interface& ci, const RouterID& rid) { - const auto& scid = ci.scid(); - RouterID rid{ci.remote_key()}; - ep.service_connid_map.emplace(scid, rid); - auto [itr, b] = ep.service_conns.emplace(rid, nullptr); - - log::critical(logcat, "Queueing BTStream to be opened..."); - auto control_stream = ci.queue_incoming_stream( [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { log::warning( logcat, - "BTRequestStream closed unexpectedly (ec:{}); closing connection...", + "BTRequestStream closed unexpectedly (ec:{}); closing inbound connection...", error_code); ep.close_connection(rid); }); - log::critical(logcat, "Queued BTStream to be opened ID:{}", control_stream->stream_id()); + log::critical(logcat, "Queued BTStream to be opened (ID:{})", control_stream->stream_id()); assert(control_stream->stream_id() == 0); register_commands(control_stream, rid); - itr->second = std::make_shared(ci.shared_from_this(), control_stream); - log::critical(logcat, "Successfully configured inbound connection fom {}...", rid); + return control_stream; } - // TODO: should we add routes here now that Router::SessionOpen is gone? void - LinkManager::on_conn_open(oxen::quic::connection_interface& ci) + LinkManager::on_inbound_conn(oxen::quic::connection_interface& ci) { - _router.loop()->call([this, &conn_interface = ci]() { - const auto rid = RouterID{conn_interface.remote_key()}; - const auto& remote = conn_interface.remote(); - const auto& scid = conn_interface.scid(); - - if (conn_interface.is_inbound()) - { - log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote); - on_inbound_conn(conn_interface); - } - else - { - log::critical(logcat, "Searching for RID:{} in pending conns...", rid); - - if (auto itr = ep.pending_conns.find(rid); itr != ep.pending_conns.end()) - { - ep.service_connid_map.emplace(scid, rid); - auto [it, b] = ep.service_conns.emplace(rid, nullptr); - it->second = std::move(itr->second); - ep.pending_conns.erase(itr); - - log::critical(logcat, "Connection to RID:{} moved from pending to active conns!", rid); - } - else - throw std::runtime_error{"Could not find newly established connection in pending conns!"}; - } + assert(_is_service_node); + RouterID rid{ci.remote_key()}; + if (auto it = ep.service_conns.find(rid); it != ep.service_conns.end()) + { + log::critical(logcat, "Configuring inbound connection from relay RID:{}", rid); + it->second = std::make_shared(ci.shared_from_this(), make_control(ci, rid)); + } + else if (auto it = ep.client_conns.find(rid); it != ep.client_conns.end()) + { + log::critical(logcat, "Configuring inbound connection from client RID:{}", rid); + it->second = + std::make_shared(ci.shared_from_this(), make_control(ci, rid), false); + } + else + { log::critical( logcat, - "SERVICE NODE (RID:{}) ESTABLISHED CONNECTION TO RID:{}", - _router.local_rid(), + "ERROR: connection accepted from RID:{} that was not logged in key verification!", rid); + } + + log::critical(logcat, "Successfully configured inbound connection fom {}...", rid); + } + + void + LinkManager::on_outbound_conn(oxen::quic::connection_interface& ci) + { + RouterID rid{ci.remote_key()}; + + if (auto it = ep.service_conns.find(rid); it != ep.service_conns.end()) + { + log::critical(logcat, "Fetched configured outbound connection to relay RID:{}", rid); + + auto& conn = it->second->conn; + auto& str = it->second->control_stream; - // check to see if this connection was established while we were attempting to queue - // messages to the remote if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end()) { log::critical(logcat, "Clearing pending queue for RID:{}", rid); @@ -370,12 +352,12 @@ namespace llarp if (msg.is_control) { log::critical(logcat, "Dispatching {} request!", *msg.endpoint); - ep.service_conns[rid]->control_stream->command( - std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); + str->command(std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); } else { - conn_interface.send_datagram(std::move(msg.body)); + log::critical(logcat, "DIspatching data message: {}", msg.body); + conn->send_datagram(std::move(msg.body)); } que.pop_front(); @@ -383,6 +365,40 @@ namespace llarp } log::warning(logcat, "Pending queue empty for RID:{}", rid); + } + else + { + log::critical( + logcat, + "ERROR: connection established to RID:{} that was not logged in key verifrication!", + rid); + } + } + + // TODO: should we add routes here now that Router::SessionOpen is gone? + void + LinkManager::on_conn_open(oxen::quic::connection_interface& ci) + { + _router.loop()->call([this, &conn_interface = ci]() { + const auto rid = RouterID{conn_interface.remote_key()}; + const auto& remote = conn_interface.remote(); + + if (conn_interface.is_inbound()) + { + log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote); + on_inbound_conn(conn_interface); + } + else + { + log::critical(logcat, "Outbound connection fom {} (remote:{})", rid, remote); + on_outbound_conn(conn_interface); + } + + log::critical( + logcat, + "SERVICE NODE (RID:{}) ESTABLISHED CONNECTION TO RID:{}", + _router.local_rid(), + rid); }); }; @@ -390,33 +406,22 @@ namespace llarp LinkManager::on_conn_closed(oxen::quic::connection_interface& ci, uint64_t ec) { _router.loop()->call( - [this, scid = ci.scid(), _rid = RouterID{ci.remote_key()}, error_code = ec]() { - log::critical(quic_cat, "Purging quic connection CID:{} (ec: {})", scid, error_code); - - // a pending connection would not be in the connid_map - if (auto v_itr = ep.pending_conns.find(_rid); v_itr != ep.pending_conns.end()) - { - ep.pending_conns.erase(v_itr); + [this, scid = ci.scid(), rid = RouterID{ci.remote_key()}, error_code = ec]() { + log::critical(quic_cat, "Purging quic connection CID:{} (ec:{})", scid, error_code); - // in case this didn't clear earlier, do it now - if (auto p_itr = pending_conn_msg_queue.find(_rid); - p_itr != pending_conn_msg_queue.end()) - pending_conn_msg_queue.erase(p_itr); + // in case this didn't clear earlier, do it now + if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) + pending_conn_msg_queue.erase(p_itr); - log::critical(quic_cat, "Pending quic connection CID:{} purged successfully", scid); + if (auto s_itr = ep.service_conns.find(rid); s_itr != ep.service_conns.end()) + { + log::critical(quic_cat, "Quic connection to relay RID:{} purged successfully", rid); + ep.service_conns.erase(s_itr); } - else if (const auto& c_itr = ep.service_connid_map.find(scid); - c_itr != ep.service_connid_map.end()) + else if (auto c_itr = ep.client_conns.find(rid); c_itr != ep.client_conns.end()) { - const auto& rid = c_itr->second; - assert(_rid == rid); // this should hold true - - if (auto m_itr = ep.service_conns.find(rid); m_itr != ep.service_conns.end()) - ep.service_conns.erase(m_itr); - - ep.service_connid_map.erase(c_itr); - - log::critical(quic_cat, "Quic connection CID:{} purged successfully", scid); + log::critical(quic_cat, "Quic connection to client RID:{} purged successfully", rid); + ep.client_conns.erase(c_itr); } else log::critical(quic_cat, "Nothing to purge for quic connection CID:{}", scid); @@ -466,19 +471,15 @@ namespace llarp f = std::move(func)]() { auto pending = PendingMessage(std::move(body), std::move(endpoint), std::move(f)); - if (auto it1 = ep.pending_conns.find(remote); it1 != ep.pending_conns.end()) + if (auto it = pending_conn_msg_queue.find(remote); it != pending_conn_msg_queue.end()) { - if (auto it2 = pending_conn_msg_queue.find(remote); it2 != pending_conn_msg_queue.end()) - { - it2->second.push_back(std::move(pending)); - log::critical( - logcat, "Connection (RID:{}) is pending; message appended to send queue!", remote); - } + it->second.push_back(std::move(pending)); + log::critical( + logcat, "Connection to RID:{} is pending; message appended to send queue!", remote); } else { - log::critical( - logcat, "Connection (RID:{}) not found in pending conns; creating send queue!", remote); + log::critical(logcat, "Connection to RID:{} is pending; creating send queue!", remote); auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); itr->second.push_back(std::move(pending)); connect_to(remote); @@ -494,7 +495,7 @@ namespace llarp if (is_stopping) return false; - if (auto conn = ep.get_conn(remote); conn) + if (auto conn = ep.get_service_conn(remote); conn) { conn->conn->send_datagram(std::move(body)); return true; @@ -539,13 +540,12 @@ namespace llarp log::warning(quic_cat, "Could not find RouterContact for connection to rid:{}", rid); } - // This function assumes the RC has already had its signature verified and connection is allowed. void LinkManager::connect_to(const RemoteRC& rc, conn_open_hook on_open, conn_closed_hook on_close) { const auto& rid = rc.router_id(); - if (ep.have_conn(rid)) + if (ep.have_service_conn(rid)) { log::warning(logcat, "We already have a connection to {}!", rid); // TODO: should implement some connection failed logic, but not the same logic that @@ -576,6 +576,12 @@ namespace llarp return ep.have_conn(remote); } + bool + LinkManager::have_service_connection_to(const RouterID& remote) const + { + return ep.have_service_conn(remote); + } + bool LinkManager::have_client_connection_to(const RouterID& remote) const { @@ -628,12 +634,6 @@ namespace llarp return get_num_connected(true); } - bool - LinkManager::get_random_connected(RemoteRC& router) const - { - return ep.get_random_connection(router); - } - bool LinkManager::is_service_node() const { @@ -666,10 +666,10 @@ namespace llarp LinkManager::connect_to_random(int num_conns, bool client_only) { auto filter = [this, client_only](const RemoteRC& rc) -> bool { - auto res = - client_only ? not ep.have_client_conn(rc.router_id()) : not ep.have_conn(rc.router_id()); + const auto& rid = rc.router_id(); + auto res = client_only ? not ep.have_client_conn(rid) : not ep.have_conn(rid); - log::debug(logcat, "RID:{} {}", rc.router_id(), res ? "ACCEPTED" : "REJECTED"); + log::debug(logcat, "RID:{} {}", rid, res ? "ACCEPTED" : "REJECTED"); return res; }; @@ -766,17 +766,19 @@ namespace llarp }; } - if (auto conn = ep.get_conn(source); conn) + const auto& rid = source.router_id(); + + if (auto conn = ep.get_service_conn(rid); conn) { conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f)); log::critical(logcat, "Dispatched bootstrap fetch request!"); return; } - log::critical(logcat, "Queuing bootstrap fetch request to {}", source.router_id()); + log::critical(logcat, "Queuing bootstrap fetch request to {}", rid); auto pending = PendingMessage(std::move(payload), "bfetch_rcs"s, std::move(f)); - auto [itr, b] = pending_conn_msg_queue.emplace(source.router_id(), MessageQueue()); + auto [itr, b] = pending_conn_msg_queue.emplace(rid, MessageQueue()); itr->second.push_back(std::move(pending)); connect_to(source); diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 30232ca3f..186e7cfcd 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -63,29 +63,31 @@ namespace llarp std::shared_ptr endpoint; LinkManager& link_manager; - // for outgoing packets, we route via RouterID; map RouterID->Connection - // for incoming packets, we get a ConnectionID; map ConnectionID->RouterID + /** Connection containers: + - service_conns: holds all connections where the remote (from the perspective + of the local lokinet instance) is a service node. This means all relay to + relay connections are held here; clients will also hold their connections to + relays here as well + - client_conns: holds all connections wehre the remote is a client. This is only + used by service nodes to store their client connections + */ std::unordered_map> service_conns; - std::unordered_map service_connid_map; - std::unordered_map> client_conns; - std::unordered_map client_connid_map; - - // for pending connections, cleared in LinkManager::on_conn_open - std::unordered_map> pending_conns; - // TODO: see which of these is actually useful and delete the other std::shared_ptr - get_conn(const RemoteRC&) const; + get_conn(const RouterID&) const; std::shared_ptr - get_conn(const RouterID&) const; + get_service_conn(const RouterID&) const; + + bool + have_conn(const RouterID& remote) const; bool have_client_conn(const RouterID& remote) const; bool - have_conn(const RouterID& remote) const; + have_service_conn(const RouterID& remote) const; std::pair num_in_out() const; @@ -93,9 +95,6 @@ namespace llarp size_t num_connected(bool clients_only) const; - bool - get_random_connection(RemoteRC& router) const; - template bool establish_connection( @@ -229,9 +228,15 @@ namespace llarp void recv_control_message(oxen::quic::message msg); + std::shared_ptr + make_control(oxen::quic::connection_interface& ci, const RouterID& rid); + void on_inbound_conn(oxen::quic::connection_interface& ci); + void + on_outbound_conn(oxen::quic::connection_interface& ci); + void on_conn_open(oxen::quic::connection_interface& ci); @@ -242,7 +247,10 @@ namespace llarp startup_endpoint(); void - register_commands(std::shared_ptr& s, const RouterID& rid); + register_commands( + std::shared_ptr& s, + const RouterID& rid, + bool client_only = false); public: const link::Endpoint& @@ -291,6 +299,9 @@ namespace llarp bool have_connection_to(const RouterID& remote) const; + bool + have_service_connection_to(const RouterID& remote) const; + bool have_client_connection_to(const RouterID& remote) const; @@ -321,9 +332,6 @@ namespace llarp size_t get_num_connected_clients() const; - bool - get_random_connected(RemoteRC& router) const; - bool is_service_node() const; @@ -444,22 +452,24 @@ namespace llarp is_snode ? ROUTER_KEEP_ALIVE : CLIENT_KEEP_ALIVE, std::forward(opts)...); - // add to pending conns - auto [itr, b] = pending_conns.emplace(rid, nullptr); + // add to service conns + auto [itr, b] = service_conns.emplace(rid, nullptr); auto control_stream = conn_interface->template open_stream( [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { log::warning( logcat, - "BTRequestStream closed unexpectedly (ec:{}); closing connection...", + "BTRequestStream closed unexpectedly (ec:{}); closing outbound connection...", error_code); close_connection(rid); }); + log::critical(logcat, "Opened BTStream (ID:{})", control_stream->stream_id()); + assert(control_stream->stream_id() == 0); link_manager.register_commands(control_stream, rid); - itr->second = std::make_shared(conn_interface, control_stream); + itr->second = std::make_shared(conn_interface, control_stream, true); - log::critical(logcat, "Connection to RID:{} added to pending connections...", rid); + log::critical(logcat, "Outbound connection to RID:{} added to service conns...", rid); return true; } catch (...) @@ -469,50 +479,4 @@ namespace llarp } } } // namespace link - } // namespace llarp - -/* -- Refactor RouterID to use gnutls info and maybe ConnectionID -- Combine routerID and connectionID to simplify mapping in llarp/link/endpoint.hpp -- Combine llarp/link/session.hpp into llarp/link/connection.hpp::Connection - -- Combine llarp/link/server.hpp::ILinkLayer into llarp/link/endpoint.hpp::Endpoint - - must maintain metadata storage, callbacks, etc - -- If: one endpoint for ipv4 and ipv6 - - Then: can potentially combine: - - llarp/link/endpoint.hpp - - llarp/link/link_manager.hpp - - llarp/link/outbound_message_handler.hpp - - llarp/link/outbound_session_maker.hpp - - -> Yields mega-combo endpoint managing object? - - Can avoid "kitchen sink" by greatly reducing complexity of implementation - - llarp/router/outbound_message_handler.hpp - - pendingsessionmessagequeue - - establish queue of messages to be sent on a connection we are creating - - upon creation, send these messages in the connection established callback - - if connection times out, flush queue - - TOCHECK: is priority used at all?? - - -std::unordered_map -rpc_commands = { - {"find_name", &handle_find_name}, - // ... -}; - -for (const auto& [name, mfn] : rpc_commands) - bparser.add_command(name, [this, mfn] (oxen::quic::message m) { - router->call([this, mfn, m=std::move(m)] mutable { - try { - std::invoke(mfn, this, std::move(m)); - } catch (const std::exception& e) { - m.respond("Error: "s + e.what(), true); - } - }); - }); - -*/ diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index d15be5b1c..2a8f3d0a0 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -287,7 +287,7 @@ namespace llarp const auto fetch_threshold = (double)union_size / num_received; - /** We are checking 2, potentially 3 things here: + /** We are checking 2 things here: 1) The ratio of received/accepted to total received is above GOOD_RID_FETCH_THRESHOLD. This tells us how well the rid source's sets of rids "agree" with one another 2) The total number received is above MIN_RID_FETCH_TOTAL. This ensures that we are @@ -790,6 +790,13 @@ namespace llarp const std::vector& greylist, const std::vector& greenlist) { + log::critical( + logcat, + "Oxend provided {}/{}/{} (white/gray/green) routers", + whitelist.size(), + greylist.size(), + greenlist.size()); + if (whitelist.empty()) return; @@ -807,7 +814,7 @@ namespace llarp log::critical( logcat, - "Oxend provided {}:{} (whitelist:registered)", + "Service node holding {}:{} (whitelist:registered) after oxend integration", _router_whitelist.size(), _registered_routers.size()); } @@ -1039,7 +1046,7 @@ namespace llarp RemoteRC rc{}; const llarp::dht::XorMetric compare(location); - VisitAll([&rc, compare](const auto& otherRC) { + visit_all([&rc, compare](const auto& otherRC) { const auto& rid = rc.router_id(); if (rid.IsZero() || compare(dht::Key_t{otherRC.router_id()}, dht::Key_t{rid})) diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 9ff0e73a3..adfabe0f5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -481,7 +481,7 @@ namespace llarp /// visit all known_rcs template void - VisitAll(Visit visit) const + visit_all(Visit visit) const { _router.loop()->call([this, visit]() { for (const auto& item : known_rcs) @@ -496,7 +496,7 @@ namespace llarp /// remove an entry given a filter that inspects the rc template void - RemoveIf(Filter visit) + remove_if(Filter visit) { _router.loop()->call([this, visit]() { std::unordered_set removed; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index c3441594a..5f8289f56 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -847,7 +847,7 @@ namespace llarp _last_tick != 0s and delta > NETWORK_RESET_SKIP_INTERVAL) { // we detected a time skip into the futre, thaw the network - log::warning(logcat, "Timeskip of {} detected, resetting network state!", delta.count()); + log::error(logcat, "Timeskip of {} detected, resetting network state!", delta.count()); Thaw(); } @@ -877,28 +877,28 @@ namespace llarp last_rc_gossip = now_timepoint; - // TESTNET: 1 to 4 minutes before testnet gossip interval - auto random_delta = + // TESTNET: 1 to 5 minutes before testnet gossip interval + auto delta = std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; // 1min to 5min before "stale time" is next gossip time // auto random_delta = // std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; - next_rc_gossip = now_timepoint + TESTNET_GOSSIP_INTERVAL - random_delta; + next_rc_gossip = now_timepoint + TESTNET_GOSSIP_INTERVAL - delta; // next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; } report_stats(); } - if (needs_initial_fetch()) + if (needs_rebootstrap() and now_timepoint > next_bootstrap_attempt) { - if (not _config->bootstrap.seednode) - node_db()->fetch_initial(is_service_node()); + node_db()->fallback_to_bootstrap(); } - else if (needs_rebootstrap() and now_timepoint > next_bootstrap_attempt) + else if (needs_initial_fetch()) { - node_db()->fallback_to_bootstrap(); + if (not _config->bootstrap.seednode) + node_db()->fetch_initial(is_service_node()); } else if (not is_snode) { @@ -918,13 +918,14 @@ namespace llarp } // remove RCs for nodes that are no longer allowed by network policy - node_db()->RemoveIf([&](const RemoteRC& rc) -> bool { + node_db()->remove_if([&](const RemoteRC& rc) -> bool { // don't purge bootstrap nodes from nodedb if (is_bootstrap_node(rc.router_id())) { log::trace(logcat, "Not removing {}: is bootstrap node", rc.router_id()); return false; } + // if for some reason we stored an RC that isn't a valid router // purge this entry if (not rc.is_public_addressable()) @@ -932,12 +933,14 @@ namespace llarp log::debug(logcat, "Removing {}: not a valid router", rc.router_id()); return true; } - /// clear out a fully expired RC + + // clear out a fully expired RC if (rc.is_expired(now)) { log::debug(logcat, "Removing {}: RC is expired", rc.router_id()); return true; } + // clients have no notion of a whilelist // we short circuit logic here so we dont remove // routers that are not whitelisted for first hops @@ -965,26 +968,6 @@ namespace llarp return false; }); - /* TODO: this behavior seems incorrect, but fixing it will require discussion - * - if (not is_snode or not whitelist_received) - { - // find all deregistered relays - std::unordered_set close_peers; - - for_each_connection([this, &close_peers](link::Connection& conn) { - const auto& pk = conn.remote_rc.router_id(); - - if (conn.remote_is_relay and not _rc_lookup_handler.is_session_allowed(pk)) - close_peers.insert(pk); - }); - - // mark peers as de-registered - for (auto& peer : close_peers) - _link_manager.close_connection(peer); - } - */ - _link_manager->check_persisting_conns(now); auto num_conns = num_router_connections(); @@ -1064,12 +1047,6 @@ namespace llarp _last_tick = llarp::time_now_ms(); } - bool - Router::GetRandomConnectedRouter(RemoteRC& result) const - { - return _link_manager->get_random_connected(result); - } - const std::set& Router::get_whitelist() const { @@ -1324,12 +1301,6 @@ namespace llarp _loop->call_later(200ms, [this] { AfterStopIssued(); }); } - bool - Router::HasSessionTo(const RouterID& remote) const - { - return _link_manager->have_connection_to(remote); - } - std::string Router::ShortName() const { diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 9434d2ad4..beeae1edf 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -540,12 +540,6 @@ namespace llarp size_t num_client_connections() const; - bool - GetRandomConnectedRouter(RemoteRC& result) const; - - bool - HasSessionTo(const RouterID& remote) const; - std::string ShortName() const; From 5be09563fa32f6ba4cb5a48a7c04c1047053cf20 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 19 Dec 2023 09:25:56 -0800 Subject: [PATCH 83/93] address parsing of deprecated opts --- llarp/config/config.cpp | 39 ++++++++++++--------------------------- llarp/router/router.cpp | 15 ++++++--------- llarp/router/router.hpp | 2 +- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index 4dbc60fc7..a26159eeb 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -933,39 +933,24 @@ namespace llarp auto parse_addr_for_link = [net_ptr](const std::string& arg) { std::optional maybe = std::nullopt; - std::string_view arg_v; + std::string_view arg_v{arg}, host; + uint16_t p{DEFAULT_LISTEN_PORT}; - // explicitly provided value - if (not arg.empty()) + if (auto pos = arg_v.find(':'); pos != arg_v.npos) { - arg_v = std::string_view{arg}; - } - - if (arg_v[0] == ':') - { - uint16_t res; - if (auto rv = llarp::parse_int(arg_v.substr(1), res); not rv) - res = DEFAULT_LISTEN_PORT; + host = arg_v.substr(0, pos); - maybe = oxen::quic::Address{""s, res}; + if (not llarp::parse_int(arg_v.substr(pos + 1), p)) + throw std::invalid_argument{"Failed to parse port in arg:{}"_format(arg)}; } - else if (auto pos = arg_v.find(':'); pos != arg_v.npos) - { - auto h = arg_v.substr(0, pos); - uint16_t p; - if (auto rv = llarp::parse_int(arg_v.substr(pos + 1), p); not rv) - p = DEFAULT_LISTEN_PORT; - maybe = oxen::quic::Address{std::string{h}, p}; + if (host.empty()) + maybe = net_ptr->get_best_public_address(true, p); + else + maybe = oxen::quic::Address{std::string{host}, p}; - if (maybe->is_loopback()) - throw std::invalid_argument{fmt::format("{} is a loopback address", arg)}; - } - if (not maybe) - // infer public address - maybe = net_ptr->get_best_public_address(true, DEFAULT_LISTEN_PORT); - else if (maybe && maybe->port() == 0) - maybe->set_port(DEFAULT_LISTEN_PORT); + if (maybe and maybe->is_loopback()) + throw std::invalid_argument{"{} is a loopback address"_format(arg)}; return maybe; }; diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 5f8289f56..fc9d17ed6 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -764,6 +764,9 @@ namespace llarp out, num_client_connections(), router_contact.time_to_expiry(now)); + + if (num_router_connections() >= _node_db->num_rcs()) + log::critical(logcat, "SERVICE NODE IS FULLY MESHED"); } else { @@ -780,6 +783,8 @@ namespace llarp log::info(logcat, "Last reported stats time {}", now - _last_stats_report); _last_stats_report = now; + + oxen::log::flush(); } std::string @@ -880,15 +885,11 @@ namespace llarp // TESTNET: 1 to 5 minutes before testnet gossip interval auto delta = std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; - // 1min to 5min before "stale time" is next gossip time - // auto random_delta = - // std::chrono::seconds{std::uniform_int_distribution{60, 300}(llarp::csrng)}; next_rc_gossip = now_timepoint + TESTNET_GOSSIP_INTERVAL - delta; - // next_rc_gossip = now_timepoint + RouterContact::STALE_AGE - random_delta; } - report_stats(); + // report_stats(); } if (needs_rebootstrap() and now_timepoint > next_bootstrap_attempt) @@ -1008,8 +1009,6 @@ namespace llarp FULL_MESH_ITERATION); _link_manager->connect_to_random(FULL_MESH_ITERATION); } - else - log::critical(logcat, "SERVICE NODE IS FULLY MESHED"); } else { @@ -1129,8 +1128,6 @@ namespace llarp _node_db->load_from_disk(); _node_db->store_bootstraps(); - oxen::log::flush(); - log::info(logcat, "Creating Introset Contacts..."); _contacts = std::make_unique(*this); diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index beeae1edf..045553551 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -59,7 +59,7 @@ namespace llarp // how big of a time skip before we reset network state inline constexpr std::chrono::milliseconds NETWORK_RESET_SKIP_INTERVAL{1min}; - inline constexpr std::chrono::milliseconds REPORT_STATS_INTERVAL{1h}; + inline constexpr std::chrono::milliseconds REPORT_STATS_INTERVAL{10s}; inline constexpr std::chrono::milliseconds DECOMM_WARNING_INTERVAL{5min}; From 30f62d2689f2bfb3b7a455d17945075cc21932bc Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 11:34:55 -0800 Subject: [PATCH 84/93] zero cost exception handling my ass - rework bootstrap loading to move all logic into BootstrapList object - ability to parse lists and dicts of bootstraps implemented - netid parsing refers to the correct fallback - cross your fingers boys here we go --- contrib/bootstrap/testnet.signed | Bin 308 -> 161 bytes llarp/bootstrap.cpp | 114 ++++++++++++++++++++++----- llarp/bootstrap.hpp | 9 +++ llarp/config/config.cpp | 27 +------ llarp/config/config.hpp | 10 --- llarp/link/link_manager.hpp | 11 +-- llarp/nodedb.cpp | 54 +++---------- llarp/nodedb.hpp | 6 +- llarp/router/router.cpp | 131 ++++++------------------------- llarp/router_contact.cpp | 12 ++- llarp/router_contact.hpp | 7 +- 11 files changed, 161 insertions(+), 220 deletions(-) diff --git a/contrib/bootstrap/testnet.signed b/contrib/bootstrap/testnet.signed index 366c5c5a0098922587845c63f3985bfce70b1d63..e80acd6cd75a5a21b4fe3c325c3e40dbdc4d0630 100644 GIT binary patch literal 161 zcmV;S0ABxOFgj^4WidK5HaZtr2Kp)BF*<2CI&@`obZ%vIF*a literal 308 zcmc~vF|3 zI5?)R{Qh3-X}qCTrjenQl~o!LSQ(m_=$V`88Jd~s85sbD3o;GO%*=o$mSq~GrUL0q zQ>*mE+}uP%t8Adz_fCCa|CDQ+Kf5LBg5m%Bxh%;&s=Ebt{Qtdgrf@Lxk)1#TfT|Q- z8P7j@SvBLYRaf_~zpdQ*1`BVsa38+As%5L+a|tI&L#v`3pz$DNk(y}?vZfShuc3*# zg@v)9fr+UJ)B%Q86*)j=m6?fE&yvyw?)#^>Ot$T**t)3t{TWlke=L&H32*N={B(U( r&drjmS;%H|Bb{UY#u-0mDZJl&KyyaO#ek_>K5hOs{h%RJYAP21S?hdC diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index b5f35c605..061e3b3b3 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -9,10 +9,25 @@ namespace llarp bool BootstrapList::bt_decode(std::string_view buf) { - try + const auto& f = buf.front(); + + switch (f) { - oxenc::bt_list_consumer btlc{buf}; + case 'l': + return bt_decode(oxenc::bt_list_consumer{buf}); + case 'd': + return bt_decode(oxenc::bt_dict_consumer{buf}); + default: + log::critical(logcat, "Unable to parse bootstrap as bt list or dict!"); + return false; + } + } + bool + BootstrapList::bt_decode(oxenc::bt_list_consumer btlc) + { + try + { while (not btlc.is_finished()) emplace(btlc.consume_dict_data()); } @@ -22,6 +37,24 @@ namespace llarp return false; } + _curr = begin(); + return true; + } + + bool + BootstrapList::bt_decode(oxenc::bt_dict_consumer btdc) + { + try + { + emplace(btdc); + } + catch (const std::exception& e) + { + log::warning(logcat, "Unable to decode bootstrap RemoteRC: {}", e.what()); + return false; + } + + _curr = begin(); return true; } @@ -55,37 +88,82 @@ namespace llarp } void - BootstrapList::read_from_file(const fs::path& fpath) + BootstrapList::populate_bootstraps( + std::vector paths, const fs::path& def, bool load_fallbacks) { - bool isListFile = false; - - if (std::ifstream inf(fpath.c_str(), std::ios::binary); inf.is_open()) + for (const auto& f : paths) { - const char ch = inf.get(); - isListFile = ch == 'l'; + // TESTNET: TODO: revise fucked config + log::debug(logcat, "Loading BootstrapRC from file at path:{}", f); + if (not read_from_file(f)) + throw std::invalid_argument{"User-provided BootstrapRC is invalid!"}; } - if (isListFile) + if (empty()) { - auto content = util::file_to_string(fpath); + log::debug( + logcat, + "BootstrapRC list empty; looking for default BootstrapRC from file at path:{}", + def); + read_from_file(def); + } - if (not bt_decode(content)) + for (auto itr = begin(); itr != end(); ++itr) + { + if (RouterContact::is_obsolete(*itr)) // can move this into ::read_from_file { - throw std::runtime_error{fmt::format("failed to read bootstrap list file '{}'", fpath)}; + log::critical(logcat, "Deleting obsolete BootstrapRC (rid:{})", itr->router_id()); + itr = erase(itr); + continue; } } - else + + if (empty() and load_fallbacks) { - RemoteRC rc; + log::critical(logcat, "BootstrapRC list empty; loading fallbacks..."); + auto fallbacks = llarp::load_bootstrap_fallbacks(); - if (not rc.read(fpath)) + if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) { - throw std::runtime_error{ - fmt::format("failed to decode bootstrap RC, file='{}', rc={}", fpath, rc.to_string())}; + log::critical( + logcat, "Loading {} default fallback bootstrap router(s)!", itr->second.size()); + merge(itr->second); + } + + if (empty()) + { + log::error( + logcat, + "No Bootstrap routers were loaded. The default Bootstrap file {} does not exist, and " + "loading fallback Bootstrap RCs failed.", + def); + + throw std::runtime_error("No Bootstrap nodes available."); } - insert(rc); } + log::critical(logcat, "We have {} Bootstrap router(s)!", size()); + _curr = begin(); + } + + bool + BootstrapList::read_from_file(const fs::path& fpath) + { + bool result = false; + + if (not fs::exists(fpath)) + { + log::critical(logcat, "Bootstrap RC file non-existant at path:{}", fpath); + return result; + } + + auto content = util::file_to_string(fpath); + result = bt_decode(content); + + log::critical( + logcat, "{}uccessfully loaded BootstrapRC file at path:{}", result ? "S" : "Un", fpath); + _curr = begin(); + return result; } } // namespace llarp diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index 7d037ff55..86cf416f0 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -23,10 +23,19 @@ namespace llarp bool bt_decode(std::string_view buf); + bool + bt_decode(oxenc::bt_list_consumer btlc); + + bool + bt_decode(oxenc::bt_dict_consumer btdc); + std::string_view bt_encode() const; void + populate_bootstraps(std::vector paths, const fs::path& def, bool load_fallbacks); + + bool read_from_file(const fs::path& fpath); bool diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index a26159eeb..eeb2fecbf 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -1068,26 +1068,6 @@ namespace llarp }); } - void - ConnectConfig::define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params) - { - (void)params; - - conf.add_undeclared_handler( - "connect", [this](std::string_view section, std::string_view name, std::string_view value) { - fs::path file{value.begin(), value.end()}; - if (not fs::exists(file)) - throw std::runtime_error{fmt::format( - "Specified bootstrap file {} specified in [{}]:{} does not exist", - value, - section, - name)}; - - routers.emplace_back(std::move(file)); - return true; - }); - } - void ApiConfig::define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params) { @@ -1204,14 +1184,12 @@ namespace llarp }, [this](std::string arg) { if (arg.empty()) - { throw std::invalid_argument("cannot use empty filename as bootstrap"); - } + files.emplace_back(std::move(arg)); + if (not fs::exists(files.back())) - { throw std::invalid_argument("file does not exist: " + arg); - } }); } @@ -1460,7 +1438,6 @@ namespace llarp router.define_config_options(conf, params); network.define_config_options(conf, params); paths.define_config_options(conf, params); - connect.define_config_options(conf, params); dns.define_config_options(conf, params); links.define_config_options(conf, params); api.define_config_options(conf, params); diff --git a/llarp/config/config.hpp b/llarp/config/config.hpp index b0a9231a9..152da9345 100644 --- a/llarp/config/config.hpp +++ b/llarp/config/config.hpp @@ -184,14 +184,6 @@ namespace llarp define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params); }; - struct ConnectConfig - { - std::vector routers; - - void - define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params); - }; - // TODO: remove oxenmq from this header struct ApiConfig { @@ -215,7 +207,6 @@ namespace llarp struct BootstrapConfig { std::vector files; - BootstrapList routers; bool seednode; void @@ -245,7 +236,6 @@ namespace llarp RouterConfig router; NetworkConfig network; PeerSelectionConfig paths; - ConnectConfig connect; DnsConfig dns; LinksConfig links; ApiConfig api; diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 186e7cfcd..105adec06 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -423,13 +423,10 @@ namespace llarp void handle_update_exit_response(oxen::quic::message); void handle_close_exit_response(oxen::quic::message); - std::unordered_map rpc_responses = { - {"find_name", &LinkManager::handle_find_name_response}, - {"publish_intro", &LinkManager::handle_publish_intro_response}, - {"find_intro", &LinkManager::handle_find_intro_response}, - {"update_exit", &LinkManager::handle_update_exit_response}, - {"obtain_exit", &LinkManager::handle_obtain_exit_response}, - {"close_exit", &LinkManager::handle_close_exit_response}}; + /** + Clients register 0 endpoints + - nobody is making requests to clients + */ }; namespace link diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 2a8f3d0a0..7b1a12bce 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -498,21 +498,13 @@ namespace llarp { if (error) { - auto& fail_count = (_using_bootstrap_fallback) ? bootstrap_attempts : fetch_failures; - auto& THRESHOLD = - (_using_bootstrap_fallback) ? MAX_BOOTSTRAP_FETCH_ATTEMPTS : MAX_FETCH_ATTEMPTS; - - // This catches three different failure cases; - // 1) bootstrap fetching and over failure threshold - // 2) bootstrap fetching and more failures to go - // 3) standard fetching and over threshold - if (++fail_count >= THRESHOLD || _using_bootstrap_fallback) + if (++fetch_failures >= MAX_FETCH_ATTEMPTS) { log::info( logcat, "RC fetching from {} reached failure threshold ({}); falling back to bootstrap...", fetch_source, - THRESHOLD); + MAX_FETCH_ATTEMPTS); fallback_to_bootstrap(); return; @@ -536,7 +528,7 @@ namespace llarp void NodeDB::fetch_rids_result(bool initial) { - if (fetch_failures > MAX_FETCH_ATTEMPTS) + if (fetch_failures >= MAX_FETCH_ATTEMPTS) { log::info( logcat, @@ -661,15 +653,18 @@ namespace llarp log::critical(logcat, "Dispatching BootstrapRC fetch request to {}", fetch_source); + auto num_needed = _router.is_service_node() ? SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT + : CLIENT_BOOTSTRAP_SOURCE_COUNT; + _router.link_manager().fetch_bootstrap_rcs( rc, - BootstrapFetchMessage::serialize(_router.router_contact, CLIENT_BOOTSTRAP_SOURCE_COUNT), - [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { + BootstrapFetchMessage::serialize(_router.router_contact, num_needed), + [this, is_snode = _router.is_service_node(), num_needed = num_needed]( + oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); if (not m) { - // ++bootstrap_attempts; log::warning( logcat, "BootstrapRC fetch request to {} failed (error {}/{})", @@ -680,7 +675,6 @@ namespace llarp return; } - // std::set rids; size_t num = 0; try @@ -700,7 +694,6 @@ namespace llarp } catch (const std::exception& e) { - // ++bootstrap_attempts; log::warning( logcat, "Failed to parse BootstrapRC fetch response from {} (error {}/{}): {}", @@ -713,19 +706,12 @@ namespace llarp return; } - // We set this to the max allowable value because if this result is bad, we won't - // try this bootstrap again. If this result is undersized, we roll right into the - // next call to fallback_to_bootstrap() and hit the base case, rotating sources - // bootstrap_attempts = MAX_BOOTSTRAP_FETCH_ATTEMPTS; - - // const auto& num = rids.size(); - log::critical( logcat, "BootstrapRC fetch response from {} returned {}/{} needed RCs", fetch_source, num, - CLIENT_BOOTSTRAP_SOURCE_COUNT); + num_needed); if (not is_snode) { @@ -739,25 +725,6 @@ namespace llarp log::critical(logcat, "Service node completed processing BootstrapRC fetch!"); post_snode_bootstrap(); } - - // FIXME: when moving to testnet, uncomment this - // if (rids.size() == BOOTSTRAP_SOURCE_COUNT) - // { - // known_rids.merge(rids); - // fetch_initial(); - // } - // else - // { - // // ++bootstrap_attempts; - // log::warning( - // logcat, - // "BootstrapRC fetch response from {} returned insufficient number of RC's (error " - // "{}/{})", - // fetch_source, - // bootstrap_attempts, - // MAX_BOOTSTRAP_FETCH_ATTEMPTS); - // fallback_to_bootstrap(); - // } }); } @@ -783,7 +750,6 @@ namespace llarp replace_subset(rid_sources, specific, known_rids, RID_SOURCE_COUNT, csrng); } - // TODO: nuke all this shit void NodeDB::set_router_whitelist( const std::vector& whitelist, diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index adfabe0f5..7a3c8333d 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -25,6 +25,7 @@ namespace llarp // TESTNET: the following constants have been shortened for testing purposes /* RC Fetch Constants */ + // fallback to bootstrap if we have less than this many RCs inline constexpr size_t MIN_ACTIVE_RCS{6}; // max number of attempts we make in non-bootstrap fetch requests inline constexpr int MAX_FETCH_ATTEMPTS{10}; @@ -34,11 +35,10 @@ namespace llarp inline constexpr double MIN_GOOD_RC_FETCH_THRESHOLD{}; /* RID Fetch Constants */ - inline constexpr size_t MIN_ACTIVE_RIDS{24}; // the number of rid sources that we make rid fetch requests to - inline constexpr size_t RID_SOURCE_COUNT{12}; + inline constexpr size_t RID_SOURCE_COUNT{8}; // upper limit on how many rid fetch requests to rid sources can fail - inline constexpr size_t MAX_RID_ERRORS{4}; + inline constexpr size_t MAX_RID_ERRORS{2}; // each returned rid must appear this number of times across all responses inline constexpr int MIN_RID_FETCH_FREQ{RID_SOURCE_COUNT - MAX_RID_ERRORS - 1}; // the total number of accepted returned rids should be above this number diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index fc9d17ed6..625b235ee 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -353,6 +353,25 @@ namespace llarp _config = std::move(c); auto& conf = *_config; + const auto& netid = conf.router.net_id; + + if (not netid.empty() and netid != llarp::LOKINET_DEFAULT_NETID) + { + _testnet = netid == llarp::LOKINET_TESTNET_NETID; + _testing_disabled = conf.lokid.disable_testing; + + RouterContact::ACTIVE_NETID = netid; + + if (_testing_disabled and not _testnet) + throw std::runtime_error{"Error: reachability testing can only be disabled on testnet!"}; + + auto err = "Lokinet network ID set to {}, NOT mainnet! {}"_format( + netid, + _testnet ? "Please ensure your local instance is configured to operate on testnet" + : "Local lokinet instance will attempt to run on the specified network"); + log::critical(logcat, err); + } + // Do logging config as early as possible to get the configured log level applied // Backwards compat: before 0.9.10 we used `type=file` with `file=|-|stdout` for print mode @@ -528,24 +547,6 @@ namespace llarp // Set netid before anything else log::debug(logcat, "Network ID set to {}", conf.router.net_id); - const auto& netid = conf.router.net_id; - - if (not netid.empty() and netid != llarp::LOKINET_DEFAULT_NETID) - { - log::critical( - logcat, - "Network ID set to {}, which is not {}! Lokinet will attempt to run on the specified " - "network", - netid, - llarp::LOKINET_DEFAULT_NETID); - - _testnet = netid == llarp::LOKINET_TESTNET_NETID; - _testing_disabled = conf.lokid.disable_testing; - - if (_testing_disabled and not _testnet) - throw std::runtime_error{"Error: reachability testing can only be disabled on testnet!"}; - } - // Router config client_router_connections = conf.router.client_router_connections; @@ -603,101 +604,15 @@ namespace llarp log::debug(logcat, "{} strict-connect routers configured", val.size()); } - std::vector configRouters = conf.connect.routers; - - configRouters.insert( - configRouters.end(), conf.bootstrap.files.begin(), conf.bootstrap.files.end()); - - // if our conf had no bootstrap files specified, try the default location of - // /bootstrap.signed. If this isn't present, leave a useful error message - // TODO: use constant - fs::path defaultBootstrapFile = conf.router.data_dir / "bootstrap.signed"; - - if (configRouters.empty() and conf.bootstrap.routers.empty()) - { - if (fs::exists(defaultBootstrapFile)) - configRouters.push_back(defaultBootstrapFile); - } - - // BootstrapList _bootstrap_rc_list; - auto& node_bstrap = _node_db->bootstrap_list(); - - auto clear_bad_rcs = [&]() mutable { - log::critical(logcat, "Clearing bad RCs..."); - // in case someone has an old bootstrap file and is trying to use a bootstrap - // that no longer exists - for (auto it = node_bstrap.begin(); it != node_bstrap.end();) - { - if (it->is_obsolete_bootstrap()) - log::critical(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); - else if (not it->verify()) - log::critical(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); - else - { - ++it; - continue; - } - - // we are in one of the above error cases that we warned about: - it = node_bstrap.erase(it); - } - }; - - for (const auto& router : configRouters) - { - log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); - node_bstrap.read_from_file(router); - // _bootstrap_rc_list.read_from_file(router); - } - - for (const auto& rc : conf.bootstrap.routers) - { - // _bootstrap_rc_list.emplace(rc); - node_bstrap.emplace(rc); - } - _bootstrap_seed = conf.bootstrap.seednode; - if (_bootstrap_seed) - log::critical(logcat, "We are a bootstrap seed node!"); - - if (node_bstrap.empty() and not _bootstrap_seed) - { - log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node"); - - auto fallbacks = llarp::load_bootstrap_fallbacks(); - - if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) - { - // _bootstrap_rc_list.merge(itr->second); - node_bstrap.merge(itr->second); - } - - if (node_bstrap.empty()) - { - // empty after trying fallback, if set - log::error( - logcat, - "No bootstrap routers were loaded. The default bootstrap file {} does not exist, and " - "loading fallback bootstrap RCs failed.", - defaultBootstrapFile); - - throw std::runtime_error("No bootstrap nodes available."); - } - - log::critical(logcat, "Loaded {} default fallback bootstrap routers!", node_bstrap.size()); - } + std::vector bootstrap_paths{std::move(conf.bootstrap.files)}; - clear_bad_rcs(); - log::critical(logcat, "We have {} bootstrap routers!", node_bstrap.size()); + fs::path default_bootstrap = conf.router.data_dir / "bootstrap.signed"; - // node_db()->set_bootstrap_routers(_bootstrap_rc_list); + auto& bootstrap = _node_db->bootstrap_list(); - // TODO: RC refactor here - // if (_is_service_node) - // init_inbounds(); - // else - // init_outbounds(); + bootstrap.populate_bootstraps(bootstrap_paths, default_bootstrap, not _bootstrap_seed); // profiling _profile_file = conf.router.data_dir / "profiles.dat"; diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index ebd87e701..2f3f5f606 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -36,7 +36,7 @@ namespace llarp }); if (not btdc.is_finished()) - throw std::runtime_error{"RouterContact has some fucked up shit at the end"}; + throw std::runtime_error{"RemoteRC has some fucked up shit at the end"}; btdc.finish(); } @@ -185,7 +185,7 @@ namespace llarp return time_to_expiry(now) <= dlt; } - static constexpr std::array obsolete_bootstraps = { + static const std::set obsolete_bootstraps{ "7a16ac0b85290bcf69b2f3b52456d7e989ac8913b4afbb980614e249a3723218"sv, "e6b3a6fe5e32c379b64212c72232d65b0b88ddf9bbaed4997409d329f8519e0b"sv, }; @@ -200,4 +200,12 @@ namespace llarp } return false; } + + bool + RouterContact::is_obsolete(const RouterContact& rc) + { + const auto& hex = rc._router_id.ToHex(); + + return obsolete_bootstraps.count(hex); + } } // namespace llarp diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index c37652fe5..76f62c31f 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -195,6 +195,9 @@ namespace llarp bool is_obsolete_bootstrap() const; + static bool + is_obsolete(const RouterContact& rc); + void bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired = false) const; @@ -304,9 +307,6 @@ namespace llarp /// the data in the constructor, eliminating the need for a ::verify method/ struct RemoteRC final : public RouterContact { - private: - explicit RemoteRC(oxenc::bt_dict_consumer btdc); - public: RemoteRC() = default; explicit RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} @@ -317,6 +317,7 @@ namespace llarp { _payload = data; } + explicit RemoteRC(oxenc::bt_dict_consumer btdc); ~RemoteRC() = default; std::string_view From 3fb9ba57e55860148ff91b1b1156aca53d701e75 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 12:35:15 -0800 Subject: [PATCH 85/93] gossip new RC's on bfetch --- llarp/link/link_manager.cpp | 2 +- llarp/nodedb.cpp | 20 +++++++++++++++++--- llarp/nodedb.hpp | 3 +++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 82ff615b0..03b65c59d 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -825,7 +825,7 @@ namespace llarp "Bootstrap seed confirmed RID:{} is registered; approving fetch request and " "saving RC!", rid); - node_db->put_rc(remote); + node_db->verify_gossip_bfetch_rc(remote); } } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 7b1a12bce..12ea089eb 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -659,8 +659,7 @@ namespace llarp _router.link_manager().fetch_bootstrap_rcs( rc, BootstrapFetchMessage::serialize(_router.router_contact, num_needed), - [this, is_snode = _router.is_service_node(), num_needed = num_needed]( - oxen::quic::message m) mutable { + [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); if (not m) @@ -711,7 +710,7 @@ namespace llarp "BootstrapRC fetch response from {} returned {}/{} needed RCs", fetch_source, num, - num_needed); + MIN_ACTIVE_RCS); if (not is_snode) { @@ -972,6 +971,21 @@ namespace llarp return false; } + void + NodeDB::verify_gossip_bfetch_rc(const RemoteRC& rc) + { + if (auto maybe = get_rc(rc.router_id())) + { + if (maybe->other_is_newer(rc)) + put_rc(rc); + } + else + { + if (put_rc(rc)) + _router.link_manager().gossip_rc(_router.local_rid(), rc); + } + } + bool NodeDB::put_rc_if_newer(RemoteRC rc) { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 7a3c8333d..8eaa077a8 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -581,6 +581,9 @@ namespace llarp bool put_rc_if_newer(RemoteRC rc); + void + verify_gossip_bfetch_rc(const RemoteRC& rc); + bool verify_store_gossip_rc(const RemoteRC& rc); }; From f6a660ae1e071adb51e2ddba8bfbd97b2af0f4c1 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Wed, 20 Dec 2023 16:43:04 -0400 Subject: [PATCH 86/93] Fix SETCAP disabling If you start without -DWITH_SETCAP=OFF, but then later re-run cmake, SETCAP is still set (and so still gets tried). This fixes it. --- daemon/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/CMakeLists.txt b/daemon/CMakeLists.txt index 05685f44a..477e32e42 100644 --- a/daemon/CMakeLists.txt +++ b/daemon/CMakeLists.txt @@ -83,7 +83,7 @@ endforeach() target_link_libraries(lokinet PRIVATE CLI11) target_link_libraries(lokinet-vpn PRIVATE CLI11) -if(SETCAP) +if(WITH_SETCAP) install(CODE "execute_process(COMMAND ${SETCAP} cap_net_admin,cap_net_bind_service=+eip ${CMAKE_INSTALL_PREFIX}/bin/lokinet)") endif() From 0d8017d61dc9a11f798f811971f1c8282dd1a514 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 12:51:56 -0800 Subject: [PATCH 87/93] okay now lets try clients --- llarp/link/link_manager.cpp | 2 +- llarp/link/link_manager.hpp | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 03b65c59d..5f5518206 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -276,7 +276,7 @@ namespace llarp }); if (_router.is_service_node()) { - e->listen(tls_creds, ROUTER_KEEP_ALIVE); + e->listen(tls_creds); } return e; } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 105adec06..f78401481 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -38,7 +38,7 @@ namespace llarp using outbound_alpns = oxen::quic::opt::outbound_alpns; inline const keep_alive ROUTER_KEEP_ALIVE{10s}; - inline const keep_alive CLIENT_KEEP_ALIVE{0s}; + inline const keep_alive CLIENT_KEEP_ALIVE{10s}; namespace alpns { @@ -422,11 +422,6 @@ namespace llarp void handle_obtain_exit_response(oxen::quic::message); void handle_update_exit_response(oxen::quic::message); void handle_close_exit_response(oxen::quic::message); - - /** - Clients register 0 endpoints - - nobody is making requests to clients - */ }; namespace link @@ -463,7 +458,10 @@ namespace llarp log::critical(logcat, "Opened BTStream (ID:{})", control_stream->stream_id()); assert(control_stream->stream_id() == 0); - link_manager.register_commands(control_stream, rid); + if (is_snode) + link_manager.register_commands(control_stream, rid); + else + log::critical(logcat, "Client NOT registering BTStream commands!"); itr->second = std::make_shared(conn_interface, control_stream, true); log::critical(logcat, "Outbound connection to RID:{} added to service conns...", rid); From eaa853d04f85797bcbdb994bf901e2484a65ddd3 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 13:15:09 -0800 Subject: [PATCH 88/93] client testing --- llarp/router/router.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 625b235ee..cb1b65350 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -372,8 +372,6 @@ namespace llarp log::critical(logcat, err); } - // Do logging config as early as possible to get the configured log level applied - // Backwards compat: before 0.9.10 we used `type=file` with `file=|-|stdout` for print mode auto log_type = conf.logging.type; @@ -942,11 +940,13 @@ namespace llarp log::critical(logcat, "Client connecting to {} random routers to keep alive", needed); _link_manager->connect_to_random(needed); } + else + { + _hidden_service_context.Tick(now); + _exit_context.Tick(now); + } } - _hidden_service_context.Tick(now); - _exit_context.Tick(now); - // save profiles if (router_profiling().ShouldSave(now) and _config->network.save_profiles) { From 4c336c9ea1dea5de9fc00e5880be665e580c4847 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 13:53:03 -0800 Subject: [PATCH 89/93] do not gossip client RCs! --- llarp/link/link_manager.cpp | 42 +++++++++++++++++++++---------------- llarp/messages/fetch.hpp | 10 ++++++--- llarp/nodedb.cpp | 10 +++++++-- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 5f5518206..51024722c 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -792,14 +792,17 @@ namespace llarp assert(_router.is_service_node()); log::critical(logcat, "Handling fetch bootstrap fetch request..."); - RemoteRC remote; + std::optional remote; size_t quantity; try { oxenc::bt_dict_consumer btdc{m.body()}; - btdc.required("local"); - remote = RemoteRC{btdc.consume_dict_data()}; + if (btdc.skip_until("local")) + remote.emplace(btdc.consume_dict_data()); + + // btdc.required("local"); + // remote = RemoteRC{btdc.consume_dict_data()}; quantity = btdc.require("quantity"); } catch (const std::exception& e) @@ -808,27 +811,30 @@ namespace llarp m.respond(messages::ERROR_RESPONSE, true); return; } - - auto is_seed = _router.is_bootstrap_seed(); - auto& rid = remote.router_id(); - - // TODO: if we are not the seed, how do we check the requester - if (is_seed) + + if (remote) { - // we already insert the - auto& registered = node_db->registered_routers(); + auto is_snode = _router.is_service_node(); + auto& rid = remote->router_id(); - if (auto itr = registered.find(rid); itr != registered.end()) + if (is_snode) { - log::critical( - logcat, - "Bootstrap seed confirmed RID:{} is registered; approving fetch request and " - "saving RC!", - rid); - node_db->verify_gossip_bfetch_rc(remote); + // we already insert the + auto& registered = node_db->registered_routers(); + + if (auto itr = registered.find(rid); itr != registered.end()) + { + log::critical( + logcat, + "Bootstrap node confirmed RID:{} is registered; approving fetch request and " + "saving RC!", + rid); + node_db->verify_gossip_bfetch_rc(*remote); + } } } + auto& src = node_db->get_known_rcs(); auto count = src.size(); diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index 95a6018aa..d80839f4a 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -62,12 +62,16 @@ namespace llarp { // the LocalRC is converted to a RemoteRC type to send to the bootstrap seed inline static std::string - serialize(const LocalRC& local_rc, size_t quantity) + serialize(std::optional local_rc, size_t quantity) { oxenc::bt_dict_producer btdp; - btdp.append_encoded("local", oxen::quic::to_sv(local_rc.view())); - log::critical(logcat, "Serializing localRC: {}", oxenc::to_hex(local_rc.view())); + + if (local_rc) + btdp.append_encoded("local", oxen::quic::to_sv(local_rc->view())); + + log::critical(logcat, "Serializing localRC: {}", oxenc::to_hex(local_rc->view())); btdp.append("quantity", quantity); + return std::move(btdp).str(); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 12ea089eb..81c51211d 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -653,12 +653,18 @@ namespace llarp log::critical(logcat, "Dispatching BootstrapRC fetch request to {}", fetch_source); - auto num_needed = _router.is_service_node() ? SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT + auto is_snode = _router.is_service_node(); + + auto num_needed = is_snode ? SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT : CLIENT_BOOTSTRAP_SOURCE_COUNT; _router.link_manager().fetch_bootstrap_rcs( rc, - BootstrapFetchMessage::serialize(_router.router_contact, num_needed), + BootstrapFetchMessage::serialize( + is_snode ? + std::make_optional(_router.router_contact) : + std::nullopt, + num_needed), [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); From 5469c9beb08a7adab8174015299b0ac5474d0305 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 14:21:10 -0800 Subject: [PATCH 90/93] HAPPY NEW YEAR --- daemon/lokinet.cpp | 3 ++ llarp/context.cpp | 1 - llarp/link/link_manager.cpp | 83 +++++++++++++++++++++------------ llarp/link/link_manager.hpp | 27 +++++------ llarp/messages/fetch.hpp | 10 ++-- llarp/nodedb.cpp | 88 ++++++++++++++++++++--------------- llarp/nodedb.hpp | 14 +++++- llarp/router/router.cpp | 91 +++++++++++++++++++------------------ llarp/router/router.hpp | 6 +-- 9 files changed, 188 insertions(+), 135 deletions(-) diff --git a/daemon/lokinet.cpp b/daemon/lokinet.cpp index 19c673780..1e4b406f4 100644 --- a/daemon/lokinet.cpp +++ b/daemon/lokinet.cpp @@ -442,6 +442,9 @@ namespace cli.exit(e); }; + // TESTNET: + oxen::log::set_level("quic", oxen::log::Level::critical); + if (configFile.has_value()) { // when we have an explicit filepath diff --git a/llarp/context.cpp b/llarp/context.cpp index 2624fb6cd..fb2418fe6 100644 --- a/llarp/context.cpp +++ b/llarp/context.cpp @@ -66,7 +66,6 @@ namespace llarp } router = makeRouter(loop); - nodedb = makeNodeDB(); if (!router->Configure(config, opts.isSNode, nodedb)) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 51024722c..ff14edab3 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -94,9 +94,15 @@ namespace llarp } size_t - Endpoint::num_connected(bool clients_only) const + Endpoint::num_client_conns() const { - return clients_only ? client_conns.size() : client_conns.size() + service_conns.size(); + return client_conns.size(); + } + + size_t + Endpoint::num_router_conns() const + { + return service_conns.size(); } void @@ -159,6 +165,16 @@ namespace llarp [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); }); + s->register_command("fetch_rcs"s, [this](oxen::quic::message m) { + _router.loop()->call( + [this, msg = std::move(m)]() mutable { handle_fetch_rcs(std::move(msg)); }); + }); + + s->register_command("fetch_rids"s, [this](oxen::quic::message m) { + _router.loop()->call( + [this, msg = std::move(m)]() mutable { handle_fetch_router_ids(std::move(msg)); }); + }); + s->register_command("path_build"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( [this, &rid, msg = std::move(m)]() mutable { handle_path_build(std::move(msg), rid); }); @@ -351,7 +367,8 @@ namespace llarp if (msg.is_control) { - log::critical(logcat, "Dispatching {} request!", *msg.endpoint); + log::critical( + logcat, "Dispatching {} request (stream ID: {})!", *msg.endpoint, str->stream_id()); str->command(std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); } else @@ -379,26 +396,27 @@ namespace llarp void LinkManager::on_conn_open(oxen::quic::connection_interface& ci) { - _router.loop()->call([this, &conn_interface = ci]() { + _router.loop()->call([this, &conn_interface = ci, is_snode = _is_service_node]() { const auto rid = RouterID{conn_interface.remote_key()}; const auto& remote = conn_interface.remote(); + log::critical( + logcat, + "{} (RID:{}) ESTABLISHED CONNECTION TO RID:{}", + is_snode ? "SERVICE NODE" : "CLIENT", + _router.local_rid(), + rid); + if (conn_interface.is_inbound()) { - log::critical(logcat, "Inbound connection fom {} (remote:{})", rid, remote); + log::critical(logcat, "Inbound connection from {} (remote:{})", rid, remote); on_inbound_conn(conn_interface); } else { - log::critical(logcat, "Outbound connection fom {} (remote:{})", rid, remote); + log::critical(logcat, "Outbound connection from {} (remote:{})", rid, remote); on_outbound_conn(conn_interface); } - - log::critical( - logcat, - "SERVICE NODE (RID:{}) ESTABLISHED CONNECTION TO RID:{}", - _router.local_rid(), - rid); }); }; @@ -623,15 +641,15 @@ namespace llarp } size_t - LinkManager::get_num_connected(bool clients_only) const + LinkManager::get_num_connected_routers() const { - return ep.num_connected(clients_only); + return ep.num_router_conns(); } size_t LinkManager::get_num_connected_clients() const { - return get_num_connected(true); + return ep.num_client_conns(); } bool @@ -704,10 +722,6 @@ namespace llarp if (rid == gossip_src or rid == last_sender) continue; - // don't gossip RCs to clients - if (not conn->remote_is_relay) - continue; - send_control_message( rid, "gossip_rc"s, @@ -800,9 +814,7 @@ namespace llarp oxenc::bt_dict_consumer btdc{m.body()}; if (btdc.skip_until("local")) remote.emplace(btdc.consume_dict_data()); - - // btdc.required("local"); - // remote = RemoteRC{btdc.consume_dict_data()}; + quantity = btdc.require("quantity"); } catch (const std::exception& e) @@ -811,7 +823,7 @@ namespace llarp m.respond(messages::ERROR_RESPONSE, true); return; } - + if (remote) { auto is_snode = _router.is_service_node(); @@ -834,7 +846,6 @@ namespace llarp } } - auto& src = node_db->get_known_rcs(); auto count = src.size(); @@ -865,7 +876,7 @@ namespace llarp } } - m.respond(std::move(btdp).str()); + m.respond(std::move(btdp).str(), count == 0); } void @@ -881,6 +892,7 @@ namespace llarp void LinkManager::handle_fetch_rcs(oxen::quic::message m) { + log::critical(logcat, "Handling FetchRC request..."); // this handler should not be registered for clients assert(_router.is_service_node()); @@ -900,7 +912,7 @@ namespace llarp } catch (const std::exception& e) { - log::info(link_cat, "Exception handling RC Fetch request: {}", e.what()); + log::critical(link_cat, "Exception handling RC Fetch request: {}", e.what()); m.respond(messages::ERROR_RESPONSE, true); return; } @@ -908,7 +920,7 @@ namespace llarp const auto& rcs = node_db->get_rcs(); oxenc::bt_dict_producer btdp; - const auto& last_time = node_db->get_last_rc_update_times(); + // const auto& last_time = node_db->get_last_rc_update_times(); { auto sublist = btdp.append_list("rcs"); @@ -920,19 +932,24 @@ namespace llarp // Initial fetch: give me all the RC's if (explicit_ids.empty()) { + log::critical(logcat, "Returning ALL locally held RCs for initial FetchRC request..."); for (const auto& rc : rcs) { - if (last_time.at(rc.router_id()) > since_time) - sublist.append_encoded(rc.view()); + sublist.append_encoded(rc.view()); } } else { + int count = 0; for (const auto& rid : explicit_ids) { if (auto maybe_rc = node_db->get_rc_by_rid(rid)) + { sublist.append_encoded(maybe_rc->view()); + ++count; + } } + log::critical(logcat, "Returning {} RCs for FetchRC request...", count); } } @@ -949,6 +966,8 @@ namespace llarp void LinkManager::handle_fetch_router_ids(oxen::quic::message m) { + log::critical(logcat, "Handling FetchRIDs request..."); + RouterID source; RouterID local = router().local_rid(); @@ -960,7 +979,9 @@ namespace llarp } catch (const std::exception& e) { - log::info(link_cat, "Error fulfilling fetch RouterIDs request: {}", e.what()); + log::critical(link_cat, "Error fulfilling FetchRIDs request: {}", e.what()); + m.respond(messages::ERROR_RESPONSE, true); + return; } // if bad request, silently fail @@ -969,6 +990,7 @@ namespace llarp if (source != local) { + log::critical(logcat, "Relaying FetchRID request to intended target RID:{}", source); send_control_message( source, "fetch_router_ids"s, @@ -999,6 +1021,7 @@ namespace llarp return sig; }); + log::critical(logcat, "Returning ALL locally held RIDs to FetchRIDs request!"); m.respond(std::move(btdp).str()); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index f78401481..028d9f3ee 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -40,6 +40,9 @@ namespace llarp inline const keep_alive ROUTER_KEEP_ALIVE{10s}; inline const keep_alive CLIENT_KEEP_ALIVE{10s}; + inline constexpr int MIN_CLIENT_ROUTER_CONNS{4}; + inline constexpr int MAX_CLIENT_ROUTER_CONNS{6}; + namespace alpns { inline const auto SN_ALPNS = "SERVICE_NODE"_us; @@ -93,7 +96,10 @@ namespace llarp num_in_out() const; size_t - num_connected(bool clients_only) const; + num_client_conns() const; + + size_t + num_router_conns() const; template bool @@ -198,12 +204,6 @@ namespace llarp // holds any messages we attempt to send while connections are establishing std::unordered_map pending_conn_msg_queue; - // when establishing a connection, the rid of the remote is placed here to be cross- - // checked by the tls verification callback - std::map rids_pending_verification; - // in the interim of verifying an outbound connection and the creation of its link::Connection - // object, we store the rid and rc here - std::map verified_rids; util::DecayingHashSet clients{path::DEFAULT_LIFETIME}; @@ -327,7 +327,7 @@ namespace llarp num_in_out() const; size_t - get_num_connected(bool clients_only = false) const; + get_num_connected_routers() const; size_t get_num_connected_clients() const; @@ -434,9 +434,11 @@ namespace llarp try { const auto& rid = rc.router_id(); - log::critical(logcat, "Establishing connection to RID:{}", rid); + const auto& is_snode = _is_service_node; - bool is_snode = link_manager.is_service_node(); + log::critical(logcat, "Establishing connection to RID:{}", rid); + // add to service conns + auto [itr, b] = service_conns.emplace(rid, nullptr); auto conn_interface = endpoint->connect( remote, @@ -444,9 +446,6 @@ namespace llarp is_snode ? ROUTER_KEEP_ALIVE : CLIENT_KEEP_ALIVE, std::forward(opts)...); - // add to service conns - auto [itr, b] = service_conns.emplace(rid, nullptr); - auto control_stream = conn_interface->template open_stream( [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { log::warning( @@ -456,8 +455,6 @@ namespace llarp close_connection(rid); }); - log::critical(logcat, "Opened BTStream (ID:{})", control_stream->stream_id()); - assert(control_stream->stream_id() == 0); if (is_snode) link_manager.register_commands(control_stream, rid); else diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index d80839f4a..af6ffe683 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -65,13 +65,15 @@ namespace llarp serialize(std::optional local_rc, size_t quantity) { oxenc::bt_dict_producer btdp; - + if (local_rc) + { + log::critical(logcat, "Serializing localRC: {}", oxenc::to_hex(local_rc->view())); btdp.append_encoded("local", oxen::quic::to_sv(local_rc->view())); - - log::critical(logcat, "Serializing localRC: {}", oxenc::to_hex(local_rc->view())); + } + btdp.append("quantity", quantity); - + return std::move(btdp).str(); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 81c51211d..be8140304 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -336,6 +336,7 @@ namespace llarp } else { + _fetching_initial = true; // Set fetch source as random selection of known active client routers fetch_source = *std::next(known_rids.begin(), csrng() % known_rids.size()); fetch_rcs(true); @@ -357,13 +358,27 @@ namespace llarp std::vector needed; const auto now = time_point_now(); - for (const auto& [rid, rc] : rc_lookup) + if (not initial) { - if (now - rc.timestamp() > RouterContact::OUTDATED_AGE) - needed.push_back(rid); + for (const auto& [rid, rc] : rc_lookup) + { + if (now - rc.timestamp() > RouterContact::OUTDATED_AGE) + needed.push_back(rid); + } } RouterID& src = fetch_source; + log::critical( + logcat, + "Sending{} FetchRCs request to {} for {} RCs", + initial ? " initial" : "", + src, + initial ? "all of the" : std::to_string(needed.size())); + + if (initial) + _router.next_initial_fetch_attempt = now + INITIAL_ATTEMPT_INTERVAL; + + _router.last_rc_fetch = now; _router.link_manager().fetch_rcs( src, @@ -371,7 +386,7 @@ namespace llarp [this, src, initial](oxen::quic::message m) mutable { if (m.timed_out) { - log::info(logcat, "RC fetch to {} timed out!", src); + log::critical(logcat, "RC fetch to {} timed out!", src); fetch_rcs_result(initial, m.timed_out); return; } @@ -382,7 +397,7 @@ namespace llarp if (m.is_error()) { auto reason = btdc.require(messages::STATUS_KEY); - log::info(logcat, "RC fetch to {} returned error: {}", src, reason); + log::critical(logcat, "RC fetch to {} returned error: {}", src, reason); fetch_rcs_result(initial, m.is_error()); return; } @@ -399,7 +414,7 @@ namespace llarp } catch (const std::exception& e) { - log::info(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); + log::critical(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); fetch_rcs_result(initial, true); return; } @@ -431,9 +446,11 @@ namespace llarp fetch_counters.clear(); RouterID& src = fetch_source; + _router.last_rid_fetch = llarp::time_point_now(); for (const auto& target : rid_sources) { + log::critical(logcat, "Sending FetchRIDs request to {} via {}", target, src); _router.link_manager().fetch_router_ids( src, FetchRIDMessage::serialize(target), @@ -442,7 +459,7 @@ namespace llarp { auto err = "RID fetch from {} via {} {}"_format( src, target, m.timed_out ? "timed out" : "failed"); - log::info(link_cat, err); + log::critical(link_cat, err); ingest_rid_fetch_responses(target); fetch_rids_result(initial); return; @@ -469,7 +486,7 @@ namespace llarp { if (s.size() != RouterID::SIZE) { - log::warning( + log::critical( link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); ingest_rid_fetch_responses(target); fetch_rids_result(initial); @@ -485,7 +502,7 @@ namespace llarp } catch (const std::exception& e) { - log::info(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); + log::critical(link_cat, "Error handling fetch RouterIDs response: {}", e.what()); ingest_rid_fetch_responses(target); fetch_rids_result(initial); } @@ -500,7 +517,7 @@ namespace llarp { if (++fetch_failures >= MAX_FETCH_ATTEMPTS) { - log::info( + log::critical( logcat, "RC fetching from {} reached failure threshold ({}); falling back to bootstrap...", fetch_source, @@ -510,6 +527,9 @@ namespace llarp return; } + if (initial) + _needs_initial_fetch = true; + // If we have passed the last last conditional, then it means we are not bootstrapping // and the current fetch_source has more attempts before being rotated. As a result, we // find new non-bootstrap RC fetch source and try again buddy @@ -520,7 +540,7 @@ namespace llarp } else { - log::debug(logcat, "Successfully fetched RC's from {}", fetch_source); + log::critical(logcat, "Successfully fetched RC's from {}", fetch_source); post_fetch_rcs(initial); } } @@ -530,7 +550,7 @@ namespace llarp { if (fetch_failures >= MAX_FETCH_ATTEMPTS) { - log::info( + log::critical( logcat, "Failed {} attempts to fetch RID's from {}; reverting to bootstrap...", MAX_FETCH_ATTEMPTS, @@ -544,7 +564,7 @@ namespace llarp if (n_responses < RID_SOURCE_COUNT) { - log::debug(logcat, "Received {}/{} fetch RID requests", n_responses, RID_SOURCE_COUNT); + log::critical(logcat, "Received {}/{} fetch RID requests", n_responses, RID_SOURCE_COUNT); return; } @@ -552,18 +572,18 @@ namespace llarp if (n_fails <= MAX_RID_ERRORS) { - log::debug( + log::critical( logcat, "RID fetching was successful ({}/{} acceptable errors)", n_fails, MAX_RID_ERRORS); // this is where the trust model will do verification based on the similarity of the sets if (process_fetched_rids()) { - log::debug(logcat, "Accumulated RID's accepted by trust model"); + log::critical(logcat, "Accumulated RID's accepted by trust model"); post_fetch_rids(initial); return; } - log::debug( + log::critical( logcat, "Accumulated RID's rejected by trust model, reselecting all RID sources..."); reselect_router_id_sources(rid_sources); ++fetch_failures; @@ -571,29 +591,24 @@ namespace llarp else { // we had 4 or more failed requests, so we will need to rotate our rid sources - log::debug( + log::critical( logcat, "RID fetching found {} failures; reselecting failed RID sources...", n_fails); ++fetch_failures; reselect_router_id_sources(fail_sources); } - fetch_rids(true); + fetch_rids(initial); } + // This function is only called after a successful FetchRC request void NodeDB::post_fetch_rcs(bool initial) { - _router.last_rc_fetch = llarp::time_point_now(); - - if (_router.is_service_node()) - { - _needs_rebootstrap = false; - _needs_initial_fetch = false; - _using_bootstrap_fallback = false; - fail_sources.clear(); - fetch_failures = 0; - return; - } + _needs_rebootstrap = false; + _needs_initial_fetch = false; + _using_bootstrap_fallback = false; + fail_sources.clear(); + fetch_failures = 0; if (initial) fetch_rids(initial); @@ -604,7 +619,6 @@ namespace llarp { fail_sources.clear(); fetch_failures = 0; - _router.last_rid_fetch = llarp::time_point_now(); fetch_counters.clear(); _needs_rebootstrap = false; _using_bootstrap_fallback = false; @@ -612,6 +626,7 @@ namespace llarp if (initial) { _needs_initial_fetch = false; + _fetching_initial = false; _initial_completed = true; } } @@ -655,16 +670,13 @@ namespace llarp auto is_snode = _router.is_service_node(); - auto num_needed = is_snode ? SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT - : CLIENT_BOOTSTRAP_SOURCE_COUNT; + auto num_needed = + is_snode ? SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT : CLIENT_BOOTSTRAP_SOURCE_COUNT; _router.link_manager().fetch_bootstrap_rcs( rc, BootstrapFetchMessage::serialize( - is_snode ? - std::make_optional(_router.router_contact) : - std::nullopt, - num_needed), + is_snode ? std::make_optional(_router.router_contact) : std::nullopt, num_needed), [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); @@ -706,7 +718,6 @@ namespace llarp bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS, e.what()); - log::critical(logcat, "DEBUG FIXME THIS IS WHAT I GOT: {}", oxenc::to_hex(m.body())); fallback_to_bootstrap(); return; } @@ -853,8 +864,11 @@ namespace llarp RemoteRC rc{}; if (not rc.read(f) or rc.is_expired(now)) + { // try loading it, purge it if it is junk or expired purge.push_back(f); + continue; + } const auto& rid = rc.router_id(); diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 8eaa077a8..d9ac04a96 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -176,7 +176,7 @@ namespace llarp std::atomic fetch_failures{0}, bootstrap_attempts{0}; std::atomic _using_bootstrap_fallback{false}, _needs_rebootstrap{false}, - _needs_initial_fetch{true}, _initial_completed{false}; + _needs_initial_fetch{true}, _fetching_initial{false}, _initial_completed{false}; bool want_rc(const RouterID& rid) const; @@ -211,6 +211,18 @@ namespace llarp std::optional get_rc_by_rid(const RouterID& rid); + bool + is_initial_fetching() const + { + return _fetching_initial; + } + + bool + initial_fetch_completed() const + { + return _initial_completed; + } + bool needs_initial_fetch() const { diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index cb1b65350..389a864af 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -391,6 +391,9 @@ namespace llarp else llarp::logRingBuffer = nullptr; + // TESTNET: + oxen::log::set_level("quic", oxen::log::Level::critical); + log::debug(logcat, "Configuring router"); _is_service_node = conf.router.is_relay; @@ -417,9 +420,7 @@ namespace llarp logcat, _is_service_node ? "Running as a relay (service node)" : "Running as a client"); if (_is_service_node) - { _rpc_client->ConnectAsync(rpc_addr); - } log::debug(logcat, "Initializing key manager"); if (not _key_manager->initialize(conf, true, isSNode)) @@ -522,7 +523,7 @@ namespace llarp size_t Router::num_router_connections() const { - return _link_manager->get_num_connected(); + return _link_manager->get_num_connected_routers(); } size_t @@ -661,35 +662,28 @@ namespace llarp { const auto now = llarp::time_now_ms(); - if (is_service_node()) - { - auto [in, out] = _link_manager->num_in_out(); - - log::critical( - logcat, - "Local Service Node has {} RCs, {} RIDs, {} bootstrap peers, {}:{} (inbound:outbound) " - "router " - "connections, and {} client connections since last RC update ({} to expiry)", - _node_db->num_rcs(), - _node_db->num_rids(), - _node_db->num_bootstraps(), - in, - out, - num_client_connections(), - router_contact.time_to_expiry(now)); - - if (num_router_connections() >= _node_db->num_rcs()) - log::critical(logcat, "SERVICE NODE IS FULLY MESHED"); - } - else + auto [in, out] = _link_manager->num_in_out(); + auto num_bootstraps = _node_db->num_bootstraps(); + auto num_rids = _node_db->num_rids(); + auto num_rcs = _node_db->num_rcs(); + auto num_router_conns = num_router_connections(); + + log::critical( + logcat, + "Local {} has {} RCs, {} RIDs, {} bootstrap peers, {}:{} (inbound:outbound) " + "conns ({} router, {} client)", + is_service_node() ? "Service Node" : "Client", + num_rcs, + num_rids, + num_bootstraps, + in, + out, + num_router_conns, + num_client_connections()); + + if (is_service_node() and num_router_connections() >= num_rcs) { - log::critical( - logcat, - "{} RCs loaded with {} RIDs, {} bootstrap peers, and {} router connections!", - _node_db->num_rcs(), - _node_db->num_rids(), - _node_db->num_bootstraps(), - num_router_connections()); + log::critical(logcat, "SERVICE NODE IS FULLY MESHED"); } if (_last_stats_report > 0s) @@ -758,6 +752,16 @@ namespace llarp const bool is_snode = is_service_node(); const bool is_decommed = appears_decommed(); + const auto& local = local_rid(); + + if (is_snode and not node_db()->registered_routers().count(local)) + { + log::critical(logcat, "We are NOT registered router, figure it out!"); + // update tick timestamp + _last_tick = llarp::time_now_ms(); + return; + } + const auto now = llarp::time_now_ms(); auto now_timepoint = std::chrono::system_clock::time_point(now); @@ -801,20 +805,18 @@ namespace llarp next_rc_gossip = now_timepoint + TESTNET_GOSSIP_INTERVAL - delta; } - - // report_stats(); } if (needs_rebootstrap() and now_timepoint > next_bootstrap_attempt) { node_db()->fallback_to_bootstrap(); } - else if (needs_initial_fetch()) + else if (needs_initial_fetch() and now_timepoint > next_initial_fetch_attempt) { if (not _config->bootstrap.seednode) node_db()->fetch_initial(is_service_node()); } - else if (not is_snode) + else if (not is_snode and node_db()->initial_fetch_completed()) { // (client-only) periodically fetch updated RCs if (now_timepoint - last_rc_fetch > RC_UPDATE_INTERVAL) @@ -884,9 +886,8 @@ namespace llarp _link_manager->check_persisting_conns(now); - auto num_conns = num_router_connections(); - - const auto& num_rcs = node_db()->num_rcs(); + auto num_router_conns = num_router_connections(); + auto num_rcs = node_db()->num_rcs(); if (is_snode) { @@ -914,7 +915,7 @@ namespace llarp } } - if (num_conns < num_rcs) + if (num_router_conns < num_rcs) { log::critical( logcat, @@ -925,7 +926,7 @@ namespace llarp } else { - size_t min_client_conns = _link_manager->client_router_connections; + size_t min_client_conns = MIN_CLIENT_ROUTER_CONNS; const auto& pinned_edges = _node_db->pinned_edges(); const auto pinned_count = pinned_edges.size(); @@ -934,16 +935,18 @@ namespace llarp // if we need more sessions to routers and we are not a service node kicked from the network // or we are a client we shall connect out to others - if (num_conns < min_client_conns) + if (num_router_conns < min_client_conns) { - size_t needed = min_client_conns - num_conns; + size_t needed = min_client_conns - num_router_conns; log::critical(logcat, "Client connecting to {} random routers to keep alive", needed); _link_manager->connect_to_random(needed); } else { - _hidden_service_context.Tick(now); - _exit_context.Tick(now); + // log::critical( + // logcat, "Client skipping hidden service exit tick or whatever the fuck that means"); + // _hidden_service_context.Tick(now); + // _exit_context.Tick(now); } } diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 045553551..7880a49aa 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -50,7 +50,8 @@ namespace llarp // TESTNET: these constants are shortened for testing purposes inline constexpr std::chrono::milliseconds TESTNET_GOSSIP_INTERVAL{10min}; - inline constexpr std::chrono::milliseconds RC_UPDATE_INTERVAL{4min}; + inline constexpr std::chrono::milliseconds RC_UPDATE_INTERVAL{5min}; + inline constexpr std::chrono::milliseconds INITIAL_ATTEMPT_INTERVAL{30s}; // as we advance towards full mesh, we try to connect to this number per tick inline constexpr int FULL_MESH_ITERATION{1}; inline constexpr std::chrono::milliseconds ROUTERID_UPDATE_INTERVAL{1h}; @@ -159,11 +160,10 @@ namespace llarp insufficient_peers() const; protected: - // bool _needs_initial_fetch{true}; - std::chrono::system_clock::time_point last_rc_gossip{ std::chrono::system_clock::time_point::min()}; std::chrono::system_clock::time_point next_rc_gossip{last_rc_gossip}; + std::chrono::system_clock::time_point next_initial_fetch_attempt{last_rc_gossip}; std::chrono::system_clock::time_point last_rc_fetch{last_rc_gossip}; std::chrono::system_clock::time_point last_rid_fetch{last_rc_gossip}; std::chrono::system_clock::time_point next_bootstrap_attempt{last_rc_gossip}; From e331f0b31f65df57d4f0f1a58f0e022c0b0c6a84 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Thu, 21 Dec 2023 11:28:24 -0800 Subject: [PATCH 91/93] No fetch only throw --- llarp/link/link_manager.cpp | 7 +++---- llarp/messages/fetch.hpp | 15 ++++++++++++--- llarp/nodedb.cpp | 37 ++++++++++++++++++++++--------------- llarp/router_id.hpp | 3 +++ 4 files changed, 40 insertions(+), 22 deletions(-) diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index ff14edab3..b850ec462 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -960,7 +960,7 @@ namespace llarp LinkManager::fetch_router_ids( const RouterID& via, std::string payload, std::function func) { - send_control_message(via, "fetch_router_ids"s, std::move(payload), std::move(func)); + send_control_message(via, "fetch_rids"s, std::move(payload), std::move(func)); } void @@ -974,8 +974,7 @@ namespace llarp try { oxenc::bt_dict_consumer btdc{m.body()}; - - source.from_string(btdc.require("source")); + source = RouterID{btdc.require("source")}; } catch (const std::exception& e) { @@ -993,7 +992,7 @@ namespace llarp log::critical(logcat, "Relaying FetchRID request to intended target RID:{}", source); send_control_message( source, - "fetch_router_ids"s, + "fetch_rids"s, m.body_str(), [source_rid = std::move(source), original = std::move(m)](oxen::quic::message m) mutable { original.respond(m.body_str(), m.is_error()); diff --git a/llarp/messages/fetch.hpp b/llarp/messages/fetch.hpp index af6ffe683..f8c194ff7 100644 --- a/llarp/messages/fetch.hpp +++ b/llarp/messages/fetch.hpp @@ -105,9 +105,18 @@ namespace llarp inline static std::string serialize(const RouterID& source) { - // serialize_response is a bit weird here, and perhaps could have a sister function - // with the same purpose but as a request, but...it works. - return messages::serialize_response({{"source", source.ToView()}}); + oxenc::bt_dict_producer btdp; + + try + { + btdp.append("source", source.ToView()); + } + catch (...) + { + log::error(link_cat, "Error: FetchRIDMessage failed to bt encode contents!"); + } + + return std::move(btdp).str(); } } // namespace FetchRIDMessage diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index be8140304..5b9eb6bce 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -233,8 +233,13 @@ namespace llarp // if we are not bootstrapping, we should check the rc's against the ones we currently hold if (not _using_bootstrap_fallback) { - if (not process_fetched_rcs(rcs)) - return false; + log::critical(logcat, "Checking returned RCs against locally held..."); + + auto success = process_fetched_rcs(rcs); + + log::critical( + logcat, "RCs returned by FetchRC {} by trust model", success ? "approved" : "rejected"); + return success; } while (!rcs.empty()) @@ -383,10 +388,10 @@ namespace llarp _router.link_manager().fetch_rcs( src, FetchRCMessage::serialize(_router.last_rc_fetch, needed), - [this, src, initial](oxen::quic::message m) mutable { + [this, source = src, initial](oxen::quic::message m) mutable { if (m.timed_out) { - log::critical(logcat, "RC fetch to {} timed out!", src); + log::critical(logcat, "RC fetch to {} timed out!", source); fetch_rcs_result(initial, m.timed_out); return; } @@ -397,7 +402,7 @@ namespace llarp if (m.is_error()) { auto reason = btdc.require(messages::STATUS_KEY); - log::critical(logcat, "RC fetch to {} returned error: {}", src, reason); + log::critical(logcat, "RC fetch to {} returned error: {}", source, reason); fetch_rcs_result(initial, m.is_error()); return; } @@ -414,7 +419,8 @@ namespace llarp } catch (const std::exception& e) { - log::critical(logcat, "Failed to parse RC fetch response from {}: {}", src, e.what()); + log::critical( + logcat, "Failed to parse RC fetch response from {}: {}", source, e.what()); fetch_rcs_result(initial, true); return; } @@ -454,11 +460,11 @@ namespace llarp _router.link_manager().fetch_router_ids( src, FetchRIDMessage::serialize(target), - [this, src, target, initial](oxen::quic::message m) mutable { + [this, source = src, target, initial](oxen::quic::message m) mutable { if (m.is_error()) { auto err = "RID fetch from {} via {} {}"_format( - src, target, m.timed_out ? "timed out" : "failed"); + target, source, m.timed_out ? "timed out" : "failed"); log::critical(link_cat, err); ingest_rid_fetch_responses(target); fetch_rids_result(initial); @@ -472,10 +478,10 @@ namespace llarp btdc.required("routers"); auto router_id_strings = btdc.consume_list>(); - btdc.require_signature("signature", [&src](ustring_view msg, ustring_view sig) { + btdc.require_signature("signature", [&source](ustring_view msg, ustring_view sig) { if (sig.size() != 64) throw std::runtime_error{"Invalid signature: not 64 bytes"}; - if (not crypto::verify(src, msg, sig)) + if (not crypto::verify(source, msg, sig)) throw std::runtime_error{ "Failed to verify signature for fetch RouterIDs response."}; }); @@ -487,7 +493,7 @@ namespace llarp if (s.size() != RouterID::SIZE) { log::critical( - link_cat, "RID fetch from {} via {} returned bad RouterID", target, src); + link_cat, "RID fetch from {} via {} returned bad RouterID", target, source); ingest_rid_fetch_responses(target); fetch_rids_result(initial); return; @@ -677,7 +683,8 @@ namespace llarp rc, BootstrapFetchMessage::serialize( is_snode ? std::make_optional(_router.router_contact) : std::nullopt, num_needed), - [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { + [this, is_snode = _router.is_service_node(), src = rc.router_id()]( + oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); if (not m) @@ -685,7 +692,7 @@ namespace llarp log::warning( logcat, "BootstrapRC fetch request to {} failed (error {}/{})", - fetch_source, + src, bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS); fallback_to_bootstrap(); @@ -714,7 +721,7 @@ namespace llarp log::warning( logcat, "Failed to parse BootstrapRC fetch response from {} (error {}/{}): {}", - fetch_source, + src, bootstrap_attempts, MAX_BOOTSTRAP_FETCH_ATTEMPTS, e.what()); @@ -725,7 +732,7 @@ namespace llarp log::critical( logcat, "BootstrapRC fetch response from {} returned {}/{} needed RCs", - fetch_source, + src, num, MIN_ACTIVE_RCS); diff --git a/llarp/router_id.hpp b/llarp/router_id.hpp index d587f8365..f82f78402 100644 --- a/llarp/router_id.hpp +++ b/llarp/router_id.hpp @@ -24,6 +24,9 @@ namespace llarp RouterID(ustring_view data) : PubKey(data.data()) {} + RouterID(std::string_view data) : RouterID(to_usv(data)) + {} + util::StatusObject ExtractStatus() const; From a5b7a7e35b27a4084469d7610f2fd716af4557e9 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Sun, 7 Jan 2024 13:15:55 -0800 Subject: [PATCH 92/93] Deprecate pending_msg_que in favor of libquic internal stream buffers --- llarp/exit/session.cpp | 64 ++++++++-------- llarp/link/link_manager.cpp | 147 ++++++++++++------------------------ llarp/link/link_manager.hpp | 96 +++++++++++++++++++---- llarp/router/router.cpp | 12 --- llarp/router/router.hpp | 6 -- 5 files changed, 163 insertions(+), 162 deletions(-) diff --git a/llarp/exit/session.cpp b/llarp/exit/session.cpp index 5f1c6192e..2802fd9ca 100644 --- a/llarp/exit/session.cpp +++ b/llarp/exit/session.cpp @@ -246,38 +246,38 @@ namespace llarp::exit bool BaseSession::FlushUpstream() { - auto now = router->now(); - auto path = PickEstablishedPath(llarp::path::ePathRoleExit); - if (path) - { - // for (auto& [i, queue] : m_Upstream) - // { - // while (queue.size()) - // { - // auto& msg = queue.front(); - // msg.sequence_number = path->NextSeqNo(); - // path->SendRoutingMessage(msg, router); - // queue.pop_front(); - // } - // } - } - else - { - // if (m_Upstream.size()) - // llarp::LogWarn("no path for exit session"); - // // discard upstream - // for (auto& [i, queue] : m_Upstream) - // queue.clear(); - // m_Upstream.clear(); - - if (numHops == 1) - { - if (const auto maybe = router->node_db()->get_rc(exit_router); maybe.has_value()) - router->connect_to(*maybe); - } - else if (UrgentBuild(now)) - BuildOneAlignedTo(exit_router); - } + // auto now = router->now(); + // auto path = PickEstablishedPath(llarp::path::ePathRoleExit); + // if (path) + // { + // for (auto& [i, queue] : m_Upstream) + // { + // while (queue.size()) + // { + // auto& msg = queue.front(); + // msg.sequence_number = path->NextSeqNo(); + // path->SendRoutingMessage(msg, router); + // queue.pop_front(); + // } + // } + // } + // else + // { + // if (m_Upstream.size()) + // llarp::LogWarn("no path for exit session"); + // // discard upstream + // for (auto& [i, queue] : m_Upstream) + // queue.clear(); + // m_Upstream.clear(); + + // if (numHops == 1) + // { + // if (const auto maybe = router->node_db()->get_rc(exit_router); maybe.has_value()) + // router->connect_to(*maybe); + // } + // else if (UrgentBuild(now)) + // BuildOneAlignedTo(exit_router); + // } return true; } diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index b850ec462..9e3db54dc 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -352,36 +352,6 @@ namespace llarp if (auto it = ep.service_conns.find(rid); it != ep.service_conns.end()) { log::critical(logcat, "Fetched configured outbound connection to relay RID:{}", rid); - - auto& conn = it->second->conn; - auto& str = it->second->control_stream; - - if (auto itr = pending_conn_msg_queue.find(rid); itr != pending_conn_msg_queue.end()) - { - log::critical(logcat, "Clearing pending queue for RID:{}", rid); - auto& que = itr->second; - - while (not que.empty()) - { - auto& msg = que.front(); - - if (msg.is_control) - { - log::critical( - logcat, "Dispatching {} request (stream ID: {})!", *msg.endpoint, str->stream_id()); - str->command(std::move(*msg.endpoint), std::move(msg.body), std::move(msg.func)); - } - else - { - log::critical(logcat, "DIspatching data message: {}", msg.body); - conn->send_datagram(std::move(msg.body)); - } - - que.pop_front(); - } - } - - log::warning(logcat, "Pending queue empty for RID:{}", rid); } else { @@ -427,10 +397,6 @@ namespace llarp [this, scid = ci.scid(), rid = RouterID{ci.remote_key()}, error_code = ec]() { log::critical(quic_cat, "Purging quic connection CID:{} (ec:{})", scid, error_code); - // in case this didn't clear earlier, do it now - if (auto p_itr = pending_conn_msg_queue.find(rid); p_itr != pending_conn_msg_queue.end()) - pending_conn_msg_queue.erase(p_itr); - if (auto s_itr = ep.service_conns.find(rid); s_itr != ep.service_conns.end()) { log::critical(quic_cat, "Quic connection to relay RID:{} purged successfully", rid); @@ -453,7 +419,12 @@ namespace llarp std::string body, std::function func) { - assert(func); // makes no sense to send control message and ignore response + // DISCUSS: revisit if this assert makes sense. If so, there's no need to if (func) the + // next logic block + assert(func); // makes no sense to send control message and ignore response (maybe gossip?) + + if (is_stopping) + return false; if (func) { @@ -463,19 +434,6 @@ namespace llarp }; } - return send_control_message_impl(remote, std::move(endpoint), std::move(body), std::move(func)); - } - - bool - LinkManager::send_control_message_impl( - const RouterID& remote, - std::string endpoint, - std::string body, - std::function func) - { - if (is_stopping) - return false; - if (auto conn = ep.get_conn(remote); conn) { conn->control_stream->command(std::move(endpoint), std::move(body), std::move(func)); @@ -487,21 +445,7 @@ namespace llarp endpoint = std::move(endpoint), body = std::move(body), f = std::move(func)]() { - auto pending = PendingMessage(std::move(body), std::move(endpoint), std::move(f)); - - if (auto it = pending_conn_msg_queue.find(remote); it != pending_conn_msg_queue.end()) - { - it->second.push_back(std::move(pending)); - log::critical( - logcat, "Connection to RID:{} is pending; message appended to send queue!", remote); - } - else - { - log::critical(logcat, "Connection to RID:{} is pending; creating send queue!", remote); - auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); - itr->second.push_back(std::move(pending)); - connect_to(remote); - } + connect_and_send(remote, std::move(endpoint), std::move(body), std::move(f)); }); return false; @@ -520,12 +464,7 @@ namespace llarp } _router.loop()->call([this, body = std::move(body), remote]() { - auto pending = PendingMessage(std::move(body)); - - auto [itr, b] = pending_conn_msg_queue.emplace(remote, MessageQueue()); - itr->second.push_back(std::move(pending)); - - connect_to(remote); + connect_and_send(remote, std::nullopt, std::move(body)); }); return false; @@ -550,12 +489,35 @@ namespace llarp } void - LinkManager::connect_to(const RouterID& rid, conn_open_hook hook) + LinkManager::connect_and_send( + const RouterID& router, + std::optional endpoint, + std::string body, + std::function func) { - if (auto rc = node_db->get_rc(rid)) - connect_to(*rc, std::move(hook)); + // by the time we have called this, we have already checked if we have a connection to this RID + // in ::send_control_message_impl, at which point we will dispatch on that stream + if (auto rc = node_db->get_rc(router)) + { + const auto& remote_addr = rc->addr(); + + if (auto rv = ep.establish_and_send( + oxen::quic::RemoteAddress{router.ToView(), remote_addr}, + *rc, + std::move(endpoint), + std::move(body), + std::move(func)); + rv) + { + log::info(quic_cat, "Begun establishing connection to {}", remote_addr); + return; + } + + log::warning(quic_cat, "Failed to begin establishing connection to {}", remote_addr); + } else - log::warning(quic_cat, "Could not find RouterContact for connection to rid:{}", rid); + log::error( + quic_cat, "Error: Could not find RC for connection to rid:{}, message not sent!", router); } void @@ -573,8 +535,6 @@ namespace llarp const auto& remote_addr = rc.addr(); - // TODO: confirm remote end is using the expected pubkey (RouterID). - // TODO: ALPN for "client" vs "relay" (could just be set on endpoint creation) if (auto rv = ep.establish_connection( oxen::quic::RemoteAddress{rid.ToView(), remote_addr}, rc, @@ -767,35 +727,28 @@ namespace llarp log::critical(link_cat, "Received known or old RC, not storing or forwarding."); } + // TODO: can probably use ::send_control_message instead. Need to discuss the potential difference + // in calling Endpoint::get_service_conn vs Endpoint::get_conn void LinkManager::fetch_bootstrap_rcs( const RemoteRC& source, std::string payload, std::function func) { - _router.loop()->call([this, source, payload, f = std::move(func)]() mutable { - if (f) - { - f = [this, func = std::move(f)](oxen::quic::message m) mutable { - _router.loop()->call( - [f = std::move(func), msg = std::move(m)]() mutable { f(std::move(msg)); }); - }; - } - - const auto& rid = source.router_id(); - - if (auto conn = ep.get_service_conn(rid); conn) - { - conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(f)); - log::critical(logcat, "Dispatched bootstrap fetch request!"); - return; - } + func = [this, f = std::move(func)](oxen::quic::message m) mutable { + _router.loop()->call( + [func = std::move(f), msg = std::move(m)]() mutable { func(std::move(msg)); }); + }; - log::critical(logcat, "Queuing bootstrap fetch request to {}", rid); - auto pending = PendingMessage(std::move(payload), "bfetch_rcs"s, std::move(f)); + const auto& rid = source.router_id(); - auto [itr, b] = pending_conn_msg_queue.emplace(rid, MessageQueue()); - itr->second.push_back(std::move(pending)); + if (auto conn = ep.get_service_conn(rid); conn) + { + conn->control_stream->command("bfetch_rcs"s, std::move(payload), std::move(func)); + log::critical(logcat, "Dispatched bootstrap fetch request!"); + return; + } - connect_to(source); + _router.loop()->call([this, source, payload, f = std::move(func), rid = rid]() mutable { + connect_and_send(rid, "bfetch_rcs"s, std::move(payload), std::move(f)); }); } diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 028d9f3ee..c73662308 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -106,6 +106,16 @@ namespace llarp establish_connection( const oxen::quic::RemoteAddress& remote, const RemoteRC& rc, Opt&&... opts); + template + bool + establish_and_send( + const oxen::quic::RemoteAddress& remote, + const RemoteRC& rc, + std::optional endpoint, + std::string body, + std::function func = nullptr, + Opt&&... opts); + void for_each_connection(std::function func); @@ -188,13 +198,6 @@ namespace llarp private: explicit LinkManager(Router& r); - bool - send_control_message_impl( - const RouterID& remote, - std::string endpoint, - std::string body, - std::function = nullptr); - friend struct link::Endpoint; std::atomic is_stopping; @@ -202,9 +205,6 @@ namespace llarp // sessions to persist -> timestamp to end persist at std::unordered_map persisting_conns; - // holds any messages we attempt to send while connections are establishing - std::unordered_map pending_conn_msg_queue; - util::DecayingHashSet clients{path::DEFAULT_LIFETIME}; std::shared_ptr node_db; @@ -225,9 +225,6 @@ namespace llarp void recv_data_message(oxen::quic::dgram_interface& dgi, bstring dgram); - void - recv_control_message(oxen::quic::message msg); - std::shared_ptr make_control(oxen::quic::connection_interface& ci, const RouterID& rid); @@ -309,10 +306,14 @@ namespace llarp test_reachability(const RouterID& rid, conn_open_hook, conn_closed_hook); void - connect_to(const RouterID& router, conn_open_hook = nullptr); + connect_to(const RemoteRC& rc, conn_open_hook = nullptr, conn_closed_hook = nullptr); void - connect_to(const RemoteRC& rc, conn_open_hook = nullptr, conn_closed_hook = nullptr); + connect_and_send( + const RouterID& router, + std::optional endpoint, + std::string body, + std::function func = nullptr); void close_connection(RouterID rid); @@ -426,6 +427,71 @@ namespace llarp namespace link { + template + bool + Endpoint::establish_and_send( + const oxen::quic::RemoteAddress& remote, + const RemoteRC& rc, + std::optional ep, + std::string body, + std::function func, + Opt&&... opts) + { + try + { + const auto& rid = rc.router_id(); + const auto& is_snode = _is_service_node; + const auto& is_control = ep.has_value(); + const auto us = is_snode ? "Relay"s : "Client"s; + + log::critical(logcat, "Establishing connection to RID:{}", rid); + // add to service conns + auto [itr, b] = service_conns.emplace(rid, nullptr); + + auto conn_interface = endpoint->connect( + remote, + link_manager.tls_creds, + is_snode ? ROUTER_KEEP_ALIVE : CLIENT_KEEP_ALIVE, + std::forward(opts)...); + + // auto + std::shared_ptr control_stream = + conn_interface->template open_stream( + [this, rid = rid](oxen::quic::Stream&, uint64_t error_code) { + log::warning( + logcat, + "BTRequestStream closed unexpectedly (ec:{}); closing outbound connection...", + error_code); + close_connection(rid); + }); + + if (is_snode) + link_manager.register_commands(control_stream, rid); + else + log::critical(logcat, "Client NOT registering BTStream commands!"); + + log::critical( + logcat, + "{} dispatching {} on outbound connection to remote (rid:{})", + us, + is_control ? "control message (ep:{})"_format(ep) : "data message", + rid); + + (is_control) ? control_stream->command(std::move(*ep), std::move(body), std::move(func)) + : conn_interface->send_datagram(std::move(body)); + + itr->second = std::make_shared(conn_interface, control_stream, true); + + log::critical(logcat, "Outbound connection to RID:{} added to service conns...", rid); + return true; + } + catch (...) + { + log::error(quic_cat, "Error: failed to establish connection to {}", remote); + return false; + } + } + template bool Endpoint::establish_connection( diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 389a864af..85e3b7ba3 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -247,18 +247,6 @@ namespace llarp loop_wakeup->Trigger(); } - void - Router::connect_to(const RouterID& rid) - { - _link_manager->connect_to(rid); - } - - void - Router::connect_to(const RemoteRC& rc) - { - _link_manager->connect_to(rc); - } - bool Router::send_data_message(const RouterID& remote, std::string payload) { diff --git a/llarp/router/router.hpp b/llarp/router/router.hpp index 7880a49aa..ea2774e15 100644 --- a/llarp/router/router.hpp +++ b/llarp/router/router.hpp @@ -202,12 +202,6 @@ namespace llarp void for_each_connection(std::function func); - void - connect_to(const RouterID& rid); - - void - connect_to(const RemoteRC& rc); - const Contacts& contacts() const { From 7421d59009d0bb9a2a6b85aa9cdf42e8aeae7b55 Mon Sep 17 00:00:00 2001 From: dr7ana Date: Tue, 16 Jan 2024 07:04:48 -0800 Subject: [PATCH 93/93] libquic vbump --- external/oxen-libquic | 2 +- llarp/link/link_manager.cpp | 22 +++++++++++----------- llarp/service/endpoint.hpp | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/external/oxen-libquic b/external/oxen-libquic index 9fe1aebbd..73d4b6f94 160000 --- a/external/oxen-libquic +++ b/external/oxen-libquic @@ -1 +1 @@ -Subproject commit 9fe1aebbd8604c58c9b58098cfd07a233f80fabb +Subproject commit 73d4b6f940c0790e6f7c3ec46e03c014f51be1fa diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp index 9e3db54dc..a1af33d64 100644 --- a/llarp/link/link_manager.cpp +++ b/llarp/link/link_manager.cpp @@ -160,39 +160,39 @@ namespace llarp { log::debug(logcat, "{} called", __PRETTY_FUNCTION__); - s->register_command("bfetch_rcs"s, [this](oxen::quic::message m) { + s->register_handler("bfetch_rcs"s, [this](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_fetch_bootstrap_rcs(std::move(msg)); }); }); - s->register_command("fetch_rcs"s, [this](oxen::quic::message m) { + s->register_handler("fetch_rcs"s, [this](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_fetch_rcs(std::move(msg)); }); }); - s->register_command("fetch_rids"s, [this](oxen::quic::message m) { + s->register_handler("fetch_rids"s, [this](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_fetch_router_ids(std::move(msg)); }); }); - s->register_command("path_build"s, [this, rid = router_id](oxen::quic::message m) { + s->register_handler("path_build"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( [this, &rid, msg = std::move(m)]() mutable { handle_path_build(std::move(msg), rid); }); }); - s->register_command("path_control"s, [this, rid = router_id](oxen::quic::message m) { + s->register_handler("path_control"s, [this, rid = router_id](oxen::quic::message m) { _router.loop()->call( [this, &rid, msg = std::move(m)]() mutable { handle_path_control(std::move(msg), rid); }); }); - s->register_command("gossip_rc"s, [this](oxen::quic::message m) { + s->register_handler("gossip_rc"s, [this](oxen::quic::message m) { _router.loop()->call( [this, msg = std::move(m)]() mutable { handle_gossip_rc(std::move(msg)); }); }); for (auto& method : direct_requests) { - s->register_command( + s->register_handler( std::string{method.first}, [this, func = std::move(method.second)](oxen::quic::message m) { _router.loop()->call([this, msg = std::move(m), func = std::move(func)]() mutable { @@ -394,8 +394,8 @@ namespace llarp LinkManager::on_conn_closed(oxen::quic::connection_interface& ci, uint64_t ec) { _router.loop()->call( - [this, scid = ci.scid(), rid = RouterID{ci.remote_key()}, error_code = ec]() { - log::critical(quic_cat, "Purging quic connection CID:{} (ec:{})", scid, error_code); + [this, ref_id = ci.reference_id(), rid = RouterID{ci.remote_key()}, error_code = ec]() { + log::critical(quic_cat, "Purging quic connection {} (ec:{})", ref_id, error_code); if (auto s_itr = ep.service_conns.find(rid); s_itr != ep.service_conns.end()) { @@ -408,7 +408,7 @@ namespace llarp ep.client_conns.erase(c_itr); } else - log::critical(quic_cat, "Nothing to purge for quic connection CID:{}", scid); + log::critical(quic_cat, "Nothing to purge for quic connection {}", ref_id); }); } @@ -1369,8 +1369,8 @@ namespace llarp } oxenc::bt_dict_consumer frame_info{payload_list.front()}; - auto hash = frame_info.require("HASH"); auto frame = frame_info.require("FRAME"); + auto hash = frame_info.require("HASH"); oxenc::bt_dict_consumer hop_dict{frame}; auto hop_payload = hop_dict.require("ENCRYPTED"); diff --git a/llarp/service/endpoint.hpp b/llarp/service/endpoint.hpp index 266c0bf1f..0ceb16df3 100644 --- a/llarp/service/endpoint.hpp +++ b/llarp/service/endpoint.hpp @@ -178,7 +178,7 @@ namespace llarp EgresPacketRouter() { return nullptr; - }; + } virtual vpn::NetworkInterface* GetVPNInterface()