From 30f62d2689f2bfb3b7a455d17945075cc21932bc Mon Sep 17 00:00:00 2001 From: dr7ana Date: Wed, 20 Dec 2023 11:34:55 -0800 Subject: [PATCH] zero cost exception handling my ass - rework bootstrap loading to move all logic into BootstrapList object - ability to parse lists and dicts of bootstraps implemented - netid parsing refers to the correct fallback - cross your fingers boys here we go --- contrib/bootstrap/testnet.signed | Bin 308 -> 161 bytes llarp/bootstrap.cpp | 114 ++++++++++++++++++++++----- llarp/bootstrap.hpp | 9 +++ llarp/config/config.cpp | 27 +------ llarp/config/config.hpp | 10 --- llarp/link/link_manager.hpp | 11 +-- llarp/nodedb.cpp | 54 +++---------- llarp/nodedb.hpp | 6 +- llarp/router/router.cpp | 131 ++++++------------------------- llarp/router_contact.cpp | 12 ++- llarp/router_contact.hpp | 7 +- 11 files changed, 161 insertions(+), 220 deletions(-) diff --git a/contrib/bootstrap/testnet.signed b/contrib/bootstrap/testnet.signed index 366c5c5a0098922587845c63f3985bfce70b1d63..e80acd6cd75a5a21b4fe3c325c3e40dbdc4d0630 100644 GIT binary patch literal 161 zcmV;S0ABxOFgj^4WidK5HaZtr2Kp)BF*<2CI&@`obZ%vIF*a literal 308 zcmc~vF|3 zI5?)R{Qh3-X}qCTrjenQl~o!LSQ(m_=$V`88Jd~s85sbD3o;GO%*=o$mSq~GrUL0q zQ>*mE+}uP%t8Adz_fCCa|CDQ+Kf5LBg5m%Bxh%;&s=Ebt{Qtdgrf@Lxk)1#TfT|Q- z8P7j@SvBLYRaf_~zpdQ*1`BVsa38+As%5L+a|tI&L#v`3pz$DNk(y}?vZfShuc3*# zg@v)9fr+UJ)B%Q86*)j=m6?fE&yvyw?)#^>Ot$T**t)3t{TWlke=L&H32*N={B(U( r&drjmS;%H|Bb{UY#u-0mDZJl&KyyaO#ek_>K5hOs{h%RJYAP21S?hdC diff --git a/llarp/bootstrap.cpp b/llarp/bootstrap.cpp index b5f35c605..061e3b3b3 100644 --- a/llarp/bootstrap.cpp +++ b/llarp/bootstrap.cpp @@ -9,10 +9,25 @@ namespace llarp bool BootstrapList::bt_decode(std::string_view buf) { - try + const auto& f = buf.front(); + + switch (f) { - oxenc::bt_list_consumer btlc{buf}; + case 'l': + return bt_decode(oxenc::bt_list_consumer{buf}); + case 'd': + return bt_decode(oxenc::bt_dict_consumer{buf}); + default: + log::critical(logcat, "Unable to parse bootstrap as bt list or dict!"); + return false; + } + } + bool + BootstrapList::bt_decode(oxenc::bt_list_consumer btlc) + { + try + { while (not btlc.is_finished()) emplace(btlc.consume_dict_data()); } @@ -22,6 +37,24 @@ namespace llarp return false; } + _curr = begin(); + return true; + } + + bool + BootstrapList::bt_decode(oxenc::bt_dict_consumer btdc) + { + try + { + emplace(btdc); + } + catch (const std::exception& e) + { + log::warning(logcat, "Unable to decode bootstrap RemoteRC: {}", e.what()); + return false; + } + + _curr = begin(); return true; } @@ -55,37 +88,82 @@ namespace llarp } void - BootstrapList::read_from_file(const fs::path& fpath) + BootstrapList::populate_bootstraps( + std::vector paths, const fs::path& def, bool load_fallbacks) { - bool isListFile = false; - - if (std::ifstream inf(fpath.c_str(), std::ios::binary); inf.is_open()) + for (const auto& f : paths) { - const char ch = inf.get(); - isListFile = ch == 'l'; + // TESTNET: TODO: revise fucked config + log::debug(logcat, "Loading BootstrapRC from file at path:{}", f); + if (not read_from_file(f)) + throw std::invalid_argument{"User-provided BootstrapRC is invalid!"}; } - if (isListFile) + if (empty()) { - auto content = util::file_to_string(fpath); + log::debug( + logcat, + "BootstrapRC list empty; looking for default BootstrapRC from file at path:{}", + def); + read_from_file(def); + } - if (not bt_decode(content)) + for (auto itr = begin(); itr != end(); ++itr) + { + if (RouterContact::is_obsolete(*itr)) // can move this into ::read_from_file { - throw std::runtime_error{fmt::format("failed to read bootstrap list file '{}'", fpath)}; + log::critical(logcat, "Deleting obsolete BootstrapRC (rid:{})", itr->router_id()); + itr = erase(itr); + continue; } } - else + + if (empty() and load_fallbacks) { - RemoteRC rc; + log::critical(logcat, "BootstrapRC list empty; loading fallbacks..."); + auto fallbacks = llarp::load_bootstrap_fallbacks(); - if (not rc.read(fpath)) + if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) { - throw std::runtime_error{ - fmt::format("failed to decode bootstrap RC, file='{}', rc={}", fpath, rc.to_string())}; + log::critical( + logcat, "Loading {} default fallback bootstrap router(s)!", itr->second.size()); + merge(itr->second); + } + + if (empty()) + { + log::error( + logcat, + "No Bootstrap routers were loaded. The default Bootstrap file {} does not exist, and " + "loading fallback Bootstrap RCs failed.", + def); + + throw std::runtime_error("No Bootstrap nodes available."); } - insert(rc); } + log::critical(logcat, "We have {} Bootstrap router(s)!", size()); + _curr = begin(); + } + + bool + BootstrapList::read_from_file(const fs::path& fpath) + { + bool result = false; + + if (not fs::exists(fpath)) + { + log::critical(logcat, "Bootstrap RC file non-existant at path:{}", fpath); + return result; + } + + auto content = util::file_to_string(fpath); + result = bt_decode(content); + + log::critical( + logcat, "{}uccessfully loaded BootstrapRC file at path:{}", result ? "S" : "Un", fpath); + _curr = begin(); + return result; } } // namespace llarp diff --git a/llarp/bootstrap.hpp b/llarp/bootstrap.hpp index 7d037ff55..86cf416f0 100644 --- a/llarp/bootstrap.hpp +++ b/llarp/bootstrap.hpp @@ -23,10 +23,19 @@ namespace llarp bool bt_decode(std::string_view buf); + bool + bt_decode(oxenc::bt_list_consumer btlc); + + bool + bt_decode(oxenc::bt_dict_consumer btdc); + std::string_view bt_encode() const; void + populate_bootstraps(std::vector paths, const fs::path& def, bool load_fallbacks); + + bool read_from_file(const fs::path& fpath); bool diff --git a/llarp/config/config.cpp b/llarp/config/config.cpp index a26159eeb..eeb2fecbf 100644 --- a/llarp/config/config.cpp +++ b/llarp/config/config.cpp @@ -1068,26 +1068,6 @@ namespace llarp }); } - void - ConnectConfig::define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params) - { - (void)params; - - conf.add_undeclared_handler( - "connect", [this](std::string_view section, std::string_view name, std::string_view value) { - fs::path file{value.begin(), value.end()}; - if (not fs::exists(file)) - throw std::runtime_error{fmt::format( - "Specified bootstrap file {} specified in [{}]:{} does not exist", - value, - section, - name)}; - - routers.emplace_back(std::move(file)); - return true; - }); - } - void ApiConfig::define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params) { @@ -1204,14 +1184,12 @@ namespace llarp }, [this](std::string arg) { if (arg.empty()) - { throw std::invalid_argument("cannot use empty filename as bootstrap"); - } + files.emplace_back(std::move(arg)); + if (not fs::exists(files.back())) - { throw std::invalid_argument("file does not exist: " + arg); - } }); } @@ -1460,7 +1438,6 @@ namespace llarp router.define_config_options(conf, params); network.define_config_options(conf, params); paths.define_config_options(conf, params); - connect.define_config_options(conf, params); dns.define_config_options(conf, params); links.define_config_options(conf, params); api.define_config_options(conf, params); diff --git a/llarp/config/config.hpp b/llarp/config/config.hpp index b0a9231a9..152da9345 100644 --- a/llarp/config/config.hpp +++ b/llarp/config/config.hpp @@ -184,14 +184,6 @@ namespace llarp define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params); }; - struct ConnectConfig - { - std::vector routers; - - void - define_config_options(ConfigDefinition& conf, const ConfigGenParameters& params); - }; - // TODO: remove oxenmq from this header struct ApiConfig { @@ -215,7 +207,6 @@ namespace llarp struct BootstrapConfig { std::vector files; - BootstrapList routers; bool seednode; void @@ -245,7 +236,6 @@ namespace llarp RouterConfig router; NetworkConfig network; PeerSelectionConfig paths; - ConnectConfig connect; DnsConfig dns; LinksConfig links; ApiConfig api; diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp index 186e7cfcd..105adec06 100644 --- a/llarp/link/link_manager.hpp +++ b/llarp/link/link_manager.hpp @@ -423,13 +423,10 @@ namespace llarp void handle_update_exit_response(oxen::quic::message); void handle_close_exit_response(oxen::quic::message); - std::unordered_map rpc_responses = { - {"find_name", &LinkManager::handle_find_name_response}, - {"publish_intro", &LinkManager::handle_publish_intro_response}, - {"find_intro", &LinkManager::handle_find_intro_response}, - {"update_exit", &LinkManager::handle_update_exit_response}, - {"obtain_exit", &LinkManager::handle_obtain_exit_response}, - {"close_exit", &LinkManager::handle_close_exit_response}}; + /** + Clients register 0 endpoints + - nobody is making requests to clients + */ }; namespace link diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 2a8f3d0a0..7b1a12bce 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -498,21 +498,13 @@ namespace llarp { if (error) { - auto& fail_count = (_using_bootstrap_fallback) ? bootstrap_attempts : fetch_failures; - auto& THRESHOLD = - (_using_bootstrap_fallback) ? MAX_BOOTSTRAP_FETCH_ATTEMPTS : MAX_FETCH_ATTEMPTS; - - // This catches three different failure cases; - // 1) bootstrap fetching and over failure threshold - // 2) bootstrap fetching and more failures to go - // 3) standard fetching and over threshold - if (++fail_count >= THRESHOLD || _using_bootstrap_fallback) + if (++fetch_failures >= MAX_FETCH_ATTEMPTS) { log::info( logcat, "RC fetching from {} reached failure threshold ({}); falling back to bootstrap...", fetch_source, - THRESHOLD); + MAX_FETCH_ATTEMPTS); fallback_to_bootstrap(); return; @@ -536,7 +528,7 @@ namespace llarp void NodeDB::fetch_rids_result(bool initial) { - if (fetch_failures > MAX_FETCH_ATTEMPTS) + if (fetch_failures >= MAX_FETCH_ATTEMPTS) { log::info( logcat, @@ -661,15 +653,18 @@ namespace llarp log::critical(logcat, "Dispatching BootstrapRC fetch request to {}", fetch_source); + auto num_needed = _router.is_service_node() ? SERVICE_NODE_BOOTSTRAP_SOURCE_COUNT + : CLIENT_BOOTSTRAP_SOURCE_COUNT; + _router.link_manager().fetch_bootstrap_rcs( rc, - BootstrapFetchMessage::serialize(_router.router_contact, CLIENT_BOOTSTRAP_SOURCE_COUNT), - [this, is_snode = _router.is_service_node()](oxen::quic::message m) mutable { + BootstrapFetchMessage::serialize(_router.router_contact, num_needed), + [this, is_snode = _router.is_service_node(), num_needed = num_needed]( + oxen::quic::message m) mutable { log::critical(logcat, "Received response to BootstrapRC fetch request..."); if (not m) { - // ++bootstrap_attempts; log::warning( logcat, "BootstrapRC fetch request to {} failed (error {}/{})", @@ -680,7 +675,6 @@ namespace llarp return; } - // std::set rids; size_t num = 0; try @@ -700,7 +694,6 @@ namespace llarp } catch (const std::exception& e) { - // ++bootstrap_attempts; log::warning( logcat, "Failed to parse BootstrapRC fetch response from {} (error {}/{}): {}", @@ -713,19 +706,12 @@ namespace llarp return; } - // We set this to the max allowable value because if this result is bad, we won't - // try this bootstrap again. If this result is undersized, we roll right into the - // next call to fallback_to_bootstrap() and hit the base case, rotating sources - // bootstrap_attempts = MAX_BOOTSTRAP_FETCH_ATTEMPTS; - - // const auto& num = rids.size(); - log::critical( logcat, "BootstrapRC fetch response from {} returned {}/{} needed RCs", fetch_source, num, - CLIENT_BOOTSTRAP_SOURCE_COUNT); + num_needed); if (not is_snode) { @@ -739,25 +725,6 @@ namespace llarp log::critical(logcat, "Service node completed processing BootstrapRC fetch!"); post_snode_bootstrap(); } - - // FIXME: when moving to testnet, uncomment this - // if (rids.size() == BOOTSTRAP_SOURCE_COUNT) - // { - // known_rids.merge(rids); - // fetch_initial(); - // } - // else - // { - // // ++bootstrap_attempts; - // log::warning( - // logcat, - // "BootstrapRC fetch response from {} returned insufficient number of RC's (error " - // "{}/{})", - // fetch_source, - // bootstrap_attempts, - // MAX_BOOTSTRAP_FETCH_ATTEMPTS); - // fallback_to_bootstrap(); - // } }); } @@ -783,7 +750,6 @@ namespace llarp replace_subset(rid_sources, specific, known_rids, RID_SOURCE_COUNT, csrng); } - // TODO: nuke all this shit void NodeDB::set_router_whitelist( const std::vector& whitelist, diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index adfabe0f5..7a3c8333d 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -25,6 +25,7 @@ namespace llarp // TESTNET: the following constants have been shortened for testing purposes /* RC Fetch Constants */ + // fallback to bootstrap if we have less than this many RCs inline constexpr size_t MIN_ACTIVE_RCS{6}; // max number of attempts we make in non-bootstrap fetch requests inline constexpr int MAX_FETCH_ATTEMPTS{10}; @@ -34,11 +35,10 @@ namespace llarp inline constexpr double MIN_GOOD_RC_FETCH_THRESHOLD{}; /* RID Fetch Constants */ - inline constexpr size_t MIN_ACTIVE_RIDS{24}; // the number of rid sources that we make rid fetch requests to - inline constexpr size_t RID_SOURCE_COUNT{12}; + inline constexpr size_t RID_SOURCE_COUNT{8}; // upper limit on how many rid fetch requests to rid sources can fail - inline constexpr size_t MAX_RID_ERRORS{4}; + inline constexpr size_t MAX_RID_ERRORS{2}; // each returned rid must appear this number of times across all responses inline constexpr int MIN_RID_FETCH_FREQ{RID_SOURCE_COUNT - MAX_RID_ERRORS - 1}; // the total number of accepted returned rids should be above this number diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index fc9d17ed6..625b235ee 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -353,6 +353,25 @@ namespace llarp _config = std::move(c); auto& conf = *_config; + const auto& netid = conf.router.net_id; + + if (not netid.empty() and netid != llarp::LOKINET_DEFAULT_NETID) + { + _testnet = netid == llarp::LOKINET_TESTNET_NETID; + _testing_disabled = conf.lokid.disable_testing; + + RouterContact::ACTIVE_NETID = netid; + + if (_testing_disabled and not _testnet) + throw std::runtime_error{"Error: reachability testing can only be disabled on testnet!"}; + + auto err = "Lokinet network ID set to {}, NOT mainnet! {}"_format( + netid, + _testnet ? "Please ensure your local instance is configured to operate on testnet" + : "Local lokinet instance will attempt to run on the specified network"); + log::critical(logcat, err); + } + // Do logging config as early as possible to get the configured log level applied // Backwards compat: before 0.9.10 we used `type=file` with `file=|-|stdout` for print mode @@ -528,24 +547,6 @@ namespace llarp // Set netid before anything else log::debug(logcat, "Network ID set to {}", conf.router.net_id); - const auto& netid = conf.router.net_id; - - if (not netid.empty() and netid != llarp::LOKINET_DEFAULT_NETID) - { - log::critical( - logcat, - "Network ID set to {}, which is not {}! Lokinet will attempt to run on the specified " - "network", - netid, - llarp::LOKINET_DEFAULT_NETID); - - _testnet = netid == llarp::LOKINET_TESTNET_NETID; - _testing_disabled = conf.lokid.disable_testing; - - if (_testing_disabled and not _testnet) - throw std::runtime_error{"Error: reachability testing can only be disabled on testnet!"}; - } - // Router config client_router_connections = conf.router.client_router_connections; @@ -603,101 +604,15 @@ namespace llarp log::debug(logcat, "{} strict-connect routers configured", val.size()); } - std::vector configRouters = conf.connect.routers; - - configRouters.insert( - configRouters.end(), conf.bootstrap.files.begin(), conf.bootstrap.files.end()); - - // if our conf had no bootstrap files specified, try the default location of - // /bootstrap.signed. If this isn't present, leave a useful error message - // TODO: use constant - fs::path defaultBootstrapFile = conf.router.data_dir / "bootstrap.signed"; - - if (configRouters.empty() and conf.bootstrap.routers.empty()) - { - if (fs::exists(defaultBootstrapFile)) - configRouters.push_back(defaultBootstrapFile); - } - - // BootstrapList _bootstrap_rc_list; - auto& node_bstrap = _node_db->bootstrap_list(); - - auto clear_bad_rcs = [&]() mutable { - log::critical(logcat, "Clearing bad RCs..."); - // in case someone has an old bootstrap file and is trying to use a bootstrap - // that no longer exists - for (auto it = node_bstrap.begin(); it != node_bstrap.end();) - { - if (it->is_obsolete_bootstrap()) - log::critical(logcat, "ignoring obsolete bootstrap RC: {}", it->router_id()); - else if (not it->verify()) - log::critical(logcat, "ignoring invalid bootstrap RC: {}", it->router_id()); - else - { - ++it; - continue; - } - - // we are in one of the above error cases that we warned about: - it = node_bstrap.erase(it); - } - }; - - for (const auto& router : configRouters) - { - log::debug(logcat, "Loading bootstrap router list from {}", defaultBootstrapFile); - node_bstrap.read_from_file(router); - // _bootstrap_rc_list.read_from_file(router); - } - - for (const auto& rc : conf.bootstrap.routers) - { - // _bootstrap_rc_list.emplace(rc); - node_bstrap.emplace(rc); - } - _bootstrap_seed = conf.bootstrap.seednode; - if (_bootstrap_seed) - log::critical(logcat, "We are a bootstrap seed node!"); - - if (node_bstrap.empty() and not _bootstrap_seed) - { - log::warning(logcat, "Warning: bootstrap list is empty and we are not a seed node"); - - auto fallbacks = llarp::load_bootstrap_fallbacks(); - - if (auto itr = fallbacks.find(RouterContact::ACTIVE_NETID); itr != fallbacks.end()) - { - // _bootstrap_rc_list.merge(itr->second); - node_bstrap.merge(itr->second); - } - - if (node_bstrap.empty()) - { - // empty after trying fallback, if set - log::error( - logcat, - "No bootstrap routers were loaded. The default bootstrap file {} does not exist, and " - "loading fallback bootstrap RCs failed.", - defaultBootstrapFile); - - throw std::runtime_error("No bootstrap nodes available."); - } - - log::critical(logcat, "Loaded {} default fallback bootstrap routers!", node_bstrap.size()); - } + std::vector bootstrap_paths{std::move(conf.bootstrap.files)}; - clear_bad_rcs(); - log::critical(logcat, "We have {} bootstrap routers!", node_bstrap.size()); + fs::path default_bootstrap = conf.router.data_dir / "bootstrap.signed"; - // node_db()->set_bootstrap_routers(_bootstrap_rc_list); + auto& bootstrap = _node_db->bootstrap_list(); - // TODO: RC refactor here - // if (_is_service_node) - // init_inbounds(); - // else - // init_outbounds(); + bootstrap.populate_bootstraps(bootstrap_paths, default_bootstrap, not _bootstrap_seed); // profiling _profile_file = conf.router.data_dir / "profiles.dat"; diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index ebd87e701..2f3f5f606 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -36,7 +36,7 @@ namespace llarp }); if (not btdc.is_finished()) - throw std::runtime_error{"RouterContact has some fucked up shit at the end"}; + throw std::runtime_error{"RemoteRC has some fucked up shit at the end"}; btdc.finish(); } @@ -185,7 +185,7 @@ namespace llarp return time_to_expiry(now) <= dlt; } - static constexpr std::array obsolete_bootstraps = { + static const std::set obsolete_bootstraps{ "7a16ac0b85290bcf69b2f3b52456d7e989ac8913b4afbb980614e249a3723218"sv, "e6b3a6fe5e32c379b64212c72232d65b0b88ddf9bbaed4997409d329f8519e0b"sv, }; @@ -200,4 +200,12 @@ namespace llarp } return false; } + + bool + RouterContact::is_obsolete(const RouterContact& rc) + { + const auto& hex = rc._router_id.ToHex(); + + return obsolete_bootstraps.count(hex); + } } // namespace llarp diff --git a/llarp/router_contact.hpp b/llarp/router_contact.hpp index c37652fe5..76f62c31f 100644 --- a/llarp/router_contact.hpp +++ b/llarp/router_contact.hpp @@ -195,6 +195,9 @@ namespace llarp bool is_obsolete_bootstrap() const; + static bool + is_obsolete(const RouterContact& rc); + void bt_verify(oxenc::bt_dict_consumer& data, bool reject_expired = false) const; @@ -304,9 +307,6 @@ namespace llarp /// the data in the constructor, eliminating the need for a ::verify method/ struct RemoteRC final : public RouterContact { - private: - explicit RemoteRC(oxenc::bt_dict_consumer btdc); - public: RemoteRC() = default; explicit RemoteRC(std::string_view data) : RemoteRC{oxenc::bt_dict_consumer{data}} @@ -317,6 +317,7 @@ namespace llarp { _payload = data; } + explicit RemoteRC(oxenc::bt_dict_consumer btdc); ~RemoteRC() = default; std::string_view