From 6984cda3a677ba806c844201e7f3aada1b026335 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 8 Jun 2020 15:15:54 -0400 Subject: [PATCH] add pybind and router hive to ci --- .drone.jsonnet | 4 ++++ contrib/ci/drone-run-router-hive.sh | 2 ++ llarp/tooling/router_hive.cpp | 2 +- llarp/util/logging/logger.hpp | 7 ------ pybind/CMakeLists.txt | 1 + pybind/common.hpp | 5 +++++ pybind/llarp/config.cpp | 35 ++++++++++++++++------------- pybind/llarp/context.cpp | 4 +++- pybind/llarp/handlers/pyhandler.hpp | 3 +++ pybind/llarp/logger.cpp | 31 +++++++++++++++++++++++++ pybind/module.cpp | 1 + test/hive/hive.py | 22 +++++++++--------- 12 files changed, 81 insertions(+), 36 deletions(-) create mode 100755 contrib/ci/drone-run-router-hive.sh create mode 100644 pybind/llarp/logger.cpp diff --git a/.drone.jsonnet b/.drone.jsonnet index ff392384f..e8aebbb63 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -158,6 +158,10 @@ local mac_builder(name, build_type='Release', werror=true, cmake_extra='', extra '../contrib/ci/drone-static-upload.sh' ]), + // integration tests + debian_pipeline("Router Hive", "ubuntu:focal", deps='python3-dev python3-pytest python3-pybind11 ' + default_deps, + cmake_extra='-DWITH_HIVE=ON', extra_cmds=['../contrib/ci/drone-run-router-hive.sh']), + // Deb builds: deb_builder("debian:sid", "sid", "debian/sid"), deb_builder("debian:buster", "buster", "debian/buster", imaginary_repo=true), diff --git a/contrib/ci/drone-run-router-hive.sh b/contrib/ci/drone-run-router-hive.sh new file mode 100755 index 000000000..29f042b4c --- /dev/null +++ b/contrib/ci/drone-run-router-hive.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +PYTHONPATH=pybind pytest-3 ../test/ diff --git a/llarp/tooling/router_hive.cpp b/llarp/tooling/router_hive.cpp index 3c5cffba3..613db5a9a 100644 --- a/llarp/tooling/router_hive.cpp +++ b/llarp/tooling/router_hive.cpp @@ -18,7 +18,7 @@ namespace tooling const std::shared_ptr& config, std::vector* routers, bool isRelay) { llarp_main* ctx = llarp_main_init_from_config(config->Copy(), isRelay); - auto result = llarp_main_setup(ctx); + auto result = llarp_main_setup(ctx, isRelay); if (result == 0) { llarp::Context::Get(ctx)->InjectHive(this); diff --git a/llarp/util/logging/logger.hpp b/llarp/util/logging/logger.hpp index f717fbfa8..26b7c37d5 100644 --- a/llarp/util/logging/logger.hpp +++ b/llarp/util/logging/logger.hpp @@ -82,21 +82,14 @@ namespace llarp /** internal */ template inline static void -#ifndef LOKINET_HIVE _Log(LogLevel lvl, const char* fname, int lineno, TArgs&&... args) noexcept -#else - _Log(LogLevel, const char*, int, TArgs&&...) noexcept -#endif { -/* nop out logging for hive mode for now */ -#ifndef LOKINET_HIVE auto& log = LogContext::Instance(); if (log.curLevel > lvl || log.logStream == nullptr) return; std::stringstream ss; LogAppend(ss, std::forward(args)...); log.logStream->AppendLog(lvl, fname, lineno, log.nodeName, ss.str()); -#endif } } // namespace llarp diff --git a/pybind/CMakeLists.txt b/pybind/CMakeLists.txt index e9e514e6a..e307f67cd 100644 --- a/pybind/CMakeLists.txt +++ b/pybind/CMakeLists.txt @@ -5,6 +5,7 @@ pybind11_add_module(pyllarp MODULE llarp/router_contact.cpp llarp/crypto/types.cpp llarp/config.cpp + llarp/logger.cpp llarp/dht/dht_types.cpp llarp/path/path_types.cpp llarp/path/path_hop_config.cpp diff --git a/pybind/common.hpp b/pybind/common.hpp index 9b6d32ac1..93faecd6e 100644 --- a/pybind/common.hpp +++ b/pybind/common.hpp @@ -5,10 +5,15 @@ #include +#include + namespace py = pybind11; namespace llarp { + void + Logger_Init(py::module& mod); + void Context_Init(py::module& mod); diff --git a/pybind/llarp/config.cpp b/pybind/llarp/config.cpp index a51077ae1..2b412da7e 100644 --- a/pybind/llarp/config.cpp +++ b/pybind/llarp/config.cpp @@ -41,19 +41,10 @@ namespace llarp [](RouterConfig& self) { return self.m_dataDir.c_str(); }, [](RouterConfig& self, std::string dir) { self.m_dataDir = dir; }) .def_readwrite("blockBogons", &RouterConfig::m_blockBogons) - .def_readwrite("publicOverride", &RouterConfig::m_publicOverride) - .def_readwrite("ip4addr", &RouterConfig::m_ip4addr) .def( "overrideAddress", - [](RouterConfig& self, std::string ip, std::string port) { - llarp::Addr addr(ip); - self.m_addrInfo.ip = *addr.addr6(); - - int portInt = stoi(port); - self.m_ip4addr.sin_port = portInt; - self.m_addrInfo.port = portInt; - - self.m_publicOverride = true; + [](RouterConfig& self, std::string addr) { + self.m_publicAddress = llarp::IpAddress(addr); }) .def_readwrite("workerThreads", &RouterConfig::m_workerThreads) .def_readwrite("numNetThreads", &RouterConfig::m_numNetThreads) @@ -67,10 +58,10 @@ namespace llarp .def_readwrite("keyfile", &NetworkConfig::m_keyfile) .def_readwrite("endpointType", &NetworkConfig::m_endpointType) .def_readwrite("reachable", &NetworkConfig::m_reachable) - .def_readwrite("hops", &NetworkConfig::m_hops) - .def_readwrite("paths", &NetworkConfig::m_paths) + .def_readwrite("hops", &NetworkConfig::m_Hops) + .def_readwrite("paths", &NetworkConfig::m_Paths) .def_readwrite("snodeBlacklist", &NetworkConfig::m_snodeBlacklist) - .def_readwrite("mapAddr", &NetworkConfig::m_mapAddr) + .def_readwrite("mapAddr", &NetworkConfig::m_mapAddrs) .def_readwrite("strictConnect", &NetworkConfig::m_strictConnect); py::class_(mod, "ConnectConfig") @@ -116,7 +107,21 @@ namespace llarp py::class_(mod, "BootstrapConfig") .def(py::init<>()) - .def_readwrite("routers", &BootstrapConfig::routers); + .def_property( + "routers", + [](BootstrapConfig& self) { + std::vector args; + for (const auto& arg : self.routers) + args.emplace_back(arg.string()); + return args; + }, + [](BootstrapConfig& self, std::vector args) { + self.routers.clear(); + for (const auto& arg : args) + { + self.routers.emplace_back(arg); + } + }); py::class_(mod, "LoggingConfig") .def(py::init<>()) diff --git a/pybind/llarp/context.cpp b/pybind/llarp/context.cpp index 02400fe90..e2e8084be 100644 --- a/pybind/llarp/context.cpp +++ b/pybind/llarp/context.cpp @@ -9,7 +9,9 @@ namespace llarp { using Context_ptr = std::shared_ptr; py::class_(mod, "Context") - .def("Setup", [](Context_ptr self) -> bool { return self->Setup() == 0; }) + .def( + "Setup", + [](Context_ptr self, bool isRelay) -> bool { return self->Setup(isRelay) == 0; }) .def("Run", [](Context_ptr self) -> int { return self->Run(llarp_main_runtime_opts{}); }) .def("Stop", [](Context_ptr self) { self->CloseAsync(); }) .def("IsUp", &Context::IsUp) diff --git a/pybind/llarp/handlers/pyhandler.hpp b/pybind/llarp/handlers/pyhandler.hpp index e659ea593..a8e406cdf 100644 --- a/pybind/llarp/handlers/pyhandler.hpp +++ b/pybind/llarp/handlers/pyhandler.hpp @@ -69,6 +69,9 @@ namespace llarp }); } + void + SendPacketToRemote(const llarp_buffer_t&) override{}; + std::string GetOurAddress() const { diff --git a/pybind/llarp/logger.cpp b/pybind/llarp/logger.cpp new file mode 100644 index 000000000..ea92324c4 --- /dev/null +++ b/pybind/llarp/logger.cpp @@ -0,0 +1,31 @@ +#include "common.hpp" +#include +#include + +namespace llarp +{ + struct PyLogger + { + std::unique_ptr shutup; + }; + + void + Logger_Init(py::module& mod) + { + py::class_(mod, "LogContext") + .def(py::init<>()) + .def_property( + "shutup", + [](PyLogger& self) { return self.shutup != nullptr; }, + [](PyLogger& self, bool shutup) { + if (self.shutup == nullptr && shutup) + { + self.shutup = std::make_unique(); + } + else if (self.shutup != nullptr && shutup == false) + { + self.shutup.reset(); + } + }); + } +} // namespace llarp diff --git a/pybind/module.cpp b/pybind/module.cpp index 28d6ea8a4..e53894179 100644 --- a/pybind/module.cpp +++ b/pybind/module.cpp @@ -16,4 +16,5 @@ PYBIND11_MODULE(pyllarp, m) llarp::handlers::PyHandler_Init(m); llarp::service::Address_Init(m); m.def("EnableDebug", []() { llarp::SetLogLevel(llarp::eLogDebug); }); + llarp::Logger_Init(m); } diff --git a/test/hive/hive.py b/test/hive/hive.py index 66cf79da0..3adda4eef 100644 --- a/test/hive/hive.py +++ b/test/hive/hive.py @@ -14,7 +14,9 @@ import traceback class RouterHive(object): - def __init__(self, n_relays=10, n_clients=10, netid="hive"): + def __init__(self, n_relays=10, n_clients=10, netid="hive", shutup=True): + self._log = pyllarp.LogContext() + self._log.shutup = shutup try: self.endpointName = "pyllarp" @@ -61,13 +63,7 @@ class RouterHive(object): config.router.dataDir = dirname config.router.netid = self.netid config.router.nickname = "Router%d" % index - config.router.publicOverride = True - config.router.overrideAddress("127.0.0.1", '{}'.format(port)) - """ - config.router.ip4addr.sin_family = AF_INET - config.router.ip4addr.sin_port = htons(port) - config.router.ip4addr.sin_addr.set("127.0.0.1") - """ + config.router.overrideAddress('127.0.0.1:{}'.format(port)) config.router.blockBogons = False config.network.enableProfiling = False @@ -176,7 +172,7 @@ class RouterHive(object): return rcs -def main(n_relays=10, n_clients=10, print_each_event=True): +def main(n_relays=10, n_clients=10, print_each_event=True, verbose=False): running = True @@ -188,7 +184,7 @@ def main(n_relays=10, n_clients=10, print_each_event=True): signal(SIGINT, handle_sigint) try: - hive = RouterHive(n_relays, n_clients) + hive = RouterHive(n_relays, n_clients, shutup=not verbose) hive.Start() except Exception as err: @@ -239,7 +235,8 @@ def main(n_relays=10, n_clients=10, print_each_event=True): pprint(event_counts) hive.events = [] - sleep(1) + for _ in range(100): + sleep(1.0 / 100) print('stopping') hive.Stop() @@ -254,5 +251,6 @@ if __name__ == '__main__': parser.add_argument('--print-events', dest="print_events", action='store_true') parser.add_argument('--relay-count', dest="relay_count", type=int, default=10) parser.add_argument('--client-count', dest="client_count", type=int, default=10) + parser.add_argument('--verbose', action='store_true', dest='verbose') args = parser.parse_args() - main(n_relays=args.relay_count, n_clients=args.client_count, print_each_event = args.print_events) + main(n_relays=args.relay_count, n_clients=args.client_count, print_each_event = args.print_events, verbose=args.verbose)