Unleak exit mode DNS via unbound DNS trampoline on (macOS)

When we enable/disable exit mode on this restarts the unbound DNS
responder with the DNS trampoline (or restores upstream, when disabling)
to properly route DNS requests through the tunnel (because libunbound's
direct requests don't get tunneled because unbound is inside the network
extension).

llarp/apple/CMakeLists.txt

@@ -15,7 +15,7 @@ find_library(COREFOUNDATION CoreFoundation REQUIRED)
 target_sources(lokinet-util PRIVATE apple_logger.cpp)
 target_link_libraries(lokinet-util PUBLIC ${FOUNDATION})
 
-target_sources(lokinet-platform PRIVATE vpn_interface.cpp route_manager.cpp context_wrapper.cpp)
+target_sources(lokinet-platform PRIVATE vpn_platform.cpp vpn_interface.cpp route_manager.cpp context_wrapper.cpp)
 
 add_executable(lokinet-extension MACOSX_BUNDLE
   PacketTunnelProvider.m

llarp/apple/DNSTrampoline.m

@@ -128,7 +128,7 @@ static void alloc_buffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b
 
 - (void) dealloc
 {
-  NSLog(@"Shutting down DNS trampoline");
+  NSLog(@"Stopping DNS trampoline");
   uv_close((uv_handle_t*) &request_socket, NULL);
   uv_close((uv_handle_t*) &write_trigger, NULL);
 }

llarp/apple/PacketTunnelProvider.m

@@ -3,10 +3,6 @@
 #include "context_wrapper.h"
 #include "DNSTrampoline.h"
 
-// Port (on localhost) for our DNS trampoline for bouncing DNS requests through the exit route when
-// in exit mode.
-const uint16_t dns_trampoline_port = 1053;
-
 @interface LLARPPacketTunnel : NEPacketTunnelProvider
 {
   void* lokinet;
@@ -239,6 +235,7 @@ static void del_default_route(void* ctx) {
       return completionHandler(start_failure);
     }
 
+    NSLog(@"Starting DNS exit mode trampoline to %@ on 127.0.0.1:%d", upstreamdns_ep, dns_trampoline_port);
     NWUDPSession* upstreamdns = [strongSelf createUDPSessionThroughTunnelToEndpoint:upstreamdns_ep fromEndpoint:nil];
     strongSelf->dns_tramp = [LLARPDNSTrampoline alloc];
     [strongSelf->dns_tramp

llarp/apple/context_wrapper.cpp

@@ -29,6 +29,8 @@ namespace
 
 }  // namespace
 
+const uint16_t dns_trampoline_port = 1053;
+
 void*
 llarp_apple_init(llarp_apple_config* appleconf)
 {

llarp/apple/context_wrapper.h

@@ -12,6 +12,10 @@ extern "C"
 #include <sys/socket.h>
 #include <uv.h>
 
+  // Port (on localhost) for our DNS trampoline for bouncing DNS requests through the exit route
+  // when in exit mode.
+  extern const uint16_t dns_trampoline_port;
+
   /// C callback function for us to invoke when we need to write a packet
   typedef void(*packet_writer_callback)(int af, const void* data, size_t size, void* ctx);
 

llarp/apple/route_manager.cpp

@@ -1,15 +1,49 @@
 #include "route_manager.hpp"
+#include <llarp/handlers/tun.hpp>
+#include <llarp/service/context.hpp>
+#include <llarp.hpp>
+#include <memory>
 
 namespace llarp::apple {
 
+void RouteManager::check_trampoline(bool enable) {
+  if (trampoline_active == enable)
+    return;
+  auto router = context.router;
+  if (!router) {
+    LogError("Cannot reconfigure to use DNS trampoline: no router");
+    return;
+  }
+
+  std::shared_ptr<llarp::handlers::TunEndpoint> tun;
+  router->hiddenServiceContext().ForEachService([&tun] (const auto& name, const auto ep) {
+      tun = std::dynamic_pointer_cast<llarp::handlers::TunEndpoint>(ep);
+      return !tun;
+  });
+
+  if (!tun) {
+    LogError("Cannot reconfigure to use DNS trampoline: no tun endpoint found (!?)");
+    return;
+  }
+
+  if (enable)
+    saved_upstream_dns = tun->ReconfigureDNS({SockAddr{127, 0, 0, 1, huint16_t{dns_trampoline_port}}});
+  else
+    tun->ReconfigureDNS(std::move(saved_upstream_dns));
+  trampoline_active = enable;
+}
+
+
 void RouteManager::AddDefaultRouteViaInterface(std::string)
 {
+  check_trampoline(true);
     if (callback_context and route_callbacks.add_default_route)
         route_callbacks.add_default_route(callback_context);
 }
 
 void RouteManager::DelDefaultRouteViaInterface(std::string)
 {
+  check_trampoline(false);
     if (callback_context and route_callbacks.del_default_route)
         route_callbacks.del_default_route(callback_context);
 }
@@ -17,6 +51,7 @@ void RouteManager::DelDefaultRouteViaInterface(std::string)
 void
 RouteManager::AddRouteViaInterface(vpn::NetworkInterface&, IPRange range)
 {
+  check_trampoline(true);
     if (callback_context)
     {
         if (range.IsV4()) {
@@ -35,6 +70,7 @@ RouteManager::AddRouteViaInterface(vpn::NetworkInterface&, IPRange range)
 void
 RouteManager::DelRouteViaInterface(vpn::NetworkInterface&, IPRange range)
 {
+  check_trampoline(false);
     if (callback_context)
     {
         if (range.IsV4()) {

llarp/apple/route_manager.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <llarp/router/abstractrouter.hpp>
 #include <llarp/ev/vpn.hpp>
 #include "context_wrapper.h"
 
@@ -7,8 +8,8 @@ namespace llarp::apple {
 
 class RouteManager final : public llarp::vpn::IRouteManager {
 public:
-    RouteManager(llarp_route_callbacks rcs, void* callback_context)
-        : route_callbacks{std::move(rcs)}, callback_context{callback_context} {}
+    RouteManager(llarp::Context& ctx, llarp_route_callbacks rcs, void* callback_context)
+        : context{ctx}, route_callbacks{std::move(rcs)}, callback_context{callback_context} {}
 
     /// These are called for poking route holes, but we don't have to do that at all on macos
     /// because the appex isn't subject to its own rules.
@@ -39,6 +40,13 @@ public:
         return ret;
     }
 
+private:
+
+    llarp::Context& context;
+    bool trampoline_active = false;
+    std::vector<llarp::SockAddr> saved_upstream_dns;
+    void check_trampoline(bool enable);
+
     void* callback_context = nullptr;
     llarp_route_callbacks route_callbacks;
 };

llarp/apple/vpn_platform.cpp

@@ -0,0 +1,22 @@
+#include "vpn_platform.hpp"
+#include "context.hpp"
+
+namespace llarp::apple
+{
+  VPNPlatform::VPNPlatform(
+      Context& ctx,
+      VPNInterface::packet_write_callback packet_writer,
+      VPNInterface::on_readable_callback on_readable,
+      llarp_route_callbacks route_callbacks,
+      void* callback_context)
+      : m_Context{ctx}
+      , m_RouteManager{ctx, std::move(route_callbacks), callback_context}
+      , m_PacketWriter{std::move(packet_writer)}
+      , m_OnReadable{std::move(on_readable)}
+  {}
+
+  std::shared_ptr<vpn::NetworkInterface> VPNPlatform::ObtainInterface(vpn::InterfaceInfo)
+  {
+    return std::make_shared<VPNInterface>(m_Context, m_PacketWriter, m_OnReadable);
+  }
+}  // namespace llarp::apple

llarp/apple/vpn_platform.hpp

@@ -14,17 +14,9 @@ namespace llarp::apple
         VPNInterface::packet_write_callback packet_writer,
         VPNInterface::on_readable_callback on_readable,
         llarp_route_callbacks route_callbacks,
-        void* callback_context)
-        : m_Context{ctx}
-        , m_RouteManager{std::move(route_callbacks), callback_context}
-        , m_PacketWriter{std::move(packet_writer)}
-        , m_OnReadable{std::move(on_readable)}
-    {}
+        void* callback_context);
 
-    std::shared_ptr<vpn::NetworkInterface> ObtainInterface(vpn::InterfaceInfo) override
-    {
-      return std::make_shared<VPNInterface>(m_Context, m_PacketWriter, m_OnReadable);
-    }
+    std::shared_ptr<vpn::NetworkInterface> ObtainInterface(vpn::InterfaceInfo) override;
 
     vpn::IRouteManager& RouteManager() override { return m_RouteManager; }
 

llarp/dns/server.cpp

@@ -70,6 +70,7 @@ namespace llarp::dns
 
     m_UnboundResolver =
         std::make_shared<UnboundResolver>(m_Loop, std::move(replyFunc), std::move(failFunc));
+    m_Resolvers.clear();
     if (not m_UnboundResolver->Init())
     {
       llarp::LogError("Failed to initialize upstream DNS resolver.");
@@ -102,9 +103,13 @@ namespace llarp::dns
       llarp::LogError("dns reply failed");
   }
 
+  bool PacketHandler::IsUpstreamResolver(const SockAddr& to, const SockAddr& from) const {
+    return m_Resolvers.count(to);
+  }
+
   bool
   PacketHandler::ShouldHandlePacket(
-      const SockAddr& to, [[maybe_unused]] const SockAddr& from, llarp_buffer_t buf) const
+      const SockAddr& to, const SockAddr& from, llarp_buffer_t buf) const
   {
     MessageHeader hdr;
     if (not hdr.Decode(&buf))
@@ -121,11 +126,7 @@ namespace llarp::dns
     if (m_QueryHandler and m_QueryHandler->ShouldHookDNSMessage(msg))
       return true;
 
-    if (m_Resolvers.find(to) != m_Resolvers.end())
-    {
-      return false;
-    }
-    return true;
+    return !IsUpstreamResolver(to, from);
   }
 
   void

llarp/dns/server.hpp

@@ -56,6 +56,10 @@ namespace llarp
       virtual void
       SendServerMessageBufferTo(const SockAddr& to, const SockAddr& from, llarp_buffer_t buf) = 0;
 
+      // Returns true if this packet is something that looks like it's going to an upstream
+      // resolver, i.e. matches a configured resolver.
+      virtual bool IsUpstreamResolver(const SockAddr& to, const SockAddr& from) const;
+
      private:
       void
       HandleUpstreamFailure(const SockAddr& from, const SockAddr& to, Message msg);

llarp/dns/unbound_resolver.cpp

@@ -140,7 +140,8 @@ namespace llarp::dns
   UnboundResolver::AddUpstreamResolver(const SockAddr& upstreamResolver)
   {
     std::stringstream ss;
-    ss << upstreamResolver.hostString();
+    auto hoststr = upstreamResolver.hostString();
+    ss << hoststr;
 
     if (const auto port = upstreamResolver.getPort(); port != 53)
       ss << "@" << port;
@@ -151,6 +152,24 @@ namespace llarp::dns
       Reset();
       return false;
     }
+
+#ifdef __APPLE__
+    // On Apple, we configure a localhost resolver to trampoline requests through the tunnel to the
+    // actual upstream (because the network extension itself cannot route through the tunnel using
+    // normal sockets but instead we "get" to use Apple's interfaces, hurray).
+    if (hoststr == "127.0.0.1") {
+        // Not at all clear why this is needed but without it we get "send failed: Can't assign
+        // requested address" when unbound tries to connect to the localhost address using a source
+        // address of 0.0.0.0.  Yay apple.
+        ub_ctx_set_option(unboundContext, "outgoing-interface:", hoststr.c_str());
+
+        // The trampoline expects just a single source port (and sends everything back to it)
+        ub_ctx_set_option(unboundContext, "outgoing-range:", "1");
+        ub_ctx_set_option(unboundContext, "outgoing-port-avoid:", "0-65535");
+        ub_ctx_set_option(unboundContext, "outgoing-port-permit:", "1253");
+    }
+#endif
+
     return true;
   }
 

llarp/handlers/tun.cpp

@@ -35,8 +35,8 @@ namespace llarp
   namespace handlers
   {
     // Intercepts DNS IP packets going to an IP on the tun interface; this is currently used on
-    // Android where binding to a DNS port (i.e. via llarp::dns::Proxy) isn't possible because of OS
-    // restrictions, but a tun interface *is* available.
+    // Android and macOS where binding to a DNS port (i.e. via llarp::dns::Proxy) isn't possible
+    // because of OS restrictions, but a tun interface *is* available.
     class DnsInterceptor : public dns::PacketHandler
     {
      public:
@@ -61,6 +61,20 @@ namespace llarp
         m_Endpoint->HandleWriteIPPacket(
             pkt.ConstBuffer(), net::ExpandV4(from.asIPv4()), net::ExpandV4(to.asIPv4()), 0);
       }
+
+#ifdef __APPLE__
+      // DNS on Apple is a bit weird because in order for the NetworkExtension itself to send data
+      // through the tunnel we have to proxy DNS requests through Apple APIs (and so our actual
+      // upstream DNS won't be set in our resolvers, which is why the vanilla IsUpstreamResolver
+      // won't work for us.  However when active the mac also only queries the main tunnel IP for
+      // DNS, so we consider anything else to be upstream-bound DNS to let it through the tunnel.
+      bool
+      IsUpstreamResolver(const SockAddr& to, const SockAddr& from) const override
+      {
+          LogError("IsUpstreamResolver? ", to.asIPv6(), " != ", m_Endpoint->GetIfAddr(), ", from=", from);
+          return to.asIPv6() != m_Endpoint->GetIfAddr();
+      }
+#endif
     };
 
     TunEndpoint::TunEndpoint(AbstractRouter* r, service::Context* parent)
@@ -136,6 +150,17 @@ namespace llarp
         m_Resolver->Restart();
     }
 
+    std::vector<SockAddr>
+    TunEndpoint::ReconfigureDNS(std::vector<SockAddr> servers)
+    {
+        std::swap(m_UpstreamResolvers, servers);
+        m_Resolver->Stop();
+        if (!m_Resolver->Start(
+                    m_LocalResolverAddr.createSockAddr(), m_UpstreamResolvers, m_hostfiles))
+            llarp::LogError(Name(), " failed to reconfigure DNS server");
+        return servers;
+    }
+
     bool
     TunEndpoint::Configure(const NetworkConfig& conf, const DnsConfig& dnsConf)
     {

llarp/handlers/tun.hpp

@@ -43,6 +43,11 @@ namespace llarp
       void
       Thaw() override;
 
+      // Reconfigures DNS servers and restarts libunbound with the new servers.  Returns the old set
+      // of configured dns servers.
+      std::vector<SockAddr>
+      ReconfigureDNS(std::vector<SockAddr> servers);
+
       bool
       Configure(const NetworkConfig& conf, const DnsConfig& dnsConf) override;
 

llarp/service/endpoint.hpp

@@ -11,7 +11,6 @@
 #include "identity.hpp"
 #include "pendingbuffer.hpp"
 #include "protocol.hpp"
-#include "quic/server.hpp"
 #include "sendcontext.hpp"
 #include "service/protocol_type.hpp"
 #include "session.hpp"