Harmonize error management

internal/crawler/crawler.go

@@ -76,8 +76,8 @@ func execute(ctx *cli.Context) error {
 
 	log.Info().Msg("Successfully initialized tdsh-crawler. Waiting for URLs")
 
-	if err := sub.QueueSubscribe(messaging.URLTodoSubject, "crawlers",
-		handleMessage(httpClient, ctx.StringSlice("allowed-ct"))); err != nil {
+	handler := handleMessage(httpClient, ctx.StringSlice("allowed-ct"))
+	if err := sub.QueueSubscribe(messaging.URLTodoSubject, "crawlers", handler); err != nil {
 		return err
 	}
 
@@ -93,8 +93,7 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) me
 
 		body, err := crawURL(httpClient, urlMsg.URL, allowedContentTypes)
 		if err != nil {
-			log.Err(err).Str("url", urlMsg.URL).Msg("Error while crawling url")
-			return err
+			return fmt.Errorf("error while crawling URL: %s", err)
 		}
 
 		// Publish resource body
@@ -103,7 +102,7 @@ func handleMessage(httpClient *fasthttp.Client, allowedContentTypes []string) me
 			Body: body,
 		}
 		if err := sub.PublishMsg(&res); err != nil {
-			log.Err(err).Msg("Error while publishing resource body")
+			return fmt.Errorf("error while publishing resource: %s", err)
 		}
 
 		return nil

internal/extractor/extractor.go

@@ -59,8 +59,8 @@ func execute(ctx *cli.Context) error {
 
 	log.Info().Msg("Successfully initialized tdsh-extractor. Waiting for resources")
 
-	if err := sub.QueueSubscribe(messaging.NewResourceSubject, "extractors",
-		handleMessage(apiClient)); err != nil {
+	handler := handleMessage(apiClient)
+	if err := sub.QueueSubscribe(messaging.NewResourceSubject, "extractors", handler); err != nil {
 		return err
 	}
 
@@ -71,7 +71,6 @@ func handleMessage(apiClient api.Client) messaging.MsgHandler {
 	return func(sub messaging.Subscriber, msg *nats.Msg) error {
 		var resMsg messaging.NewResourceMsg
 		if err := sub.ReadMsg(msg, &resMsg); err != nil {
-			log.Err(err).Msg("Error while reading message")
 			return err
 		}
 
@@ -80,15 +79,13 @@ func handleMessage(apiClient api.Client) messaging.MsgHandler {
 		// Extract & process resource
 		resDto, urls, err := extractResource(resMsg)
 		if err != nil {
-			log.Err(err).Msg("Error while extracting resource")
-			return err
+			return fmt.Errorf("error while extracting resource: %s", err)
 		}
 
 		// Submit to the API
 		_, err = apiClient.AddResource(resDto)
 		if err != nil {
-			log.Err(err).Msg("Error while adding resource")
-			return err
+			return fmt.Errorf("error while adding resource (%s): %s", resDto.URL, err)
 		}
 
 		// Finally push found URLs

internal/messaging/subscriber.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/nats-io/nats.go"
+	"github.com/rs/zerolog/log"
 )
 
 // MsgHandler represent an handler for a NATS subscriber
@@ -51,11 +52,13 @@ func (s *subscriber) QueueSubscribe(subject, queue string, handler MsgHandler) e
 		// Read incoming message
 		msg, err := sub.NextMsgWithContext(context.Background())
 		if err != nil {
+			log.Warn().Str("err", err.Error()).Msg("error while reading incoming message, skipping it")
 			continue
 		}
 
 		// ... And process it
 		if err := handler(s, msg); err != nil {
+			log.Err(err).Msg("error while processing message")
 			continue
 		}
 	}

internal/scheduler/scheduler.go

@@ -67,8 +67,8 @@ func execute(ctx *cli.Context) error {
 
 	log.Info().Msg("Successfully initialized tdsh-scheduler. Waiting for URLs")
 
-	callback := handleMessage(apiClient, refreshDelay, ctx.StringSlice("forbidden-extensions"))
-	if err := sub.QueueSubscribe(messaging.URLFoundSubject, "schedulers", callback); err != nil {
+	handler := handleMessage(apiClient, refreshDelay, ctx.StringSlice("forbidden-extensions"))
+	if err := sub.QueueSubscribe(messaging.URLFoundSubject, "schedulers", handler); err != nil {
 		return err
 	}
 
@@ -86,14 +86,13 @@ func handleMessage(apiClient api.Client, refreshDelay time.Duration, forbiddenEx
 
 		u, err := url.Parse(urlMsg.URL)
 		if err != nil {
-			log.Err(err).Msg("Error while parsing URL")
-			return err
+			return fmt.Errorf("error while parsing URL: %s", err)
 		}
 
 		// Make sure URL is valid .onion
 		if !strings.Contains(u.Host, ".onion") {
 			log.Trace().Stringer("url", u).Msg("URL is not a valid hidden service")
-			return fmt.Errorf("%s is not a valid .onion", u.Host)
+			return nil // Technically not an error
 		}
 
 		// Make sure extension is not forbidden
@@ -103,7 +102,7 @@ func handleMessage(apiClient api.Client, refreshDelay time.Duration, forbiddenEx
 					Stringer("url", u).
 					Str("ext", ext).
 					Msg("Skipping URL with forbidden extension")
-				return fmt.Errorf("%s contains invalid extension .%s", u, ext)
+				return nil // Technically not an error
 			}
 		}
 
@@ -116,8 +115,7 @@ func handleMessage(apiClient api.Client, refreshDelay time.Duration, forbiddenEx
 
 		_, count, err := apiClient.SearchResources(u.String(), "", time.Time{}, endDate, 1, 1)
 		if err != nil {
-			log.Err(err).Msg("Error while searching URL")
-			return err
+			return fmt.Errorf("error while searching resource (%s): %s", u, err)
 		}
 
 		// No matches: schedule!

internal/scheduler/scheduler_test.go

@@ -42,7 +42,7 @@ func TestHandleMessageNotOnion(t *testing.T) {
 		SetArg(1, messaging.URLFoundMsg{URL: "https://example.org"}).
 		Return(nil)
 
-	if err := handleMessage(apiClientMock, -1, []string{})(subscriberMock, &msg); err == nil {
+	if err := handleMessage(apiClientMock, -1, []string{})(subscriberMock, &msg); err != nil {
 		t.FailNow()
 	}
 }
@@ -82,7 +82,7 @@ func TestHandleMessageForbiddenExtensions(t *testing.T) {
 		SetArg(1, messaging.URLFoundMsg{URL: "https://example.onion/image.png"}).
 		Return(nil)
 
-	if err := handleMessage(apiClientMock, -1, []string{"png"})(subscriberMock, &msg); err == nil {
+	if err := handleMessage(apiClientMock, -1, []string{"png"})(subscriberMock, &msg); err != nil {
 		t.FailNow()
 	}
 }