last fixes

internal/cache/redis.go

@@ -34,7 +34,12 @@ func (rc *redisCache) SetBytes(key string, value []byte, TTL time.Duration) erro
 }
 
 func (rc *redisCache) GetInt64(key string) (int64, error) {
-	return rc.client.Get(context.Background(), key).Int64()
+	val, err := rc.client.Get(context.Background(), key).Int64()
+	if err == redis.Nil {
+		err = ErrNIL
+	}
+
+	return val, err
 }
 
 func (rc *redisCache) SetInt64(key string, value int64, TTL time.Duration) error {

internal/configapi/configapi.go

@@ -62,7 +62,7 @@ func (state *State) Initialize(provider process.Provider) error {
 		}
 	}
 	if len(defaultValues) > 0 {
-		if err := setDefaultValues(nil, defaultValues); err != nil {
+		if err := setDefaultValues(configCache, defaultValues); err != nil {
 			return err
 		}
 	}

internal/indexer/indexer.go

@@ -204,12 +204,7 @@ func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg even
 	log.Info().Str("url", evt.URL).Msg("Successfully indexed resource")
 
 	// Finally push found URLs
-	publishedURLS := map[string]string{}
 	for _, u := range urls {
-		if _, exist := publishedURLS[u]; exist {
-			continue
-		}
-
 		// make sure url has not been published (yet)
 		count, err := state.urlCache.GetInt64(fmt.Sprintf("urls:%s", u))
 		if err != nil && err != cache.ErrNIL {
@@ -250,10 +245,6 @@ func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg even
 			log.Err(err).Msg("error while updating URL cache")
 		}
 
-		log.Trace().
-			Str("url", u).
-			Msg("Publishing found URL")
-
 		if err := subscriber.PublishEvent(&event.FoundURLEvent{URL: u}); err != nil {
 			log.Warn().
 				Str("url", u).
@@ -261,7 +252,9 @@ func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg even
 				Msg("Error while publishing URL")
 		}
 
-		publishedURLS[u] = u
+		log.Trace().
+			Str("url", u).
+			Msg("Published found URL")
 	}
 
 	return nil

internal/indexer/indexer_test.go

@@ -492,6 +492,8 @@ Thanks to https://help.facebook.onion/ for the hosting :D
 		PageNumber: 1,
 	}).Return(int64(0), nil)
 
+	urlCacheMock.EXPECT().GetInt64("urls:https://example.org").Return(int64(1), nil)
+
 	urlCacheMock.EXPECT().GetInt64("urls:https://help.facebook.onion").Return(int64(1), nil)
 
 	urlCacheMock.EXPECT().GetInt64("urls:https://google.com/test?test=test").Return(int64(0), cache.ErrNIL)