Archives
/
bathyscaphe
mirror of https://github.com/creekorful/bathyscaphe
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #132 from creekorful/120-blacklister-final-tweaks

Browse Source 
Blacklister final tweaks
pull/133/head
Aloïs Micard 3 years ago committed by GitHub
parent b84e6d28ac 9b83830d95
commit bbdba2546f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 68 additions and 39 deletions
Show Stats Download Patch File Download Diff File

4
deployments/docker/docker-compose.yml
Unescape Escape View File

@ -89,8 +89,8 @@ services:
--redis redis:6379
--default-value forbidden-hostnames="[]"
--default-value allowed-mime-types="[{\"content-type\":\"text/\",\"extensions\":[\"html\",\"php\",\"aspx\", \"htm\"]}]"
--default-value refresh-delay="{\"delay\": -1}"
--default-value blacklist-threshold="{\"threshold\": 5}"
--default-value refresh-delay="{\"delay\": 0}"
--default-value blacklist-config="{\"threshold\": 5, \"ttl\": 1200}"
restart: always
depends_on:
- rabbitmq

26
internal/blacklister/blacklister.go
Unescape Escape View File

@ -45,7 +45,7 @@ func (state *State) Initialize(provider process.Provider) error {
}
state.hostnameCache = hostnameCache
configClient, err := provider.ConfigClient([]string{configapi.ForbiddenHostnamesKey, configapi.BlackListThresholdKey})
configClient, err := provider.ConfigClient([]string{configapi.ForbiddenHostnamesKey, configapi.BlackListConfigKey})
if err != nil {
return err
}
@ -104,7 +104,22 @@ func (state *State) handleTimeoutURLEvent(subscriber event.Subscriber, msg event
// Check by ourselves if the hostname doesn't respond
_, err = state.httpClient.Get(fmt.Sprintf("%s://%s", u.Scheme, u.Host))
if err == nil || err != chttp.ErrTimeout {
if err != nil && err != chttp.ErrTimeout {
return err
}
cacheKey := u.Hostname()
if err == nil {
log.Debug().
Str("hostname", u.Hostname()).
Msg("Response received.")
// Host is not down, remove it from cache
if err := state.hostnameCache.Remove(cacheKey); err != nil {
return err
}
return nil
}
@ -112,19 +127,18 @@ func (state *State) handleTimeoutURLEvent(subscriber event.Subscriber, msg event
Str("hostname", u.Hostname()).
Msg("Timeout confirmed")
threshold, err := state.configClient.GetBlackListThreshold()
blackListConfig, err := state.configClient.GetBlackListConfig()
if err != nil {
return err
}
cacheKey := u.Hostname()
count, err := state.hostnameCache.GetInt64(cacheKey)
if err != nil {
return err
}
count++
if count >= threshold.Threshold {
if count >= blackListConfig.Threshold {
forbiddenHostnames, err := state.configClient.GetForbiddenHostnames()
if err != nil {
return err
@ -155,7 +169,7 @@ func (state *State) handleTimeoutURLEvent(subscriber event.Subscriber, msg event
}
// Update count
if err := state.hostnameCache.SetInt64(cacheKey, count, cache.NoTTL); err != nil {
if err := state.hostnameCache.SetInt64(cacheKey, count, blackListConfig.TTL); err != nil {
return err
}

22
internal/blacklister/blacklister_test.go
Unescape Escape View File

@ -2,7 +2,6 @@ package blacklister
import (
"errors"
"github.com/creekorful/trandoshan/internal/cache"
"github.com/creekorful/trandoshan/internal/cache_mock"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/configapi/client_mock"
@ -15,6 +14,7 @@ import (
"github.com/creekorful/trandoshan/internal/test"
"github.com/golang/mock/gomock"
"testing"
"time"
)
func TestState_Name(t *testing.T) {
@ -37,7 +37,7 @@ func TestState_CustomFlags(t *testing.T) {
func TestState_Initialize(t *testing.T) {
test.CheckInitialize(t, &State{}, func(p *process_mock.MockProviderMockRecorder) {
p.Cache("down-hostname")
p.ConfigClient([]string{configapi.ForbiddenHostnamesKey, configapi.BlackListThresholdKey})
p.ConfigClient([]string{configapi.ForbiddenHostnamesKey, configapi.BlackListConfigKey})
p.HTTPClient()
})
}
@ -69,6 +69,8 @@ func TestHandleTimeoutURLEventNoTimeout(t *testing.T) {
httpClientMock.EXPECT().Get("https://down-example.onion:8080").Return(httpResponseMock, nil)
configClientMock.EXPECT().GetForbiddenHostnames().Return([]configapi.ForbiddenHostname{}, nil)
hostnameCacheMock.EXPECT().Remove("down-example.onion")
s := State{configClient: configClientMock, hostnameCache: hostnameCacheMock, httpClient: httpClientMock}
if err := s.handleTimeoutURLEvent(subscriberMock, msg); err != nil {
t.Fail()
@ -94,10 +96,13 @@ func TestHandleTimeoutURLEventNoDispatch(t *testing.T) {
httpClientMock.EXPECT().Get("https://down-example.onion").Return(httpResponseMock, http.ErrTimeout)
configClientMock.EXPECT().GetForbiddenHostnames().Return([]configapi.ForbiddenHostname{}, nil)
configClientMock.EXPECT().GetBlackListThreshold().Return(configapi.BlackListThreshold{Threshold: 10}, nil)
configClientMock.EXPECT().GetBlackListConfig().Return(configapi.BlackListConfig{
Threshold: 10,
TTL: 5,
}, nil)
hostnameCacheMock.EXPECT().GetInt64("down-example.onion").Return(int64(0), nil)
hostnameCacheMock.EXPECT().SetInt64("down-example.onion", int64(1), cache.NoTTL).Return(nil)
hostnameCacheMock.EXPECT().SetInt64("down-example.onion", int64(1), time.Duration(5)).Return(nil)
s := State{configClient: configClientMock, hostnameCache: hostnameCacheMock, httpClient: httpClientMock}
if err := s.handleTimeoutURLEvent(subscriberMock, msg); err != nil {
@ -124,7 +129,10 @@ func TestHandleTimeoutURLEvent(t *testing.T) {
httpClientMock.EXPECT().Get("https://down-example.onion").Return(httpResponseMock, http.ErrTimeout)
configClientMock.EXPECT().GetForbiddenHostnames().Return([]configapi.ForbiddenHostname{}, nil)
configClientMock.EXPECT().GetBlackListThreshold().Return(configapi.BlackListThreshold{Threshold: 10}, nil)
configClientMock.EXPECT().GetBlackListConfig().Return(configapi.BlackListConfig{
Threshold: 10,
TTL: 5,
}, nil)
hostnameCacheMock.EXPECT().GetInt64("down-example.onion").Return(int64(9), nil)
@ -138,7 +146,9 @@ func TestHandleTimeoutURLEvent(t *testing.T) {
}).
Return(nil)
hostnameCacheMock.EXPECT().SetInt64("down-example.onion", int64(10), cache.NoTTL).Return(nil)
hostnameCacheMock.EXPECT().
SetInt64("down-example.onion", int64(10), time.Duration(5)).
Return(nil)
s := State{configClient: configClientMock, hostnameCache: hostnameCacheMock, httpClient: httpClientMock}
if err := s.handleTimeoutURLEvent(subscriberMock, msg); err != nil {

2
internal/cache/cache.go vendored
Unescape Escape View File

@ -21,4 +21,6 @@ type Cache interface {
GetManyInt64(keys []string) (map[string]int64, error)
SetManyInt64(values map[string]int64, TTL time.Duration) error
Remove(key string) error
}

4
internal/cache/redis.go vendored
Unescape Escape View File

@ -92,6 +92,10 @@ func (rc *redisCache) SetManyInt64(values map[string]int64, TTL time.Duration) e
return err
}
func (rc *redisCache) Remove(key string) error {
return rc.client.Del(context.Background(), rc.getKey(key)).Err()
}
func (rc *redisCache) getKey(key string) string {
if rc.keyPrefix == "" {
return key

40
internal/configapi/client/client.go
Unescape Escape View File

@ -21,8 +21,8 @@ const (
ForbiddenHostnamesKey = "forbidden-hostnames"
// RefreshDelayKey is the key to access the refresh delay config
RefreshDelayKey = "refresh-delay"
// BlackListThresholdKey is the key to access the blacklist threshold config
BlackListThresholdKey = "blacklist-threshold"
// BlackListConfigKey is the key to access the blacklist configuration
BlackListConfigKey = "blacklist-config"
)
// MimeType is the mime type as represented in the config
@ -43,9 +43,10 @@ type RefreshDelay struct {
Delay time.Duration `json:"delay"`
}
// BlackListThreshold is the threshold to reach before blacklisting domain
type BlackListThreshold struct {
Threshold int64 `json:"threshold"`
// BlackListConfig is the config used for hostname blacklisting
type BlackListConfig struct {
Threshold int64 `json:"threshold"`
TTL time.Duration `json:"ttl"`
}
// Client is a nice client interface for the ConfigAPI
@ -53,7 +54,7 @@ type Client interface {
GetAllowedMimeTypes() ([]MimeType, error)
GetForbiddenHostnames() ([]ForbiddenHostname, error)
GetRefreshDelay() (RefreshDelay, error)
GetBlackListThreshold() (BlackListThreshold, error)
GetBlackListConfig() (BlackListConfig, error)
Set(key string, value interface{}) error
}
@ -68,7 +69,7 @@ type client struct {
allowedMimeTypes []MimeType
forbiddenHostnames []ForbiddenHostname
refreshDelay RefreshDelay
blackListThreshold BlackListThreshold
blackListConfig BlackListConfig
}
// NewConfigClient create a new client for the ConfigAPI.
@ -150,18 +151,21 @@ func (c *client) setRefreshDelay(value RefreshDelay) error {
return nil
}
func (c *client) GetBlackListThreshold() (BlackListThreshold, error) {
c.mutexes[BlackListThresholdKey].RLock()
defer c.mutexes[BlackListThresholdKey].RUnlock()
func (c *client) GetBlackListConfig() (BlackListConfig, error) {
c.mutexes[BlackListConfigKey].RLock()
defer c.mutexes[BlackListConfigKey].RUnlock()
return c.blackListThreshold, nil
return c.blackListConfig, nil
}
func (c *client) setBlackListThreshold(value BlackListThreshold) error {
c.mutexes[BlackListThresholdKey].Lock()
defer c.mutexes[BlackListThresholdKey].Unlock()
func (c *client) setBlackListConfig(value BlackListConfig) error {
c.mutexes[BlackListConfigKey].Lock()
defer c.mutexes[BlackListConfigKey].Unlock()
c.blackListThreshold = value
c.blackListConfig = BlackListConfig{
Threshold: value.Threshold,
TTL: value.TTL * time.Second, // TTL is in seconds
}
return nil
}
@ -232,12 +236,12 @@ func (c *client) setValue(key string, value []byte) error {
return err
}
break
case BlackListThresholdKey:
var val BlackListThreshold
case BlackListConfigKey:
var val BlackListConfig
if err := json.Unmarshal(value, &val); err != nil {
return err
}
if err := c.setBlackListThreshold(val); err != nil {
if err := c.setBlackListConfig(val); err != nil {
return err
}
break

7
internal/scheduler/scheduler.go
Unescape Escape View File

@ -121,13 +121,8 @@ func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg even
return err
}
ttl := delay.Delay
if ttl == -1 {
ttl = cache.NoTTL
}
// Update values in batch
if err := state.urlCache.SetManyInt64(urlCache, ttl); err != nil {
if err := state.urlCache.SetManyInt64(urlCache, delay.Delay); err != nil {
return err
}

2
internal/scheduler/scheduler_test.go
Unescape Escape View File

@ -233,7 +233,7 @@ This domain is blacklisted: https://m.fbi.onion/test.php
Return([]client.ForbiddenHostname{
{Hostname: "fbi.onion"},
}, nil)
configClientMock.EXPECT().GetRefreshDelay().Return(client.RefreshDelay{Delay: -1}, nil)
configClientMock.EXPECT().GetRefreshDelay().Return(client.RefreshDelay{Delay: 0}, nil)
subscriberMock.EXPECT().PublishEvent(&event.NewURLEvent{
URL: "https://facebook.onion/test.php?id=1",

Write Preview
Loading…
Cancel
Save