|
|
|
@ -12,8 +12,9 @@ import (
|
|
|
|
|
"github.com/creekorful/trandoshan/internal/process_mock"
|
|
|
|
|
"github.com/creekorful/trandoshan/internal/test"
|
|
|
|
|
"github.com/golang/mock/gomock"
|
|
|
|
|
"hash/fnv"
|
|
|
|
|
"strconv"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestState_Name(t *testing.T) {
|
|
|
|
@ -23,9 +24,9 @@ func TestState_Name(t *testing.T) {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestState_CommonFlags(t *testing.T) {
|
|
|
|
|
func TestState_Features(t *testing.T) {
|
|
|
|
|
s := State{}
|
|
|
|
|
test.CheckProcessCommonFlags(t, &s, []string{process.HubURIFlag, process.ConfigAPIURIFlag, process.RedisURIFlag})
|
|
|
|
|
test.CheckProcessFeatures(t, &s, []process.Feature{process.EventFeature, process.ConfigFeature, process.CacheFeature})
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestState_CustomFlags(t *testing.T) {
|
|
|
|
@ -66,7 +67,7 @@ func TestProcessURL_NotDotOnion(t *testing.T) {
|
|
|
|
|
|
|
|
|
|
for _, url := range urls {
|
|
|
|
|
state := State{}
|
|
|
|
|
if err := state.processURL(url, nil); !errors.Is(err, errNotOnionHostname) {
|
|
|
|
|
if err := state.processURL(url, nil, nil); !errors.Is(err, errNotOnionHostname) {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -80,7 +81,7 @@ func TestProcessURL_ProtocolForbidden(t *testing.T) {
|
|
|
|
|
|
|
|
|
|
for _, url := range urls {
|
|
|
|
|
state := State{}
|
|
|
|
|
if err := state.processURL(url, nil); !errors.Is(err, errProtocolNotAllowed) {
|
|
|
|
|
if err := state.processURL(url, nil, nil); !errors.Is(err, errProtocolNotAllowed) {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -98,7 +99,7 @@ func TestProcessURL_ExtensionForbidden(t *testing.T) {
|
|
|
|
|
configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"html", "php"}}}, nil)
|
|
|
|
|
|
|
|
|
|
state := State{configClient: configClientMock}
|
|
|
|
|
if err := state.processURL(url, nil); !errors.Is(err, errExtensionNotAllowed) {
|
|
|
|
|
if err := state.processURL(url, nil, nil); !errors.Is(err, errExtensionNotAllowed) {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -139,7 +140,7 @@ func TestProcessURL_HostnameForbidden(t *testing.T) {
|
|
|
|
|
configClientMock.EXPECT().GetForbiddenHostnames().Return(tst.forbiddenHostnames, nil)
|
|
|
|
|
|
|
|
|
|
state := State{configClient: configClientMock}
|
|
|
|
|
if err := state.processURL(tst.url, nil); !errors.Is(err, errHostnameNotAllowed) {
|
|
|
|
|
if err := state.processURL(tst.url, nil, nil); !errors.Is(err, errHostnameNotAllowed) {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -149,42 +150,102 @@ func TestProcessURL_AlreadyScheduled(t *testing.T) {
|
|
|
|
|
mockCtrl := gomock.NewController(t)
|
|
|
|
|
defer mockCtrl.Finish()
|
|
|
|
|
|
|
|
|
|
urlCacheMock := cache_mock.NewMockCache(mockCtrl)
|
|
|
|
|
configClientMock := client_mock.NewMockClient(mockCtrl)
|
|
|
|
|
|
|
|
|
|
urlCacheMock.EXPECT().GetInt64("https://facebookcorewwi.onion/test.php?id=12").Return(int64(1), nil)
|
|
|
|
|
configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"html", "php"}}}, nil)
|
|
|
|
|
configClientMock.EXPECT().GetForbiddenHostnames().Return([]client.ForbiddenHostname{}, nil)
|
|
|
|
|
|
|
|
|
|
state := State{urlCache: urlCacheMock, configClient: configClientMock}
|
|
|
|
|
if err := state.processURL("https://facebookcorewwi.onion/test.php?id=12", nil); !errors.Is(err, errAlreadyScheduled) {
|
|
|
|
|
urlCache := map[string]int64{"3056224523184958": 1}
|
|
|
|
|
state := State{configClient: configClientMock}
|
|
|
|
|
if err := state.processURL("https://facebookcorewwi.onion/test.php?id=12", nil, urlCache); !errors.Is(err, errAlreadyScheduled) {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestHandleNewResourceEvent(t *testing.T) {
|
|
|
|
|
func TestProcessURL(t *testing.T) {
|
|
|
|
|
mockCtrl := gomock.NewController(t)
|
|
|
|
|
defer mockCtrl.Finish()
|
|
|
|
|
|
|
|
|
|
urlCacheMock := cache_mock.NewMockCache(mockCtrl)
|
|
|
|
|
configClientMock := client_mock.NewMockClient(mockCtrl)
|
|
|
|
|
pubMock := event_mock.NewMockPublisher(mockCtrl)
|
|
|
|
|
|
|
|
|
|
urls := []string{"https://example.onion/index.php", "http://google.onion/admin.secret/login.html",
|
|
|
|
|
"https://example.onion", "https://www.facebookcorewwwi.onion/recover.now/initiate?ars=facebook_login"}
|
|
|
|
|
|
|
|
|
|
// pre fill cache
|
|
|
|
|
urlCache := map[string]int64{}
|
|
|
|
|
for _, url := range urls {
|
|
|
|
|
urlCacheMock.EXPECT().GetInt64(url).Return(int64(0), cache.ErrNIL)
|
|
|
|
|
configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"html", "php"}}}, nil)
|
|
|
|
|
configClientMock.EXPECT().GetForbiddenHostnames().Return([]client.ForbiddenHostname{}, nil)
|
|
|
|
|
configClientMock.EXPECT().GetRefreshDelay().Return(client.RefreshDelay{Delay: 10 * time.Hour}, nil)
|
|
|
|
|
|
|
|
|
|
urlCacheMock.EXPECT().SetInt64(url, int64(1), time.Duration(10*time.Hour)).Return(nil)
|
|
|
|
|
pubMock.EXPECT().PublishEvent(&event.NewURLEvent{URL: url}).Return(nil)
|
|
|
|
|
|
|
|
|
|
state := State{urlCache: urlCacheMock, configClient: configClientMock}
|
|
|
|
|
if err := state.processURL(url, pubMock); err != nil {
|
|
|
|
|
state := State{configClient: configClientMock}
|
|
|
|
|
if err := state.processURL(url, pubMock, urlCache); err != nil {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Compute url hash
|
|
|
|
|
c := fnv.New64()
|
|
|
|
|
if _, err := c.Write([]byte(url)); err != nil {
|
|
|
|
|
t.Error(err)
|
|
|
|
|
}
|
|
|
|
|
urlHash := strconv.FormatUint(c.Sum64(), 10)
|
|
|
|
|
|
|
|
|
|
if val, exist := urlCache[urlHash]; !exist || val != 1 {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestHandleNewResourceEvent(t *testing.T) {
|
|
|
|
|
mockCtrl := gomock.NewController(t)
|
|
|
|
|
defer mockCtrl.Finish()
|
|
|
|
|
|
|
|
|
|
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
|
|
|
|
|
urlCacheMock := cache_mock.NewMockCache(mockCtrl)
|
|
|
|
|
configClientMock := client_mock.NewMockClient(mockCtrl)
|
|
|
|
|
|
|
|
|
|
msg := event.RawMessage{}
|
|
|
|
|
subscriberMock.EXPECT().
|
|
|
|
|
Read(&msg, &event.NewResourceEvent{}).
|
|
|
|
|
SetArg(1, event.NewResourceEvent{
|
|
|
|
|
URL: "https://l.facebookcorewwwi.onion/test.php",
|
|
|
|
|
Body: `
|
|
|
|
|
<a href=\"https://facebook.onion/test.php?id=1\">This is a little test</a>.
|
|
|
|
|
Check out https://google.onion. This is an image https://example.onion/test.png
|
|
|
|
|
This domain is blacklisted: https://m.fbi.onion/test.php
|
|
|
|
|
`,
|
|
|
|
|
}).
|
|
|
|
|
Return(nil)
|
|
|
|
|
|
|
|
|
|
urlCacheMock.EXPECT().
|
|
|
|
|
GetManyInt64([]string{"15038381360563270096", "17173291053643777680", "14332094874591870497", "5985629257333875968"}).
|
|
|
|
|
Return(map[string]int64{
|
|
|
|
|
"17173291053643777680": 1,
|
|
|
|
|
}, nil)
|
|
|
|
|
|
|
|
|
|
configClientMock.EXPECT().GetAllowedMimeTypes().
|
|
|
|
|
Times(4).
|
|
|
|
|
Return([]client.MimeType{{Extensions: []string{"php"}}}, nil)
|
|
|
|
|
configClientMock.EXPECT().GetForbiddenHostnames().
|
|
|
|
|
Times(3).
|
|
|
|
|
Return([]client.ForbiddenHostname{
|
|
|
|
|
{Hostname: "fbi.onion"},
|
|
|
|
|
}, nil)
|
|
|
|
|
configClientMock.EXPECT().GetRefreshDelay().Return(client.RefreshDelay{Delay: 0}, nil)
|
|
|
|
|
|
|
|
|
|
subscriberMock.EXPECT().PublishEvent(&event.NewURLEvent{
|
|
|
|
|
URL: "https://facebook.onion/test.php?id=1",
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
urlCacheMock.EXPECT().SetManyInt64(map[string]int64{
|
|
|
|
|
"17173291053643777680": 1,
|
|
|
|
|
"15038381360563270096": 1,
|
|
|
|
|
}, cache.NoTTL).Return(nil)
|
|
|
|
|
|
|
|
|
|
s := State{urlCache: urlCacheMock, configClient: configClientMock}
|
|
|
|
|
if err := s.handleNewResourceEvent(subscriberMock, msg); err != nil {
|
|
|
|
|
t.Fail()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|