Refactor to use github.com/creekorful/event

devel
Aloïs Micard 3 years ago
parent c141fbf058
commit 5bc7444494

@ -6,6 +6,7 @@ require (
github.com/PuerkitoBio/goquery v1.6.0
github.com/PuerkitoBio/purell v1.1.1
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/creekorful/event v0.2.0
github.com/go-redis/redis/v8 v8.4.4
github.com/golang/mock v1.4.4
github.com/gorilla/mux v1.8.0

@ -15,6 +15,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creekorful/event v0.1.0 h1:/Pnh6rAvEnCssS0z6u8esFi9uqo51a6aBeRgPwEjeB4=
github.com/creekorful/event v0.1.0/go.mod h1:Za4oteYTv0QhYTqKCH1kJKryy5fzNgpMsd5C/6BeW44=
github.com/creekorful/event v0.2.0 h1:5ZpPqZXxwcCAc2nmieBxg6nqECqnMQSLYOgW1DvAciM=
github.com/creekorful/event v0.2.0/go.mod h1:Za4oteYTv0QhYTqKCH1kJKryy5fzNgpMsd5C/6BeW44=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=

@ -2,9 +2,10 @@ package blacklister
import (
"fmt"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache"
configapi "github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
chttp "github.com/darkspot-org/bathyscaphe/internal/http"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
@ -76,7 +77,7 @@ func (state *State) Initialize(provider process.Provider) error {
// Subscribers return the process subscribers
func (state *State) Subscribers() []process.SubscriberDef {
return []process.SubscriberDef{
{Exchange: event.TimeoutURLExchange, Queue: "blacklistingQueue", Handler: state.handleTimeoutURLEvent},
{Exchange: eventdef.TimeoutURLExchange, Queue: "blacklistingQueue", Handler: state.handleTimeoutURLEvent},
}
}
@ -85,9 +86,9 @@ func (state *State) HTTPHandler() http.Handler {
return nil
}
func (state *State) handleTimeoutURLEvent(subscriber event.Subscriber, msg event.RawMessage) error {
var evt event.TimeoutURLEvent
if err := subscriber.Read(&msg, &evt); err != nil {
func (state *State) handleTimeoutURLEvent(subscriber event.Subscriber, msg *event.RawMessage) error {
var evt eventdef.TimeoutURLEvent
if err := subscriber.Read(msg, &evt); err != nil {
return err
}

@ -2,10 +2,11 @@ package blacklister
import (
"errors"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache_mock"
configapi "github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client_mock"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/event_mock"
"github.com/darkspot-org/bathyscaphe/internal/http"
"github.com/darkspot-org/bathyscaphe/internal/http_mock"
@ -59,10 +60,10 @@ func TestHandleTimeoutURLEventNoTimeout(t *testing.T) {
httpClientMock := http_mock.NewMockClient(mockCtrl)
httpResponseMock := http_mock.NewMockResponse(mockCtrl)
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.TimeoutURLEvent{}).
SetArg(1, event.TimeoutURLEvent{
Read(msg, &eventdef.TimeoutURLEvent{}).
SetArg(1, eventdef.TimeoutURLEvent{
URL: "https://down-example.onion:8080/reset-password?username=test",
}).Return(nil)
@ -87,10 +88,10 @@ func TestHandleTimeoutURLEventNoDispatch(t *testing.T) {
httpClientMock := http_mock.NewMockClient(mockCtrl)
httpResponseMock := http_mock.NewMockResponse(mockCtrl)
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.TimeoutURLEvent{}).
SetArg(1, event.TimeoutURLEvent{
Read(msg, &eventdef.TimeoutURLEvent{}).
SetArg(1, eventdef.TimeoutURLEvent{
URL: "https://down-example.onion/login.php",
}).Return(nil)
@ -120,10 +121,10 @@ func TestHandleTimeoutURLEvent(t *testing.T) {
httpClientMock := http_mock.NewMockClient(mockCtrl)
httpResponseMock := http_mock.NewMockResponse(mockCtrl)
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.TimeoutURLEvent{}).
SetArg(1, event.TimeoutURLEvent{
Read(msg, &eventdef.TimeoutURLEvent{}).
SetArg(1, eventdef.TimeoutURLEvent{
URL: "https://down-example.onion/test.html",
}).Return(nil)
@ -164,10 +165,10 @@ func TestHandleTimeoutURLEventNoDuplicates(t *testing.T) {
configClientMock := client_mock.NewMockClient(mockCtrl)
hostnameCacheMock := cache_mock.NewMockCache(mockCtrl)
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.TimeoutURLEvent{}).
SetArg(1, event.TimeoutURLEvent{
Read(msg, &eventdef.TimeoutURLEvent{}).
SetArg(1, eventdef.TimeoutURLEvent{
URL: "https://facebookcorewwwi.onion/morning-routine.php?id=12",
}).Return(nil)

@ -6,7 +6,8 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/creekorful/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/rs/zerolog/log"
"io/ioutil"
"net/http"
@ -96,7 +97,7 @@ func NewConfigClient(configAPIURL string, subscriber event.Subscriber, keys []st
}
// Subscribe for config changed
if err := client.sub.SubscribeAll(event.ConfigExchange, client.handleConfigEvent); err != nil {
if err := client.sub.SubscribeAll(eventdef.ConfigExchange, client.handleConfigEvent); err != nil {
return nil, err
}
@ -254,7 +255,7 @@ func (c *client) setValue(key string, value []byte) error {
return nil
}
func (c *client) handleConfigEvent(_ event.Subscriber, msg event.RawMessage) error {
func (c *client) handleConfigEvent(_ event.Subscriber, msg *event.RawMessage) error {
// Make sure we have the header
configKey, ok := msg.Headers["Config-Key"].(string)
if !ok {

@ -1,7 +1,7 @@
package client
import (
"github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/event_mock"
"github.com/golang/mock/gomock"
"sync"
@ -34,7 +34,7 @@ func TestClient(t *testing.T) {
t.Fail()
}
msg := event.RawMessage{
msg := &event.RawMessage{
Body: []byte("[{\"content-type\": \"application/json\", \"extensions\": [\"json\"]}]"),
Headers: map[string]interface{}{"Config-Key": AllowedMimeTypesKey},
}

@ -2,8 +2,9 @@ package configapi
import (
"fmt"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/gorilla/mux"
"github.com/rs/zerolog/log"
@ -134,7 +135,7 @@ func (state *State) setConfiguration(w http.ResponseWriter, r *http.Request) {
}
// publish event to notify config changed
if err := state.pub.PublishJSON(event.ConfigExchange, event.RawMessage{
if err := state.pub.PublishRaw(eventdef.ConfigExchange, &event.RawMessage{
Body: b,
Headers: map[string]interface{}{"Config-Key": key},
}); err != nil {

@ -1,9 +1,9 @@
package configapi
import (
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache"
"github.com/darkspot-org/bathyscaphe/internal/cache_mock"
"github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/event_mock"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/darkspot-org/bathyscaphe/internal/process_mock"
@ -87,7 +87,7 @@ func TestSetConfiguration(t *testing.T) {
pubMock := event_mock.NewMockPublisher(mockCtrl)
configCacheMock.EXPECT().SetBytes("hello", []byte("{\"ttl\": \"10s\"}"), cache.NoTTL).Return(nil)
pubMock.EXPECT().PublishJSON("config", event.RawMessage{
pubMock.EXPECT().PublishRaw("config", &event.RawMessage{
Body: []byte("{\"ttl\": \"10s\"}"),
Headers: map[string]interface{}{"Config-Key": "hello"},
}).Return(nil)

@ -2,10 +2,11 @@ package crawler
import (
"fmt"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/clock"
configapi "github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/constraint"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
chttp "github.com/darkspot-org/bathyscaphe/internal/http"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
@ -79,7 +80,7 @@ func (state *State) Initialize(provider process.Provider) error {
// Subscribers return the process subscribers
func (state *State) Subscribers() []process.SubscriberDef {
return []process.SubscriberDef{
{Exchange: event.NewURLExchange, Queue: "crawlingQueue", Handler: state.handleNewURLEvent},
{Exchange: eventdef.NewURLExchange, Queue: "crawlingQueue", Handler: state.handleNewURLEvent},
}
}
@ -88,9 +89,9 @@ func (state *State) HTTPHandler() http.Handler {
return nil
}
func (state *State) handleNewURLEvent(subscriber event.Subscriber, msg event.RawMessage) error {
var evt event.NewURLEvent
if err := subscriber.Read(&msg, &evt); err != nil {
func (state *State) handleNewURLEvent(subscriber event.Subscriber, msg *event.RawMessage) error {
var evt eventdef.NewURLEvent
if err := subscriber.Read(msg, &evt); err != nil {
return err
}
@ -107,7 +108,7 @@ func (state *State) handleNewURLEvent(subscriber event.Subscriber, msg event.Raw
if err != nil {
if err == chttp.ErrTimeout {
// indicate that crawling has failed
_ = subscriber.PublishEvent(&event.TimeoutURLEvent{URL: evt.URL})
_ = subscriber.PublishEvent(&eventdef.TimeoutURLEvent{URL: evt.URL})
}
return err
@ -140,7 +141,7 @@ func (state *State) handleNewURLEvent(subscriber event.Subscriber, msg event.Raw
return err
}
res := event.NewResourceEvent{
res := eventdef.NewResourceEvent{
URL: evt.URL,
Body: string(b),
Headers: r.Headers(),

@ -2,10 +2,11 @@ package crawler
import (
"errors"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/clock_mock"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client_mock"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/event_mock"
"github.com/darkspot-org/bathyscaphe/internal/http"
"github.com/darkspot-org/bathyscaphe/internal/http_mock"
@ -133,17 +134,17 @@ func TestHandleNewURLEvent(t *testing.T) {
}
for _, test := range tests {
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewURLEvent{}).
SetArg(1, event.NewURLEvent{URL: test.url}).
Read(msg, &eventdef.NewURLEvent{}).
SetArg(1, eventdef.NewURLEvent{URL: test.url}).
Return(nil)
// mock crawling
switch test.err {
case http.ErrTimeout:
httpClientMock.EXPECT().Get(test.url).Return(httpResponseMock, http.ErrTimeout)
subscriberMock.EXPECT().PublishEvent(&event.TimeoutURLEvent{URL: test.url}).Return(nil)
subscriberMock.EXPECT().PublishEvent(&eventdef.TimeoutURLEvent{URL: test.url}).Return(nil)
break
default:
httpResponseMock.EXPECT().Headers().Return(test.responseHeaders)
@ -164,7 +165,7 @@ func TestHandleNewURLEvent(t *testing.T) {
clockMock.EXPECT().Now().Return(tn)
// if test should pass expect event publishing
subscriberMock.EXPECT().PublishEvent(&event.NewResourceEvent{
subscriberMock.EXPECT().PublishEvent(&eventdef.NewResourceEvent{
URL: test.url,
Body: test.responseBody,
Headers: test.responseHeaders,
@ -193,10 +194,10 @@ func TestHandleNewURLEventHostnameForbidden(t *testing.T) {
configClient: configClientMock,
}
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewURLEvent{}).
SetArg(1, event.NewURLEvent{URL: "https://l.facebookcorewwwi.onion/test.php"}).
Read(msg, &eventdef.NewURLEvent{}).
SetArg(1, eventdef.NewURLEvent{URL: "https://l.facebookcorewwwi.onion/test.php"}).
Return(nil)
configClientMock.EXPECT().GetForbiddenHostnames().

@ -1,6 +1,6 @@
package event
//go:generate mockgen -destination=../event_mock/event_mock.go -package=event_mock . Publisher,Subscriber
//go:generate mockgen -destination=../event_mock/event_mock.go -package=event_mock github.com/creekorful/event Publisher,Subscriber
import "time"

@ -1,57 +0,0 @@
package event
import (
"encoding/json"
"fmt"
"github.com/streadway/amqp"
)
// Publisher is something that push an event
type Publisher interface {
PublishEvent(event Event) error
PublishJSON(exchange string, msg RawMessage) error
Close() error
}
type publisher struct {
channel *amqp.Channel
}
// NewPublisher create a new Publisher instance
func NewPublisher(amqpURI string) (Publisher, error) {
conn, err := amqp.Dial(amqpURI)
if err != nil {
return nil, err
}
c, err := conn.Channel()
if err != nil {
return nil, err
}
return &publisher{
channel: c,
}, nil
}
func (p *publisher) PublishEvent(event Event) error {
evtBytes, err := json.Marshal(event)
if err != nil {
return fmt.Errorf("error while encoding event: %s", err)
}
return p.PublishJSON(event.Exchange(), RawMessage{Body: evtBytes})
}
func (p *publisher) PublishJSON(exchange string, msg RawMessage) error {
return p.channel.Publish(exchange, "", false, false, amqp.Publishing{
ContentType: "application/json",
Body: msg.Body,
DeliveryMode: amqp.Persistent,
Headers: msg.Headers,
})
}
func (p *publisher) Close() error {
return p.channel.Close()
}

@ -1,172 +0,0 @@
package event
import (
"encoding/json"
"fmt"
"github.com/rs/zerolog/log"
"github.com/streadway/amqp"
)
// RawMessage is a raw message as viewed by the messaging system
type RawMessage struct {
Body []byte
Headers map[string]interface{}
}
// Handler represent an event handler
type Handler func(Subscriber, RawMessage) error
// Subscriber is something that read msg from an event queue
type Subscriber interface {
Publisher
// Read RawMessage and deserialize it into proper Event
Read(msg *RawMessage, event Event) error
// Subscribe to named exchange with unique consuming guaranty
Subscribe(exchange, queue string, handler Handler) error
// SubscribeAll subscribe to given exchange but ensure everyone on the exchange receive the messages
SubscribeAll(exchange string, handler Handler) error
}
// Subscriber represent a subscriber
type subscriber struct {
channel *amqp.Channel
}
// NewSubscriber create a new subscriber and connect it to given server
func NewSubscriber(amqpURI string, prefetch int) (Subscriber, error) {
conn, err := amqp.Dial(amqpURI)
if err != nil {
return nil, err
}
c, err := conn.Channel()
if err != nil {
return nil, err
}
if err := c.Qos(prefetch, 0, false); err != nil {
return nil, err
}
return &subscriber{
channel: c,
}, nil
}
func (s *subscriber) PublishEvent(event Event) error {
evtBytes, err := json.Marshal(event)
if err != nil {
return fmt.Errorf("error while encoding event: %s", err)
}
return s.PublishJSON(event.Exchange(), RawMessage{Body: evtBytes})
}
func (s *subscriber) PublishJSON(exchange string, msg RawMessage) error {
return s.channel.Publish(exchange, "", false, false, amqp.Publishing{
ContentType: "application/json",
Body: msg.Body,
DeliveryMode: amqp.Persistent,
Headers: msg.Headers,
})
}
func (s *subscriber) Close() error {
return s.channel.Close()
}
func (s *subscriber) Read(msg *RawMessage, event Event) error {
if err := json.Unmarshal(msg.Body, event); err != nil {
return err
}
return nil
}
func (s *subscriber) Subscribe(exchange, queue string, handler Handler) error {
// First of all declare the exchange
if err := s.channel.ExchangeDeclare(exchange, amqp.ExchangeFanout, true, false, false, false, nil); err != nil {
return err
}
// Then declare the queue
q, err := s.channel.QueueDeclare(queue, true, false, false, false, nil)
if err != nil {
return err
}
// Bind the queue to the exchange
if err := s.channel.QueueBind(q.Name, "", exchange, false, nil); err != nil {
return err
}
// Start consuming asynchronously
deliveries, err := s.channel.Consume(q.Name, "", false, false, false, false, nil)
if err != nil {
return err
}
go func() {
for delivery := range deliveries {
msg := RawMessage{
Body: delivery.Body,
Headers: delivery.Headers,
}
if err := handler(s, msg); err != nil {
log.Err(err).Msg("error while processing event")
}
// Ack no matter what happen since we doesn't care about failing event (yet?)
if err := delivery.Ack(false); err != nil {
log.Err(err).Msg("error while acknowledging event")
}
}
}()
return nil
}
func (s *subscriber) SubscribeAll(exchange string, handler Handler) error {
// First of all declare the exchange
if err := s.channel.ExchangeDeclare(exchange, amqp.ExchangeFanout, true, false, false, false, nil); err != nil {
return err
}
// Then declare the queue
q, err := s.channel.QueueDeclare("", false, true, true, false, nil)
if err != nil {
return err
}
// Bind the queue to the exchange
if err := s.channel.QueueBind(q.Name, "", exchange, false, nil); err != nil {
return err
}
// Start consuming asynchronously
deliveries, err := s.channel.Consume(q.Name, "", false, false, false, false, nil)
if err != nil {
return err
}
go func() {
for delivery := range deliveries {
msg := RawMessage{
Body: delivery.Body,
Headers: delivery.Headers,
}
if err := handler(s, msg); err != nil {
log.Err(err).Msg("error while processing event")
}
// Ack no matter what happen since we doesn't care about failing event (yet?)
if err := delivery.Ack(false); err != nil {
log.Err(err).Msg("error while acknowledging event")
}
}
}()
return nil
}

@ -2,9 +2,10 @@ package indexer
import (
"fmt"
"github.com/creekorful/event"
configapi "github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/constraint"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/indexer/index"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
@ -82,7 +83,7 @@ func (state *State) Initialize(provider process.Provider) error {
// Subscribers return the process subscribers
func (state *State) Subscribers() []process.SubscriberDef {
return []process.SubscriberDef{
{Exchange: event.NewResourceExchange, Queue: fmt.Sprintf("%sIndexingQueue", state.indexDriver), Handler: state.handleNewResourceEvent},
{Exchange: eventdef.NewResourceExchange, Queue: fmt.Sprintf("%sIndexingQueue", state.indexDriver), Handler: state.handleNewResourceEvent},
}
}
@ -91,9 +92,9 @@ func (state *State) HTTPHandler() http.Handler {
return nil
}
func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg event.RawMessage) error {
var evt event.NewResourceEvent
if err := subscriber.Read(&msg, &evt); err != nil {
func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg *event.RawMessage) error {
var evt eventdef.NewResourceEvent
if err := subscriber.Read(msg, &evt); err != nil {
return err
}

@ -2,9 +2,10 @@ package indexer
import (
"errors"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client_mock"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/event_mock"
"github.com/darkspot-org/bathyscaphe/internal/indexer/index"
"github.com/darkspot-org/bathyscaphe/internal/indexer/index_mock"
@ -80,10 +81,10 @@ Thanks to https://help.facebook.onion/ for the hosting :D
tn := time.Now()
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewResourceEvent{}).
SetArg(1, event.NewResourceEvent{
Read(msg, &eventdef.NewResourceEvent{}).
SetArg(1, eventdef.NewResourceEvent{
URL: "https://example.onion",
Body: body,
Headers: map[string]string{"Server": "Traefik", "Content-Type": "application/html"},
@ -126,10 +127,10 @@ Thanks to https://help.facebook.onion/ for the hosting :D
tn := time.Now()
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewResourceEvent{}).
SetArg(1, event.NewResourceEvent{
Read(msg, &eventdef.NewResourceEvent{}).
SetArg(1, eventdef.NewResourceEvent{
URL: "https://example.onion",
Body: body,
Headers: map[string]string{"Server": "Traefik", "Content-Type": "application/html"},
@ -183,10 +184,10 @@ Thanks to https://help.facebook.onion/ for the hosting :D
tn := time.Now()
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewResourceEvent{}).
SetArg(1, event.NewResourceEvent{
Read(msg, &eventdef.NewResourceEvent{}).
SetArg(1, eventdef.NewResourceEvent{
URL: "https://example.onion",
Body: body,
Headers: map[string]string{"Server": "Traefik", "Content-Type": "application/html"},
@ -241,10 +242,10 @@ This is sparta (hosted on https://example.org)
tn := time.Now()
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewResourceEvent{}).
SetArg(1, event.NewResourceEvent{
Read(msg, &eventdef.NewResourceEvent{}).
SetArg(1, eventdef.NewResourceEvent{
URL: "https://example.onion",
Body: body,
Headers: map[string]string{"Server": "Traefik", "Content-Type": "application/html"},

@ -6,10 +6,10 @@ import (
"context"
"crypto/tls"
"fmt"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache"
"github.com/darkspot-org/bathyscaphe/internal/clock"
configapi "github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/event"
chttp "github.com/darkspot-org/bathyscaphe/internal/http"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"

@ -4,10 +4,11 @@ import (
"errors"
"fmt"
"github.com/PuerkitoBio/purell"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache"
configapi "github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/constraint"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
@ -81,7 +82,7 @@ func (state *State) Initialize(provider process.Provider) error {
// Subscribers return the process subscribers
func (state *State) Subscribers() []process.SubscriberDef {
return []process.SubscriberDef{
{Exchange: event.NewResourceExchange, Queue: "schedulingQueue", Handler: state.handleNewResourceEvent},
{Exchange: eventdef.NewResourceExchange, Queue: "schedulingQueue", Handler: state.handleNewResourceEvent},
}
}
@ -90,9 +91,9 @@ func (state *State) HTTPHandler() http.Handler {
return nil
}
func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg event.RawMessage) error {
var evt event.NewResourceEvent
if err := subscriber.Read(&msg, &evt); err != nil {
func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg *event.RawMessage) error {
var evt eventdef.NewResourceEvent
if err := subscriber.Read(msg, &evt); err != nil {
return err
}
@ -212,14 +213,14 @@ func (state *State) processURL(rawURL string, pub event.Publisher, urlCache map[
urlCache[urlHash]++
if err := pub.PublishEvent(&event.NewURLEvent{URL: rawURL}); err != nil {
if err := pub.PublishEvent(&eventdef.NewURLEvent{URL: rawURL}); err != nil {
return fmt.Errorf("error while publishing URL: %s", err)
}
return nil
}
func extractURLS(msg *event.NewResourceEvent) ([]string, error) {
func extractURLS(msg *eventdef.NewResourceEvent) ([]string, error) {
// Extract & normalize URLs
xu := xurls.Strict()
urls := xu.FindAllString(msg.Body, -1)

@ -2,11 +2,12 @@ package scheduler
import (
"errors"
"github.com/creekorful/event"
"github.com/darkspot-org/bathyscaphe/internal/cache"
"github.com/darkspot-org/bathyscaphe/internal/cache_mock"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client"
"github.com/darkspot-org/bathyscaphe/internal/configapi/client_mock"
"github.com/darkspot-org/bathyscaphe/internal/event"
eventdef "github.com/darkspot-org/bathyscaphe/internal/event"
"github.com/darkspot-org/bathyscaphe/internal/event_mock"
"github.com/darkspot-org/bathyscaphe/internal/process"
"github.com/darkspot-org/bathyscaphe/internal/process_mock"
@ -178,7 +179,7 @@ func TestProcessURL(t *testing.T) {
configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"html", "php"}}}, nil)
configClientMock.EXPECT().GetForbiddenHostnames().Return([]client.ForbiddenHostname{}, nil)
pubMock.EXPECT().PublishEvent(&event.NewURLEvent{URL: url}).Return(nil)
pubMock.EXPECT().PublishEvent(&eventdef.NewURLEvent{URL: url}).Return(nil)
state := State{configClient: configClientMock}
if err := state.processURL(url, pubMock, urlCache); err != nil {
@ -206,10 +207,10 @@ func TestHandleNewResourceEvent(t *testing.T) {
urlCacheMock := cache_mock.NewMockCache(mockCtrl)
configClientMock := client_mock.NewMockClient(mockCtrl)
msg := event.RawMessage{}
msg := &event.RawMessage{}
subscriberMock.EXPECT().
Read(&msg, &event.NewResourceEvent{}).
SetArg(1, event.NewResourceEvent{
Read(msg, &eventdef.NewResourceEvent{}).
SetArg(1, eventdef.NewResourceEvent{
URL: "https://l.facebookcorewwwi.onion/test.php",
Body: `
<a href=\"https://facebook.onion/test.php?id=1\">This is a little test</a>.
@ -235,7 +236,7 @@ This domain is blacklisted: https://m.fbi.onion/test.php
}, nil)
configClientMock.EXPECT().GetRefreshDelay().Return(client.RefreshDelay{Delay: 0}, nil)
subscriberMock.EXPECT().PublishEvent(&event.NewURLEvent{
subscriberMock.EXPECT().PublishEvent(&eventdef.NewURLEvent{
URL: "https://facebook.onion/test.php?id=1",
})

Loading…
Cancel
Save