Archives
/
bathyscaphe
mirror of https://github.com/creekorful/bathyscaphe
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor to use api.API everywhere

Browse Source
pull/81/head
Aloïs Micard 3 years ago
parent 004253f5e0
commit b04c8a9e32
No known key found for this signature in database
GPG Key ID: 1A0EB82F071F5EFE
11 changed files with 104 additions and 86 deletions
Show Stats Download Patch File Download Diff File

51
api/api.go
Unescape Escape View File

@ -8,7 +8,7 @@ import (
"time"
)
//go:generate mockgen -destination=../api_mock/api_mock.go -package=api_mock . Client
//go:generate mockgen -destination=../api_mock/api_mock.go -package=api_mock . API
const (
// PaginationPageHeader is the header to determinate current page in paginated endpoint
@ -40,10 +40,22 @@ type CredentialsDto struct {
Password string `json:"password"`
}
// Client is the interface to interact with the API component
type Client interface {
SearchResources(url, keyword string, startDate, endDate time.Time,
paginationPage, paginationSize int) ([]ResourceDto, int64, error)
// ResSearchParams is the search params used
type ResSearchParams struct {
URL string
Keyword string
StartDate time.Time
EndDate time.Time
WithBody bool
PageSize int
PageNumber int
// TODO allow searching by meta
// TODO allow searching by headers
}
// API is the interface to interact with the API component
type API interface {
SearchResources(params *ResSearchParams) ([]ResourceDto, int64, error)
AddResource(res ResourceDto) (ResourceDto, error)
ScheduleURL(url string) error
}
@ -53,34 +65,33 @@ type client struct {
baseURL string
}
func (c *client) SearchResources(url, keyword string,
startDate, endDate time.Time, paginationPage, paginationSize int) ([]ResourceDto, int64, error) {
func (c *client) SearchResources(params *ResSearchParams) ([]ResourceDto, int64, error) {
targetEndpoint := fmt.Sprintf("%s/v1/resources?", c.baseURL)
req := c.httpClient.R()
if url != "" {
b64URL := base64.URLEncoding.EncodeToString([]byte(url))
if params.URL != "" {
b64URL := base64.URLEncoding.EncodeToString([]byte(params.URL))
req.SetQueryParam("url", b64URL)
}
if keyword != "" {
req.SetQueryParam("keyword", keyword)
if params.Keyword != "" {
req.SetQueryParam("keyword", params.Keyword)
}
if !startDate.IsZero() {
req.SetQueryParam("start-date", startDate.Format(time.RFC3339))
if !params.StartDate.IsZero() {
req.SetQueryParam("start-date", params.StartDate.Format(time.RFC3339))
}
if !endDate.IsZero() {
req.SetQueryParam("end-date", endDate.Format(time.RFC3339))
if !params.EndDate.IsZero() {
req.SetQueryParam("end-date", params.EndDate.Format(time.RFC3339))
}
if paginationPage != 0 {
req.Header.Set(PaginationPageHeader, strconv.Itoa(paginationPage))
if params.PageNumber != 0 {
req.Header.Set(PaginationPageHeader, strconv.Itoa(params.PageNumber))
}
if paginationSize != 0 {
req.Header.Set(PaginationSizeHeader, strconv.Itoa(paginationSize))
if params.PageSize != 0 {
req.Header.Set(PaginationSizeHeader, strconv.Itoa(params.PageSize))
}
var resources []ResourceDto
@ -124,7 +135,7 @@ func (c *client) ScheduleURL(url string) error {
}
// NewClient create a new API client using given details
func NewClient(baseURL, token string) Client {
func NewClient(baseURL, token string) API {
httpClient := resty.New()
httpClient.SetAuthScheme("Bearer")
httpClient.SetAuthToken(token)

24
internal/api/database/database.go
Unescape Escape View File

@ -3,6 +3,7 @@ package database
import (
"context"
"encoding/json"
"github.com/creekorful/trandoshan/api"
"github.com/olivere/elastic/v7"
"github.com/rs/zerolog/log"
"time"
@ -23,24 +24,11 @@ type ResourceIdx struct {
Headers map[string]string `json:"headers"`
}
// ResSearchParams is the search params used
type ResSearchParams struct {
URL string
Keyword string
StartDate time.Time
EndDate time.Time
WithBody bool
PageSize int
PageNumber int
// TODO allow searching by meta
// TODO allow searching by headers
}
// Database is the interface used to abstract communication
// with the persistence unit
type Database interface {
SearchResources(params *ResSearchParams) ([]ResourceIdx, error)
CountResources(params *ResSearchParams) (int64, error)
SearchResources(params *api.ResSearchParams) ([]ResourceIdx, error)
CountResources(params *api.ResSearchParams) (int64, error)
AddResource(res ResourceIdx) error
}
@ -72,7 +60,7 @@ func NewElasticDB(uri string) (Database, error) {
}, nil
}
func (e *elasticSearchDB) SearchResources(params *ResSearchParams) ([]ResourceIdx, error) {
func (e *elasticSearchDB) SearchResources(params *api.ResSearchParams) ([]ResourceIdx, error) {
q := buildSearchQuery(params)
from := (params.PageNumber - 1) * params.PageSize
@ -105,7 +93,7 @@ func (e *elasticSearchDB) SearchResources(params *ResSearchParams) ([]ResourceId
return resources, nil
}
func (e *elasticSearchDB) CountResources(params *ResSearchParams) (int64, error) {
func (e *elasticSearchDB) CountResources(params *api.ResSearchParams) (int64, error) {
q := buildSearchQuery(params)
count, err := e.client.Count(resourcesIndex).Query(q).Do(context.Background())
@ -124,7 +112,7 @@ func (e *elasticSearchDB) AddResource(res ResourceIdx) error {
return err
}
func buildSearchQuery(params *ResSearchParams) elastic.Query {
func buildSearchQuery(params *api.ResSearchParams) elastic.Query {
var queries []elastic.Query
if params.URL != "" {
log.Trace().Str("url", params.URL).Msg("SearchQuery: Setting url")

13
internal/api/rest/rest.go
Unescape Escape View File

@ -3,7 +3,6 @@ package rest
import (
"encoding/base64"
"github.com/creekorful/trandoshan/api"
"github.com/creekorful/trandoshan/internal/api/database"
"github.com/creekorful/trandoshan/internal/api/service"
"github.com/labstack/echo/v4"
"net/http"
@ -18,7 +17,7 @@ var (
)
// SearchResources allows to search resources
func SearchResources(s service.Service) echo.HandlerFunc {
func SearchResources(s *service.Service) echo.HandlerFunc {
return func(c echo.Context) error {
searchParams, err := newSearchParams(c)
if err != nil {
@ -37,7 +36,7 @@ func SearchResources(s service.Service) echo.HandlerFunc {
}
// AddResource persist a new resource
func AddResource(s service.Service) echo.HandlerFunc {
func AddResource(s *service.Service) echo.HandlerFunc {
return func(c echo.Context) error {
var res api.ResourceDto
if err := c.Bind(&res); err != nil {
@ -54,7 +53,7 @@ func AddResource(s service.Service) echo.HandlerFunc {
}
// ScheduleURL schedule given URL for crawling
func ScheduleURL(s service.Service) echo.HandlerFunc {
func ScheduleURL(s *service.Service) echo.HandlerFunc {
return func(c echo.Context) error {
var url string
if err := c.Bind(&url); err != nil {
@ -82,14 +81,14 @@ func readPagination(c echo.Context) (int, int) {
return paginationPage, paginationSize
}
func writePagination(c echo.Context, s *database.ResSearchParams, totalCount int64) {
func writePagination(c echo.Context, s *api.ResSearchParams, totalCount int64) {
c.Response().Header().Set(api.PaginationPageHeader, strconv.Itoa(s.PageNumber))
c.Response().Header().Set(api.PaginationSizeHeader, strconv.Itoa(s.PageSize))
c.Response().Header().Set(api.PaginationCountHeader, strconv.FormatInt(totalCount, 10))
}
func newSearchParams(c echo.Context) (*database.ResSearchParams, error) {
params := &database.ResSearchParams{}
func newSearchParams(c echo.Context) (*api.ResSearchParams, error) {
params := &api.ResSearchParams{}
params.Keyword = c.QueryParam("keyword")

29
internal/api/service/service.go
Unescape Escape View File

@ -11,22 +11,15 @@ import (
"time"
)
// Service is the functionality the API provide
type Service interface {
SearchResources(params *database.ResSearchParams) ([]api.ResourceDto, int64, error)
AddResource(res api.ResourceDto) (api.ResourceDto, error)
ScheduleURL(url string) error
Close()
}
type svc struct {
// Service represent the functionality the API expose
type Service struct {
db database.Database
pub event.Publisher
refreshDelay time.Duration
}
// New create a new Service instance
func New(c *cli.Context) (Service, error) {
func New(c *cli.Context) (*Service, error) {
// Connect to the messaging server
pub, err := event.NewPublisher(c.String("hub-uri"))
if err != nil {
@ -41,14 +34,15 @@ func New(c *cli.Context) (Service, error) {
refreshDelay := duration.ParseDuration(c.String("refresh-delay"))
return &svc{
return &Service{
db: db,
pub: pub,
refreshDelay: refreshDelay,
}, nil
}
func (s *svc) SearchResources(params *database.ResSearchParams) ([]api.ResourceDto, int64, error) {
// SearchResources allows to search resources using given params
func (s *Service) SearchResources(params *api.ResSearchParams) ([]api.ResourceDto, int64, error) {
totalCount, err := s.db.CountResources(params)
if err != nil {
log.Err(err).Msg("Error while counting on ES")
@ -74,7 +68,8 @@ func (s *svc) SearchResources(params *database.ResSearchParams) ([]api.ResourceD
return resources, totalCount, nil
}
func (s *svc) AddResource(res api.ResourceDto) (api.ResourceDto, error) {
// AddResource allows to add given resource
func (s *Service) AddResource(res api.ResourceDto) (api.ResourceDto, error) {
log.Debug().Str("url", res.URL).Msg("Saving resource")
// Hacky stuff to prevent from adding 'duplicate resource'
@ -88,7 +83,7 @@ func (s *svc) AddResource(res api.ResourceDto) (api.ResourceDto, error) {
endDate = time.Now().Add(-s.refreshDelay)
}
count, err := s.db.CountResources(&database.ResSearchParams{
count, err := s.db.CountResources(&api.ResSearchParams{
URL: res.URL,
EndDate: endDate,
PageSize: 1,
@ -125,7 +120,8 @@ func (s *svc) AddResource(res api.ResourceDto) (api.ResourceDto, error) {
return res, nil
}
func (s *svc) ScheduleURL(url string) error {
// ScheduleURL schedule given url for crawling
func (s *Service) ScheduleURL(url string) error {
// Publish the URL
if err := s.pub.Publish(&event.FoundURLEvent{URL: url}); err != nil {
log.Err(err).Msg("Unable to publish URL")
@ -136,6 +132,7 @@ func (s *svc) ScheduleURL(url string) error {
return nil
}
func (s *svc) Close() {
// Close disconnect the service consumer
func (s *Service) Close() {
s.pub.Close()
}

22
internal/api/service/service_test.go
Unescape Escape View File

@ -15,7 +15,7 @@ func TestSearchResources(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
params := &database.ResSearchParams{Keyword: "example"}
params := &api.ResSearchParams{Keyword: "example"}
dbMock := database_mock.NewMockDatabase(mockCtrl)
@ -35,7 +35,7 @@ func TestSearchResources(t *testing.T) {
},
}, nil)
s := svc{db: dbMock}
s := Service{db: dbMock}
res, count, err := s.SearchResources(params)
if err != nil {
@ -55,7 +55,7 @@ func TestAddResource(t *testing.T) {
dbMock := database_mock.NewMockDatabase(mockCtrl)
dbMock.EXPECT().CountResources(&searchParamsMatcher{target: database.ResSearchParams{
dbMock.EXPECT().CountResources(&searchParamsMatcher{target: api.ResSearchParams{
URL: "https://example.onion",
PageSize: 1,
PageNumber: 1,
@ -71,7 +71,7 @@ func TestAddResource(t *testing.T) {
Headers: map[string]string{"Content-Type": "application/html", "Server": "Traefik"},
})
s := svc{db: dbMock, refreshDelay: 5 * time.Hour}
s := Service{db: dbMock, refreshDelay: 5 * time.Hour}
res, err := s.AddResource(api.ResourceDto{
URL: "https://example.onion",
@ -118,13 +118,13 @@ func TestAddResourceDuplicateNotAllowed(t *testing.T) {
dbMock := database_mock.NewMockDatabase(mockCtrl)
dbMock.EXPECT().CountResources(&searchParamsMatcher{target: database.ResSearchParams{
dbMock.EXPECT().CountResources(&searchParamsMatcher{target: api.ResSearchParams{
URL: "https://example.onion",
PageSize: 1,
PageNumber: 1,
}, endDateZero: true}).Return(int64(1), nil)
s := svc{db: dbMock, refreshDelay: -1}
s := Service{db: dbMock, refreshDelay: -1}
_, err := s.AddResource(api.ResourceDto{
URL: "https://example.onion",
@ -146,14 +146,14 @@ func TestAddResourceTooYoung(t *testing.T) {
dbMock := database_mock.NewMockDatabase(mockCtrl)
dbMock.EXPECT().CountResources(&searchParamsMatcher{target: database.ResSearchParams{
dbMock.EXPECT().CountResources(&searchParamsMatcher{target: api.ResSearchParams{
URL: "https://example.onion",
EndDate: time.Now().Add(-10 * time.Minute),
PageSize: 1,
PageNumber: 1,
}}).Return(int64(1), nil)
s := svc{db: dbMock, refreshDelay: -10 * time.Minute}
s := Service{db: dbMock, refreshDelay: -10 * time.Minute}
_, err := s.AddResource(api.ResourceDto{
URL: "https://example.onion",
@ -175,7 +175,7 @@ func TestScheduleURL(t *testing.T) {
pubMock := event_mock.NewMockPublisher(mockCtrl)
s := svc{pub: pubMock}
s := Service{pub: pubMock}
pubMock.EXPECT().Publish(&event.FoundURLEvent{URL: "https://example.onion"})
@ -187,12 +187,12 @@ func TestScheduleURL(t *testing.T) {
// custom matcher to ignore time field when doing comparison ;(
// todo: do less crappy?
type searchParamsMatcher struct {
target database.ResSearchParams
target api.ResSearchParams
endDateZero bool
}
func (sm *searchParamsMatcher) Matches(x interface{}) bool {
arg := x.(*database.ResSearchParams)
arg := x.(*api.ResSearchParams)
return arg.URL == sm.target.URL && arg.PageSize == sm.target.PageSize && arg.PageNumber == sm.target.PageNumber &&
sm.endDateZero == arg.EndDate.IsZero()
}

2
internal/extractor/extractor.go
Unescape Escape View File

@ -75,7 +75,7 @@ func execute(ctx *cli.Context) error {
}
type state struct {
apiClient api.Client
apiClient api.API
}
func (state *state) handleNewResourceEvent(subscriber event.Subscriber, body io.Reader) error {

2
internal/extractor/extractor_test.go
Unescape Escape View File

@ -91,7 +91,7 @@ This is sparta (hosted on https://example.org)
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClientMock := api_mock.NewMockClient(mockCtrl)
apiClientMock := api_mock.NewMockAPI(mockCtrl)
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
tn := time.Now()

11
internal/scheduler/scheduler.go
Unescape Escape View File

@ -100,7 +100,7 @@ func execute(ctx *cli.Context) error {
}
type state struct {
apiClient api.Client
apiClient api.API
refreshDelay time.Duration
forbiddenExtensions []string
}
@ -142,7 +142,14 @@ func (state *state) handleURLFoundEvent(subscriber event.Subscriber, body io.Rea
endDate = time.Now().Add(-state.refreshDelay)
}
_, count, err := state.apiClient.SearchResources(u.String(), "", time.Time{}, endDate, 1, 1)
params := api.ResSearchParams{
URL: u.String(),
EndDate: endDate,
WithBody: false,
PageSize: 1,
PageNumber: 1,
}
_, count, err := state.apiClient.SearchResources(&params)
if err != nil {
return fmt.Errorf("error while searching resource (%s): %s", u, err)
}

25
internal/scheduler/scheduler_test.go
Unescape Escape View File

@ -10,14 +10,13 @@ import (
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/golang/mock/gomock"
"testing"
"time"
)
func TestHandleMessageNotOnion(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClientMock := api_mock.NewMockClient(mockCtrl)
apiClientMock := api_mock.NewMockAPI(mockCtrl)
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
msg := bytes.NewReader(nil)
@ -41,7 +40,7 @@ func TestHandleMessageWrongProtocol(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClientMock := api_mock.NewMockClient(mockCtrl)
apiClientMock := api_mock.NewMockAPI(mockCtrl)
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
msg := bytes.NewReader(nil)
@ -68,7 +67,7 @@ func TestHandleMessageAlreadyCrawled(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClientMock := api_mock.NewMockClient(mockCtrl)
apiClientMock := api_mock.NewMockAPI(mockCtrl)
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
msg := bytes.NewReader(nil)
@ -77,8 +76,13 @@ func TestHandleMessageAlreadyCrawled(t *testing.T) {
SetArg(1, event.FoundURLEvent{URL: "https://example.onion"}).
Return(nil)
params := api.ResSearchParams{
URL: "https://example.onion",
PageSize: 1,
PageNumber: 1,
}
apiClientMock.EXPECT().
SearchResources("https://example.onion", "", time.Time{}, time.Time{}, 1, 1).
SearchResources(&params).
Return([]api.ResourceDto{}, int64(1), nil)
s := state{
@ -96,7 +100,7 @@ func TestHandleMessageForbiddenExtensions(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClientMock := api_mock.NewMockClient(mockCtrl)
apiClientMock := api_mock.NewMockAPI(mockCtrl)
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
msg := bytes.NewReader(nil)
@ -120,7 +124,7 @@ func TestHandleMessage(t *testing.T) {
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
apiClientMock := api_mock.NewMockClient(mockCtrl)
apiClientMock := api_mock.NewMockAPI(mockCtrl)
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
msg := bytes.NewReader(nil)
@ -129,8 +133,13 @@ func TestHandleMessage(t *testing.T) {
SetArg(1, event.FoundURLEvent{URL: "https://example.onion"}).
Return(nil)
params := api.ResSearchParams{
URL: "https://example.onion",
PageSize: 1,
PageNumber: 1,
}
apiClientMock.EXPECT().
SearchResources("https://example.onion", "", time.Time{}, time.Time{}, 1, 1).
SearchResources(&params).
Return([]api.ResourceDto{}, int64(0), nil)
subscriberMock.EXPECT().

9
internal/trandoshanctl/trandoshanctl.go
Unescape Escape View File

@ -2,6 +2,7 @@ package trandoshanctl
import (
"fmt"
"github.com/creekorful/trandoshan/api"
"github.com/creekorful/trandoshan/internal/logging"
"github.com/creekorful/trandoshan/internal/util"
"github.com/olekukonko/tablewriter"
@ -75,7 +76,13 @@ func search(c *cli.Context) error {
// Create the API client
apiClient := util.GetAPIClient(c)
res, count, err := apiClient.SearchResources("", keyword, time.Time{}, time.Time{}, 1, 10)
params := api.ResSearchParams{
Keyword: keyword,
WithBody: false,
PageSize: 1,
PageNumber: 10,
}
res, count, err := apiClient.SearchResources(&params)
if err != nil {
log.Err(err).Str("keyword", keyword).Msg("Unable to search resources")
return err

2
internal/util/api.go
Unescape Escape View File

@ -24,6 +24,6 @@ func GetAPIURIFlag() *cli.StringFlag {
}
// GetAPIClient return a new configured API client
func GetAPIClient(c *cli.Context) api.Client {
func GetAPIClient(c *cli.Context) api.API {
return api.NewClient(c.String("api-uri"), c.String("api-token"))
}

Write Preview
Loading…
Cancel
Save