Big improvements
- Reduce debug noise - Create scripts to blacklist 'famous' legit hostnames - Make blacklister more resilient - Merge archiver & indexer together - Better prefix for cache key - Rework scheduling process - Update architecture.png - Remove trandoshanctl - Improve testingpull/121/head
Aloïs Micard
3 years ago
parent
2d7499f7e2
commit
ec3357be5d
@ -1,24 +0,0 @@
|
|||||||
# build image
|
|
||||||
FROM golang:1.15.0-alpine as builder
|
|
||||||
|
|
||||||
RUN apk update && apk upgrade && \
|
|
||||||
apk add --no-cache bash git openssh
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Copy and download dependencies to cache them and faster build time
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# Test then build app
|
|
||||||
RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-archiver
|
|
||||||
|
|
||||||
# runtime image
|
|
||||||
FROM alpine:latest
|
|
||||||
COPY --from=builder /app/tdsh-archiver /app/
|
|
||||||
|
|
||||||
WORKDIR /app/
|
|
||||||
|
|
||||||
ENTRYPOINT ["./tdsh-archiver"]
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
# build image
|
|
||||||
FROM golang:1.15.0-alpine as builder
|
|
||||||
|
|
||||||
RUN apk update && apk upgrade && \
|
|
||||||
apk add --no-cache bash git openssh
|
|
||||||
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
# Copy and download dependencies to cache them and faster build time
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# Test then build app
|
|
||||||
RUN go build -v github.com/creekorful/trandoshan/cmd/trandoshanctl
|
|
||||||
|
|
||||||
# runtime image
|
|
||||||
FROM alpine:latest
|
|
||||||
COPY --from=builder /app/trandoshanctl /app/
|
|
||||||
|
|
||||||
WORKDIR /app/
|
|
||||||
|
|
||||||
ENTRYPOINT ["./trandoshanctl"]
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/creekorful/trandoshan/internal/archiver"
|
|
||||||
"github.com/creekorful/trandoshan/internal/process"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
app := process.MakeApp(&archiver.State{})
|
|
||||||
if err := app.Run(os.Args); err != nil {
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,13 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/creekorful/trandoshan/internal/trandoshanctl"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
app := trandoshanctl.GetApp()
|
|
||||||
if err := app.Run(os.Args); err != nil {
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 54 KiB After Width: | Height: | Size: 59 KiB |
@ -1,99 +0,0 @@
|
|||||||
package archiver
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"github.com/creekorful/trandoshan/internal/archiver/storage"
|
|
||||||
"github.com/creekorful/trandoshan/internal/event"
|
|
||||||
"github.com/creekorful/trandoshan/internal/process"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// State represent the application state
|
|
||||||
type State struct {
|
|
||||||
storage storage.Storage
|
|
||||||
}
|
|
||||||
|
|
||||||
// Name return the process name
|
|
||||||
func (state *State) Name() string {
|
|
||||||
return "archiver"
|
|
||||||
}
|
|
||||||
|
|
||||||
// CommonFlags return process common flags
|
|
||||||
func (state *State) CommonFlags() []string {
|
|
||||||
return []string{process.HubURIFlag}
|
|
||||||
}
|
|
||||||
|
|
||||||
// CustomFlags return process custom flags
|
|
||||||
func (state *State) CustomFlags() []cli.Flag {
|
|
||||||
return []cli.Flag{
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "storage-dir",
|
|
||||||
Usage: "Path to the storage directory",
|
|
||||||
Required: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize the process
|
|
||||||
func (state *State) Initialize(provider process.Provider) error {
|
|
||||||
st, err := storage.NewLocalStorage(provider.GetValue("storage-dir"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
state.storage = st
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Subscribers return the process subscribers
|
|
||||||
func (state *State) Subscribers() []process.SubscriberDef {
|
|
||||||
return []process.SubscriberDef{
|
|
||||||
{Exchange: event.NewIndexExchange, Queue: "archivingQueue", Handler: state.handleNewIndexEvent},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HTTPHandler returns the HTTP API the process expose
|
|
||||||
func (state *State) HTTPHandler(provider process.Provider) http.Handler {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *State) handleNewIndexEvent(subscriber event.Subscriber, msg event.RawMessage) error {
|
|
||||||
var evt event.NewIndexEvent
|
|
||||||
if err := subscriber.Read(&msg, &evt); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := formatResource(&evt)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("error while formatting resource: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := state.storage.Store(evt.URL, evt.Time, res); err != nil {
|
|
||||||
return fmt.Errorf("error while storing resource: %s", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debug().Str("url", evt.URL).Msg("Successfully archived resource")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func formatResource(evt *event.NewIndexEvent) ([]byte, error) {
|
|
||||||
builder := strings.Builder{}
|
|
||||||
|
|
||||||
// First URL
|
|
||||||
builder.WriteString(fmt.Sprintf("%s\n\n", evt.URL))
|
|
||||||
|
|
||||||
// Then headers
|
|
||||||
for key, value := range evt.Headers {
|
|
||||||
builder.WriteString(fmt.Sprintf("%s: %s\n", key, value))
|
|
||||||
}
|
|
||||||
builder.WriteString("\n")
|
|
||||||
|
|
||||||
// Then body
|
|
||||||
builder.WriteString(evt.Body)
|
|
||||||
|
|
||||||
return []byte(builder.String()), nil
|
|
||||||
}
|
|
||||||
@ -1,55 +0,0 @@
|
|||||||
package archiver
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/creekorful/trandoshan/internal/archiver/storage_mock"
|
|
||||||
"github.com/creekorful/trandoshan/internal/event"
|
|
||||||
"github.com/creekorful/trandoshan/internal/event_mock"
|
|
||||||
"github.com/golang/mock/gomock"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHandleNewResourceEvent(t *testing.T) {
|
|
||||||
mockCtrl := gomock.NewController(t)
|
|
||||||
defer mockCtrl.Finish()
|
|
||||||
|
|
||||||
subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
|
|
||||||
storageMock := storage_mock.NewMockStorage(mockCtrl)
|
|
||||||
|
|
||||||
tn := time.Now()
|
|
||||||
|
|
||||||
msg := event.RawMessage{}
|
|
||||||
subscriberMock.EXPECT().
|
|
||||||
Read(&msg, &event.NewIndexEvent{}).
|
|
||||||
SetArg(1, event.NewIndexEvent{
|
|
||||||
URL: "https://example.onion",
|
|
||||||
Body: "Hello, world",
|
|
||||||
Headers: map[string]string{"Server": "Traefik", "Content-Type": "application/html"},
|
|
||||||
Time: tn,
|
|
||||||
}).Return(nil)
|
|
||||||
|
|
||||||
storageMock.EXPECT().Store("https://example.onion", tn, []byte("https://example.onion\n\nServer: Traefik\nContent-Type: application/html\n\nHello, world")).Return(nil)
|
|
||||||
|
|
||||||
s := State{storage: storageMock}
|
|
||||||
if err := s.handleNewIndexEvent(subscriberMock, msg); err != nil {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatResource(t *testing.T) {
|
|
||||||
evt := &event.NewIndexEvent{
|
|
||||||
URL: "https://google.com",
|
|
||||||
Body: "Hello, world",
|
|
||||||
Headers: map[string]string{"Server": "Traefik", "Content-Type": "text/html"},
|
|
||||||
Time: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := formatResource(evt)
|
|
||||||
if err != nil {
|
|
||||||
t.FailNow()
|
|
||||||
}
|
|
||||||
|
|
||||||
if string(res) != "https://google.com\n\nServer: Traefik\nContent-Type: text/html\n\nHello, world" {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
package storage
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
//go:generate mockgen -destination=../storage_mock/storage_mock.go -package=storage_mock . Storage
|
|
||||||
|
|
||||||
// Storage is a abstraction layer where we store resource
|
|
||||||
type Storage interface {
|
|
||||||
// Store the resource
|
|
||||||
Store(url string, time time.Time, body []byte) error
|
|
||||||
}
|
|
||||||
@ -0,0 +1,15 @@
|
|||||||
|
package cache
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestRedisCache_GetKey(t *testing.T) {
|
||||||
|
rc := redisCache{}
|
||||||
|
if got := rc.getKey("user"); got != "user" {
|
||||||
|
t.Errorf("got %s want %s", got, "user")
|
||||||
|
}
|
||||||
|
|
||||||
|
rc.keyPrefix = "config"
|
||||||
|
if got := rc.getKey("user"); got != "config:user" {
|
||||||
|
t.Errorf("got %s want %s", got, "config:user")
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,124 +0,0 @@
|
|||||||
package auth
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"github.com/dgrijalva/jwt-go"
|
|
||||||
"github.com/gorilla/mux"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
type key int
|
|
||||||
|
|
||||||
const (
|
|
||||||
usernameKey key = iota
|
|
||||||
)
|
|
||||||
|
|
||||||
// Token is the authentication token used by processes when dialing with the API
|
|
||||||
type Token struct {
|
|
||||||
// Username used for logging purposes
|
|
||||||
Username string `json:"username"`
|
|
||||||
|
|
||||||
// Rights that the token provides
|
|
||||||
// Format is: METHOD - list of paths
|
|
||||||
Rights map[string][]string `json:"rights"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Middleware is the authentication middleware
|
|
||||||
type Middleware struct {
|
|
||||||
signingKey []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewMiddleware create a new Middleware instance with given secret token signing key
|
|
||||||
func NewMiddleware(signingKey []byte) *Middleware {
|
|
||||||
return &Middleware{signingKey: signingKey}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Middleware return an net/http compatible middleware func to use
|
|
||||||
func (m *Middleware) Middleware() mux.MiddlewareFunc {
|
|
||||||
return func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// Extract authorization header
|
|
||||||
tokenStr := r.Header.Get("Authorization")
|
|
||||||
if tokenStr == "" {
|
|
||||||
log.Warn().Msg("missing token")
|
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
tokenStr = strings.TrimPrefix(tokenStr, "Bearer ")
|
|
||||||
|
|
||||||
// Decode the JWT token
|
|
||||||
token, err := jwt.Parse(tokenStr, func(t *jwt.Token) (interface{}, error) {
|
|
||||||
// Validate expected alg
|
|
||||||
if v, ok := t.Method.(*jwt.SigningMethodHMAC); !ok || v.Name != "HS256" {
|
|
||||||
return nil, fmt.Errorf("unexpected signing method: %s", t.Header["alg"])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return signing secret
|
|
||||||
return m.signingKey, nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
log.Err(err).Msg("error while decoding JWT token")
|
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// From here we have a valid JWT token, extract claims
|
|
||||||
claims, ok := token.Claims.(jwt.MapClaims)
|
|
||||||
if !ok {
|
|
||||||
log.Err(err).Msg("error while decoding token claims")
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rights := map[string][]string{}
|
|
||||||
for method, paths := range claims["rights"].(map[string]interface{}) {
|
|
||||||
for _, path := range paths.([]interface{}) {
|
|
||||||
rights[method] = append(rights[method], path.(string))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t := Token{
|
|
||||||
Username: claims["username"].(string),
|
|
||||||
Rights: rights,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate rights
|
|
||||||
paths, contains := t.Rights[r.Method]
|
|
||||||
if !contains {
|
|
||||||
log.Warn().
|
|
||||||
Str("username", t.Username).
|
|
||||||
Str("method", r.Method).
|
|
||||||
Str("resource", r.URL.Path).
|
|
||||||
Msg("Access to resources is unauthorized")
|
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
authorized := false
|
|
||||||
for _, path := range paths {
|
|
||||||
if path == r.URL.Path {
|
|
||||||
authorized = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !authorized {
|
|
||||||
log.Warn().
|
|
||||||
Str("username", t.Username).
|
|
||||||
Str("method", r.Method).
|
|
||||||
Str("resource", r.URL.Path).
|
|
||||||
Msg("Access to resources is unauthorized")
|
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Everything's fine, call next handler ;D
|
|
||||||
ctx := context.WithValue(r.Context(), usernameKey, t.Username)
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,85 +0,0 @@
|
|||||||
package auth
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io/ioutil"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMiddleware_NoTokenShouldReturnUnauthorized(t *testing.T) {
|
|
||||||
m := (&Middleware{signingKey: []byte("test")}).Middleware()(okHandler())
|
|
||||||
|
|
||||||
// no token shouldn't be able to access
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/users", nil)
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
|
|
||||||
m.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusUnauthorized {
|
|
||||||
t.Errorf("StatusUnauthorized was expected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMiddleware_InvalidTokenShouldReturnUnauthorized(t *testing.T) {
|
|
||||||
m := (&Middleware{signingKey: []byte("test")}).Middleware()(okHandler())
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/users", nil)
|
|
||||||
req.Header.Add("Authorization", "zarBR")
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
|
|
||||||
m.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusUnauthorized {
|
|
||||||
t.Errorf("StatusUnauthorized was expected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMiddleware_BadRightsShouldReturnUnauthorized(t *testing.T) {
|
|
||||||
m := (&Middleware{signingKey: []byte("test")}).Middleware()(okHandler())
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "/users", nil)
|
|
||||||
req.Header.Add("Authorization", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6IkpvaG4gRG9lIiwicmlnaHRzIjp7IkdFVCI6WyIvdXNlcnMiXSwiUE9TVCI6WyIvc2VhcmNoIl19fQ.fRx0Q66ZgnY_rKCf-9Vaz6gzGKH_tKSgkVHhoQMtKfM")
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
|
|
||||||
m.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusUnauthorized {
|
|
||||||
t.Errorf("StatusUnauthorized was expected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMiddleware(t *testing.T) {
|
|
||||||
m := (&Middleware{signingKey: []byte("test")}).Middleware()(okHandler())
|
|
||||||
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "/users?id=10", nil)
|
|
||||||
req.Header.Add("Authorization", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6IkpvaG4gRG9lIiwicmlnaHRzIjp7IkdFVCI6WyIvdXNlcnMiXSwiUE9TVCI6WyIvc2VhcmNoIl19fQ.fRx0Q66ZgnY_rKCf-9Vaz6gzGKH_tKSgkVHhoQMtKfM")
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
|
|
||||||
m.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
t.Errorf("StatusUnauthorized was expected")
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := ioutil.ReadAll(rec.Body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
if string(b) != "Hello, John Doe" {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func okHandler() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if username := r.Context().Value(usernameKey).(string); username != "" {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
_, _ = w.Write([]byte(fmt.Sprintf("Hello, %s", username)))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,141 +0,0 @@
|
|||||||
package client
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/base64"
|
|
||||||
"fmt"
|
|
||||||
"github.com/go-resty/resty/v2"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
//go:generate mockgen -destination=../client_mock/client_mock.go -package=client_mock . Client
|
|
||||||
|
|
||||||
const (
|
|
||||||
// PaginationPageHeader is the header to determinate current page in paginated endpoint
|
|
||||||
PaginationPageHeader = "X-Pagination-Page"
|
|
||||||
// PaginationSizeHeader is the header to determinate page size in paginated endpoint
|
|
||||||
PaginationSizeHeader = "X-Pagination-Size"
|
|
||||||
// PaginationCountHeader is the header to determinate total count of element in paginated endpoint
|
|
||||||
PaginationCountHeader = "X-Pagination-Count"
|
|
||||||
// PaginationPageQueryParam is the query parameter used to set current page in paginated endpoint
|
|
||||||
PaginationPageQueryParam = "pagination-page"
|
|
||||||
// PaginationSizeQueryParam is the query parameter used to set page size in paginated endpoint
|
|
||||||
PaginationSizeQueryParam = "pagination-size"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ResourceDto represent a resource as given by the API
|
|
||||||
type ResourceDto struct {
|
|
||||||
URL string `json:"url"`
|
|
||||||
Body string `json:"body"`
|
|
||||||
Time time.Time `json:"time"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Meta map[string]string `json:"meta"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
Headers map[string]string `json:"headers"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// CredentialsDto represent the credential when logging in the API
|
|
||||||
type CredentialsDto struct {
|
|
||||||
Username string `json:"username"`
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ResSearchParams is the search params used
|
|
||||||
type ResSearchParams struct {
|
|
||||||
URL string
|
|
||||||
Keyword string
|
|
||||||
StartDate time.Time
|
|
||||||
EndDate time.Time
|
|
||||||
WithBody bool
|
|
||||||
PageSize int
|
|
||||||
PageNumber int
|
|
||||||
// TODO allow searching by meta
|
|
||||||
// TODO allow searching by headers
|
|
||||||
}
|
|
||||||
|
|
||||||
// Client is the interface to interact with the indexer API
|
|
||||||
type Client interface {
|
|
||||||
SearchResources(params *ResSearchParams) ([]ResourceDto, int64, error)
|
|
||||||
ScheduleURL(url string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type client struct {
|
|
||||||
httpClient *resty.Client
|
|
||||||
baseURL string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *client) SearchResources(params *ResSearchParams) ([]ResourceDto, int64, error) {
|
|
||||||
targetEndpoint := fmt.Sprintf("%s/v1/resources?", c.baseURL)
|
|
||||||
|
|
||||||
req := c.httpClient.R()
|
|
||||||
|
|
||||||
if params.URL != "" {
|
|
||||||
b64URL := base64.URLEncoding.EncodeToString([]byte(params.URL))
|
|
||||||
req.SetQueryParam("url", b64URL)
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.Keyword != "" {
|
|
||||||
req.SetQueryParam("keyword", params.Keyword)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !params.StartDate.IsZero() {
|
|
||||||
req.SetQueryParam("start-date", params.StartDate.Format(time.RFC3339))
|
|
||||||
}
|
|
||||||
|
|
||||||
if !params.EndDate.IsZero() {
|
|
||||||
req.SetQueryParam("end-date", params.EndDate.Format(time.RFC3339))
|
|
||||||
}
|
|
||||||
|
|
||||||
if params.PageNumber != 0 {
|
|
||||||
req.Header.Set(PaginationPageHeader, strconv.Itoa(params.PageNumber))
|
|
||||||
}
|
|
||||||
if params.PageSize != 0 {
|
|
||||||
req.Header.Set(PaginationSizeHeader, strconv.Itoa(params.PageSize))
|
|
||||||
}
|
|
||||||
|
|
||||||
var resources []ResourceDto
|
|
||||||
req.SetResult(&resources)
|
|
||||||
|
|
||||||
res, err := req.Get(targetEndpoint)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
count, err := strconv.ParseInt(res.Header().Get(PaginationCountHeader), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return resources, count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *client) ScheduleURL(url string) error {
|
|
||||||
targetEndpoint := fmt.Sprintf("%s/v1/urls", c.baseURL)
|
|
||||||
|
|
||||||
req := c.httpClient.R()
|
|
||||||
req.SetHeader("Content-Type", "application/json")
|
|
||||||
req.SetBody(fmt.Sprintf("\"%s\"", url))
|
|
||||||
|
|
||||||
_, err := req.Post(targetEndpoint)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewClient create a new API client using given details
|
|
||||||
func NewClient(baseURL, token string) Client {
|
|
||||||
httpClient := resty.New()
|
|
||||||
httpClient.SetAuthScheme("Bearer")
|
|
||||||
httpClient.SetAuthToken(token)
|
|
||||||
httpClient.OnAfterResponse(func(c *resty.Client, r *resty.Response) error {
|
|
||||||
if r.StatusCode() > 302 {
|
|
||||||
return fmt.Errorf("error when making HTTP request: %s", r.Status())
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
client := &client{
|
|
||||||
httpClient: httpClient,
|
|
||||||
baseURL: baseURL,
|
|
||||||
}
|
|
||||||
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
package index
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/creekorful/trandoshan/internal/event"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtractResource(t *testing.T) {
|
||||||
|
body := `
|
||||||
|
<title>Creekorful Inc</title>
|
||||||
|
|
||||||
|
This is sparta
|
||||||
|
|
||||||
|
<a href="https://google.com/test?test=test#12">
|
||||||
|
|
||||||
|
<meta name="Description" content="Zhello world">
|
||||||
|
<meta property="og:url" content="https://example.org">
|
||||||
|
`
|
||||||
|
|
||||||
|
msg := event.NewResourceEvent{
|
||||||
|
URL: "https://example.org/300",
|
||||||
|
Body: body,
|
||||||
|
}
|
||||||
|
|
||||||
|
resDto, err := extractResource("https://example.org/300", time.Time{}, body, map[string]string{"Content-Type": "application/json"})
|
||||||
|
if err != nil {
|
||||||
|
t.FailNow()
|
||||||
|
}
|
||||||
|
|
||||||
|
if resDto.URL != "https://example.org/300" {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
if resDto.Title != "Creekorful Inc" {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
if resDto.Body != msg.Body {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
|
||||||
|
if resDto.Description != "Zhello world" {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
|
||||||
|
if resDto.Meta["description"] != "Zhello world" {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
|
||||||
|
if resDto.Meta["og:url"] != "https://example.org" {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
|
||||||
|
if resDto.Headers["content-type"] != "application/json" {
|
||||||
|
t.Fail()
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,27 +1,33 @@
|
|||||||
package index
|
package index
|
||||||
|
|
||||||
|
//go:generate mockgen -destination=../index_mock/index_mock.go -package=index_mock . Index
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/creekorful/trandoshan/internal/indexer/client"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:generate mockgen -destination=../index_mock/index_mock.go -package=index_mock . Index
|
const (
|
||||||
|
// Elastic is an Index backed by ES instance
|
||||||
// ResourceIdx represent a resource as stored in elasticsearch
|
Elastic = "elastic"
|
||||||
type ResourceIdx struct {
|
// Local is an Index backed by local FS instance
|
||||||
URL string `json:"url"`
|
Local = "local"
|
||||||
Body string `json:"body"`
|
)
|
||||||
Time time.Time `json:"time"`
|
|
||||||
Title string `json:"title"`
|
|
||||||
Meta map[string]string `json:"meta"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
Headers map[string]string `json:"headers"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Index is the interface used to abstract communication
|
// Index is the interface used to abstract communication
|
||||||
// with the persistence unit
|
// with the persistence unit
|
||||||
type Index interface {
|
type Index interface {
|
||||||
SearchResources(params *client.ResSearchParams) ([]ResourceIdx, error)
|
IndexResource(url string, time time.Time, body string, headers map[string]string) error
|
||||||
CountResources(params *client.ResSearchParams) (int64, error)
|
}
|
||||||
AddResource(res ResourceIdx) error
|
|
||||||
|
// NewIndex create a new index using given driver, destination
|
||||||
|
func NewIndex(driver string, dest string) (Index, error) {
|
||||||
|
switch driver {
|
||||||
|
case Elastic:
|
||||||
|
return newElasticIndex(dest)
|
||||||
|
case Local:
|
||||||
|
return newLocalIndex(dest)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("no driver named %s found", driver)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,31 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GetLogFlag return the CLI flag parameter used to setup application log level
|
|
||||||
func GetLogFlag() *cli.StringFlag {
|
|
||||||
return &cli.StringFlag{
|
|
||||||
Name: "log-level",
|
|
||||||
Usage: "Set the application log level",
|
|
||||||
Value: "info",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConfigureLogger configure the logger using given log level (read from cli context)
|
|
||||||
func ConfigureLogger(ctx *cli.Context) {
|
|
||||||
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
|
||||||
|
|
||||||
// Set application log level
|
|
||||||
if lvl, err := zerolog.ParseLevel(ctx.String("log-level")); err == nil {
|
|
||||||
zerolog.SetGlobalLevel(lvl)
|
|
||||||
} else {
|
|
||||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Debug().Stringer("lvl", zerolog.GlobalLevel()).Msg("Setting log level")
|
|
||||||
}
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
package logging
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestGetLogFlag(t *testing.T) {
|
|
||||||
flag := GetLogFlag()
|
|
||||||
if flag.Name != "log-level" {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
if flag.Usage != "Set the application log level" {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
if flag.Value != "info" {
|
|
||||||
t.Fail()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -0,0 +1,68 @@
|
|||||||
|
package test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/creekorful/trandoshan/internal/process"
|
||||||
|
"github.com/creekorful/trandoshan/internal/process_mock"
|
||||||
|
"github.com/golang/mock/gomock"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SubscriberDef is use to test subscriber definition
|
||||||
|
type SubscriberDef struct {
|
||||||
|
Queue string
|
||||||
|
Exchange string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckProcessCommonFlags check process defined common flags
|
||||||
|
func CheckProcessCommonFlags(t *testing.T, p process.Process, wantFlags []string) {
|
||||||
|
if !checkListEquals(p.CommonFlags(), wantFlags) {
|
||||||
|
t.Errorf("Differents flags: %v %v", p.CommonFlags(), wantFlags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckProcessCustomFlags check process defined custom flags
|
||||||
|
func CheckProcessCustomFlags(t *testing.T, p process.Process, wantFlags []string) {
|
||||||
|
var names []string
|
||||||
|
for _, customFlag := range p.CustomFlags() {
|
||||||
|
names = append(names, customFlag.Names()[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
if !checkListEquals(names, wantFlags) {
|
||||||
|
t.Errorf("Differents flags: %v %v", names, wantFlags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckInitialize check process initialization phase
|
||||||
|
func CheckInitialize(t *testing.T, p process.Process, callback func(provider *process_mock.MockProviderMockRecorder)) {
|
||||||
|
mockCtrl := gomock.NewController(t)
|
||||||
|
defer mockCtrl.Finish()
|
||||||
|
|
||||||
|
providerMock := process_mock.NewMockProvider(mockCtrl)
|
||||||
|
callback(providerMock.EXPECT())
|
||||||
|
|
||||||
|
if err := p.Initialize(providerMock); err != nil {
|
||||||
|
t.Errorf("Error while Initializing process: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckProcessSubscribers check process defined subscribers
|
||||||
|
func CheckProcessSubscribers(t *testing.T, p process.Process, subscribers []SubscriberDef) {
|
||||||
|
var defs []SubscriberDef
|
||||||
|
for _, sub := range p.Subscribers() {
|
||||||
|
defs = append(defs, SubscriberDef{
|
||||||
|
Queue: sub.Queue,
|
||||||
|
Exchange: sub.Exchange,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(defs, subscribers) {
|
||||||
|
t.Errorf("Differents subscribers: %v %v", defs, subscribers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO HTTPHandler
|
||||||
|
|
||||||
|
func checkListEquals(a []string, b []string) bool {
|
||||||
|
return reflect.DeepEqual(a, b)
|
||||||
|
}
|
||||||
@ -1,119 +0,0 @@
|
|||||||
package trandoshanctl
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"github.com/creekorful/trandoshan/internal/indexer/client"
|
|
||||||
"github.com/creekorful/trandoshan/internal/logging"
|
|
||||||
"github.com/olekukonko/tablewriter"
|
|
||||||
"github.com/rs/zerolog/log"
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GetApp returns the Trandoshan CLI app
|
|
||||||
func GetApp() *cli.App {
|
|
||||||
return &cli.App{
|
|
||||||
Name: "trandoshanctl",
|
|
||||||
Version: "0.9.0",
|
|
||||||
Usage: "Trandoshan CLI",
|
|
||||||
Flags: []cli.Flag{
|
|
||||||
logging.GetLogFlag(),
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "api-uri",
|
|
||||||
Usage: "URI to the API server",
|
|
||||||
Value: "http://localhost:15005",
|
|
||||||
Required: false,
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Name: "api-token",
|
|
||||||
Usage: "Token to use to authenticate against the API",
|
|
||||||
Required: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Commands: []*cli.Command{
|
|
||||||
{
|
|
||||||
Name: "schedule",
|
|
||||||
Usage: "Schedule crawling for given URL",
|
|
||||||
Action: schedule,
|
|
||||||
ArgsUsage: "URL",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "search",
|
|
||||||
Usage: "Search for specific resources",
|
|
||||||
ArgsUsage: "keyword",
|
|
||||||
Action: search,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Before: before,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func before(ctx *cli.Context) error {
|
|
||||||
logging.ConfigureLogger(ctx)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func schedule(c *cli.Context) error {
|
|
||||||
if c.NArg() == 0 {
|
|
||||||
return fmt.Errorf("missing argument URL")
|
|
||||||
}
|
|
||||||
|
|
||||||
url := c.Args().First()
|
|
||||||
|
|
||||||
// Create the API client
|
|
||||||
apiClient := client.NewClient(c.String("api-uri"), c.String("api-token"))
|
|
||||||
|
|
||||||
if err := apiClient.ScheduleURL(url); err != nil {
|
|
||||||
log.Err(err).Str("url", url).Msg("Unable to schedule crawling for URL")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Info().Str("url", url).Msg("Successfully schedule crawling")
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func search(c *cli.Context) error {
|
|
||||||
keyword := c.Args().First()
|
|
||||||
|
|
||||||
// Create the API client
|
|
||||||
apiClient := client.NewClient(c.String("api-uri"), c.String("api-token"))
|
|
||||||
|
|
||||||
params := client.ResSearchParams{
|
|
||||||
Keyword: keyword,
|
|
||||||
WithBody: false,
|
|
||||||
PageSize: 1,
|
|
||||||
PageNumber: 10,
|
|
||||||
}
|
|
||||||
res, count, err := apiClient.SearchResources(¶ms)
|
|
||||||
if err != nil {
|
|
||||||
log.Err(err).Str("keyword", keyword).Msg("Unable to search resources")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(res) == 0 {
|
|
||||||
fmt.Println("No resources crawled (yet).")
|
|
||||||
}
|
|
||||||
|
|
||||||
table := tablewriter.NewWriter(os.Stdout)
|
|
||||||
table.SetHeader([]string{"Time", "URL", "Title"})
|
|
||||||
|
|
||||||
for _, v := range res {
|
|
||||||
table.Append([]string{v.Time.Format(time.RFC822), shortenURL(v.URL), v.Title})
|
|
||||||
}
|
|
||||||
table.Render()
|
|
||||||
|
|
||||||
fmt.Printf("Total: %d\n", count)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func shortenURL(url string) string {
|
|
||||||
if len(url) > 125 {
|
|
||||||
url := url[0:125]
|
|
||||||
return url + "..."
|
|
||||||
}
|
|
||||||
|
|
||||||
return url
|
|
||||||
}
|
|
||||||
@ -0,0 +1,56 @@
|
|||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
# This script is used to import list of hostnames to 'blacklist'
|
||||||
|
# it will pull hostnames from the CT log source (see url variable) & custom define ones
|
||||||
|
# and blacklist them to prevent useless crawling
|
||||||
|
|
||||||
|
url = "https://raw.githubusercontent.com/alecmuffett/real-world-onion-sites/master/ct-log.txt"
|
||||||
|
custom_hostnames = [
|
||||||
|
'gamebombfak3pwnh.onion', # gaming forum, lot of noise
|
||||||
|
'metagerv65pwclop2rsfzg4jwowpavpwd6grhhlvdgsswvo6ii4akgyd.onion' # search engine, lot of noise
|
||||||
|
]
|
||||||
|
config_api_uri = sys.argv[1]
|
||||||
|
|
||||||
|
|
||||||
|
def add_if_not_exist(a: List[dict], b: str):
|
||||||
|
found = False
|
||||||
|
for i in a:
|
||||||
|
if i['hostname'] == b:
|
||||||
|
found = True
|
||||||
|
|
||||||
|
if not found:
|
||||||
|
a.append({'hostname': b})
|
||||||
|
|
||||||
|
|
||||||
|
# Get up-to-date list of real-world / legit .onion
|
||||||
|
r = requests.get(url)
|
||||||
|
new_hostnames = []
|
||||||
|
for hostname in r.text.splitlines():
|
||||||
|
new_hostnames.append({'hostname': hostname})
|
||||||
|
print("pulled {} real world hostnames from ct-log.txt".format(len(new_hostnames)))
|
||||||
|
|
||||||
|
# Append custom hostnames ignore list
|
||||||
|
for custom_hostname in custom_hostnames:
|
||||||
|
add_if_not_exist(new_hostnames, custom_hostname)
|
||||||
|
print("added {} custom hostnames".format(len(custom_hostnames)))
|
||||||
|
|
||||||
|
# Query existing blacklisted hostnames from ConfigAPI
|
||||||
|
r = requests.get(config_api_uri + "/config/forbidden-hostnames")
|
||||||
|
forbidden_hostnames = r.json()
|
||||||
|
print("there is {} forbidden hostnames defined in ConfigAPI".format(len(forbidden_hostnames)))
|
||||||
|
|
||||||
|
# Merge the lists while preventing duplicates
|
||||||
|
for forbidden_hostname in forbidden_hostnames:
|
||||||
|
add_if_not_exist(new_hostnames, forbidden_hostname['hostname'])
|
||||||
|
print("there is {} forbidden hostnames now".format(len(forbidden_hostnames)))
|
||||||
|
|
||||||
|
# Update ConfigAPI
|
||||||
|
headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
|
||||||
|
r = requests.put(config_api_uri + "/config/forbidden-hostnames", json.dumps(forbidden_hostnames), headers=headers)
|
||||||
|
|
||||||
|
if r.ok:
|
||||||
|
print("successfully updated forbidden hostnames")
|
||||||
Loading…
Reference in New Issue