Merge pull request #52 from creekorful/35-better-acl-for-api

Finalize ACL implementation
pull/41/head
Aloïs Micard 4 years ago committed by GitHub
commit 09f38acf71
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -45,13 +45,11 @@ type Client interface {
paginationPage, paginationSize int) ([]ResourceDto, int64, error)
AddResource(res ResourceDto) (ResourceDto, error)
ScheduleURL(url string) error
Authenticate(credentials CredentialsDto) (string, error)
}
type client struct {
httpClient *resty.Client
baseURL string
token string
}
func (c *client) SearchResources(url, keyword string,
@ -59,7 +57,6 @@ func (c *client) SearchResources(url, keyword string,
targetEndpoint := fmt.Sprintf("%s/v1/resources?", c.baseURL)
req := c.httpClient.R()
req.SetAuthToken(c.token)
if url != "" {
b64URL := base64.URLEncoding.EncodeToString([]byte(url))
@ -105,7 +102,6 @@ func (c *client) AddResource(res ResourceDto) (ResourceDto, error) {
targetEndpoint := fmt.Sprintf("%s/v1/resources", c.baseURL)
req := c.httpClient.R()
req.SetAuthToken(c.token)
req.SetBody(res)
var resourceDto ResourceDto
@ -119,7 +115,6 @@ func (c *client) ScheduleURL(url string) error {
targetEndpoint := fmt.Sprintf("%s/v1/urls", c.baseURL)
req := c.httpClient.R()
req.SetAuthToken(c.token)
req.SetHeader("Content-Type", "application/json")
req.SetBody(fmt.Sprintf("\"%s\"", url))
@ -127,23 +122,11 @@ func (c *client) ScheduleURL(url string) error {
return err
}
func (c *client) Authenticate(credentials CredentialsDto) (string, error) {
targetEndpoint := fmt.Sprintf("%s/v1/sessions", c.baseURL)
req := c.httpClient.R()
req.SetBody(credentials)
var token string
req.SetResult(&token)
_, err := req.Post(targetEndpoint)
return token, err
}
// NewAuthenticatedClient create a new Client & authenticate it against the API
func NewAuthenticatedClient(baseURL string, credentials CredentialsDto) (Client, error) {
// NewClient create a new API client using given details
func NewClient(baseURL, token string) Client {
httpClient := resty.New()
httpClient.SetAuthScheme("Bearer")
httpClient.SetAuthToken(token)
httpClient.OnAfterResponse(func(c *resty.Client, r *resty.Response) error {
if r.StatusCode() > 302 {
return fmt.Errorf("error when making HTTP request: %s", r.Status())
@ -156,11 +139,5 @@ func NewAuthenticatedClient(baseURL string, credentials CredentialsDto) (Client,
baseURL: baseURL,
}
token, err := client.Authenticate(credentials)
if err != nil {
return nil, err
}
client.token = token
return client, nil
return client
}

@ -20,28 +20,48 @@ services:
- 15004:5601
crawler:
image: creekorful/tdsh-crawler:latest
command: --log-level debug --nats-uri nats --tor-uri torproxy:9050
command: >
--log-level debug
--nats-uri nats
--tor-uri torproxy:9050
restart: always
depends_on:
- nats
- torproxy
scheduler:
image: creekorful/tdsh-scheduler:latest
command: --log-level debug --nats-uri nats --api-uri http://api:8080 --api-login scheduler:ZjDXeaLGj4EEUGu6 --forbidden-extensions jpg --forbidden-extensions png
command: >
--log-level debug
--nats-uri nats
--api-uri http://api:8080
--api-token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6InNjaGVkdWxlciIsInJpZ2h0cyI6eyJHRVQiOlsiL3YxL3Jlc291cmNlcyJdfX0.dBR6KLQp2h2srY-By3zikEznhQplLCtDrvOkcXP6USY
--forbidden-extensions png
--forbidden-extensions gif
--forbidden-extensions jpg
--forbidden-extensions jpeg
--forbidden-extensions bmp
restart: always
depends_on:
- nats
- api
extractor:
image: creekorful/tdsh-extractor:latest
command: --log-level debug --nats-uri nats --api-uri http://api:8080 --api-login extractor:hWx2KsrhWVQb5vxg
command: >
--log-level debug
--nats-uri nats
--api-uri http://api:8080
--api-token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImV4dHJhY3RvciIsInJpZ2h0cyI6eyJQT1NUIjpbIi92MS9yZXNvdXJjZXMiXX19.mytGd_9zyK8y_T3fsWAmH8FnaBNr6qWefwCPDOx4in0
restart: always
depends_on:
- nats
- api
api:
image: creekorful/tdsh-api:latest
command: --log-level debug --nats-uri nats --elasticsearch-uri http://elasticsearch:9200 --signing-key K==M5RsU_DQa4_XSbkX?L27s^xWmde25 --users extractor:hWx2KsrhWVQb5vxg --users scheduler:ZjDXeaLGj4EEUGu6 --users demo:demo
command: >
--log-level debug
--nats-uri nats
--elasticsearch-uri http://elasticsearch:9200
--signing-key K==M5RsU_DQa4_XSbkX?L27s^xWmde25
restart: always
depends_on:
- elasticsearch

@ -1,10 +1,10 @@
package api
import (
"github.com/creekorful/trandoshan/internal/api/auth"
"github.com/creekorful/trandoshan/internal/logging"
"github.com/creekorful/trandoshan/internal/util"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)
@ -47,6 +47,10 @@ func execute(c *cli.Context) error {
logging.ConfigureLogger(c)
e := echo.New()
e.HTTPErrorHandler = func(err error, c echo.Context) {
log.Err(err).Msg("error while processing API call")
e.DefaultHTTPErrorHandler(err, c)
}
e.HideBanner = true
log.Info().Str("ver", c.App.Version).
@ -57,20 +61,20 @@ func execute(c *cli.Context) error {
signingKey := []byte(c.String("signing-key"))
// Create the service
svc, err := newService(c, signingKey)
svc, err := newService(c)
if err != nil {
log.Err(err).Msg("Unable to start API")
return err
}
// Setup middlewares
jwtMiddleware := middleware.JWT(signingKey)
authMiddleware := auth.NewMiddleware(signingKey)
e.Use(authMiddleware.Middleware())
// Add endpoints
e.GET("/v1/resources", searchResourcesEndpoint(svc), jwtMiddleware)
e.POST("/v1/resources", addResourceEndpoint(svc), jwtMiddleware)
e.POST("/v1/urls", scheduleURLEndpoint(svc), jwtMiddleware)
e.POST("/v1/sessions", authenticateEndpoint(svc))
e.GET("/v1/resources", searchResourcesEndpoint(svc))
e.POST("/v1/resources", addResourceEndpoint(svc))
e.POST("/v1/urls", scheduleURLEndpoint(svc))
log.Info().Msg("Successfully initialized tdsh-api. Waiting for requests")

@ -0,0 +1,112 @@
package auth
import (
"fmt"
"github.com/dgrijalva/jwt-go"
"github.com/labstack/echo/v4"
"net/http"
"strings"
)
// ErrInvalidOrMissingAuth is returned if the authorization header is absent or invalid
var ErrInvalidOrMissingAuth = &echo.HTTPError{
Code: http.StatusUnauthorized,
Message: "Invalid or missing `Authorization` header",
}
// ErrAccessUnauthorized is returned if the token doesn't grant access to the current resource
var ErrAccessUnauthorized = &echo.HTTPError{
Code: http.StatusUnauthorized,
Message: "Access to the resource is not authorized",
}
// Token is the authentication token used by processes when dialing with the API
type Token struct {
// Username used for logging purposes
Username string `json:"username"`
// Rights that the token provides
// Format is: METHOD - list of paths
Rights map[string][]string `json:"rights"`
}
// Middleware is the authentication middleware
type Middleware struct {
signingKey []byte
}
// NewMiddleware create a new Middleware instance with given secret token signing key
func NewMiddleware(signingKey []byte) *Middleware {
return &Middleware{signingKey: signingKey}
}
// Middleware return an echo compatible middleware func to use
func (m *Middleware) Middleware() echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
// Extract authorization header
tokenStr := c.Request().Header.Get(echo.HeaderAuthorization)
if tokenStr == "" {
return ErrInvalidOrMissingAuth
}
tokenStr = strings.TrimPrefix(tokenStr, "Bearer ")
// Decode the JWT token
token, err := jwt.Parse(tokenStr, func(t *jwt.Token) (interface{}, error) {
// Validate expected alg
if v, ok := t.Method.(*jwt.SigningMethodHMAC); !ok || v.Name != "HS256" {
return nil, fmt.Errorf("unexpected signing method: %s", t.Header["alg"])
}
// Return signing secret
return m.signingKey, nil
})
if err != nil {
return ErrInvalidOrMissingAuth
}
// From here we have a valid JWT token, extract claims
claims, ok := token.Claims.(jwt.MapClaims)
if !ok {
return fmt.Errorf("error while parsing token claims")
}
rights := map[string][]string{}
for method, paths := range claims["rights"].(map[string]interface{}) {
for _, path := range paths.([]interface{}) {
rights[method] = append(rights[method], path.(string))
}
}
t := Token{
Username: claims["username"].(string),
Rights: rights,
}
// Validate rights
paths, contains := t.Rights[c.Request().Method]
if !contains {
return ErrAccessUnauthorized
}
authorized := false
for _, path := range paths {
if path == c.Request().URL.Path {
authorized = true
break
}
}
if !authorized {
return ErrAccessUnauthorized
}
// Set user context
c.Set("username", t.Username)
// Everything's fine, call next handler ;D
return next(c)
}
}
}

@ -0,0 +1,88 @@
package auth
import (
"fmt"
"github.com/labstack/echo/v4"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
)
func TestMiddleware_NoTokenShouldReturnUnauthorized(t *testing.T) {
e := echo.New()
m := (&Middleware{signingKey: []byte("test")}).Middleware()(okHandler())
// no token shouldn't be able to access
req := httptest.NewRequest(http.MethodGet, "/users", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
if err := m(c); err != ErrInvalidOrMissingAuth {
t.Errorf("ErrInvalidOrMissingAuth was expected")
}
}
func TestMiddleware_InvalidTokenShouldReturnUnauthorized(t *testing.T) {
e := echo.New()
m := (&Middleware{signingKey: []byte("test")}).Middleware()
req := httptest.NewRequest(http.MethodGet, "/users", nil)
req.Header.Add(echo.HeaderAuthorization, "zarBR")
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
if err := m(okHandler())(c); err != ErrInvalidOrMissingAuth {
t.Errorf("ErrInvalidOrMissingAuth was expected")
}
}
func TestMiddleware_BadRightsShouldReturnUnauthorized(t *testing.T) {
e := echo.New()
m := (&Middleware{signingKey: []byte("test")}).Middleware()
req := httptest.NewRequest(http.MethodPost, "/users", nil)
req.Header.Add(echo.HeaderAuthorization, "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6IkpvaG4gRG9lIiwicmlnaHRzIjp7IkdFVCI6WyIvdXNlcnMiXSwiUE9TVCI6WyIvc2VhcmNoIl19fQ.fRx0Q66ZgnY_rKCf-9Vaz6gzGKH_tKSgkVHhoQMtKfM")
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
if err := m(okHandler())(c); err != ErrAccessUnauthorized {
t.Errorf("ErrAccessUnauthorized was expected")
}
}
func TestMiddleware(t *testing.T) {
e := echo.New()
m := (&Middleware{signingKey: []byte("test")}).Middleware()
req := httptest.NewRequest(http.MethodGet, "/users?id=10", nil)
req.Header.Add(echo.HeaderAuthorization, "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6IkpvaG4gRG9lIiwicmlnaHRzIjp7IkdFVCI6WyIvdXNlcnMiXSwiUE9TVCI6WyIvc2VhcmNoIl19fQ.fRx0Q66ZgnY_rKCf-9Vaz6gzGKH_tKSgkVHhoQMtKfM")
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
_ = m(okHandler())(c)
if rec.Code != http.StatusOK {
t.Fail()
}
b, err := ioutil.ReadAll(rec.Body)
if err != nil {
t.Fail()
}
if string(b) != "Hello, John Doe" {
t.Fail()
}
if token := c.Get("username").(string); token != "John Doe" {
t.Fail()
}
}
func okHandler() echo.HandlerFunc {
return func(c echo.Context) error {
if username := c.Get("username").(string); username != "" {
return c.String(http.StatusOK, fmt.Sprintf("Hello, %s", username))
}
return c.NoContent(http.StatusNoContent)
}
}

@ -56,23 +56,6 @@ func scheduleURLEndpoint(s service) echo.HandlerFunc {
}
}
func authenticateEndpoint(s service) echo.HandlerFunc {
return func(c echo.Context) error {
// Validate provided credentials
var credentials api.CredentialsDto
if err := c.Bind(&credentials); err != nil {
return err
}
token, err := s.authenticate(credentials)
if err != nil {
return err
}
return c.JSON(http.StatusOK, token)
}
}
func readPagination(c echo.Context) (int, int) {
paginationPage, err := strconv.Atoi(c.QueryParam(api.PaginationPageQueryParam))
if err != nil {

@ -4,47 +4,23 @@ import (
"github.com/creekorful/trandoshan/api"
"github.com/creekorful/trandoshan/internal/api/database"
"github.com/creekorful/trandoshan/internal/messaging"
"github.com/dgrijalva/jwt-go"
"github.com/labstack/echo/v4"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"golang.org/x/crypto/bcrypt"
"net/http"
"strings"
)
type service interface {
searchResources(params *database.ResSearchParams) ([]api.ResourceDto, int64, error)
addResource(res api.ResourceDto) (api.ResourceDto, error)
scheduleURL(url string) error
authenticate(credentials api.CredentialsDto) (string, error)
close()
}
type svc struct {
users map[string][]byte
signingKey []byte
db database.Database
pub messaging.Publisher
db database.Database
pub messaging.Publisher
}
func newService(c *cli.Context, signingKey []byte) (service, error) {
users := map[string][]byte{}
for _, userEntry := range c.StringSlice("users") {
parts := strings.Split(userEntry, ":")
user := parts[0]
pass := parts[1]
passBytes, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost)
if err != nil {
log.Err(err).Msg("Unable to generate user password")
return nil, err
}
log.Debug().Str("username", user).Msg("Register new user")
users[user] = passBytes
}
func newService(c *cli.Context) (service, error) {
// Connect to the NATS server
pub, err := messaging.NewPublisher(c.String("nats-uri"))
if err != nil {
@ -60,10 +36,8 @@ func newService(c *cli.Context, signingKey []byte) (service, error) {
}
return &svc{
db: db,
users: users,
signingKey: signingKey,
pub: pub,
db: db,
pub: pub,
}, nil
}
@ -126,37 +100,6 @@ func (s *svc) scheduleURL(url string) error {
return nil
}
func (s *svc) authenticate(credentials api.CredentialsDto) (string, error) {
if credentials.Username == "" || credentials.Password == "" {
log.Warn().Msg("Invalid credentials supplied")
return "", echo.NewHTTPError(http.StatusUnprocessableEntity)
}
// Try to find the user
pass, exists := s.users[credentials.Username]
if !exists {
log.Warn().Str("username", credentials.Username).Msg("No user found")
return "", echo.NewHTTPError(http.StatusUnprocessableEntity)
}
// Validate provided password
if err := bcrypt.CompareHashAndPassword(pass, []byte(credentials.Password)); err != nil {
log.Warn().Str("username", credentials.Username).Msg("Invalid password")
return "", echo.NewHTTPError(http.StatusUnauthorized)
}
log.Debug().Str("username", credentials.Username).Msg("Successfully logged-in")
// Build JWT token
claims := jwt.MapClaims{
"username": credentials.Username,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
// Sign JWT token
return token.SignedString(s.signingKey)
}
func (s *svc) close() {
s.pub.Close()
}

@ -6,7 +6,6 @@ import (
"github.com/creekorful/trandoshan/internal/api/database_mock"
"github.com/creekorful/trandoshan/internal/messaging"
"github.com/creekorful/trandoshan/internal/messaging_mock"
"github.com/dgrijalva/jwt-go"
"github.com/golang/mock/gomock"
"testing"
"time"
@ -113,52 +112,3 @@ func TestScheduleURL(t *testing.T) {
t.FailNow()
}
}
func TestAuthenticateInvalidCredentials(t *testing.T) {
s := svc{}
if _, err := s.authenticate(api.CredentialsDto{}); err == nil {
t.FailNow()
}
}
func TestAuthenticateWrongCredentials(t *testing.T) {
s := svc{users: map[string][]byte{"creekorful": []byte("")}}
if _, err := s.authenticate(api.CredentialsDto{Username: "johndoe", Password: "test"}); err == nil {
t.FailNow()
}
if _, err := s.authenticate(api.CredentialsDto{Username: "creekorful", Password: "tes"}); err == nil {
t.FailNow()
}
}
func TestAuthenticate(t *testing.T) {
s := svc{
users: map[string][]byte{
"creekorful": []byte("$2a$10$aLX2t8JsTOoy9iRLBNm.RuPMmcA8MCXijuzhLvUwUbSlh.C/D2eLm")},
signingKey: []byte("secret"),
}
tokenStr, err := s.authenticate(api.CredentialsDto{Username: "creekorful", Password: "test"})
if err != nil {
t.FailNow()
}
claims := jwt.MapClaims{}
token, err := jwt.ParseWithClaims(tokenStr, claims, func(token *jwt.Token) (interface{}, error) {
return []byte("secret"), nil
})
if err != nil {
t.Error(err)
t.FailNow()
}
if token.Header["alg"] != jwt.SigningMethodHS256.Alg() {
t.Errorf("Invalid alg: %s", token.Header["alg"])
}
if claims["username"] != "creekorful" {
t.Errorf("Invalid username: %s", claims["username"])
}
}

@ -31,7 +31,7 @@ func GetApp() *cli.App {
logging.GetLogFlag(),
util.GetNATSURIFlag(),
util.GetAPIURIFlag(),
util.GetAPILoginFlag(),
util.GetAPITokenFlag(),
},
Action: execute,
}
@ -46,10 +46,7 @@ func execute(ctx *cli.Context) error {
Str("api-uri", ctx.String("api-uri")).
Msg("Starting tdsh-extractor")
apiClient, err := util.GetAPIAuthenticatedClient(ctx)
if err != nil {
return err
}
apiClient := util.GetAPIClient(ctx)
// Create the NATS subscriber
sub, err := messaging.NewSubscriber(ctx.String("nats-uri"))

@ -25,7 +25,7 @@ func GetApp() *cli.App {
logging.GetLogFlag(),
util.GetNATSURIFlag(),
util.GetAPIURIFlag(),
util.GetAPILoginFlag(),
util.GetAPITokenFlag(),
&cli.StringFlag{
Name: "refresh-delay",
Usage: "Duration before allowing crawl of existing resource (none = never)",
@ -53,10 +53,7 @@ func execute(ctx *cli.Context) error {
Msg("Starting tdsh-scheduler")
// Create the API client
apiClient, err := util.GetAPIAuthenticatedClient(ctx)
if err != nil {
return err
}
apiClient := util.GetAPIClient(ctx)
// Create the NATS subscriber
sub, err := messaging.NewSubscriber(ctx.String("nats-uri"))

@ -79,7 +79,7 @@ func TestHandleMessageForbiddenExtensions(t *testing.T) {
msg := nats.Msg{}
subscriberMock.EXPECT().
ReadMsg(&msg, &messaging.URLFoundMsg{}).
SetArg(1, messaging.URLFoundMsg{URL: "https://example.onion/image.png"}).
SetArg(1, messaging.URLFoundMsg{URL: "https://example.onion/image.png?id=12&test=2"}).
Return(nil)
if err := handleMessage(apiClientMock, -1, []string{"png"})(subscriberMock, &msg); err != nil {

@ -17,10 +17,6 @@ func GetApp() *cli.App {
apiFlag.Value = "http://localhost:15005"
apiFlag.Required = false
apiLoginFlag := util.GetAPILoginFlag()
apiLoginFlag.Value = "demo:demo"
apiLoginFlag.Required = false
return &cli.App{
Name: "trandoshanctl",
Version: "0.5.1",
@ -28,7 +24,7 @@ func GetApp() *cli.App {
Flags: []cli.Flag{
logging.GetLogFlag(),
apiFlag,
apiLoginFlag,
util.GetAPITokenFlag(),
},
Commands: []*cli.Command{
{
@ -61,11 +57,7 @@ func schedule(c *cli.Context) error {
url := c.Args().First()
// Create the API client
apiClient, err := util.GetAPIAuthenticatedClient(c)
if err != nil {
log.Err(err).Msg("Error while creating API client")
return err
}
apiClient := util.GetAPIClient(c)
if err := apiClient.ScheduleURL(url); err != nil {
log.Err(err).Str("url", url).Msg("Unable to schedule crawling for URL")
@ -81,11 +73,7 @@ func search(c *cli.Context) error {
keyword := c.Args().First()
// Create the API client
apiClient, err := util.GetAPIAuthenticatedClient(c)
if err != nil {
log.Err(err).Msg("Error while creating API client")
return err
}
apiClient := util.GetAPIClient(c)
res, count, err := apiClient.SearchResources("", keyword, time.Time{}, time.Time{}, 1, 10)
if err != nil {

@ -1,17 +1,15 @@
package util
import (
"fmt"
"github.com/creekorful/trandoshan/api"
"github.com/urfave/cli/v2"
"strings"
)
// GetAPILoginFlag return the cli flag to set api credentials
func GetAPILoginFlag() *cli.StringFlag {
// GetAPITokenFlag return the cli flag to provide API token
func GetAPITokenFlag() *cli.StringFlag {
return &cli.StringFlag{
Name: "api-login",
Usage: "Login to use when dialing with the API",
Name: "api-token",
Usage: "Token to use to authenticate against the API",
Required: true,
}
}
@ -25,31 +23,7 @@ func GetAPIURIFlag() *cli.StringFlag {
}
}
// GetAPILogin return the credentials from cli flag
func GetAPILogin(c *cli.Context) (api.CredentialsDto, error) {
if c.String("api-login") == "" {
return api.CredentialsDto{}, fmt.Errorf("missing credentials")
}
credentials := strings.Split(c.String("api-login"), ":")
if len(credentials) != 2 {
return api.CredentialsDto{}, fmt.Errorf("wrong credentials format")
}
return api.CredentialsDto{Username: credentials[0], Password: credentials[1]}, nil
}
// GetAPIAuthenticatedClient return the authenticated api client
func GetAPIAuthenticatedClient(c *cli.Context) (api.Client, error) {
// Create the API client
credentials, err := GetAPILogin(c)
if err != nil {
return nil, err
}
apiClient, err := api.NewAuthenticatedClient(c.String("api-uri"), credentials)
if err != nil {
return nil, err
}
return apiClient, nil
// GetAPIClient return a new configured API client
func GetAPIClient(c *cli.Context) api.Client {
return api.NewClient(c.String("api-uri"), c.String("api-token"))
}

Loading…
Cancel
Save