make scheduler use whitelisting instead of blacklisting

deployments/docker/docker-compose.yml

@@ -81,8 +81,7 @@ services:
       --hub-uri amqp://guest:guest@rabbitmq:5672
       --db-uri redis:6379
       --default-value forbidden-hostnames="[{\"hostname\": \"facebookcorewwwi.onion\"}]"
-      --default-value forbidden-mime-types="[{\"content-type\":\"image/png\",\"extensions\":[\"png\"]},{\"content-type\":\"image/gif\",\"extensions\":[\"gif\"]},{\"content-type\":\"image/jpeg\",\"extensions\":[\"jpg\",\"jpeg\"]},{\"content-type\":\"image/bmp\",\"extensions\":[\"bmp\"]},{\"content-type\":\"image/svg+xml\",\"extensions\":[\"svg\"]},{\"content-type\":\"text/css\",\"extensions\":[\"css\"]},{\"content-type\":\"application/javascript\",\"extensions\":[\"js\"]}]"
-      --default-value allowed-mime-types="[{\"content-type\":\"text/\",\"extensions\":[]}]"
+      --default-value allowed-mime-types="[{\"content-type\":\"text/\",\"extensions\":[\"html\",\"php\",\"aspx\"]}]"
       --default-value refresh-delay="{\"delay\": -1}"
     restart: always
     depends_on:

internal/configapi/client/client.go

@@ -15,8 +15,6 @@ import (
 //go:generate mockgen -destination=../client_mock/client_mock.go -package=client_mock . Client
 
 const (
-	// ForbiddenMimeTypesKey is the key to access the forbidden mime types config
-	ForbiddenMimeTypesKey = "forbidden-mime-types"
 	// AllowedMimeTypesKey is the key to access the allowed mime types config
 	AllowedMimeTypesKey = "allowed-mime-types"
 	// ForbiddenHostnamesKey is the key to access the forbidden hostnames config
@@ -45,7 +43,6 @@ type RefreshDelay struct {
 
 // Client is a nice client interface for the ConfigAPI
 type Client interface {
-	GetForbiddenMimeTypes() ([]MimeType, error)
 	GetAllowedMimeTypes() ([]MimeType, error)
 	GetForbiddenHostnames() ([]ForbiddenHostname, error)
 	GetRefreshDelay() (RefreshDelay, error)
@@ -96,22 +93,6 @@ func NewConfigClient(configAPIURL string, subscriber event.Subscriber, keys []st
 	return client, nil
 }
 
-func (c *client) GetForbiddenMimeTypes() ([]MimeType, error) {
-	c.mutexes[ForbiddenMimeTypesKey].RLock()
-	defer c.mutexes[ForbiddenMimeTypesKey].RUnlock()
-
-	return c.forbiddenMimeTypes, nil
-}
-
-func (c *client) setForbiddenMimeTypes(values []MimeType) error {
-	c.mutexes[ForbiddenMimeTypesKey].Lock()
-	defer c.mutexes[ForbiddenMimeTypesKey].Unlock()
-
-	c.forbiddenMimeTypes = values
-
-	return nil
-}
-
 func (c *client) GetAllowedMimeTypes() ([]MimeType, error) {
 	c.mutexes[AllowedMimeTypesKey].RLock()
 	defer c.mutexes[AllowedMimeTypesKey].RUnlock()
@@ -199,15 +180,6 @@ func (c *client) get(key string) ([]byte, error) {
 
 func (c *client) setValue(key string, value []byte) error {
 	switch key {
-	case ForbiddenMimeTypesKey:
-		var val []MimeType
-		if err := json.Unmarshal(value, &val); err != nil {
-			return err
-		}
-		if err := c.setForbiddenMimeTypes(val); err != nil {
-			return err
-		}
-		break
 	case AllowedMimeTypesKey:
 		var val []MimeType
 		if err := json.Unmarshal(value, &val); err != nil {

internal/configapi/client/client_test.go

@@ -19,14 +19,14 @@ func TestClient(t *testing.T) {
 	client := &client{
 		configAPIURL:       "",
 		sub:                subMock,
-		mutexes:            map[string]*sync.RWMutex{ForbiddenMimeTypesKey: {}},
-		keys:               []string{ForbiddenMimeTypesKey},
+		mutexes:            map[string]*sync.RWMutex{AllowedMimeTypesKey: {}},
+		keys:               []string{AllowedMimeTypesKey},
 		forbiddenMimeTypes: []MimeType{},
 		forbiddenHostnames: nil,
 		refreshDelay:       RefreshDelay{},
 	}
 
-	val, err := client.GetForbiddenMimeTypes()
+	val, err := client.GetAllowedMimeTypes()
 	if err != nil {
 		t.FailNow()
 	}
@@ -36,14 +36,14 @@ func TestClient(t *testing.T) {
 
 	msg := event.RawMessage{
 		Body:    []byte("[{\"content-type\": \"application/json\", \"extensions\": [\"json\"]}]"),
-		Headers: map[string]interface{}{"Config-Key": ForbiddenMimeTypesKey},
+		Headers: map[string]interface{}{"Config-Key": AllowedMimeTypesKey},
 	}
 
 	if err := client.handleConfigEvent(subMock, msg); err != nil {
 		t.FailNow()
 	}
 
-	val, err = client.GetForbiddenMimeTypes()
+	val, err = client.GetAllowedMimeTypes()
 	if err != nil {
 		t.FailNow()
 	}

internal/scheduler/scheduler.go

@@ -11,6 +11,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strings"
 )
 
@@ -19,6 +20,8 @@ var (
 	errProtocolNotAllowed  = errors.New("protocol is not allowed")
 	errExtensionNotAllowed = errors.New("extension is not allowed")
 	errHostnameNotAllowed  = errors.New("hostname is not allowed")
+
+	extensionRegex = regexp.MustCompile("\\.[\\w]+")
 )
 
 // State represent the application state
@@ -43,7 +46,7 @@ func (state *State) CustomFlags() []cli.Flag {
 
 // Initialize the process
 func (state *State) Initialize(provider process.Provider) error {
-	keys := []string{configapi.ForbiddenMimeTypesKey, configapi.ForbiddenHostnamesKey}
+	keys := []string{configapi.AllowedMimeTypesKey, configapi.ForbiddenHostnamesKey}
 	configClient, err := provider.ConfigClient(keys)
 	if err != nil {
 		return err
@@ -88,17 +91,37 @@ func (state *State) handleURLFoundEvent(subscriber event.Subscriber, msg event.R
 		return fmt.Errorf("%s %w", u, errProtocolNotAllowed)
 	}
 
-	// Make sure extension is not forbidden
-	if mimeTypes, err := state.configClient.GetForbiddenMimeTypes(); err == nil {
+	// Make sure extension is allowed
+	allowed := false
+	if mimeTypes, err := state.configClient.GetAllowedMimeTypes(); err == nil {
 		for _, mimeType := range mimeTypes {
 			for _, ext := range mimeType.Extensions {
 				if strings.HasSuffix(strings.ToLower(u.Path), "."+ext) {
-					return fmt.Errorf("%s (.%s) %w", u, ext, errExtensionNotAllowed)
+					allowed = true
 				}
 			}
 		}
 	}
 
+	// Handle case no extension present
+	if !allowed {
+		components := strings.Split(u.Path, "/")
+
+		lastIdx := 0
+		if size := len(components); size > 0 {
+			lastIdx = size - 1
+		}
+
+		// generally no extension means text/* content-type
+		if !extensionRegex.MatchString(components[lastIdx]) {
+			allowed = true
+		}
+	}
+
+	if !allowed {
+		return fmt.Errorf("%s %w", u, errExtensionNotAllowed)
+	}
+
 	// Make sure hostname is not forbidden
 	if allowed, err := constraint.CheckHostnameAllowed(state.configClient, evt.URL); err != nil {
 		return err

internal/scheduler/scheduler_test.go

@@ -69,7 +69,7 @@ func TestHandleMessageForbiddenExtensions(t *testing.T) {
 	subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
 	configClientMock := client_mock.NewMockClient(mockCtrl)
 
-	urls := []string{"https://example.onion/image.png?id=12&test=2", "https://example.onion/image.PNG"}
+	urls := []string{"https://example.onion/image.PNG?id=12&test=2", "https://example.onion/favicon.ico"}
 
 	for _, url := range urls {
 		msg := event.RawMessage{}
@@ -78,7 +78,7 @@ func TestHandleMessageForbiddenExtensions(t *testing.T) {
 			SetArg(1, event.FoundURLEvent{URL: url}).
 			Return(nil)
 
-		configClientMock.EXPECT().GetForbiddenMimeTypes().Return([]client.MimeType{{Extensions: []string{"png"}}}, nil)
+		configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"php", "html"}}}, nil)
 
 		s := State{
 			configClient: configClientMock,
@@ -128,7 +128,7 @@ func TestHandleMessageHostnameForbidden(t *testing.T) {
 			SetArg(1, event.FoundURLEvent{URL: test.url}).
 			Return(nil)
 
-		configClientMock.EXPECT().GetForbiddenMimeTypes().Return([]client.MimeType{}, nil)
+		configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"png", "php"}}}, nil)
 		configClientMock.EXPECT().GetForbiddenHostnames().Return(test.forbiddenHostnames, nil)
 
 		s := State{
@@ -148,24 +148,29 @@ func TestHandleMessage(t *testing.T) {
 	subscriberMock := event_mock.NewMockSubscriber(mockCtrl)
 	configClientMock := client_mock.NewMockClient(mockCtrl)
 
-	msg := event.RawMessage{}
-	subscriberMock.EXPECT().
-		Read(&msg, &event.FoundURLEvent{}).
-		SetArg(1, event.FoundURLEvent{URL: "https://www.facebookcorewwwi.onion/recover/initiate?ars=facebook_login"}).
-		Return(nil)
+	urls := []string{"https://example.onion/index.php", "http://google.onion/admin/login.html",
+		"https://example.onion", "https://www.facebookcorewwwi.onion/recover/initiate?ars=facebook_login"}
 
-	subscriberMock.EXPECT().
-		PublishEvent(&event.NewURLEvent{URL: "https://www.facebookcorewwwi.onion/recover/initiate?ars=facebook_login"}).
-		Return(nil)
+	for _, u := range urls {
+		msg := event.RawMessage{}
+		subscriberMock.EXPECT().
+			Read(&msg, &event.FoundURLEvent{}).
+			SetArg(1, event.FoundURLEvent{URL: u}).
+			Return(nil)
 
-	configClientMock.EXPECT().GetForbiddenMimeTypes().Return([]client.MimeType{}, nil)
-	configClientMock.EXPECT().GetForbiddenHostnames().Return([]client.ForbiddenHostname{}, nil)
+		subscriberMock.EXPECT().
+			PublishEvent(&event.NewURLEvent{URL: u}).
+			Return(nil)
 
-	s := State{
-		configClient: configClientMock,
-	}
+		configClientMock.EXPECT().GetAllowedMimeTypes().Return([]client.MimeType{{Extensions: []string{"html", "php"}}}, nil)
+		configClientMock.EXPECT().GetForbiddenHostnames().Return([]client.ForbiddenHostname{}, nil)
+
+		s := State{
+			configClient: configClientMock,
+		}
 
-	if err := s.handleURLFoundEvent(subscriberMock, msg); err != nil {
-		t.FailNow()
+		if err := s.handleURLFoundEvent(subscriberMock, msg); err != nil {
+			t.FailNow()
+		}
 	}
 }