Use new storage format

internal/archiver/archiver.go

@@ -83,13 +83,14 @@ func (state *State) handleNewIndexEvent(subscriber event.Subscriber, msg event.R
 func formatResource(evt *event.NewIndexEvent) ([]byte, error) {
 	builder := strings.Builder{}
 
-	// First headers
+	// First URL
+	builder.WriteString(fmt.Sprintf("%s\n\n", evt.URL))
+
+	// Then headers
 	for key, value := range evt.Headers {
-		builder.WriteString(fmt.Sprintf("%s: %s\r\n", key, value))
+		builder.WriteString(fmt.Sprintf("%s: %s\n", key, value))
 	}
-
-	// Then separator for body
-	builder.WriteString("\r\n")
+	builder.WriteString("\n")
 
 	// Then body
 	builder.WriteString(evt.Body)

internal/archiver/archiver_test.go

@@ -28,7 +28,7 @@ func TestHandleNewResourceEvent(t *testing.T) {
 			Time:    tn,
 		}).Return(nil)
 
-	storageMock.EXPECT().Store("https://example.onion", tn, []byte("Server: Traefik\r\nContent-Type: application/html\r\n\r\nHello, world")).Return(nil)
+	storageMock.EXPECT().Store("https://example.onion", tn, []byte("https://example.onion\n\nServer: Traefik\nContent-Type: application/html\n\nHello, world")).Return(nil)
 
 	s := State{storage: storageMock}
 	if err := s.handleNewIndexEvent(subscriberMock, msg); err != nil {
@@ -49,7 +49,7 @@ func TestFormatResource(t *testing.T) {
 		t.FailNow()
 	}
 
-	if string(res) != "Server: Traefik\r\nContent-Type: text/html\r\n\r\nHello, world" {
+	if string(res) != "https://google.com\n\nServer: Traefik\nContent-Type: text/html\n\nHello, world" {
 		t.Fail()
 	}
 }

internal/archiver/storage/local.go

@@ -2,10 +2,12 @@ package storage
 
 import (
 	"fmt"
+	"hash/fnv"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -55,9 +57,14 @@ func formatPath(rawURL string, time time.Time) (string, error) {
 	b.WriteString(u.Host)
 	b.WriteRune(os.PathSeparator)
 
-	// Write path
 	if uri := u.RequestURI(); uri != "/" {
-		b.WriteString(strings.TrimPrefix(u.RequestURI(), "/"))
+		// Write path (hash it to prevent too long filename)
+		c := fnv.New64()
+		if _, err := c.Write([]byte(strings.TrimPrefix(u.RequestURI(), "/"))); err != nil {
+			return "", err
+		}
+
+		b.WriteString(strconv.FormatUint(c.Sum64(), 10))
 		b.WriteRune(os.PathSeparator)
 	}
 

internal/archiver/storage/local_test.go

@@ -26,12 +26,12 @@ func TestFormatPath(t *testing.T) {
 		{
 			url:  "http://facebook.com/admin/login.php?username=admin",
 			time: ti,
-			path: "http/facebook.com/admin/login.php?username=admin/1603973049",
+			path: "http/facebook.com/16609974401560122507/1603973049",
 		},
 		{
 			url:  "http://thisisalonghostname.onion/admin/tools/list-accounts.php?token=123223453&username=test",
 			time: ti,
-			path: "http/thisisalonghostname.onion/admin/tools/list-accounts.php?token=123223453&username=test/1603973049",
+			path: "http/thisisalonghostname.onion/7883137132857825203/1603973049",
 		},
 	}