Rename project

pull/134/head
Aloïs Micard 3 years ago
parent 3cc47a071a
commit 96c75a85b1
No known key found for this signature in database
GPG Key ID: 1A0EB82F071F5EFE

@ -1,17 +1,8 @@
# Trandoshan dark web crawler
# Bathyscaphe dark web crawler
![CI](https://github.com/creekorful/trandoshan/workflows/CI/badge.svg)
![CI](https://github.com/creekorful/bathyscaphe/workflows/CI/badge.svg)
This repository is a complete rewrite of the Trandoshan dark web crawler. Everything has been written inside a single
Git repository to ease maintenance.
## Why a rewrite?
The first version of Trandoshan [(available here)](https://github.com/trandoshan-io) is working great but not really
professional, the code start to be a mess, hard to manage since split in multiple repositories, etc.
I have therefore decided to create & maintain the project in this specific repository, where all components code will be
available (as a Go module).
Bathyscaphe is a Go written, fast, highly configurable, cloud-native dark web crawler.
# How to start the crawler
@ -30,7 +21,8 @@ and wait for all containers to start.
# How to initiate crawling
One can use the RabbitMQ dashhboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue**.
One can use the RabbitMQ dashhboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue**
.
The object should look like this:

@ -0,0 +1,24 @@
# build image
FROM golang:1.15.0-alpine as builder
RUN apk update && apk upgrade && \
apk add --no-cache bash git openssh
WORKDIR /app
# Copy and download dependencies to cache them and faster build time
COPY go.mod go.sum ./
RUN go mod download
COPY . .
# Test then build app
RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-blacklister
# runtime image
FROM alpine:latest
COPY --from=builder /app/bs-blacklister /app/
WORKDIR /app/
ENTRYPOINT ["./bs-blacklister"]

@ -13,12 +13,12 @@ RUN go mod download
COPY . .
# Test then build app
RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-scheduler
RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-configapi
# runtime image
FROM alpine:latest
COPY --from=builder /app/tdsh-scheduler /app/
COPY --from=builder /app/bs-configapi /app/
WORKDIR /app/
ENTRYPOINT ["./tdsh-scheduler"]
ENTRYPOINT ["./bs-configapi"]

@ -13,12 +13,12 @@ RUN go mod download
COPY . .
# Test then build app
RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-crawler
RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-crawler
# runtime image
FROM alpine:latest
COPY --from=builder /app/tdsh-crawler /app/
COPY --from=builder /app/bs-crawler /app/
WORKDIR /app/
ENTRYPOINT ["./tdsh-crawler"]
ENTRYPOINT ["./bs-crawler"]

@ -13,12 +13,12 @@ RUN go mod download
COPY . .
# Test then build app
RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-indexer
RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-indexer
# runtime image
FROM alpine:latest
COPY --from=builder /app/tdsh-indexer /app/
COPY --from=builder /app/bs-indexer /app/
WORKDIR /app/
ENTRYPOINT ["./tdsh-indexer"]
ENTRYPOINT ["./bs-indexer"]

@ -13,12 +13,12 @@ RUN go mod download
COPY . .
# Test then build app
RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-configapi
RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-scheduler
# runtime image
FROM alpine:latest
COPY --from=builder /app/tdsh-configapi /app/
COPY --from=builder /app/bs-scheduler /app/
WORKDIR /app/
ENTRYPOINT ["./tdsh-configapi"]
ENTRYPOINT ["./bs-scheduler"]

@ -1,24 +0,0 @@
# build image
FROM golang:1.15.0-alpine as builder
RUN apk update && apk upgrade && \
apk add --no-cache bash git openssh
WORKDIR /app
# Copy and download dependencies to cache them and faster build time
COPY go.mod go.sum ./
RUN go mod download
COPY . .
# Test then build app
RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-blacklister
# runtime image
FROM alpine:latest
COPY --from=builder /app/tdsh-blacklister /app/
WORKDIR /app/
ENTRYPOINT ["./tdsh-blacklister"]

@ -1,8 +1,8 @@
package main
import (
"github.com/creekorful/trandoshan/internal/blacklister"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/blacklister"
"github.com/creekorful/bathyscaphe/internal/process"
"os"
)

@ -1,8 +1,8 @@
package main
import (
"github.com/creekorful/trandoshan/internal/configapi"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/configapi"
"github.com/creekorful/bathyscaphe/internal/process"
"os"
)

@ -1,8 +1,8 @@
package main
import (
"github.com/creekorful/trandoshan/internal/crawler"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/crawler"
"github.com/creekorful/bathyscaphe/internal/process"
"os"
)

@ -1,8 +1,8 @@
package main
import (
"github.com/creekorful/trandoshan/internal/indexer"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/indexer"
"github.com/creekorful/bathyscaphe/internal/process"
"os"
)

@ -1,8 +1,8 @@
package main
import (
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/scheduler"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/scheduler"
"os"
)

@ -29,7 +29,7 @@ services:
volumes:
- redisdata:/data
crawler:
image: creekorful/tdsh-crawler:latest
image: creekorful/bs-crawler:latest
command: >
--log-level debug
--event-srv amqp://guest:guest@rabbitmq:5672
@ -41,7 +41,7 @@ services:
- torproxy
- configapi
scheduler:
image: creekorful/tdsh-scheduler:latest
image: creekorful/bs-scheduler:latest
command: >
--log-level debug
--event-srv amqp://guest:guest@rabbitmq:5672
@ -54,7 +54,7 @@ services:
- configapi
- redis
indexer-local:
image: creekorful/tdsh-indexer:latest
image: creekorful/bs-indexer:latest
command: >
--log-level debug
--event-srv amqp://guest:guest@rabbitmq:5672
@ -68,7 +68,7 @@ services:
- rabbitmq
- configapi
indexer-es:
image: creekorful/tdsh-indexer:latest
image: creekorful/bs-indexer:latest
command: >
--log-level debug
--event-srv amqp://guest:guest@rabbitmq:5672
@ -82,7 +82,7 @@ services:
- elasticsearch
- configapi
configapi:
image: creekorful/tdsh-configapi:latest
image: creekorful/bs-configapi:latest
command: >
--log-level debug
--event-srv amqp://guest:guest@rabbitmq:5672
@ -98,7 +98,7 @@ services:
ports:
- 15006:8080
blacklister:
image: creekorful/tdsh-blacklister:latest
image: creekorful/bs-blacklister:latest
command: >
--log-level debug
--event-srv amqp://guest:guest@rabbitmq:5672

@ -1,4 +1,4 @@
module github.com/creekorful/trandoshan
module github.com/creekorful/bathyscaphe
go 1.14
@ -6,12 +6,9 @@ require (
github.com/PuerkitoBio/goquery v1.6.0
github.com/PuerkitoBio/purell v1.1.1
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/go-redis/redis/v8 v8.4.4
github.com/go-resty/resty/v2 v2.3.0
github.com/golang/mock v1.4.4
github.com/gorilla/mux v1.8.0
github.com/olekukonko/tablewriter v0.0.4
github.com/olivere/elastic/v7 v7.0.20
github.com/rs/zerolog v1.20.0
github.com/streadway/amqp v1.0.0

@ -17,8 +17,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSY
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@ -28,8 +26,6 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc=
github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY=
github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So=
github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -66,12 +62,8 @@ github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8=
github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
github.com/olivere/elastic/v7 v7.0.20 h1:5FFpGPVJlBSlWBOdict406Y3yNTIpVpAiUvdFZeSbAo=
github.com/olivere/elastic/v7 v7.0.20/go.mod h1:Kh7iIsXIBl5qRQOBFoylCsXVTtye3keQU2Y/YbR7HD8=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -138,8 +130,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=

@ -2,11 +2,11 @@ package blacklister
import (
"fmt"
"github.com/creekorful/trandoshan/internal/cache"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/event"
chttp "github.com/creekorful/trandoshan/internal/http"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/cache"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/event"
chttp "github.com/creekorful/bathyscaphe/internal/http"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"net/http"

@ -2,16 +2,16 @@ package blacklister
import (
"errors"
"github.com/creekorful/trandoshan/internal/cache_mock"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/configapi/client_mock"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/creekorful/trandoshan/internal/http"
"github.com/creekorful/trandoshan/internal/http_mock"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/process_mock"
"github.com/creekorful/trandoshan/internal/test"
"github.com/creekorful/bathyscaphe/internal/cache_mock"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/configapi/client_mock"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/event_mock"
"github.com/creekorful/bathyscaphe/internal/http"
"github.com/creekorful/bathyscaphe/internal/http_mock"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/process_mock"
"github.com/creekorful/bathyscaphe/internal/test"
"github.com/golang/mock/gomock"
"testing"
"time"

@ -6,7 +6,7 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/rs/zerolog/log"
"io/ioutil"
"net/http"

@ -1,8 +1,8 @@
package client
import (
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/event_mock"
"github.com/golang/mock/gomock"
"sync"
"testing"

@ -2,9 +2,9 @@ package configapi
import (
"fmt"
"github.com/creekorful/trandoshan/internal/cache"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/cache"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/gorilla/mux"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"

@ -1,13 +1,13 @@
package configapi
import (
"github.com/creekorful/trandoshan/internal/cache"
"github.com/creekorful/trandoshan/internal/cache_mock"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/process_mock"
"github.com/creekorful/trandoshan/internal/test"
"github.com/creekorful/bathyscaphe/internal/cache"
"github.com/creekorful/bathyscaphe/internal/cache_mock"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/event_mock"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/process_mock"
"github.com/creekorful/bathyscaphe/internal/test"
"github.com/golang/mock/gomock"
"github.com/gorilla/mux"
"io/ioutil"

@ -1,7 +1,7 @@
package constraint
import (
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"net/url"
"strings"
)

@ -1,8 +1,8 @@
package constraint
import (
"github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/configapi/client_mock"
"github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/configapi/client_mock"
"github.com/golang/mock/gomock"
"testing"
)

@ -2,12 +2,12 @@ package crawler
import (
"fmt"
"github.com/creekorful/trandoshan/internal/clock"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/constraint"
"github.com/creekorful/trandoshan/internal/event"
chttp "github.com/creekorful/trandoshan/internal/http"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/clock"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/constraint"
"github.com/creekorful/bathyscaphe/internal/event"
chttp "github.com/creekorful/bathyscaphe/internal/http"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"io/ioutil"

@ -2,16 +2,16 @@ package crawler
import (
"errors"
"github.com/creekorful/trandoshan/internal/clock_mock"
"github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/configapi/client_mock"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/creekorful/trandoshan/internal/http"
"github.com/creekorful/trandoshan/internal/http_mock"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/process_mock"
"github.com/creekorful/trandoshan/internal/test"
"github.com/creekorful/bathyscaphe/internal/clock_mock"
"github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/configapi/client_mock"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/event_mock"
"github.com/creekorful/bathyscaphe/internal/http"
"github.com/creekorful/bathyscaphe/internal/http_mock"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/process_mock"
"github.com/creekorful/bathyscaphe/internal/test"
"github.com/golang/mock/gomock"
"strings"
"testing"

@ -1,7 +1,7 @@
package index
import (
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/bathyscaphe/internal/event"
"testing"
"time"
)

@ -2,11 +2,11 @@ package indexer
import (
"fmt"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/constraint"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/indexer/index"
"github.com/creekorful/trandoshan/internal/process"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/constraint"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/indexer/index"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"net/http"

@ -2,15 +2,15 @@ package indexer
import (
"errors"
"github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/configapi/client_mock"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/creekorful/trandoshan/internal/indexer/index"
"github.com/creekorful/trandoshan/internal/indexer/index_mock"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/process_mock"
"github.com/creekorful/trandoshan/internal/test"
"github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/configapi/client_mock"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/event_mock"
"github.com/creekorful/bathyscaphe/internal/indexer/index"
"github.com/creekorful/bathyscaphe/internal/indexer/index_mock"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/process_mock"
"github.com/creekorful/bathyscaphe/internal/test"
"github.com/golang/mock/gomock"
"reflect"
"testing"

@ -6,11 +6,11 @@ import (
"context"
"crypto/tls"
"fmt"
"github.com/creekorful/trandoshan/internal/cache"
"github.com/creekorful/trandoshan/internal/clock"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/event"
chttp "github.com/creekorful/trandoshan/internal/http"
"github.com/creekorful/bathyscaphe/internal/cache"
"github.com/creekorful/bathyscaphe/internal/clock"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/event"
chttp "github.com/creekorful/bathyscaphe/internal/http"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
@ -135,7 +135,7 @@ type SubscriberDef struct {
Handler event.Handler
}
// Process is a component of Trandoshan
// Process is a component of Bathyscaphe
type Process interface {
Name() string
Features() []Feature
@ -148,9 +148,9 @@ type Process interface {
// MakeApp return cli.App corresponding for given Process
func MakeApp(process Process) *cli.App {
app := &cli.App{
Name: fmt.Sprintf("tdsh-%s", process.Name()),
Name: fmt.Sprintf("bs-%s", process.Name()),
Version: version,
Usage: fmt.Sprintf("Trandoshan %s component", process.Name()),
Usage: fmt.Sprintf("Bathyscaphe %s component", process.Name()),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "log-level",
@ -158,6 +158,12 @@ func MakeApp(process Process) *cli.App {
Value: "info",
},
},
Authors: []*cli.Author{
{
Name: "Aloïs Micard",
Email: "alois@micard.lu",
},
},
Action: execute(process),
}

@ -4,11 +4,11 @@ import (
"errors"
"fmt"
"github.com/PuerkitoBio/purell"
"github.com/creekorful/trandoshan/internal/cache"
configapi "github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/constraint"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/bathyscaphe/internal/cache"
configapi "github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/constraint"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"hash/fnv"
@ -92,7 +92,7 @@ func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg even
}
// We are working using URL hash to reduce memory consumption.
// See: https://github.com/creekorful/trandoshan/issues/130
// See: https://github.com/creekorful/bathyscaphe/issues/130
var urlHashes []string
for _, u := range urls {
c := fnv.New64()

@ -2,15 +2,15 @@ package scheduler
import (
"errors"
"github.com/creekorful/trandoshan/internal/cache"
"github.com/creekorful/trandoshan/internal/cache_mock"
"github.com/creekorful/trandoshan/internal/configapi/client"
"github.com/creekorful/trandoshan/internal/configapi/client_mock"
"github.com/creekorful/trandoshan/internal/event"
"github.com/creekorful/trandoshan/internal/event_mock"
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/process_mock"
"github.com/creekorful/trandoshan/internal/test"
"github.com/creekorful/bathyscaphe/internal/cache"
"github.com/creekorful/bathyscaphe/internal/cache_mock"
"github.com/creekorful/bathyscaphe/internal/configapi/client"
"github.com/creekorful/bathyscaphe/internal/configapi/client_mock"
"github.com/creekorful/bathyscaphe/internal/event"
"github.com/creekorful/bathyscaphe/internal/event_mock"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/process_mock"
"github.com/creekorful/bathyscaphe/internal/test"
"github.com/golang/mock/gomock"
"hash/fnv"
"strconv"

@ -1,8 +1,8 @@
package test
import (
"github.com/creekorful/trandoshan/internal/process"
"github.com/creekorful/trandoshan/internal/process_mock"
"github.com/creekorful/bathyscaphe/internal/process"
"github.com/creekorful/bathyscaphe/internal/process_mock"
"github.com/golang/mock/gomock"
"reflect"
"testing"

@ -9,5 +9,5 @@ fi
# build docker images
for path in build/docker/Dockerfile.*; do
name=$(echo "$path" | cut -d'.' -f2)
docker build . -f "$path" -t "creekorful/$name:$tag"
docker build . -f "$path" -t "creekorful/bs-$name:$tag"
done

Loading…
Cancel
Save