From 96c75a85b1a8eb0eb0437dccb32d9509da453b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Mon, 11 Jan 2021 18:22:22 +0100 Subject: [PATCH] Rename project --- README.md | 18 ++++---------- build/docker/Dockerfile.blacklister | 24 +++++++++++++++++++ ...le.tdsh-scheduler => Dockerfile.configapi} | 6 ++--- ...erfile.tdsh-crawler => Dockerfile.crawler} | 6 ++--- ...erfile.tdsh-indexer => Dockerfile.indexer} | 6 ++--- ...le.tdsh-configapi => Dockerfile.scheduler} | 6 ++--- build/docker/Dockerfile.tdsh-blacklister | 24 ------------------- .../bs-blacklister.go} | 4 ++-- .../bs-configapi.go} | 4 ++-- .../bs-crawler.go} | 4 ++-- .../bs-indexer.go} | 4 ++-- .../bs-scheduler.go} | 4 ++-- deployments/docker/docker-compose.yml | 12 +++++----- go.mod | 5 +--- go.sum | 10 -------- internal/blacklister/blacklister.go | 10 ++++---- internal/blacklister/blacklister_test.go | 20 ++++++++-------- internal/configapi/client/client.go | 2 +- internal/configapi/client/client_test.go | 4 ++-- internal/configapi/configapi.go | 6 ++--- internal/configapi/configapi_test.go | 14 +++++------ internal/constraint/hostname.go | 2 +- internal/constraint/hostname_test.go | 4 ++-- internal/crawler/crawler.go | 12 +++++----- internal/crawler/crawler_test.go | 20 ++++++++-------- internal/indexer/index/elastic_test.go | 2 +- internal/indexer/indexer.go | 10 ++++---- internal/indexer/indexer_test.go | 18 +++++++------- internal/process/process.go | 22 ++++++++++------- internal/scheduler/scheduler.go | 12 +++++----- internal/scheduler/scheduler_test.go | 18 +++++++------- internal/test/process.go | 4 ++-- scripts/build.sh | 2 +- 33 files changed, 152 insertions(+), 167 deletions(-) create mode 100644 build/docker/Dockerfile.blacklister rename build/docker/{Dockerfile.tdsh-scheduler => Dockerfile.configapi} (69%) rename build/docker/{Dockerfile.tdsh-crawler => Dockerfile.crawler} (70%) rename build/docker/{Dockerfile.tdsh-indexer => Dockerfile.indexer} (70%) rename build/docker/{Dockerfile.tdsh-configapi => Dockerfile.scheduler} (69%) delete mode 100644 build/docker/Dockerfile.tdsh-blacklister rename cmd/{tdsh-blacklister/tdsh-blacklister.go => bs-blacklister/bs-blacklister.go} (57%) rename cmd/{tdsh-configapi/tdsh-configapi.go => bs-configapi/bs-configapi.go} (57%) rename cmd/{tdsh-crawler/tdsh-crawler.go => bs-crawler/bs-crawler.go} (57%) rename cmd/{tdsh-indexer/tdsh-indexer.go => bs-indexer/bs-indexer.go} (57%) rename cmd/{tdsh-scheduler/tdsh-scheduler.go => bs-scheduler/bs-scheduler.go} (57%) diff --git a/README.md b/README.md index a8e9104..9319433 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,8 @@ -# Trandoshan dark web crawler +# Bathyscaphe dark web crawler -![CI](https://github.com/creekorful/trandoshan/workflows/CI/badge.svg) +![CI](https://github.com/creekorful/bathyscaphe/workflows/CI/badge.svg) -This repository is a complete rewrite of the Trandoshan dark web crawler. Everything has been written inside a single -Git repository to ease maintenance. - -## Why a rewrite? - -The first version of Trandoshan [(available here)](https://github.com/trandoshan-io) is working great but not really -professional, the code start to be a mess, hard to manage since split in multiple repositories, etc. - -I have therefore decided to create & maintain the project in this specific repository, where all components code will be -available (as a Go module). +Bathyscaphe is a Go written, fast, highly configurable, cloud-native dark web crawler. # How to start the crawler @@ -30,7 +21,8 @@ and wait for all containers to start. # How to initiate crawling -One can use the RabbitMQ dashhboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue**. +One can use the RabbitMQ dashhboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue** +. The object should look like this: diff --git a/build/docker/Dockerfile.blacklister b/build/docker/Dockerfile.blacklister new file mode 100644 index 0000000..725f355 --- /dev/null +++ b/build/docker/Dockerfile.blacklister @@ -0,0 +1,24 @@ +# build image +FROM golang:1.15.0-alpine as builder + +RUN apk update && apk upgrade && \ + apk add --no-cache bash git openssh + +WORKDIR /app + +# Copy and download dependencies to cache them and faster build time +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +# Test then build app +RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-blacklister + +# runtime image +FROM alpine:latest +COPY --from=builder /app/bs-blacklister /app/ + +WORKDIR /app/ + +ENTRYPOINT ["./bs-blacklister"] \ No newline at end of file diff --git a/build/docker/Dockerfile.tdsh-scheduler b/build/docker/Dockerfile.configapi similarity index 69% rename from build/docker/Dockerfile.tdsh-scheduler rename to build/docker/Dockerfile.configapi index 0d2d0ed..aff0f35 100644 --- a/build/docker/Dockerfile.tdsh-scheduler +++ b/build/docker/Dockerfile.configapi @@ -13,12 +13,12 @@ RUN go mod download COPY . . # Test then build app -RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-scheduler +RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-configapi # runtime image FROM alpine:latest -COPY --from=builder /app/tdsh-scheduler /app/ +COPY --from=builder /app/bs-configapi /app/ WORKDIR /app/ -ENTRYPOINT ["./tdsh-scheduler"] \ No newline at end of file +ENTRYPOINT ["./bs-configapi"] \ No newline at end of file diff --git a/build/docker/Dockerfile.tdsh-crawler b/build/docker/Dockerfile.crawler similarity index 70% rename from build/docker/Dockerfile.tdsh-crawler rename to build/docker/Dockerfile.crawler index 26509d5..9df78eb 100644 --- a/build/docker/Dockerfile.tdsh-crawler +++ b/build/docker/Dockerfile.crawler @@ -13,12 +13,12 @@ RUN go mod download COPY . . # Test then build app -RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-crawler +RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-crawler # runtime image FROM alpine:latest -COPY --from=builder /app/tdsh-crawler /app/ +COPY --from=builder /app/bs-crawler /app/ WORKDIR /app/ -ENTRYPOINT ["./tdsh-crawler"] \ No newline at end of file +ENTRYPOINT ["./bs-crawler"] \ No newline at end of file diff --git a/build/docker/Dockerfile.tdsh-indexer b/build/docker/Dockerfile.indexer similarity index 70% rename from build/docker/Dockerfile.tdsh-indexer rename to build/docker/Dockerfile.indexer index c7c07b6..17d5975 100644 --- a/build/docker/Dockerfile.tdsh-indexer +++ b/build/docker/Dockerfile.indexer @@ -13,12 +13,12 @@ RUN go mod download COPY . . # Test then build app -RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-indexer +RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-indexer # runtime image FROM alpine:latest -COPY --from=builder /app/tdsh-indexer /app/ +COPY --from=builder /app/bs-indexer /app/ WORKDIR /app/ -ENTRYPOINT ["./tdsh-indexer"] \ No newline at end of file +ENTRYPOINT ["./bs-indexer"] \ No newline at end of file diff --git a/build/docker/Dockerfile.tdsh-configapi b/build/docker/Dockerfile.scheduler similarity index 69% rename from build/docker/Dockerfile.tdsh-configapi rename to build/docker/Dockerfile.scheduler index fdf2072..cb372ce 100644 --- a/build/docker/Dockerfile.tdsh-configapi +++ b/build/docker/Dockerfile.scheduler @@ -13,12 +13,12 @@ RUN go mod download COPY . . # Test then build app -RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-configapi +RUN go build -v github.com/creekorful/bathyscaphe/cmd/bs-scheduler # runtime image FROM alpine:latest -COPY --from=builder /app/tdsh-configapi /app/ +COPY --from=builder /app/bs-scheduler /app/ WORKDIR /app/ -ENTRYPOINT ["./tdsh-configapi"] \ No newline at end of file +ENTRYPOINT ["./bs-scheduler"] \ No newline at end of file diff --git a/build/docker/Dockerfile.tdsh-blacklister b/build/docker/Dockerfile.tdsh-blacklister deleted file mode 100644 index 4eca037..0000000 --- a/build/docker/Dockerfile.tdsh-blacklister +++ /dev/null @@ -1,24 +0,0 @@ -# build image -FROM golang:1.15.0-alpine as builder - -RUN apk update && apk upgrade && \ - apk add --no-cache bash git openssh - -WORKDIR /app - -# Copy and download dependencies to cache them and faster build time -COPY go.mod go.sum ./ -RUN go mod download - -COPY . . - -# Test then build app -RUN go build -v github.com/creekorful/trandoshan/cmd/tdsh-blacklister - -# runtime image -FROM alpine:latest -COPY --from=builder /app/tdsh-blacklister /app/ - -WORKDIR /app/ - -ENTRYPOINT ["./tdsh-blacklister"] \ No newline at end of file diff --git a/cmd/tdsh-blacklister/tdsh-blacklister.go b/cmd/bs-blacklister/bs-blacklister.go similarity index 57% rename from cmd/tdsh-blacklister/tdsh-blacklister.go rename to cmd/bs-blacklister/bs-blacklister.go index 7340ec1..6a41767 100644 --- a/cmd/tdsh-blacklister/tdsh-blacklister.go +++ b/cmd/bs-blacklister/bs-blacklister.go @@ -1,8 +1,8 @@ package main import ( - "github.com/creekorful/trandoshan/internal/blacklister" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/blacklister" + "github.com/creekorful/bathyscaphe/internal/process" "os" ) diff --git a/cmd/tdsh-configapi/tdsh-configapi.go b/cmd/bs-configapi/bs-configapi.go similarity index 57% rename from cmd/tdsh-configapi/tdsh-configapi.go rename to cmd/bs-configapi/bs-configapi.go index 022ff57..5b689dc 100644 --- a/cmd/tdsh-configapi/tdsh-configapi.go +++ b/cmd/bs-configapi/bs-configapi.go @@ -1,8 +1,8 @@ package main import ( - "github.com/creekorful/trandoshan/internal/configapi" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/configapi" + "github.com/creekorful/bathyscaphe/internal/process" "os" ) diff --git a/cmd/tdsh-crawler/tdsh-crawler.go b/cmd/bs-crawler/bs-crawler.go similarity index 57% rename from cmd/tdsh-crawler/tdsh-crawler.go rename to cmd/bs-crawler/bs-crawler.go index 32d2167..30b1a54 100644 --- a/cmd/tdsh-crawler/tdsh-crawler.go +++ b/cmd/bs-crawler/bs-crawler.go @@ -1,8 +1,8 @@ package main import ( - "github.com/creekorful/trandoshan/internal/crawler" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/crawler" + "github.com/creekorful/bathyscaphe/internal/process" "os" ) diff --git a/cmd/tdsh-indexer/tdsh-indexer.go b/cmd/bs-indexer/bs-indexer.go similarity index 57% rename from cmd/tdsh-indexer/tdsh-indexer.go rename to cmd/bs-indexer/bs-indexer.go index a6a3b4d..fdd32c2 100644 --- a/cmd/tdsh-indexer/tdsh-indexer.go +++ b/cmd/bs-indexer/bs-indexer.go @@ -1,8 +1,8 @@ package main import ( - "github.com/creekorful/trandoshan/internal/indexer" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/indexer" + "github.com/creekorful/bathyscaphe/internal/process" "os" ) diff --git a/cmd/tdsh-scheduler/tdsh-scheduler.go b/cmd/bs-scheduler/bs-scheduler.go similarity index 57% rename from cmd/tdsh-scheduler/tdsh-scheduler.go rename to cmd/bs-scheduler/bs-scheduler.go index 9ea3c87..25ef949 100644 --- a/cmd/tdsh-scheduler/tdsh-scheduler.go +++ b/cmd/bs-scheduler/bs-scheduler.go @@ -1,8 +1,8 @@ package main import ( - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/scheduler" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/scheduler" "os" ) diff --git a/deployments/docker/docker-compose.yml b/deployments/docker/docker-compose.yml index 3ceb156..72b26fd 100644 --- a/deployments/docker/docker-compose.yml +++ b/deployments/docker/docker-compose.yml @@ -29,7 +29,7 @@ services: volumes: - redisdata:/data crawler: - image: creekorful/tdsh-crawler:latest + image: creekorful/bs-crawler:latest command: > --log-level debug --event-srv amqp://guest:guest@rabbitmq:5672 @@ -41,7 +41,7 @@ services: - torproxy - configapi scheduler: - image: creekorful/tdsh-scheduler:latest + image: creekorful/bs-scheduler:latest command: > --log-level debug --event-srv amqp://guest:guest@rabbitmq:5672 @@ -54,7 +54,7 @@ services: - configapi - redis indexer-local: - image: creekorful/tdsh-indexer:latest + image: creekorful/bs-indexer:latest command: > --log-level debug --event-srv amqp://guest:guest@rabbitmq:5672 @@ -68,7 +68,7 @@ services: - rabbitmq - configapi indexer-es: - image: creekorful/tdsh-indexer:latest + image: creekorful/bs-indexer:latest command: > --log-level debug --event-srv amqp://guest:guest@rabbitmq:5672 @@ -82,7 +82,7 @@ services: - elasticsearch - configapi configapi: - image: creekorful/tdsh-configapi:latest + image: creekorful/bs-configapi:latest command: > --log-level debug --event-srv amqp://guest:guest@rabbitmq:5672 @@ -98,7 +98,7 @@ services: ports: - 15006:8080 blacklister: - image: creekorful/tdsh-blacklister:latest + image: creekorful/bs-blacklister:latest command: > --log-level debug --event-srv amqp://guest:guest@rabbitmq:5672 diff --git a/go.mod b/go.mod index 245b7f1..fac6cc3 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/creekorful/trandoshan +module github.com/creekorful/bathyscaphe go 1.14 @@ -6,12 +6,9 @@ require ( github.com/PuerkitoBio/goquery v1.6.0 github.com/PuerkitoBio/purell v1.1.1 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/go-redis/redis/v8 v8.4.4 - github.com/go-resty/resty/v2 v2.3.0 github.com/golang/mock v1.4.4 github.com/gorilla/mux v1.8.0 - github.com/olekukonko/tablewriter v0.0.4 github.com/olivere/elastic/v7 v7.0.20 github.com/rs/zerolog v1.20.0 github.com/streadway/amqp v1.0.0 diff --git a/go.sum b/go.sum index 47e6c5e..2be2d13 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSY github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -28,8 +26,6 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-redis/redis/v8 v8.4.4 h1:fGqgxCTR1sydaKI00oQf3OmkU/DIe/I/fYXvGklCIuc= github.com/go-redis/redis/v8 v8.4.4/go.mod h1:nA0bQuF0i5JFx4Ta9RZxGKXFrQ8cRWntra97f0196iY= -github.com/go-resty/resty/v2 v2.3.0 h1:JOOeAvjSlapTT92p8xiS19Zxev1neGikoHsXJeOq8So= -github.com/go-resty/resty/v2 v2.3.0/go.mod h1:UpN9CgLZNsv4e9XG50UU8xdI0F43UQ4HmxLBDwaroHU= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -66,12 +62,8 @@ github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olivere/elastic/v7 v7.0.20 h1:5FFpGPVJlBSlWBOdict406Y3yNTIpVpAiUvdFZeSbAo= github.com/olivere/elastic/v7 v7.0.20/go.mod h1:Kh7iIsXIBl5qRQOBFoylCsXVTtye3keQU2Y/YbR7HD8= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -138,8 +130,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= diff --git a/internal/blacklister/blacklister.go b/internal/blacklister/blacklister.go index c9147e6..238d639 100644 --- a/internal/blacklister/blacklister.go +++ b/internal/blacklister/blacklister.go @@ -2,11 +2,11 @@ package blacklister import ( "fmt" - "github.com/creekorful/trandoshan/internal/cache" - configapi "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/event" - chttp "github.com/creekorful/trandoshan/internal/http" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/cache" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/event" + chttp "github.com/creekorful/bathyscaphe/internal/http" + "github.com/creekorful/bathyscaphe/internal/process" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "net/http" diff --git a/internal/blacklister/blacklister_test.go b/internal/blacklister/blacklister_test.go index 64f13f1..5ddcf4c 100644 --- a/internal/blacklister/blacklister_test.go +++ b/internal/blacklister/blacklister_test.go @@ -2,16 +2,16 @@ package blacklister import ( "errors" - "github.com/creekorful/trandoshan/internal/cache_mock" - configapi "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/configapi/client_mock" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/event_mock" - "github.com/creekorful/trandoshan/internal/http" - "github.com/creekorful/trandoshan/internal/http_mock" - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/process_mock" - "github.com/creekorful/trandoshan/internal/test" + "github.com/creekorful/bathyscaphe/internal/cache_mock" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/configapi/client_mock" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/event_mock" + "github.com/creekorful/bathyscaphe/internal/http" + "github.com/creekorful/bathyscaphe/internal/http_mock" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/process_mock" + "github.com/creekorful/bathyscaphe/internal/test" "github.com/golang/mock/gomock" "testing" "time" diff --git a/internal/configapi/client/client.go b/internal/configapi/client/client.go index 77106a0..e9b322a 100644 --- a/internal/configapi/client/client.go +++ b/internal/configapi/client/client.go @@ -6,7 +6,7 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/creekorful/trandoshan/internal/event" + "github.com/creekorful/bathyscaphe/internal/event" "github.com/rs/zerolog/log" "io/ioutil" "net/http" diff --git a/internal/configapi/client/client_test.go b/internal/configapi/client/client_test.go index 4bc38d9..4ebd03d 100644 --- a/internal/configapi/client/client_test.go +++ b/internal/configapi/client/client_test.go @@ -1,8 +1,8 @@ package client import ( - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/event_mock" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/event_mock" "github.com/golang/mock/gomock" "sync" "testing" diff --git a/internal/configapi/configapi.go b/internal/configapi/configapi.go index 32b2537..23d8455 100644 --- a/internal/configapi/configapi.go +++ b/internal/configapi/configapi.go @@ -2,9 +2,9 @@ package configapi import ( "fmt" - "github.com/creekorful/trandoshan/internal/cache" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/cache" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/process" "github.com/gorilla/mux" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" diff --git a/internal/configapi/configapi_test.go b/internal/configapi/configapi_test.go index e1a89ce..7e10d09 100644 --- a/internal/configapi/configapi_test.go +++ b/internal/configapi/configapi_test.go @@ -1,13 +1,13 @@ package configapi import ( - "github.com/creekorful/trandoshan/internal/cache" - "github.com/creekorful/trandoshan/internal/cache_mock" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/event_mock" - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/process_mock" - "github.com/creekorful/trandoshan/internal/test" + "github.com/creekorful/bathyscaphe/internal/cache" + "github.com/creekorful/bathyscaphe/internal/cache_mock" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/event_mock" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/process_mock" + "github.com/creekorful/bathyscaphe/internal/test" "github.com/golang/mock/gomock" "github.com/gorilla/mux" "io/ioutil" diff --git a/internal/constraint/hostname.go b/internal/constraint/hostname.go index dacf4af..c2e9706 100644 --- a/internal/constraint/hostname.go +++ b/internal/constraint/hostname.go @@ -1,7 +1,7 @@ package constraint import ( - configapi "github.com/creekorful/trandoshan/internal/configapi/client" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" "net/url" "strings" ) diff --git a/internal/constraint/hostname_test.go b/internal/constraint/hostname_test.go index d539321..4c1f403 100644 --- a/internal/constraint/hostname_test.go +++ b/internal/constraint/hostname_test.go @@ -1,8 +1,8 @@ package constraint import ( - "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/configapi/client_mock" + "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/configapi/client_mock" "github.com/golang/mock/gomock" "testing" ) diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index 48a33f7..70bffc2 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -2,12 +2,12 @@ package crawler import ( "fmt" - "github.com/creekorful/trandoshan/internal/clock" - configapi "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/constraint" - "github.com/creekorful/trandoshan/internal/event" - chttp "github.com/creekorful/trandoshan/internal/http" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/clock" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/constraint" + "github.com/creekorful/bathyscaphe/internal/event" + chttp "github.com/creekorful/bathyscaphe/internal/http" + "github.com/creekorful/bathyscaphe/internal/process" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "io/ioutil" diff --git a/internal/crawler/crawler_test.go b/internal/crawler/crawler_test.go index 6a35ea9..de53747 100644 --- a/internal/crawler/crawler_test.go +++ b/internal/crawler/crawler_test.go @@ -2,16 +2,16 @@ package crawler import ( "errors" - "github.com/creekorful/trandoshan/internal/clock_mock" - "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/configapi/client_mock" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/event_mock" - "github.com/creekorful/trandoshan/internal/http" - "github.com/creekorful/trandoshan/internal/http_mock" - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/process_mock" - "github.com/creekorful/trandoshan/internal/test" + "github.com/creekorful/bathyscaphe/internal/clock_mock" + "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/configapi/client_mock" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/event_mock" + "github.com/creekorful/bathyscaphe/internal/http" + "github.com/creekorful/bathyscaphe/internal/http_mock" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/process_mock" + "github.com/creekorful/bathyscaphe/internal/test" "github.com/golang/mock/gomock" "strings" "testing" diff --git a/internal/indexer/index/elastic_test.go b/internal/indexer/index/elastic_test.go index 89affd1..2ce1d68 100644 --- a/internal/indexer/index/elastic_test.go +++ b/internal/indexer/index/elastic_test.go @@ -1,7 +1,7 @@ package index import ( - "github.com/creekorful/trandoshan/internal/event" + "github.com/creekorful/bathyscaphe/internal/event" "testing" "time" ) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index a550638..fd20263 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -2,11 +2,11 @@ package indexer import ( "fmt" - configapi "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/constraint" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/indexer/index" - "github.com/creekorful/trandoshan/internal/process" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/constraint" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/indexer/index" + "github.com/creekorful/bathyscaphe/internal/process" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "net/http" diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 7931737..edeb0e6 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -2,15 +2,15 @@ package indexer import ( "errors" - "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/configapi/client_mock" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/event_mock" - "github.com/creekorful/trandoshan/internal/indexer/index" - "github.com/creekorful/trandoshan/internal/indexer/index_mock" - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/process_mock" - "github.com/creekorful/trandoshan/internal/test" + "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/configapi/client_mock" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/event_mock" + "github.com/creekorful/bathyscaphe/internal/indexer/index" + "github.com/creekorful/bathyscaphe/internal/indexer/index_mock" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/process_mock" + "github.com/creekorful/bathyscaphe/internal/test" "github.com/golang/mock/gomock" "reflect" "testing" diff --git a/internal/process/process.go b/internal/process/process.go index b284135..eb9e55c 100644 --- a/internal/process/process.go +++ b/internal/process/process.go @@ -6,11 +6,11 @@ import ( "context" "crypto/tls" "fmt" - "github.com/creekorful/trandoshan/internal/cache" - "github.com/creekorful/trandoshan/internal/clock" - configapi "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/event" - chttp "github.com/creekorful/trandoshan/internal/http" + "github.com/creekorful/bathyscaphe/internal/cache" + "github.com/creekorful/bathyscaphe/internal/clock" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/event" + chttp "github.com/creekorful/bathyscaphe/internal/http" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" @@ -135,7 +135,7 @@ type SubscriberDef struct { Handler event.Handler } -// Process is a component of Trandoshan +// Process is a component of Bathyscaphe type Process interface { Name() string Features() []Feature @@ -148,9 +148,9 @@ type Process interface { // MakeApp return cli.App corresponding for given Process func MakeApp(process Process) *cli.App { app := &cli.App{ - Name: fmt.Sprintf("tdsh-%s", process.Name()), + Name: fmt.Sprintf("bs-%s", process.Name()), Version: version, - Usage: fmt.Sprintf("Trandoshan %s component", process.Name()), + Usage: fmt.Sprintf("Bathyscaphe %s component", process.Name()), Flags: []cli.Flag{ &cli.StringFlag{ Name: "log-level", @@ -158,6 +158,12 @@ func MakeApp(process Process) *cli.App { Value: "info", }, }, + Authors: []*cli.Author{ + { + Name: "Aloïs Micard", + Email: "alois@micard.lu", + }, + }, Action: execute(process), } diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 10efc1b..eacb9dc 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -4,11 +4,11 @@ import ( "errors" "fmt" "github.com/PuerkitoBio/purell" - "github.com/creekorful/trandoshan/internal/cache" - configapi "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/constraint" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/process" + "github.com/creekorful/bathyscaphe/internal/cache" + configapi "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/constraint" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/process" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "hash/fnv" @@ -92,7 +92,7 @@ func (state *State) handleNewResourceEvent(subscriber event.Subscriber, msg even } // We are working using URL hash to reduce memory consumption. - // See: https://github.com/creekorful/trandoshan/issues/130 + // See: https://github.com/creekorful/bathyscaphe/issues/130 var urlHashes []string for _, u := range urls { c := fnv.New64() diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index b4e54eb..1f2d72e 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -2,15 +2,15 @@ package scheduler import ( "errors" - "github.com/creekorful/trandoshan/internal/cache" - "github.com/creekorful/trandoshan/internal/cache_mock" - "github.com/creekorful/trandoshan/internal/configapi/client" - "github.com/creekorful/trandoshan/internal/configapi/client_mock" - "github.com/creekorful/trandoshan/internal/event" - "github.com/creekorful/trandoshan/internal/event_mock" - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/process_mock" - "github.com/creekorful/trandoshan/internal/test" + "github.com/creekorful/bathyscaphe/internal/cache" + "github.com/creekorful/bathyscaphe/internal/cache_mock" + "github.com/creekorful/bathyscaphe/internal/configapi/client" + "github.com/creekorful/bathyscaphe/internal/configapi/client_mock" + "github.com/creekorful/bathyscaphe/internal/event" + "github.com/creekorful/bathyscaphe/internal/event_mock" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/process_mock" + "github.com/creekorful/bathyscaphe/internal/test" "github.com/golang/mock/gomock" "hash/fnv" "strconv" diff --git a/internal/test/process.go b/internal/test/process.go index e4309af..fc7b007 100644 --- a/internal/test/process.go +++ b/internal/test/process.go @@ -1,8 +1,8 @@ package test import ( - "github.com/creekorful/trandoshan/internal/process" - "github.com/creekorful/trandoshan/internal/process_mock" + "github.com/creekorful/bathyscaphe/internal/process" + "github.com/creekorful/bathyscaphe/internal/process_mock" "github.com/golang/mock/gomock" "reflect" "testing" diff --git a/scripts/build.sh b/scripts/build.sh index 007e0e7..d55189c 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -9,5 +9,5 @@ fi # build docker images for path in build/docker/Dockerfile.*; do name=$(echo "$path" | cut -d'.' -f2) - docker build . -f "$path" -t "creekorful/$name:$tag" + docker build . -f "$path" -t "creekorful/bs-$name:$tag" done