From ce88e57726f9668ec738af77bf91448ee00118db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alo=C3=AFs=20Micard?= Date: Tue, 12 Jan 2021 23:59:32 +0100 Subject: [PATCH] Initial commit --- deployments/k8s/blacklister.yaml | 30 +++++++++++++++ deployments/k8s/configapi.yaml | 47 +++++++++++++++++++++++ deployments/k8s/crawler.yaml | 28 ++++++++++++++ deployments/k8s/helm/rabbitmq-values.yaml | 3 ++ deployments/k8s/indexer-es.yaml | 32 +++++++++++++++ deployments/k8s/scheduler.yaml | 30 +++++++++++++++ scripts/docker/delete.sh | 2 +- scripts/docker/log.sh | 2 +- scripts/docker/ps.sh | 2 +- scripts/docker/scale.sh | 2 +- scripts/docker/start.sh | 2 +- scripts/docker/stop.sh | 2 +- scripts/k8s/bootstrap.sh | 16 ++++---- 13 files changed, 185 insertions(+), 13 deletions(-) create mode 100644 deployments/k8s/blacklister.yaml create mode 100644 deployments/k8s/configapi.yaml create mode 100644 deployments/k8s/crawler.yaml create mode 100644 deployments/k8s/helm/rabbitmq-values.yaml create mode 100644 deployments/k8s/indexer-es.yaml create mode 100644 deployments/k8s/scheduler.yaml diff --git a/deployments/k8s/blacklister.yaml b/deployments/k8s/blacklister.yaml new file mode 100644 index 0000000..6c657c6 --- /dev/null +++ b/deployments/k8s/blacklister.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: blacklister-deployment + labels: + app: blacklister +spec: + replicas: 5 + selector: + matchLabels: + app: blacklister + template: + metadata: + labels: + app: blacklister + spec: + containers: + - name: blacklister + image: creekorful/bs-blacklister:latest + args: + - --log-level + - debug + - --event-srv + - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672 + - --config-api + - http://configapi-service + - --cache-srv + - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379 + - --tor-proxy + - torproxy-service:9050 \ No newline at end of file diff --git a/deployments/k8s/configapi.yaml b/deployments/k8s/configapi.yaml new file mode 100644 index 0000000..7790711 --- /dev/null +++ b/deployments/k8s/configapi.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: configapi-deployment + labels: + app: configapi +spec: + replicas: 1 + selector: + matchLabels: + app: configapi + template: + metadata: + labels: + app: configapi + spec: + containers: + - name: configapi + image: creekorful/bs-configapi:latest + args: + - --log-level + - debug + - --event-srv + - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672 + - --cache-srv + - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379 + - --default-value + - forbidden-hostnames=[] + - --default-value + - allowed-mime-types=[{"content-type":"text","extensions":["html","php","aspx", "htm"]}] + - --default-value + - refresh-delay={"delay":0} + - --default-value + - blacklist-config={"threshold":5, "ttl":1200} + +--- +apiVersion: v1 +kind: Service +metadata: + name: configapi-service +spec: + selector: + app: configapi + ports: + - protocol: TCP + port: 80 + targetPort: 8080 \ No newline at end of file diff --git a/deployments/k8s/crawler.yaml b/deployments/k8s/crawler.yaml new file mode 100644 index 0000000..403aed8 --- /dev/null +++ b/deployments/k8s/crawler.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: crawler-deployment + labels: + app: crawler +spec: + replicas: 20 + selector: + matchLabels: + app: crawler + template: + metadata: + labels: + app: crawler + spec: + containers: + - name: crawler + image: creekorful/bs-crawler:latest + args: + - --log-level + - debug + - --event-srv + - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672 + - --tor-proxy + - torproxy-service:9050 + - --config-api + - http://configapi-service \ No newline at end of file diff --git a/deployments/k8s/helm/rabbitmq-values.yaml b/deployments/k8s/helm/rabbitmq-values.yaml new file mode 100644 index 0000000..053158d --- /dev/null +++ b/deployments/k8s/helm/rabbitmq-values.yaml @@ -0,0 +1,3 @@ +auth: + username: bathyscaphe + password: nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC \ No newline at end of file diff --git a/deployments/k8s/indexer-es.yaml b/deployments/k8s/indexer-es.yaml new file mode 100644 index 0000000..de330d7 --- /dev/null +++ b/deployments/k8s/indexer-es.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-es-deployment + labels: + app: indexer-es +spec: + replicas: 2 + selector: + matchLabels: + app: indexer-es + template: + metadata: + labels: + app: indexer-es + spec: + containers: + - name: indexer + image: creekorful/bs-indexer:latest + args: + - --log-level + - debug + - --event-srv + - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672 + - --event-prefetch + - '20' + - --config-api + - http://configapi-service + - --index-driver + - elastic + - --index-dest + - http://elasticsearch-master:9200 \ No newline at end of file diff --git a/deployments/k8s/scheduler.yaml b/deployments/k8s/scheduler.yaml new file mode 100644 index 0000000..d518ed5 --- /dev/null +++ b/deployments/k8s/scheduler.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: scheduler-deployment + labels: + app: scheduler +spec: + replicas: 2 + selector: + matchLabels: + app: scheduler + template: + metadata: + labels: + app: scheduler + spec: + containers: + - name: scheduler + image: creekorful/bs-scheduler:latest + args: + - --log-level + - debug + - --event-srv + - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672 + - --event-prefetch + - '20' + - --config-api + - http://configapi-service + - --cache-srv + - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379 \ No newline at end of file diff --git a/scripts/docker/delete.sh b/scripts/docker/delete.sh index a78440b..140acb7 100755 --- a/scripts/docker/delete.sh +++ b/scripts/docker/delete.sh @@ -1,3 +1,3 @@ #!/bin/bash -./scripts/exec.sh down "$@" +./scripts/docker/exec.sh down "$@" diff --git a/scripts/docker/log.sh b/scripts/docker/log.sh index b8f1761..258b69c 100755 --- a/scripts/docker/log.sh +++ b/scripts/docker/log.sh @@ -1,3 +1,3 @@ #!/bin/bash -./scripts/exec.sh logs "$@" +./scripts/docker/exec.sh logs "$@" diff --git a/scripts/docker/ps.sh b/scripts/docker/ps.sh index 25420b9..0dbab97 100755 --- a/scripts/docker/ps.sh +++ b/scripts/docker/ps.sh @@ -1,3 +1,3 @@ #!/bin/bash -./scripts/exec.sh ps "$@" +./scripts/docker/exec.sh ps "$@" diff --git a/scripts/docker/scale.sh b/scripts/docker/scale.sh index 0f0f03d..a32f5ca 100755 --- a/scripts/docker/scale.sh +++ b/scripts/docker/scale.sh @@ -1,3 +1,3 @@ #!/bin/bash -./scripts/exec.sh scale "$@" +./scripts/docker/exec.sh scale "$@" diff --git a/scripts/docker/start.sh b/scripts/docker/start.sh index 22d0d15..693f490 100755 --- a/scripts/docker/start.sh +++ b/scripts/docker/start.sh @@ -1,3 +1,3 @@ #!/bin/bash -./scripts/exec.sh up "$@" +./scripts/docker/exec.sh up "$@" diff --git a/scripts/docker/stop.sh b/scripts/docker/stop.sh index 4c7d11f..9b5eedf 100755 --- a/scripts/docker/stop.sh +++ b/scripts/docker/stop.sh @@ -1,3 +1,3 @@ #!/bin/bash -./scripts/exec.sh stop "$@" +./scripts/docker/exec.sh stop "$@" diff --git a/scripts/k8s/bootstrap.sh b/scripts/k8s/bootstrap.sh index 367da6b..80cfb2a 100755 --- a/scripts/k8s/bootstrap.sh +++ b/scripts/k8s/bootstrap.sh @@ -1,17 +1,19 @@ #!/bin/bash -# Configure bitnami repository (requires for production ready charts) helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add elastic https://helm.elastic.co -# Create the namespace kubectl create namespace bathyscaphe -# Install Redis helm install --namespace bathyscaphe redis -f deployments/k8s/helm/redis-values.yaml bitnami/redis - -# Install RabbitMQ -# helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq +helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq +helm install --namespace bathyscaphe elasticsearch elastic/elasticsearch +helm install --namespace bathyscaphe kibana elastic/kibana # Install our resources kubectl -n bathyscaphe apply -f deployments/k8s/torproxy.yaml -# kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml +kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml +kubectl -n bathyscaphe apply -f deployments/k8s/crawler.yaml +kubectl -n bathyscaphe apply -f deployments/k8s/scheduler.yaml +kubectl -n bathyscaphe apply -f deployments/k8s/blacklister.yaml +kubectl -n bathyscaphe apply -f deployments/k8s/indexer-es.yaml