Initial commit

deployments/k8s/blacklister.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: blacklister-deployment
+  labels:
+    app: blacklister
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: blacklister
+  template:
+    metadata:
+      labels:
+        app: blacklister
+    spec:
+      containers:
+        - name: blacklister
+          image: creekorful/bs-blacklister:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --config-api
+            - http://configapi-service
+            - --cache-srv
+            - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
+            - --tor-proxy
+            - torproxy-service:9050

deployments/k8s/configapi.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: configapi-deployment
+  labels:
+    app: configapi
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: configapi
+  template:
+    metadata:
+      labels:
+        app: configapi
+    spec:
+      containers:
+        - name: configapi
+          image: creekorful/bs-configapi:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --cache-srv
+            - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
+            - --default-value
+            - forbidden-hostnames=[]
+            - --default-value
+            - allowed-mime-types=[{"content-type":"text","extensions":["html","php","aspx", "htm"]}]
+            - --default-value
+            - refresh-delay={"delay":0}
+            - --default-value
+            - blacklist-config={"threshold":5, "ttl":1200}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: configapi-service
+spec:
+  selector:
+    app: configapi
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080

deployments/k8s/crawler.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawler-deployment
+  labels:
+    app: crawler
+spec:
+  replicas: 20
+  selector:
+    matchLabels:
+      app: crawler
+  template:
+    metadata:
+      labels:
+        app: crawler
+    spec:
+      containers:
+        - name: crawler
+          image: creekorful/bs-crawler:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --tor-proxy
+            - torproxy-service:9050
+            - --config-api
+            - http://configapi-service

deployments/k8s/helm/rabbitmq-values.yaml

@@ -0,0 +1,3 @@
+auth:
+  username: bathyscaphe
+  password: nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC

deployments/k8s/indexer-es.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: indexer-es-deployment
+  labels:
+    app: indexer-es
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: indexer-es
+  template:
+    metadata:
+      labels:
+        app: indexer-es
+    spec:
+      containers:
+        - name: indexer
+          image: creekorful/bs-indexer:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --event-prefetch
+            - '20'
+            - --config-api
+            - http://configapi-service
+            - --index-driver
+            - elastic
+            - --index-dest
+            - http://elasticsearch-master:9200

deployments/k8s/scheduler.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scheduler-deployment
+  labels:
+    app: scheduler
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: scheduler
+  template:
+    metadata:
+      labels:
+        app: scheduler
+    spec:
+      containers:
+        - name: scheduler
+          image: creekorful/bs-scheduler:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --event-prefetch
+            - '20'
+            - --config-api
+            - http://configapi-service
+            - --cache-srv
+            - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379

scripts/docker/delete.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./scripts/exec.sh down "$@"
+./scripts/docker/exec.sh down "$@"

scripts/docker/log.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./scripts/exec.sh logs "$@"
+./scripts/docker/exec.sh logs "$@"

scripts/docker/ps.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./scripts/exec.sh ps "$@"
+./scripts/docker/exec.sh ps "$@"

scripts/docker/scale.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./scripts/exec.sh scale "$@"
+./scripts/docker/exec.sh scale "$@"

scripts/docker/start.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./scripts/exec.sh up "$@"
+./scripts/docker/exec.sh up "$@"

scripts/docker/stop.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-./scripts/exec.sh stop "$@"
+./scripts/docker/exec.sh stop "$@"

scripts/k8s/bootstrap.sh

@@ -1,17 +1,19 @@
 #!/bin/bash
 
-# Configure bitnami repository (requires for production ready charts)
 helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo add elastic https://helm.elastic.co
 
-# Create the namespace
 kubectl create namespace bathyscaphe
 
-# Install Redis
 helm install --namespace bathyscaphe redis -f deployments/k8s/helm/redis-values.yaml bitnami/redis
-
-# Install RabbitMQ
-# helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq
+helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq
+helm install --namespace bathyscaphe elasticsearch elastic/elasticsearch
+helm install --namespace bathyscaphe kibana elastic/kibana
 
 # Install our resources
 kubectl -n bathyscaphe apply -f deployments/k8s/torproxy.yaml
-# kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/crawler.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/scheduler.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/blacklister.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/indexer-es.yaml