Initial commit
parent
66481a3f55
commit
ce88e57726
@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: blacklister-deployment
|
||||||
|
labels:
|
||||||
|
app: blacklister
|
||||||
|
spec:
|
||||||
|
replicas: 5
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: blacklister
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: blacklister
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: blacklister
|
||||||
|
image: creekorful/bs-blacklister:latest
|
||||||
|
args:
|
||||||
|
- --log-level
|
||||||
|
- debug
|
||||||
|
- --event-srv
|
||||||
|
- amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
|
||||||
|
- --config-api
|
||||||
|
- http://configapi-service
|
||||||
|
- --cache-srv
|
||||||
|
- redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
|
||||||
|
- --tor-proxy
|
||||||
|
- torproxy-service:9050
|
@ -0,0 +1,47 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: configapi-deployment
|
||||||
|
labels:
|
||||||
|
app: configapi
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: configapi
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: configapi
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: configapi
|
||||||
|
image: creekorful/bs-configapi:latest
|
||||||
|
args:
|
||||||
|
- --log-level
|
||||||
|
- debug
|
||||||
|
- --event-srv
|
||||||
|
- amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
|
||||||
|
- --cache-srv
|
||||||
|
- redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
|
||||||
|
- --default-value
|
||||||
|
- forbidden-hostnames=[]
|
||||||
|
- --default-value
|
||||||
|
- allowed-mime-types=[{"content-type":"text","extensions":["html","php","aspx", "htm"]}]
|
||||||
|
- --default-value
|
||||||
|
- refresh-delay={"delay":0}
|
||||||
|
- --default-value
|
||||||
|
- blacklist-config={"threshold":5, "ttl":1200}
|
||||||
|
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: configapi-service
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: configapi
|
||||||
|
ports:
|
||||||
|
- protocol: TCP
|
||||||
|
port: 80
|
||||||
|
targetPort: 8080
|
@ -0,0 +1,28 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: crawler-deployment
|
||||||
|
labels:
|
||||||
|
app: crawler
|
||||||
|
spec:
|
||||||
|
replicas: 20
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: crawler
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: crawler
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: crawler
|
||||||
|
image: creekorful/bs-crawler:latest
|
||||||
|
args:
|
||||||
|
- --log-level
|
||||||
|
- debug
|
||||||
|
- --event-srv
|
||||||
|
- amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
|
||||||
|
- --tor-proxy
|
||||||
|
- torproxy-service:9050
|
||||||
|
- --config-api
|
||||||
|
- http://configapi-service
|
@ -0,0 +1,3 @@
|
|||||||
|
auth:
|
||||||
|
username: bathyscaphe
|
||||||
|
password: nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC
|
@ -0,0 +1,32 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: indexer-es-deployment
|
||||||
|
labels:
|
||||||
|
app: indexer-es
|
||||||
|
spec:
|
||||||
|
replicas: 2
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: indexer-es
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: indexer-es
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: indexer
|
||||||
|
image: creekorful/bs-indexer:latest
|
||||||
|
args:
|
||||||
|
- --log-level
|
||||||
|
- debug
|
||||||
|
- --event-srv
|
||||||
|
- amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
|
||||||
|
- --event-prefetch
|
||||||
|
- '20'
|
||||||
|
- --config-api
|
||||||
|
- http://configapi-service
|
||||||
|
- --index-driver
|
||||||
|
- elastic
|
||||||
|
- --index-dest
|
||||||
|
- http://elasticsearch-master:9200
|
@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: scheduler-deployment
|
||||||
|
labels:
|
||||||
|
app: scheduler
|
||||||
|
spec:
|
||||||
|
replicas: 2
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: scheduler
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: scheduler
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: scheduler
|
||||||
|
image: creekorful/bs-scheduler:latest
|
||||||
|
args:
|
||||||
|
- --log-level
|
||||||
|
- debug
|
||||||
|
- --event-srv
|
||||||
|
- amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
|
||||||
|
- --event-prefetch
|
||||||
|
- '20'
|
||||||
|
- --config-api
|
||||||
|
- http://configapi-service
|
||||||
|
- --cache-srv
|
||||||
|
- redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
|
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
./scripts/exec.sh down "$@"
|
./scripts/docker/exec.sh down "$@"
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
./scripts/exec.sh logs "$@"
|
./scripts/docker/exec.sh logs "$@"
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
./scripts/exec.sh ps "$@"
|
./scripts/docker/exec.sh ps "$@"
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
./scripts/exec.sh scale "$@"
|
./scripts/docker/exec.sh scale "$@"
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
./scripts/exec.sh up "$@"
|
./scripts/docker/exec.sh up "$@"
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
./scripts/exec.sh stop "$@"
|
./scripts/docker/exec.sh stop "$@"
|
||||||
|
@ -1,17 +1,19 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# Configure bitnami repository (requires for production ready charts)
|
|
||||||
helm repo add bitnami https://charts.bitnami.com/bitnami
|
helm repo add bitnami https://charts.bitnami.com/bitnami
|
||||||
|
helm repo add elastic https://helm.elastic.co
|
||||||
|
|
||||||
# Create the namespace
|
|
||||||
kubectl create namespace bathyscaphe
|
kubectl create namespace bathyscaphe
|
||||||
|
|
||||||
# Install Redis
|
|
||||||
helm install --namespace bathyscaphe redis -f deployments/k8s/helm/redis-values.yaml bitnami/redis
|
helm install --namespace bathyscaphe redis -f deployments/k8s/helm/redis-values.yaml bitnami/redis
|
||||||
|
helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq
|
||||||
# Install RabbitMQ
|
helm install --namespace bathyscaphe elasticsearch elastic/elasticsearch
|
||||||
# helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq
|
helm install --namespace bathyscaphe kibana elastic/kibana
|
||||||
|
|
||||||
# Install our resources
|
# Install our resources
|
||||||
kubectl -n bathyscaphe apply -f deployments/k8s/torproxy.yaml
|
kubectl -n bathyscaphe apply -f deployments/k8s/torproxy.yaml
|
||||||
# kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml
|
kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml
|
||||||
|
kubectl -n bathyscaphe apply -f deployments/k8s/crawler.yaml
|
||||||
|
kubectl -n bathyscaphe apply -f deployments/k8s/scheduler.yaml
|
||||||
|
kubectl -n bathyscaphe apply -f deployments/k8s/blacklister.yaml
|
||||||
|
kubectl -n bathyscaphe apply -f deployments/k8s/indexer-es.yaml
|
||||||
|
Loading…
Reference in New Issue