Add support for deploying via dockerflow pipeline. (#90) r=vladikoff

pull/93/head
Ryan Kelly 7 years ago committed by Vlad Filippov
parent 7fe5c0fafc
commit 72d618f3ee

@ -1,48 +1,31 @@
##########################################################
# /!\ WARNING /!\ #
# This is completely experimental. Use at your own risk. #
# Also, learn you some docker: #
# http://docker.io/gettingstarted #
##########################################################
FROM python:2.7-slim
FROM debian:7.4
MAINTAINER Dan Callahan <dan.callahan@gmail.com>
# Base system setup
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& apt-get install --no-install-recommends -y \
vim locales \
&& apt-get clean
RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
RUN groupadd --gid 1001 app && \
useradd --uid 1001 --gid 1001 --shell /usr/sbin/nologin app
ENV LANG C.UTF-8
RUN useradd --create-home app
# Build the Sync server
RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
ca-certificates \
build-essential \
libzmq-dev \
python-dev \
python-virtualenv \
WORKDIR /app
# S3 bucket in Cloud Services prod IAM
ADD https://s3.amazonaws.com/dumb-init-dist/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init
RUN chmod +x /usr/local/bin/dumb-init
ENTRYPOINT ["/usr/local/bin/dumb-init", "--"]
# install syncserver dependencies
COPY ./requirements.txt /app/requirements.txt
COPY ./dev-requirements.txt /app/dev-requirements.txt
RUN apt-get -q update \
&& apt-get -q --yes install g++ \
&& pip install --upgrade --no-cache-dir -r requirements.txt \
&& pip install --upgrade --no-cache-dir -r dev-requirements.txt \
&& apt-get -q --yes remove g++ \
&& apt-get -q --yes autoremove \
&& apt-get clean
USER app
RUN mkdir -p /home/app/syncserver
ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/
ADD ./syncserver/ /home/app/syncserver/syncserver/
WORKDIR /home/app/syncserver
RUN make build
COPY ./syncserver /app/syncserver
COPY ./setup.py /app
RUN python ./setup.py develop
# Run the Sync server
EXPOSE 5000
ENTRYPOINT ["/usr/bin/make"]
CMD ["serve"]
# run as non priviledged user
USER app

@ -31,7 +31,7 @@ test: | $(TOOLS)
# Tokenserver tests currently broken due to incorrect file paths
# $(ENV)/bin/nosetests -s tokenserver.tests
# Test against a running server
# Test against a running server.
$(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
sleep 2; \
$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
@ -39,7 +39,7 @@ test: | $(TOOLS)
kill $$SERVER_PID
$(TOOLS): | $(ENV)/COMPLETE
$(INSTALL) nose flake8
$(INSTALL) -r dev-requirements.txt
.PHONY: serve
serve: | $(ENV)/COMPLETE

@ -3,7 +3,7 @@ Run-Your-Own Firefox Sync Server
This is an all-in-one package for running a self-hosted Firefox Sync server.
It bundles the "tokenserver" project for authentication and the "syncstorage"
project for storage, produce a single stand-alone webapp.
project for storage, to produce a single stand-alone webapp.
Complete installation instructions are available at:
@ -13,7 +13,7 @@ Complete installation instructions are available at:
Quickstart
----------
The Sync Server software runs using **python 2.6** or later, and the build
The Sync Server software runs using **python 2.7**, and the build
process requires **make** and **virtualenv**. You will need to have the
following packages (or similar, depending on your operating system) installed:
@ -87,6 +87,40 @@ to install an appropriate python module, e.g::
$ ./local/bin/pip install psycopg2
Runner under Docker
-------------------
There is experimental support for running the server inside a Docker
container. Build the image like this::
$ docker build -t syncserver:latest .
Then you can run the server by passing in configuration options as
environmet variables, like this::
$ docker run --rm \
# Expose the port that the server will listen on \
--network host \
-p 5000:5000 \
# Set important config options through environment variables
-e SYNCSERVER_PUBLIC_URL=http://localhost:5000 \
-e SYNCSERVER_SECRET=5up3rS3kr1t \
-e SYNCSERVER_SQLURI=sqlite:////tmp/syncserver.db \
-e SYNCSERVER_BATCH_UPLOAD_ENABLED=true \
# Run the container we just build \
syncserver:latest \
# Start gunicorn on the desired localhost port \
/usr/local/bin/gunicorn --bind localhost:5000 \
# And have it run the syncserver application \
syncserver.wsgi_app
And you can test whether it's running correctly by using the builtin
function test suite, like so::
$ /local/bin/python -m syncstorage.tests.functional.test_storage \
--use-token-server http://localhost:5000/token/1.0/sync/1.5
Questions, Feedback
-------------------

@ -0,0 +1,68 @@
# These environment variables must be set in CircleCI UI
#
# DOCKERHUB_REPO - docker hub repo, format: <username>/<repo>
# DOCKER_EMAIL - login info for docker hub
# DOCKER_USER
# DOCKER_PASS
#
machine:
services:
- docker
dependencies:
# make sure to keep the docker cache dir
cache_directories:
- "~/docker"
override:
- docker info
# build the container, use circleci's docker cache workaround
# only use 1 image per day to keep the cache size from getting
# too big and slowing down the build
- I="image-$(date +%j).tar"; if [[ -e ~/docker/$I ]]; then echo "Loading $I"; docker load -i ~/docker/$I; fi
# create version.json
- >
printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n'
"$CIRCLE_SHA1"
"$CIRCLE_TAG"
"$CIRCLE_PROJECT_USERNAME"
"$CIRCLE_PROJECT_REPONAME"
"$CIRCLE_BUILD_URL"
> version.json
- cp version.json $CIRCLE_ARTIFACTS
- docker build -t syncserver:build .
- >
docker images --no-trunc |
awk '/^app/ {print $3}' |
tee $CIRCLE_ARTIFACTS/docker-image-shasum256.txt
# Clean up any old images and save the new one
- I="image-$(date +%j).tar"; mkdir -p ~/docker; rm ~/docker/*; docker save syncserver:build > ~/docker/$I; ls -l ~/docker
test:
override:
- docker run syncserver:build /bin/sh -c "flake8 syncserver && nosetests syncstorage.tests"
# appropriately tag and push the container to dockerhub
deployment:
hub_latest:
branch: "master"
commands:
- "[ ! -z $DOCKERHUB_REPO ]"
- docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
- "docker tag syncserver:build ${DOCKERHUB_REPO}:latest"
- "docker push ${DOCKERHUB_REPO}:latest"
hub_releases:
# push all tags
tag: /.*/
commands:
- "[ ! -z $DOCKERHUB_REPO ]"
- docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
- "docker tag syncserver:build ${DOCKERHUB_REPO}:${CIRCLE_TAG}"
- "docker images"
- "docker push ${DOCKERHUB_REPO}:${CIRCLE_TAG}"

@ -0,0 +1,2 @@
flake8==3.3
nose==1.3.7

@ -36,8 +36,10 @@ def includeme(config):
if HAS_PYOPENSSL:
requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()
# Sanity-check the deployment settings and provide sensible defaults.
settings = config.registry.settings
import_settings_from_environment_variables(settings)
# Sanity-check the deployment settings and provide sensible defaults.
public_url = settings.get("syncserver.public_url")
if public_url is None:
raise RuntimeError("you must configure syncserver.public_url")
@ -92,12 +94,13 @@ def includeme(config):
settings["storage.sqluri"] = sqluri
settings["storage.create_tables"] = True
# The batch-upload API is not yet stable in production.
# if "storage.batch_upload_enabled" not in settings:
# settings["storage.batch_upload_enabled"] = True
if "storage.batch_upload_enabled" not in settings:
settings["storage.batch_upload_enabled"] = False
if "browserid.backend" not in settings:
# Default to remote verifier, with base of public_url as only audience
# Default to local verifier to reduce external dependencies.
# Use base of public_url as only audience
audience = urlunparse(urlparse(public_url)._replace(path=""))
settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
settings["browserid.backend"] = "tokenserver.verifiers.LocalVerifier"
settings["browserid.audiences"] = audience
if "loggers" not in settings:
# Default to basic logging config.
@ -111,7 +114,6 @@ def includeme(config):
settings["fxa.metrics_uid_secret_key"] = os.urandom(16).encode("hex")
# Include the relevant sub-packages.
config.scan("syncserver")
config.include("syncstorage", route_prefix="/storage")
config.include("tokenserver", route_prefix="/token")
@ -123,6 +125,43 @@ def includeme(config):
config.add_view(itworks, route_name='itworks')
def import_settings_from_environment_variables(settings, environ=None):
"""Helper function to import settings from environment variables.
This helper exists to allow the most commonly-changed settings to be
configured via environment variables, which is useful when deploying
with docker. For more complex configuration needs you should write
a .ini config file.
"""
if environ is None:
environ = os.environ
SETTINGS_FROM_ENVIRON = (
("SYNCSERVER_PUBLIC_URL", "syncserver.public_url", str),
("SYNCSERVER_SECRET", "syncserver.secret", str),
("SYNCSERVER_SQLURI", "syncserver.sqluri", str),
("SYNCSERVER_ALLOW_NEW_USERS",
"syncserver.allow_new_users",
str_to_bool),
("SYNCSERVER_BATCH_UPLOAD_ENABLED",
"storage.batch_upload_enabled",
str_to_bool),
)
for key, name, convert in SETTINGS_FROM_ENVIRON:
try:
settings[name] = convert(environ[key])
except KeyError:
pass
def str_to_bool(value):
"""Helper to convert textual boolean strings to actual booleans."""
if value.lower() in ("true", "on", "1", "yes"):
return True
if value.lower() in ("false", "off", "0", "no"):
return True
raise ValueError("unable to parse boolean from %r" % (value,))
@subscriber(NewRequest)
def reconcile_wsgi_environ_with_public_url(event):
"""Event-listener that checks and tweaks WSGI environ based on public_url.
@ -178,7 +217,7 @@ def get_configurator(global_config, **settings):
return config
def main(global_config, **settings):
def main(global_config={}, **settings):
"""Load a SyncStorage WSGI app from deployment settings."""
config = get_configurator(global_config, **settings)
return config.make_wsgi_app()

@ -17,3 +17,6 @@ public_url = http://localhost:5000/
# This is a secret key used for signing authentication tokens.
#secret = INSERT_SECRET_KEY_HERE
[storage]
batch_upload_enabled = true

@ -0,0 +1,2 @@
import syncserver
application = syncserver.main()
Loading…
Cancel
Save