diff --git a/Dockerfile b/Dockerfile index b66c21f..574560a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,48 +1,31 @@ -########################################################## -# /!\ WARNING /!\ # -# This is completely experimental. Use at your own risk. # -# Also, learn you some docker: # -# http://docker.io/gettingstarted # -########################################################## +FROM python:2.7-slim -FROM debian:7.4 -MAINTAINER Dan Callahan - -# Base system setup - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install --no-install-recommends -y \ - vim locales \ - && apt-get clean - -RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale +RUN groupadd --gid 1001 app && \ + useradd --uid 1001 --gid 1001 --shell /usr/sbin/nologin app ENV LANG C.UTF-8 -RUN useradd --create-home app - -# Build the Sync server - -RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - build-essential \ - libzmq-dev \ - python-dev \ - python-virtualenv \ +WORKDIR /app + +# S3 bucket in Cloud Services prod IAM +ADD https://s3.amazonaws.com/dumb-init-dist/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init +RUN chmod +x /usr/local/bin/dumb-init +ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] + +# install syncserver dependencies +COPY ./requirements.txt /app/requirements.txt +COPY ./dev-requirements.txt /app/dev-requirements.txt +RUN apt-get -q update \ + && apt-get -q --yes install g++ \ + && pip install --upgrade --no-cache-dir -r requirements.txt \ + && pip install --upgrade --no-cache-dir -r dev-requirements.txt \ + && apt-get -q --yes remove g++ \ + && apt-get -q --yes autoremove \ && apt-get clean -USER app - -RUN mkdir -p /home/app/syncserver -ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/ -ADD ./syncserver/ /home/app/syncserver/syncserver/ -WORKDIR /home/app/syncserver - -RUN make build +COPY ./syncserver /app/syncserver +COPY ./setup.py /app +RUN python ./setup.py develop -# Run the Sync server - -EXPOSE 5000 - -ENTRYPOINT ["/usr/bin/make"] -CMD ["serve"] +# run as non priviledged user +USER app diff --git a/Makefile b/Makefile index 2efa163..604b013 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ test: | $(TOOLS) # Tokenserver tests currently broken due to incorrect file paths # $(ENV)/bin/nosetests -s tokenserver.tests - # Test against a running server + # Test against a running server. $(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \ sleep 2; \ $(ENV)/bin/python -m syncstorage.tests.functional.test_storage \ @@ -39,7 +39,7 @@ test: | $(TOOLS) kill $$SERVER_PID $(TOOLS): | $(ENV)/COMPLETE - $(INSTALL) nose flake8 + $(INSTALL) -r dev-requirements.txt .PHONY: serve serve: | $(ENV)/COMPLETE diff --git a/README.rst b/README.rst index ab0eb9b..c35e2f1 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ Run-Your-Own Firefox Sync Server This is an all-in-one package for running a self-hosted Firefox Sync server. It bundles the "tokenserver" project for authentication and the "syncstorage" -project for storage, produce a single stand-alone webapp. +project for storage, to produce a single stand-alone webapp. Complete installation instructions are available at: @@ -13,7 +13,7 @@ Complete installation instructions are available at: Quickstart ---------- -The Sync Server software runs using **python 2.6** or later, and the build +The Sync Server software runs using **python 2.7**, and the build process requires **make** and **virtualenv**. You will need to have the following packages (or similar, depending on your operating system) installed: @@ -87,6 +87,40 @@ to install an appropriate python module, e.g:: $ ./local/bin/pip install psycopg2 +Runner under Docker +------------------- + +There is experimental support for running the server inside a Docker +container. Build the image like this:: + + $ docker build -t syncserver:latest . + +Then you can run the server by passing in configuration options as +environmet variables, like this:: + + $ docker run --rm \ + # Expose the port that the server will listen on \ + --network host \ + -p 5000:5000 \ + # Set important config options through environment variables + -e SYNCSERVER_PUBLIC_URL=http://localhost:5000 \ + -e SYNCSERVER_SECRET=5up3rS3kr1t \ + -e SYNCSERVER_SQLURI=sqlite:////tmp/syncserver.db \ + -e SYNCSERVER_BATCH_UPLOAD_ENABLED=true \ + # Run the container we just build \ + syncserver:latest \ + # Start gunicorn on the desired localhost port \ + /usr/local/bin/gunicorn --bind localhost:5000 \ + # And have it run the syncserver application \ + syncserver.wsgi_app + +And you can test whether it's running correctly by using the builtin +function test suite, like so:: + + $ /local/bin/python -m syncstorage.tests.functional.test_storage \ + --use-token-server http://localhost:5000/token/1.0/sync/1.5 + + Questions, Feedback ------------------- diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..3c49176 --- /dev/null +++ b/circle.yml @@ -0,0 +1,68 @@ +# These environment variables must be set in CircleCI UI +# +# DOCKERHUB_REPO - docker hub repo, format: / +# DOCKER_EMAIL - login info for docker hub +# DOCKER_USER +# DOCKER_PASS +# +machine: + services: + - docker + +dependencies: + # make sure to keep the docker cache dir + cache_directories: + - "~/docker" + + override: + - docker info + + # build the container, use circleci's docker cache workaround + # only use 1 image per day to keep the cache size from getting + # too big and slowing down the build + - I="image-$(date +%j).tar"; if [[ -e ~/docker/$I ]]; then echo "Loading $I"; docker load -i ~/docker/$I; fi + + # create version.json + - > + printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' + "$CIRCLE_SHA1" + "$CIRCLE_TAG" + "$CIRCLE_PROJECT_USERNAME" + "$CIRCLE_PROJECT_REPONAME" + "$CIRCLE_BUILD_URL" + > version.json + - cp version.json $CIRCLE_ARTIFACTS + + - docker build -t syncserver:build . + + - > + docker images --no-trunc | + awk '/^app/ {print $3}' | + tee $CIRCLE_ARTIFACTS/docker-image-shasum256.txt + + # Clean up any old images and save the new one + - I="image-$(date +%j).tar"; mkdir -p ~/docker; rm ~/docker/*; docker save syncserver:build > ~/docker/$I; ls -l ~/docker + +test: + override: + - docker run syncserver:build /bin/sh -c "flake8 syncserver && nosetests syncstorage.tests" + +# appropriately tag and push the container to dockerhub +deployment: + hub_latest: + branch: "master" + commands: + - "[ ! -z $DOCKERHUB_REPO ]" + - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + - "docker tag syncserver:build ${DOCKERHUB_REPO}:latest" + - "docker push ${DOCKERHUB_REPO}:latest" + + hub_releases: + # push all tags + tag: /.*/ + commands: + - "[ ! -z $DOCKERHUB_REPO ]" + - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + - "docker tag syncserver:build ${DOCKERHUB_REPO}:${CIRCLE_TAG}" + - "docker images" + - "docker push ${DOCKERHUB_REPO}:${CIRCLE_TAG}" diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..1b1ad03 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,2 @@ +flake8==3.3 +nose==1.3.7 diff --git a/syncserver/__init__.py b/syncserver/__init__.py index 64ddc3a..1239867 100644 --- a/syncserver/__init__.py +++ b/syncserver/__init__.py @@ -36,8 +36,10 @@ def includeme(config): if HAS_PYOPENSSL: requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3() - # Sanity-check the deployment settings and provide sensible defaults. settings = config.registry.settings + import_settings_from_environment_variables(settings) + + # Sanity-check the deployment settings and provide sensible defaults. public_url = settings.get("syncserver.public_url") if public_url is None: raise RuntimeError("you must configure syncserver.public_url") @@ -92,12 +94,13 @@ def includeme(config): settings["storage.sqluri"] = sqluri settings["storage.create_tables"] = True # The batch-upload API is not yet stable in production. - # if "storage.batch_upload_enabled" not in settings: - # settings["storage.batch_upload_enabled"] = True + if "storage.batch_upload_enabled" not in settings: + settings["storage.batch_upload_enabled"] = False if "browserid.backend" not in settings: - # Default to remote verifier, with base of public_url as only audience + # Default to local verifier to reduce external dependencies. + # Use base of public_url as only audience audience = urlunparse(urlparse(public_url)._replace(path="")) - settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier" + settings["browserid.backend"] = "tokenserver.verifiers.LocalVerifier" settings["browserid.audiences"] = audience if "loggers" not in settings: # Default to basic logging config. @@ -111,7 +114,6 @@ def includeme(config): settings["fxa.metrics_uid_secret_key"] = os.urandom(16).encode("hex") # Include the relevant sub-packages. - config.scan("syncserver") config.include("syncstorage", route_prefix="/storage") config.include("tokenserver", route_prefix="/token") @@ -123,6 +125,43 @@ def includeme(config): config.add_view(itworks, route_name='itworks') +def import_settings_from_environment_variables(settings, environ=None): + """Helper function to import settings from environment variables. + + This helper exists to allow the most commonly-changed settings to be + configured via environment variables, which is useful when deploying + with docker. For more complex configuration needs you should write + a .ini config file. + """ + if environ is None: + environ = os.environ + SETTINGS_FROM_ENVIRON = ( + ("SYNCSERVER_PUBLIC_URL", "syncserver.public_url", str), + ("SYNCSERVER_SECRET", "syncserver.secret", str), + ("SYNCSERVER_SQLURI", "syncserver.sqluri", str), + ("SYNCSERVER_ALLOW_NEW_USERS", + "syncserver.allow_new_users", + str_to_bool), + ("SYNCSERVER_BATCH_UPLOAD_ENABLED", + "storage.batch_upload_enabled", + str_to_bool), + ) + for key, name, convert in SETTINGS_FROM_ENVIRON: + try: + settings[name] = convert(environ[key]) + except KeyError: + pass + + +def str_to_bool(value): + """Helper to convert textual boolean strings to actual booleans.""" + if value.lower() in ("true", "on", "1", "yes"): + return True + if value.lower() in ("false", "off", "0", "no"): + return True + raise ValueError("unable to parse boolean from %r" % (value,)) + + @subscriber(NewRequest) def reconcile_wsgi_environ_with_public_url(event): """Event-listener that checks and tweaks WSGI environ based on public_url. @@ -178,7 +217,7 @@ def get_configurator(global_config, **settings): return config -def main(global_config, **settings): +def main(global_config={}, **settings): """Load a SyncStorage WSGI app from deployment settings.""" config = get_configurator(global_config, **settings) return config.make_wsgi_app() diff --git a/syncserver/tests.ini b/syncserver/tests.ini index fc9ea4b..eb2c71d 100644 --- a/syncserver/tests.ini +++ b/syncserver/tests.ini @@ -17,3 +17,6 @@ public_url = http://localhost:5000/ # This is a secret key used for signing authentication tokens. #secret = INSERT_SECRET_KEY_HERE + +[storage] +batch_upload_enabled = true diff --git a/syncserver/wsgi_app.py b/syncserver/wsgi_app.py new file mode 100644 index 0000000..190ba34 --- /dev/null +++ b/syncserver/wsgi_app.py @@ -0,0 +1,2 @@ +import syncserver +application = syncserver.main()