From 8533fea555da8b1ff592aa27a98d24e739eb9407 Mon Sep 17 00:00:00 2001 From: Collin Green Date: Sun, 17 Jan 2016 21:31:08 -0800 Subject: [PATCH 1/7] add sslify (disabled by default) --- create_djeroku_project.py | 4 ++-- project/settings/common.py | 7 +++++++ project/settings/prod.py | 5 +++++ reqs/common.txt | 1 + reqs/dev.txt | 2 +- reqs/prod.txt | 2 +- 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/create_djeroku_project.py b/create_djeroku_project.py index 9514442..a4610a2 100644 --- a/create_djeroku_project.py +++ b/create_djeroku_project.py @@ -29,10 +29,10 @@ 'virtualenv_folder': 'venv', 'valid_project_name_regex': r'^[a-zA-Z]+[a-zA-Z0-9_-]*$', 'temp_project_path_format': '_djeroku_temp_project_%d', - 'django_pip_version': '"django>=1.8,<1.9"', + 'django_pip_version': '"django>=1.9,<1.10"', # point this to a local folder if you cloned djeroku locally 'djeroku_template_path': - 'https://github.com/djeroku/djeroku/archive/master.zip', + '/Users/collingreen/development/djeroku/v2/djeroku', 'dependencies': { 'pip': 'pip -V', 'virtualenv': 'virtualenv --version', diff --git a/project/settings/common.py b/project/settings/common.py index d2157a0..310018f 100644 --- a/project/settings/common.py +++ b/project/settings/common.py @@ -152,6 +152,9 @@ # MIDDLEWARE CONFIGURATION # See: https://docs.djangoproject.com/en/dev/ref/settings/#middleware-classes MIDDLEWARE_CLASSES = ( + # SSLify middleware (disabled by default) + 'sslify.middleware.SSLifyMiddleware', + # Default Django middleware. 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -239,6 +242,10 @@ setup_loader() # END CELERY CONFIGURATION +# SSLIFY +SSLIFY_DISABLE = environ.get('SSLIFY_DISABLE', True) +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +# END SSLIFY # WSGI CONFIGURATION # See: https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application diff --git a/project/settings/prod.py b/project/settings/prod.py index e42ad47..1e5a43c 100644 --- a/project/settings/prod.py +++ b/project/settings/prod.py @@ -114,3 +114,8 @@ # ADDITIONAL MIDDLEWARE MIDDLEWARE_CLASSES += () # END ADDITIONAL MIDDLEWARE + +# SSLIFY +# change to False to force all requests to use https +SSLIFY_DISABLE = environ.get('SSLIFY_DISABLE', False) +# END SSLIFY diff --git a/reqs/common.txt b/reqs/common.txt index 688af33..89f5483 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -4,3 +4,4 @@ redis==2.10.3 dj-static==0.0.6 django-celery==3.1.16 django-extensions==1.5.5 +django-sslify>=0.2.7 diff --git a/reqs/dev.txt b/reqs/dev.txt index 8771555..0f60c5f 100644 --- a/reqs/dev.txt +++ b/reqs/dev.txt @@ -1,5 +1,5 @@ -r common.txt -django-debug-toolbar==1.3.0 +django-debug-toolbar==1.4.0 pylint flake8 diff --git a/reqs/prod.txt b/reqs/prod.txt index 5733b8c..42b266e 100644 --- a/reqs/prod.txt +++ b/reqs/prod.txt @@ -1,5 +1,5 @@ -r common.txt -django-heroku-memcacheify==0.3 +django-heroku-memcacheify==1.0.0 django-heroku-redisify==0.2.1 dj-database-url==0.3.0 django-postgrespool==0.3.0 From c3dd91986e53ff084b90585be5bc28148395861e Mon Sep 17 00:00:00 2001 From: Collin Green Date: Sun, 17 Jan 2016 21:34:02 -0800 Subject: [PATCH 2/7] replace gunicorn with uwsgi gunicorn on heroku seems to end up with locked up workers a little too often - replacing with uwsgi as an experiment Closes #22 --- Procfile | 2 +- project/uwsgi.ini | 8 ++++++++ reqs/prod.txt | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 project/uwsgi.ini diff --git a/Procfile b/Procfile index 5cf18ae..ad515e3 100644 --- a/Procfile +++ b/Procfile @@ -1,4 +1,4 @@ -web: gunicorn project.wsgi:application +web: uwsgi uwsgi.ini scheduler: python manage.py celery worker -B -E --concurrency=2 --maxtasksperchild=1000 worker: python manage.py celery worker -E --concurrency=2 --maxtasksperchild=1000 multiworker: python manage.py celery worker -E --concurrency=2 --maxtasksperchild=1000 diff --git a/project/uwsgi.ini b/project/uwsgi.ini new file mode 100644 index 0000000..1f65d45 --- /dev/null +++ b/project/uwsgi.ini @@ -0,0 +1,8 @@ +[uwsgi] +http-socket = :$(PORT) +master = true +processes = 4 +die-on-term = true +module = wsgi:application +max-requests = 5000 +memory-report = true diff --git a/reqs/prod.txt b/reqs/prod.txt index 42b266e..3f6c469 100644 --- a/reqs/prod.txt +++ b/reqs/prod.txt @@ -4,8 +4,8 @@ django-heroku-redisify==0.2.1 dj-database-url==0.3.0 django-postgrespool==0.3.0 hiredis==0.2.0 -gevent==1.0.2 -gunicorn==19.3.0 newrelic==2.52.0.40 psycopg2==2.6 mandrill>=1.0.57,<2.0 +uwsgi==2.0.11.2 +werkzeug==0.11.3 From 7088d1889f7f2e0428158d0f0cff247ea3f0f7fc Mon Sep 17 00:00:00 2001 From: Collin Green Date: Sun, 17 Jan 2016 21:35:45 -0800 Subject: [PATCH 3/7] replace dj-static with whitenoise dj-static was not playing nicely with uwsgi, while whitenoise works out of the box and seems to be the current preference for serving static files from django on heroku Closes #23 --- project/wsgi.py | 13 ++++++++++--- reqs/common.txt | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/project/wsgi.py b/project/wsgi.py index 6005615..4ea2a81 100644 --- a/project/wsgi.py +++ b/project/wsgi.py @@ -24,9 +24,16 @@ """ import os from django.core.wsgi import get_wsgi_application -from dj_static import Cling +from whitenoise.django import DjangoWhiteNoise os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings.dev") -# wrap wsgi with dj-static cling middleware -application = Cling(get_wsgi_application()) +# This application object is used by any WSGI server configured to use this +# file. This includes Django's development server, if the WSGI_APPLICATION +# setting points here. + +application = DjangoWhiteNoise(get_wsgi_application()) + +# Apply WSGI middleware here. +# from helloworld.wsgi import HelloWorldApplication +# application = HelloWorldApplication(application) diff --git a/reqs/common.txt b/reqs/common.txt index 89f5483..7532bb1 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -1,7 +1,7 @@ Django>=1.8.0,<1.9 Fabric==1.10.1 redis==2.10.3 -dj-static==0.0.6 django-celery==3.1.16 django-extensions==1.5.5 django-sslify>=0.2.7 +whitenoise==2.0.6 From 65f002cd6319cc58f8e48427f2e3d25dfa2d4a33 Mon Sep 17 00:00:00 2001 From: Collin Green Date: Sun, 17 Jan 2016 21:38:24 -0800 Subject: [PATCH 4/7] remove newrelic monitoring --- fabfile.py | 1 - reqs/prod.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/fabfile.py b/fabfile.py index 19aa17a..802ad67 100644 --- a/fabfile.py +++ b/fabfile.py @@ -69,7 +69,6 @@ 'scheduler:standard', 'redistogo:nano', 'memcachier:dev', - 'newrelic:wayne', 'mandrill:starter', 'papertrail:choklad' ) diff --git a/reqs/prod.txt b/reqs/prod.txt index 3f6c469..baa657f 100644 --- a/reqs/prod.txt +++ b/reqs/prod.txt @@ -4,7 +4,6 @@ django-heroku-redisify==0.2.1 dj-database-url==0.3.0 django-postgrespool==0.3.0 hiredis==0.2.0 -newrelic==2.52.0.40 psycopg2==2.6 mandrill>=1.0.57,<2.0 uwsgi==2.0.11.2 From c4dd98c7dd5d6b46084c453556329b87ceebb7fa Mon Sep 17 00:00:00 2001 From: Collin Green Date: Mon, 18 Jan 2016 09:22:59 -0800 Subject: [PATCH 5/7] upgrade django to 1.9 --- reqs/common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reqs/common.txt b/reqs/common.txt index 7532bb1..43b11cb 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -1,7 +1,7 @@ -Django>=1.8.0,<1.9 Fabric==1.10.1 redis==2.10.3 django-celery==3.1.16 django-extensions==1.5.5 django-sslify>=0.2.7 +Django>=1.9.0,<1.10.0 whitenoise==2.0.6 From fbfcf03463e85cecbfeee6f550dee99188a623fc Mon Sep 17 00:00:00 2001 From: Collin Green Date: Mon, 18 Jan 2016 21:38:31 -0800 Subject: [PATCH 6/7] update requirements to latest versions --- reqs/common.txt | 10 +++++----- reqs/prod.txt | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/reqs/common.txt b/reqs/common.txt index 43b11cb..d5f7c18 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -1,7 +1,7 @@ -Fabric==1.10.1 -redis==2.10.3 -django-celery==3.1.16 -django-extensions==1.5.5 -django-sslify>=0.2.7 Django>=1.9.0,<1.10.0 +Fabric==1.10.2 +redis==2.10.5 whitenoise==2.0.6 +django-celery==3.1.17 +django-extensions==1.6.1 +django-sslify>=0.2.7 diff --git a/reqs/prod.txt b/reqs/prod.txt index baa657f..7dc193f 100644 --- a/reqs/prod.txt +++ b/reqs/prod.txt @@ -4,7 +4,7 @@ django-heroku-redisify==0.2.1 dj-database-url==0.3.0 django-postgrespool==0.3.0 hiredis==0.2.0 -psycopg2==2.6 +psycopg2==2.6.1 mandrill>=1.0.57,<2.0 uwsgi==2.0.11.2 werkzeug==0.11.3 From ba97bae31f5f3006502695d12efac485953e2f14 Mon Sep 17 00:00:00 2001 From: Collin Green Date: Tue, 19 Jan 2016 00:04:14 -0800 Subject: [PATCH 7/7] remove fabric, mandril - create djeroku.py - Replace fabfile with djeroku.py - Remove mandril (no longer offers free addon) - Stop calling syncdb - Update for python3 - Remove pipelines until the new version stabilizes on heroku Closes #15 Closes #24 --- README.md | 14 +- create_djeroku_project.py | 23 ++- deployment_example.md | 53 +++-- djeroku.py | 419 ++++++++++++++++++++++++++++++++++++++ fabfile.py | 316 ---------------------------- project/settings/prod.py | 12 +- reqs/common.txt | 1 - reqs/prod.txt | 1 - 8 files changed, 471 insertions(+), 368 deletions(-) create mode 100644 djeroku.py delete mode 100644 fabfile.py diff --git a/README.md b/README.md index 7cc792f..ad02ba9 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ applications, plus all the tools to easily deploy between them. ### Prerequisites -Djeroku requires pip, virtualenv, git, and fabric. Install them however makes +Djeroku requires pip, virtualenv, and git. Install them however makes sense on your system (brew, apt-get, etc). @@ -21,16 +21,16 @@ and leave you with some instructions and tools for what you can do next. #### Provisioning your Staging and Production heroku apps -Djeroku projects have a fabfile.py that provides some helpful management +Djeroku projects have a djeroku.py that provides some helpful management commands, including the automatic creating and setup of your staging and -production apps on heroku. You will need fabric installed to use them. +production apps on heroku. -`fab heroku_setup` +`python djeroku.py heroku_setup` -#### Other Fabric Commands -You can see all the commands available via the fabric file by running: -`fab --list` +#### Other Djeroku Commands +You can see all the commands available via djeroku.py by running: +`python djeroku.py --help` #### Creating a new app diff --git a/create_djeroku_project.py b/create_djeroku_project.py index a4610a2..1901915 100644 --- a/create_djeroku_project.py +++ b/create_djeroku_project.py @@ -32,7 +32,7 @@ 'django_pip_version': '"django>=1.9,<1.10"', # point this to a local folder if you cloned djeroku locally 'djeroku_template_path': - '/Users/collingreen/development/djeroku/v2/djeroku', + 'https://github.com/djeroku/djeroku/archive/master.zip', 'dependencies': { 'pip': 'pip -V', 'virtualenv': 'virtualenv --version', @@ -150,7 +150,7 @@ def _check_dependency(command): def run_django_setup(project_name): logging.info('running django setup commands') - venv(project_name, 'python %s/manage.py syncdb' % project_name) + venv(project_name, 'python %s/manage.py makemigrations' % project_name) venv(project_name, 'python %s/manage.py migrate' % project_name) venv( project_name, @@ -159,11 +159,11 @@ def run_django_setup(project_name): def print_welcome_message(project_name): - print """ + print(""" %(project_name)s project created successfully! Thanks for using djeroku! You now have an empty django project skeleton in -%(project_name)s/ ready for you to use. There is a new fabfile.py in your +%(project_name)s/ ready for you to use. There is a new djeroku.py in your project folder that contains helpful commands you will likely use while developing. @@ -173,27 +173,32 @@ def print_welcome_message(project_name): %(venv_command)s # Run the one-time setup script to create your heroku projects: - fab heroku_setup + python djeroku.py heroku_setup # Create a new app inside the project/apps folder: mkdir project/apps/newappname django-admin.py startapp newappname project/apps/newappname # Run the dev server to view your project in your browser (localhost:8000) - fab serve + python djeroku.py serve + + # Check out the other djeroku.py helper commands + python djeroku.py --help Remember, while developing, make sure you activate your virtualenvironment -first or you will get errors about django or other libraries not being found. +first or you will get errors about django or other libraries not being found +(djeroku.py does this for you automatically). When you add new libraries to your project, be sure to add them to reqs/common.txt so heroku correctly includes them in your builds. Please file any issues at https://github.com/collingreen/djeroku. -Happy coding!""" % dict( +Happy coding!""".format( project_name=project_name, venv_command=get_venv_command() - ) + )) + def main(): parser = argparse.ArgumentParser() diff --git a/deployment_example.md b/deployment_example.md index 58c8c8c..3614aa1 100644 --- a/deployment_example.md +++ b/deployment_example.md @@ -17,15 +17,14 @@ also need to sign up for a heroku account. ## Setting up the staging and production heroku applications -The fabric script in the top project directory has a useful function that +The djeroku.py script in the top project directory has a useful function that basically handles the entire heroku setup process for you, including creating -two (nearly) identical environments (staging and production), a bunch of free -addons for each, and a deployment pipeline from staging to production. You -should really have a look in the file to see what is happening under the hood. -You can easily edit the addons or environment variables you want set inside the -file. +two (nearly) identical environments (staging and production), setting everything +up, and provisioning a bunch of free addons for each. You should really have a +look in the file to see what is happening under the hood. You can easily edit +the addons or environment variables you want set inside the file. -`fab heroku_setup` +`python djeroku.py heroku_setup` There are several prompts along the way and it will warn you if something fails. The most frustrating part of the entire process (assuming it works) is that @@ -92,9 +91,9 @@ urlpatterns = [ It's time to run the project locally. First, navigate back up to the djeroku_site folder that holds manage.py -Run the djeroku fabric `serve` command -- this just runs the standard -django syncdb, migrate, collecstatic, and then runserver commands -`fab serve` +Run the djeroku `serve` command -- this just runs the standard +django migrate, collecstatic, and then runserver commands +`python djeroku.py serve` Test it by going to 127.0.0.1:8000 and confirming you can see the Djeroku test content we created above (or whatever view you routed from '/'). @@ -103,13 +102,13 @@ TODO: insert screenshot ### Commit it? -Look at all that amazing work we did. Better commit it. The `fab setup_heroku` -command earlier created a git repository for us, so all we need to do is add and -commit. +Look at all that amazing work we did. Better commit it. The +`python djeroku.py setup_heroku` command earlier created a git repository for +us, so all we need to do is add and commit. ~~~ git add . -git commit -m"Initial Djeroku Template Commit" +git commit -m"initial djeroku template commit" ~~~ @@ -119,10 +118,10 @@ uses git pushes for deployment, so you can just call `git push staging master`. You may also need to run database creation or migration scripts, and you may need to collect your static assets (`heroku run python manage.py --app yourapp-staging`). You can also -just use the djeroku fabric file again which wraps all of those up into one +just use the djeroku.py script again which wraps all of those up into one command: -`fab deploy_staging` +`python djeroku.py deploy staging` That's it! Heroku will churn away and crunch our commit into a working 'app slug' and turn our new staging app on. @@ -146,21 +145,15 @@ fail while DEBUG is false. Update your settings if this is happening to you. ## Deploying to Production -Now that we feel confident that the site is working in staging, we can take -advantage of the pipeline created by the heroku_setup script and simply promote -the slug from staging downstream to production. -`heroku pipeline:promote --app=djeroku-site-staging` +Now that we feel confident that the site is working in staging, we can deploy +to production in exactly the same way as before: +`git push production master` -You can also just call `fab promote_production` +You can also just call `python djeroku.py deploy production` Now the production site is up as well, at the production url, djeroku-site.herokuapp.com. -NOTE: you CAN push directly to production, just like staging: -`git push production master` -or -`fab deploy_production` - Now, if this was a public app, I would leave it up (and probably set up a real domain pointing to it, not just the .herokuapp.com one), but for now I put both @@ -188,16 +181,20 @@ my-development-folder |-- reqs (holds the requirements files for dev vs production) |-- dev.txt (the python packages you need for development) |-- prod.txt (the python packages you need for production) + |-- common.txt (the python packages shared for both dev and production) |-- Procfile (defines the web, scheduler, and worker processes - also how heroku knows this is a python project) |-- wsgi.py (the python wsgi file that gunicorn actually uses) - |-- fabfile.py (the fabric script that helps set everything up on heroku) + |-- uwsgi.ini (config for uwsgi) + |-- djeroku.py (the script that helps set everything up on heroku) + |-- urls.py (top level django url management) + |-- tox.ini (tox config file) |-- manage.py (the django script that, well, manages everything) |-- project (the folder containing the code) |-- settings (all the settings for dev vs production) |-- common.py (settings used in both dev and production) |-- dev.py (settings for local development - generally easy setup) |-- prod.py (settings for production use - real settings and services) - |-- static (this is where all the static assets will be collected - generally don't add anything here directly) + |-- celery.py (config for celery - mainly for auto-task discovery) |-- apps (the folder that will contain all the individual apps you write) |-- djeroku_app (finally, the app that actually drives the site -- will have all the models, tests, templates etc) ~~~ diff --git a/djeroku.py b/djeroku.py new file mode 100644 index 0000000..a487c6b --- /dev/null +++ b/djeroku.py @@ -0,0 +1,419 @@ +#! env/python + +"""Usage: + djeroku.py [...] + djeroku.py help + djeroku.py -h | --help +{command_list} + +Options: + -h --help show this help message and exit + +See 'djeroku help ' for more information on a specific command. +""" +import sys +import logging +import string +import random +import platform +import subprocess +import re + +logging.basicConfig(format='') + + +HEROKU_ADDONS = ( + 'heroku-postgresql:hobby-dev', + 'scheduler:standard', + 'redistogo:nano', + 'memcachier:dev', + 'mailgun:starter', + 'papertrail:choklad' +) + +HEROKU_CONFIGS = ( + 'DJANGO_SETTINGS_MODULE=project.settings.prod', +) + +STAGING_REMOTE = 'staging' +PRODUCTION_REMOTE = 'production' + + +# Tools for generating help content and managing commands + +def abort(message, code=1): + logging.error(message) + sys.exit(code) + + +def command(func): + """ + Command decorator. Wrap functions that should become commands. Docstrings + are automatically used to create help text. + + @command + def command_name(x, y, z): + pass + """ + func._is_command = True + return func + + +@command +def help(command_name=None, *args): + if command_name is None or command_name == 'help': + abort("Help Error - Specify a command name for more information") + + try: + command = find_command(command_name) + except AttributeError: + abort("Could not find command {command_name}".format( + command_name=command_name + )) + + doc = command.__doc__ + if doc is None: + print( + "No help information found for command {command_name}".format( + command_name=command_name + ) + ) + else: + print(doc) + + +def get_all_commands(): + this_script = sys.modules[__name__] + + return { + func_name: getattr(this_script, func_name) + for func_name in dir(this_script) + if hasattr(getattr(this_script, func_name), '_is_command') + } + + +def find_command(command_name): + commands = get_all_commands() + if command_name in commands.keys(): + return commands[command_name] + raise AttributeError('No command found for %s' % command_name) + +# End Tools + + +@command +def heroku_setup(): + """ + heroku_setup + + One-time setup with everything you need on heroku. Creates a production app + (remote: production) and a matching staging app (remote: staging) and + does the following: + + - Initialize a local git repository. + - Create new Heroku applications and set them up as git remotes. + - Install all `HEROKU_ADDONS`. + - Set all `HEROKU_CONFIGS`. + + https://devcenter.heroku.com/articles/multiple-environments + + NOTE: the production app will have ENVIRONMENT_TYPE=production while + staging will have ENVIRONMENT_TYPE=staging if the code needs to know which + environment it is running in (for example, so staging can use a + non-production db follower) + + """ + app_name = prompt( + 'What name should this heroku app use?', + default='{{project}}' + ) + staging_name = '%s-staging' % app_name + + # create git repository + run('git init') + + # create the apps on heroku + cont( + 'heroku apps:create %s --remote %s --addons %s' % + (staging_name, STAGING_REMOTE, ','.join(HEROKU_ADDONS)), + 'Failed to create the staging app on heroku. Continue anyway?' + ) + + cont( + 'heroku apps:create %s --remote %s --addons %s' % + (app_name, PRODUCTION_REMOTE, ','.join(HEROKU_ADDONS)), + 'Failed to create the production app on heroku. Continue anyway?' + ) + + # set configs + for config in HEROKU_CONFIGS: + cont( + 'heroku config:set %s --app=%s' % (config, staging_name), + 'Failed to set %s on Staging. Continue anyway?' % config + ) + cont( + 'heroku config:set %s --app=%s' % (config, app_name), + 'Failed to set %s on Production. Continue anyway?' % config + ) + + # set debug + cont( + 'heroku config:set DEBUG=True --app=%s' % staging_name, + 'Failed to set DEBUG on Staging. Continue anyway?' + ) + cont( + 'heroku config:set DEBUG=False --app=%s' % app_name, + 'Failed to set DEBUG on Production. Continue anyway?' + ) + + # set environment type + cont( + 'heroku config:set ENVIRONMENT_TYPE=staging --app=%s' % staging_name, + 'Failed to set ENVIRONMENT_TYPE on Staging. Continue anyway?' + ) + cont( + 'heroku config:set ENVIRONMENT_TYPE=production --app=%s' % app_name, + 'Failed to set ENVIRONMENT_TYPE on Production. Continue anyway?' + ) + + # set secret key on production + cont( + 'heroku config:set SECRET_KEY="%s" --app=%s' % ( + generate_secret_key(), + app_name + ), + 'Failed to set SECRET_KEY on Production. Continue anyway?' + ) + + # set git to default to staging + run('git config heroku.remote staging') + + # create git remotes + run('heroku git:remote -r staging --app=%s' % staging_name) + run('heroku git:remote -r production --app=%s' % app_name) + + +@command +def migrate(): + """ + migrate + + Runs any pending migrations by calling `python manage.py migrate` + """ + venv('python manage.py migrate') + + +@command +def collect_static(): + """ + collect_static + + Collects all the static assets for your apps by calling + `python manage.py collectstatic --noinput` + """ + venv('python manage.py collectstatic --noinput') + + +@command +def serve(): + """ + serve + + Runs any pending migrations, collects static assets, then + runs the local development server by calling `python manage.py runserver` + """ + migrate() + collect_static() + venv('python manage.py runserver 0.0.0.0:8000') + + +@command +def web(): + """ + web + + Same as serve, but runs the web process using foreman instead of the django + development server directly. Can sometimes simulate the production + environment better than the debug server. You'll probably need to install + some of the production requirements. + """ + migrate() + collect_static() + venv('foreman start web') + + +@command +def worker(): + """ + worker + + Runs a celery worker to process background tasks your application creates. + """ + venv('python manage.py celery worker') + + +@command +def test(): + """ + test + + Runs your tests using the django test runner. + """ + venv('python manage.py test') + + +@command +def lint(): + """ + lint + + Runs flake8 on everything inside the project folder. + """ + venv('flake8 project') + + +@command +def deploy(remote_name='staging'): + """ + deploy + + Deploys the current local master branch to the target remote (default: + staging) by calling `git push REMOTE master`, then migrates and + collects static. + """ + run('git push {remote} master'.format(remote=remote_name)) + after_deploy(remote_name) + + +# Helper functions + +def venv(cmd): + if platform.system == 'Windows': + # untested - good luck, windows people! (submit a working PR) + return run('venv/bin/activate.bat && ' + cmd) + return run('. venv/bin/activate && ' + cmd) + + +def after_deploy(remote): + app_name = get_heroku_app_names()[remote] + heroku_run('python manage.py migrate', app_name) + heroku_run('python manage.py collectstatic --noinput', app_name) + + +def cont(cmd, message): + result = run(cmd, capture=True) + print(result['stdout'].decode()) + if message and result['failure']: + logging.error(result['stderr'].decode()) + if not confirm(message): + abort('Stopped execution per user request.') + return False + return True + + +def confirm(message): + print(message) + print("Y/N") + choice = input() + return choice.lower() in ['y', 'yes'] + + +def prompt(message, default=None): + if default is not None: + print("{message} (default: {default}):".format( + message=message, + default=default + )) + else: + print(message) + return input() + + +def heroku_run(command, app_name): + run('heroku run {command} --app={app_name}'.format( + command=command, + app_name=app_name + )) + + +def run(cmd, capture=False): + if capture: + proc = subprocess.Popen( + cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + failure = proc.returncode > 0 + + return dict( + stdout=stdout, + stderr=stderr, + returncode=proc.returncode, + failure=failure, + success=not failure + ) + + else: + return subprocess.call(cmd, shell=True) + + +def get_heroku_app_names(): + """Expects the default setup above.""" + proc = subprocess.Popen(["git", "remote", "-v"], stdout=subprocess.PIPE) + stdout, stderr = proc.communicate() + lines = stdout.split("\n") + + pattern = r'(.*)\t.*heroku\.com/(.*)\.git (\(.*\))' + pattern2 = r'(.*)\s+.*heroku.*:(.*)\.git \(.*\)' + remotes = {} + for line in lines: + match = re.match(pattern, line) + if match: + remotes[match.group(1)] = match.group(2) + else: + match2 = re.match(pattern2, line) + if match2: + remotes[match2.group(1)] = match2.group(2) + + return remotes + + +def generate_secret_key(key_length=64): + """ + Randomly generate a 64 character key you can stick in your + settings/environment + """ + options = string.digits + string.ascii_letters + ".,!@#$%^&*()-_+={}" + return ''.join([random.choice(options) for i in range(key_length)]) + + +def print_usage(): + usage = __doc__ + + commands = get_all_commands() + commands_list = """ +Commands:""" + + for command_name in commands.keys(): + commands_list += """ + {command_name}""".format(command_name=command_name) + + print(usage.format( + command_list=commands_list + )) + + +# End Helpers + + +if __name__ == '__main__': + command_name, arguments = 'usage', [] + invocation = sys.argv[0] + if len(sys.argv) > 1: + command_name = sys.argv[1] + arguments = sys.argv[2:] + + if command_name in ['-h', '--help', 'usage']: + print_usage() + sys.exit(0) + + command_function = find_command(command_name) + command_function(*arguments) diff --git a/fabfile.py b/fabfile.py deleted file mode 100644 index 802ad67..0000000 --- a/fabfile.py +++ /dev/null @@ -1,316 +0,0 @@ -""" -Djeroku project fabfile. - -Includes helpful commands for managing a project. - -- heroku_setup - One time setup for your project. Run this to create your heroku deployment - targets. Creates both a staging and a production app on heroku, including - all the necessary settings and free addons. Also creates a pipeline from - staging to production and sets up the local git repository and heroku - remotes. - -- serve - Syncs the db, runs any pending migrations, collects static assets, then runs - the local development server by calling `python manage.py runserver` - -- web - Same as serve, but runs the web process using foreman instead of the django - development server directly. Can sometimes simulate the production - environment better than the debug server. You'll probably need to install - some of the production requirements. - -- worker - Runs a celery worker to process background tasks your application creates. - -- test - Runs your tests using the django test runner. - -- deploy_staging - Deploys the current local master branch to staging by calling - `git push staging master`, then syncs, migrates, and collects static. - -- deploy_production - Deploys the current local master branch to production by calling - `git push production master`, then syncs, migrates, and collects static. - -- promote_production - Promotes the currently deployed staging environment to production by calling - `heroku pipeline:promote`. This is a slightly better deployment process - - first deploy to staging, then test that everything is working as expected, - then promote it to move that exact slug to the production environment. - -- lint - Runs flake8 on everything inside the project folder. - - -Deployment: - You can deploy your app to staging by pushing master to the staging remote: - `git push staging master`. This will build your project and make it - accessible on staging-.herokuapp.com. You almost certainly - will want to then call `heroku run python manage.py syncdb` - - When the code on staging is ready for production, you can promote the staging - slug to your production app by calling `heroku pipeline:promote` or `fab - promote_production`. - -""" - -from fabric.contrib.console import confirm, prompt -from fabric.api import abort, local, settings, task -import string -import random -import platform -import subprocess -import re - -HEROKU_ADDONS = ( - 'heroku-postgresql:dev', - 'scheduler:standard', - 'redistogo:nano', - 'memcachier:dev', - 'mandrill:starter', - 'papertrail:choklad' -) - -HEROKU_CONFIGS = ( - 'DJANGO_SETTINGS_MODULE=project.settings.prod', -) - -STAGING_REMOTE = 'staging' -PRODUCTION_REMOTE = 'production' - - -@task -def heroku_setup(): - """ - Set up everything you need on heroku. Creates a production app - (remote: production) and a matching staging app (remote: staging) and - does the following: - - - Create new Heroku applications. - - Install all `HEROKU_ADDONS`. - - Set all `HEROKU_CONFIGS`. - - Initialize New Relic's monitoring add-on. - - https://devcenter.heroku.com/articles/multiple-environments - - NOTE: the production app will have ENVIRONMENT_TYPE=production while - staging will have ENVIRONMENT_TYPE=staging if the code needs to know which - environment it is running in (for example, so staging can use a - non-production db follower) - """ - app_name = prompt( - 'What name should this heroku app use?', - default='server' - ) - staging_name = '%s-staging' % app_name - - # create the apps on heroku - cont( - 'heroku apps:create %s --remote %s --addons %s' % - (staging_name, STAGING_REMOTE, ','.join(HEROKU_ADDONS)), - 'Failed to create the staging app on heroku. Continue anyway?' - ) - cont( - 'heroku apps:create %s --remote %s --addons %s' % - (app_name, PRODUCTION_REMOTE, ','.join(HEROKU_ADDONS)), - 'Failed to create the production app on heroku. Continue anyway?' - ) - - # set configs - for config in HEROKU_CONFIGS: - cont( - 'heroku config:set %s --app=%s' % (config, staging_name), - 'Failed to set %s on Staging. Continue anyway?' % config - ) - cont( - 'heroku config:set %s --app=%s' % (config, app_name), - 'Failed to set %s on Production. Continue anyway?' % config - ) - - # set debug - cont( - 'heroku config:set DEBUG=True --app=%s' % staging_name, - 'Failed to set DEBUG on Staging. Continue anyway?' - ) - cont( - 'heroku config:set DEBUG=False --app=%s' % app_name, - 'Failed to set DEBUG on Production. Continue anyway?' - ) - - # set environment type - cont( - 'heroku config:set ENVIRONMENT_TYPE=staging --app=%s' % staging_name, - 'Failed to set ENVIRONMENT_TYPE on Staging. Continue anyway?' - ) - cont( - 'heroku config:set ENVIRONMENT_TYPE=production --app=%s' % app_name, - 'Failed to set ENVIRONMENT_TYPE on Production. Continue anyway?' - ) - - # set secret key on production - cont( - 'heroku config:set SECRET_KEY="%s" --app=%s' % ( - generate_secret_key(), - app_name - ), - 'Failed to set SECRET_KEY on Production. Continue anyway?' - ) - - # create a pipeline from staging to production - pipelines_enabled = cont( - 'heroku labs:enable pipelines', - 'Failed to enable Pipelines. Continue anyway?' - ) - if pipelines_enabled: - pipeline_plugin = cont( - 'heroku plugins:install ' + - 'git://github.com/heroku/heroku-pipeline.git', - 'Failed to install pipelines plugin. Continue anyway?' - ) - if pipeline_plugin: - cont( - 'heroku pipeline:add -a %s %s' % (staging_name, app_name), - 'Failed to create pipeline from Staging to Production. ' + - 'Continue anyway?' - ) - - # set git to default to staging - local('git init') - local('git config heroku.remote staging') - - # create git remotes - local('heroku git:remote -r staging --app=%s' % staging_name) - local('heroku git:remote -r production --app=%s' % app_name) - - -@task -def deploy_staging(): - """ - Deploys the current local master branch to staging by calling - `git push staging master`, then syncs, migrates, and collects static. - """ - local('git push staging master') - after_deploy('staging') - - -@task -def deploy_production(): - """ - Deploys the current local master branch to production by calling - `git push production master`, then syncs, migrates, and collects static. - """ - local('git push production master') - after_deploy('production') - - -@task -def promote_production(): - """Promotes the staging slug to production.""" - local('heroku pipeline:promote') - - -@task -def serve(): - """ - Sync db, migrate, collect static, and run the django - development server. - """ - venv('python manage.py syncdb') - venv('python manage.py migrate') - venv('python manage.py collectstatic --noinput') - venv('python manage.py runserver 0.0.0.0:8000') - - -@task -def web(): - """ - Sync db, migrate, collect static, and run the web - process using foreman. - """ - venv('python manage.py syncdb') - venv('python manage.py migrate') - venv('python manage.py collectstatic --noinput') - venv('foreman start web') - - -@task -def worker(): - """Run a task queue worker.""" - venv('python manage.py celery worker') - - -@task -def test(): - """Run the django tests.""" - venv('python manage.py test') - - -@task -def lint(): - """Run flake8.""" - venv('flake8 project') - - -# HELPERS -def venv(cmd): - if platform.system == 'Windows': - # untested - good luck, windows people! (submit a working PR) - return run('venv/bin/activate.bat && ' + cmd) - return run('source venv/bin/activate && ' + cmd) - - -def after_deploy(remote): - app_name = get_heroku_app_names()[remote] - run('heroku run python manage.py syncdb --app=%s' % app_name) - run('heroku run python manage.py migrate --app=%s' % app_name) - run( - 'heroku run python manage.py collectstatic --noinput --app=%s' % - app_name - ) - - -def cont(cmd, message): - with settings(warn_only=True): - result = local(cmd, capture=True) - - if message and result.failed: - print result.stderr - if not confirm(message): - abort('Stopped execution per user request.') - return False - return True - - -def run(cmd): - return subprocess.call(cmd, shell=True) - - -def get_heroku_app_names(): - """Expects the default setup above.""" - proc = subprocess.Popen(["git", "remote", "-v"], stdout=subprocess.PIPE) - stdout, stderr = proc.communicate() - lines = stdout.split("\n") - - pattern = r'(.*)\t.*heroku\.com/(.*)\.git (\(.*\))' - pattern2 = r'(.*)\s+.*heroku.*:(.*)\.git \(.*\)' - remotes = {} - for line in lines: - match = re.match(pattern, line) - if match: - remotes[match.group(1)] = match.group(2) - else: - match2 = re.match(pattern2, line) - if match2: - remotes[match2.group(1)] = match2.group(2) - - return remotes - - -def generate_secret_key(key_length=64): - """Randomly generate a 64 character key you can stick in your - settings/environment""" - options = string.digits + string.letters + ".,!@#$%^&*()-_+={}" - return ''.join([random.choice(options) for i in range(key_length)]) -# END HELPERS diff --git a/project/settings/prod.py b/project/settings/prod.py index 1e5a43c..517cc36 100644 --- a/project/settings/prod.py +++ b/project/settings/prod.py @@ -4,7 +4,7 @@ Debug OFF Djeroku Defaults: - Mandrill Email -- Requires Mandrill addon + Mailgun Email -- Requires Mailgun addon dj_database_url and django-postgrespool for heroku postgres configuration memcachify for heroku memcache configuration Commented out by default - redisify for heroku redis cache configuration @@ -19,7 +19,7 @@ automatically by during project creation by the djeroku setup) Email: - Defaults to mandril, which is already set up when added to your app + Defaults to mailgun, which is already set up when added to your app There is also a commented version that uses your gmail address. For more control, you can set any of the following keys in your @@ -62,10 +62,10 @@ # See: https://docs.djangoproject.com/en/dev/ref/settings/#email-port # See: https://docs.djangoproject.com/en/dev/ref/settings/#email-use-tls -EMAIL_HOST = environ.get('EMAIL_HOST', 'smtp.mandrillapp.com') -EMAIL_HOST_PASSWORD = environ.get('MANDRILL_APIKEY', '') -EMAIL_HOST_USER = environ.get('MANDRILL_USERNAME', '') -EMAIL_PORT = environ.get('EMAIL_PORT', 587) +EMAIL_HOST = environ.get('MAILGUN_SMTP_SERVER', 'smtp.mailgun.com') +EMAIL_HOST_PASSWORD = environ.get('MAILGUN_SMTP_PASSWORD', '') +EMAIL_HOST_USER = environ.get('MAILGUN_SMTP_LOGIN', '') +EMAIL_PORT = environ.get('MAILGUN_SMTP_PORT', 587) EMAIL_USE_TLS = True # use this to channel your emails through a gmail powered account instead diff --git a/reqs/common.txt b/reqs/common.txt index d5f7c18..1a9e677 100644 --- a/reqs/common.txt +++ b/reqs/common.txt @@ -1,5 +1,4 @@ Django>=1.9.0,<1.10.0 -Fabric==1.10.2 redis==2.10.5 whitenoise==2.0.6 django-celery==3.1.17 diff --git a/reqs/prod.txt b/reqs/prod.txt index 7dc193f..6fa73bb 100644 --- a/reqs/prod.txt +++ b/reqs/prod.txt @@ -5,6 +5,5 @@ dj-database-url==0.3.0 django-postgrespool==0.3.0 hiredis==0.2.0 psycopg2==2.6.1 -mandrill>=1.0.57,<2.0 uwsgi==2.0.11.2 werkzeug==0.11.3