WIP solution to bug-11197: adding asynchronous task execution with celery and rabbitmq
PATCH 1/1 WIP solution to bug-11197: adding asynchronous task execution with celery and rabbitmq
- updated readme to include RabbitMQ as a dependency
- updated settings.py to include RABBIT_BROKER and RABBIT_BACKEND
- added celery to requirements.txt
- defined celery tasks in layerindex/tasks.py
- separated layerindex/update#main into two functions: main (which parses sys.argv) and do_update (which accepts options as keyword arguments)
- updated layerindex/views.py to use the send_email task to asynchronously message anyone able to publish layers when a new layer is submitted
--- README | 5 +- layerindex/tasks.py | 31 +++++++++++ layerindex/update.py | 143 +++++++++++++++++++++++++++------------------------ layerindex/views.py | 7 ++- requirements.txt | 1 + settings.py | 4 ++ 6 files changed, 120 insertions(+), 71 deletions(-) create mode 100644 layerindex/tasks.py
README +4 -1
@@ -14,6 +14,7 @@ In order to make use of this application you will need: * Python 3.4+ * Django 1.8.x - tested with 1.8.17; newer versions may work, but the application has not been tested with 1.9 or newer. +* RabbitMQ 3.6.x - tested with 3.6.10. * For production usage, a web server set up to host Django applications (not needed for local-only testing) * A database supported by Django (SQLite, MySQL, etc.). Django takes
@@ -41,7 +42,9 @@ Setup instructions: 1. Edit settings.py to specify a database, EMAIL_HOST, SECRET_KEY and other settings specific to your installation. Ensure you set LAYER_FETCH_DIR to an absolute path to a location with sufficient - space for fetching layer repositories. + space for fetching layer repositories. Modify RABBIT_BROKER + and RABBIT_BACKEND to reflect the settings used by your RabbitMQ + server. 2. Run the following commands within the layerindex-web directory to initialise the database:
layerindex/tasks.py new file +31 -0
@@ -0,0 +1,31 @@ +from celery import Celery +from django.core.mail import EmailMessage +import os +import time + +try: + from update import do_update +except ImportError: + from .update import do_update + +try: + import settings +except ImportError: + # not in a full django env, so settings is inaccessible. + # setup django to access settings. + from utils import setup_django + setup_django() + import settings + +tasks = Celery('layerindex', + broker=settings.RABBIT_BROKER, + backend=settings.RABBIT_BACKEND) + [email protected] +def update(**options): + return do_update(**options) + [email protected] +def send_email(subject, text_content, from_email=settings.DEFAULT_FROM_EMAIL, to_emails=[]): + msg = EmailMessage(subject, text_content, from_email, to_emails) + msg.send()
layerindex/update.py +77 -66
221 lines not on this page · raw message
layerindex/views.py +3 -4
@@ -19,7 +19,6 @@ from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm from django.db import transaction from django.contrib.auth.models import User, Permission from django.db.models import Q, Count, Sum -from django.core.mail import EmailMessage from django.template.loader import get_template from django.template import Context from django.utils.decorators import method_decorator
@@ -28,6 +27,7 @@ from django.contrib import messages from reversion.models import Revision from . import utils from . import simplesearch +from . import tasks import settings from django.dispatch import receiver import reversion
@@ -163,7 +163,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None): # Send email plaintext = get_template('layerindex/submitemail.txt') perm = Permission.objects.get(codename='publish_layer') - users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) ).distinct() + users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) | Q(is_superuser=True) ).distinct() for user in users: if user.first_name: user_name = user.first_name
@@ -178,8 +178,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None): from_email = settings.SUBMIT_EMAIL_FROM to_email = user.email text_content = plaintext.render(d) - msg = EmailMessage(subject, text_content, from_email, [to_email]) - msg.send() + tasks.send_email(subject, text_content, from_email, [to_email]) return HttpResponseRedirect(reverse('submit_layer_thanks')) messages.success(request, 'Layer %s saved successfully.' % layeritem.name) if return_url:
requirements.txt +1 -0
@@ -1,3 +1,4 @@ +celery==3.1.25 Django==1.8.17 django-cors-headers==1.1.0 django-nvd3==0.9.7
settings.py +4 -0
@@ -217,3 +217,7 @@ UPDATE_PURGE_DAYS = 30 # Settings for layer submission feature SUBMIT_EMAIL_FROM = '[email protected]' SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission' + +# RabbitMQ settings +RABBIT_BROKER = 'amqp://admin:password@localhost:5672/admin_vhost' +RABBIT_BACKEND = 'rpc://'
Signature · 1 line
2.7.4