Asynchronous email notifications, task execution
PATCH v2 1/1 Asynchronous email notifications, task execution
This patch adds asynchronous task execution using a Celery backend and RabbitMQ task queue.
It updates the README to reflect the installation and configuration of a basic RabbitMQ setup, adds a 'tasks.py' file to contain task definitions, updates the 'edit_layer_view' function to send emails to administrators about new and updated layers asynchronously, and modifies the 'settings.py' to include a default configuration for a RabbitMQ connection.
--- README | 10 +++++++++- TODO | 1 - layerindex/tasks.py | 24 ++++++++++++++++++++++++ layerindex/views.py | 7 +++---- requirements.txt | 1 + settings.py | 4 ++++ 6 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 layerindex/tasks.py
README +9 -1
@@ -14,6 +14,7 @@ In order to make use of this application you will need: * Python 3.4+ * Django 1.8.x - tested with 1.8.17; newer versions may work, but the application has not been tested with 1.9 or newer. +* RabbitMQ 3.6.x - tested with 3.6.10. * For production usage, a web server set up to host Django applications (not needed for local-only testing) * A database supported by Django (SQLite, MySQL, etc.). Django takes
@@ -41,7 +42,9 @@ Setup instructions: 1. Edit settings.py to specify a database, EMAIL_HOST, SECRET_KEY and other settings specific to your installation. Ensure you set LAYER_FETCH_DIR to an absolute path to a location with sufficient - space for fetching layer repositories. + space for fetching layer repositories. Modify RABBIT_BROKER + and RABBIT_BACKEND to reflect the settings used by your RabbitMQ + server. 2. Run the following commands within the layerindex-web directory to initialise the database:
@@ -64,6 +67,11 @@ Setup instructions: production you need to use a proper web server and have DEBUG set to False. + 3.1. In order to process asynchronous tasks like sending email, + you will need to run a Celery worker: + + celery -A layerindex.tasks worker --loglevel=info + 4. You'll need to add at least the openembedded-core layer to the database, or some equivalent that contains conf/bitbake.conf for the base system configuration. To add this, follow these steps:
TODO +0 -1
@@ -27,7 +27,6 @@ Other * Show layer type in layer detail? * Usage links in list page? * Subdirs in list page? -* Prevent SMTP failures from breaking submission process * Query backend service i.e. special URL to query information for external apps/scripts * Add comparison to duplicates page * Create simple script to check for unlisted layer subdirectories in all repos
layerindex/tasks.py new file +24 -0
@@ -0,0 +1,24 @@ +from celery import Celery +from django.core.mail import EmailMessage +from . import utils +import os +import time + +try: + import settings +except ImportError: + # not in a full django env, so settings is inaccessible. + # setup django to access settings. + utils.setup_django() + import settings + +tasks = Celery('layerindex', + broker=settings.RABBIT_BROKER, + backend=settings.RABBIT_BACKEND) + [email protected] +def send_email(subject, text_content, from_email=settings.DEFAULT_FROM_EMAIL, to_emails=[]): + # We seem to need to run this within the task + utils.setup_django() + msg = EmailMessage(subject, text_content, from_email, to_emails) + msg.send()
layerindex/views.py +3 -4
@@ -19,7 +19,6 @@ from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm from django.db import transaction from django.contrib.auth.models import User, Permission from django.db.models import Q, Count, Sum -from django.core.mail import EmailMessage from django.template.loader import get_template from django.template import Context from django.utils.decorators import method_decorator
@@ -28,6 +27,7 @@ from django.contrib import messages from reversion.models import Revision from . import utils from . import simplesearch +from . import tasks import settings from django.dispatch import receiver import reversion
@@ -163,7 +163,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None): # Send email plaintext = get_template('layerindex/submitemail.txt') perm = Permission.objects.get(codename='publish_layer') - users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) ).distinct() + users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) | Q(is_superuser=True)).distinct() for user in users: if user.first_name: user_name = user.first_name
@@ -181,8 +181,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None): from_email = settings.SUBMIT_EMAIL_FROM to_email = user.email text_content = plaintext.render(d) - msg = EmailMessage(subject, text_content, from_email, [to_email]) - msg.send() + tasks.send_email.apply_async((subject, text_content, from_email, [to_email])) return HttpResponseRedirect(reverse('submit_layer_thanks')) messages.success(request, 'Layer %s saved successfully.' % layeritem.name) if return_url:
requirements.txt +1 -0
@@ -1,3 +1,4 @@ +celery==3.1.25 Django==1.8.17 django-cors-headers==1.1.0 django-nvd3==0.9.7
settings.py +4 -0
@@ -224,3 +224,7 @@ FORCE_REVIEW_HTTPS = False # Settings for layer submission feature SUBMIT_EMAIL_FROM = '[email protected]' SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission' + +# RabbitMQ settings +RABBIT_BROKER = 'amqp://' +RABBIT_BACKEND = 'rpc://'
Signature · 1 line
2.7.4
Hi Diana,
Quoted 9 lines · NZST Diana Thayer
This patch adds asynchronous task execution using a Celery backend and RabbitMQ task queue.
It updates the README to reflect the installation and configuration of a basic RabbitMQ setup, adds a '
tasks.py' file to contain task definitions, updates the 'edit_layer_view' function to send emails to administrators about new and updated layers asynchronously, and modifies the 'settings.py' to include a default configuration for a RabbitMQ connection.
This summarises what the patch does, which is useful, but it misses out the most important bit - why we're making this change, i.e. to allow layer submission process to proceed even in the event that sending the notification email fails, and establishing an asynchronous execution mechanism that we can use in future e.g. for triggering parse operations from the web UI. If you could also include a reference to the bug in the form [YOCTO #11197] that would be great.
Please also add your Signed-off-by which is missing.
One other comment below.
Quoted 20 lines
....
layerindex/views.py +1
-1@@ -19,7 +19,6 @@ from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm from django.db import transaction from django.contrib.auth.models import User, Permission from django.db.models import Q, Count, Sum -from django.core.mail import EmailMessage from django.template.loader import get_template from django.template import Context from django.utils.decorators import method_decorator@@ -28,6 +27,7 @@ from django.contrib import messages from reversion.models import Revision from . import utils from . import simplesearch +from . import tasks import settings from django.dispatch import receiver import reversion
layerindex/views.py def edit_layer_view(request, template_name, branch='master', slug=None): · line 166
Quoted 5 lines
layerindex/views.py +1
-1@@ -163,7 +163,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None): # Send email plaintext = get_template('layerindex/submitemail.txt') perm = Permission.objects.get(codename='publish_layer') - users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) ).distinct() + users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) | Q(is_superuser=True)).distinct()
I forgot to mention this earlier - can you make this a separate patch? It's not directly related. (It was mostly intentional, but I can see that it's a permission and therefore it can be argued that a superuser should logically have it, so I'm OK with it.)
Thanks,
Paul
Signature · 2 lines
Paul Eggleton Intel Open Source Technology Centre