Skip to contentSkip to conversation

WIP solution to bug-11197: adding asynchronous task execution with celery and rabbitmq

PATCH New Diana Thayer 1 participant 1 message Jun 16, 2017

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 WIP solution to bug-11197: adding asynchronous task execution with celery and rabbitmq

- updated readme to include RabbitMQ as a dependency
- updated settings.py to include RABBIT_BROKER and RABBIT_BACKEND
- added celery to requirements.txt
- defined celery tasks in layerindex/tasks.py
- separated layerindex/update#main into two functions: main (which parses sys.argv) and do_update (which accepts options as keyword arguments)
- updated layerindex/views.py to use the send_email task to asynchronously message anyone able to publish layers when a new layer is submitted

---
 README               |   5 +-
 layerindex/tasks.py  |  31 +++++++++++
 layerindex/update.py | 143 +++++++++++++++++++++++++++------------------------
 layerindex/views.py  |   7 ++-
 requirements.txt     |   1 +
 settings.py          |   4 ++
 6 files changed, 120 insertions(+), 71 deletions(-)
 create mode 100644 layerindex/tasks.py
README +4 -1
index 62f739d..cfcc37f 100644
@@ -14,6 +14,7 @@ In order to make use of this application you will need:
 * Python 3.4+
 * Django 1.8.x - tested with 1.8.17; newer versions may work, but
   the application has not been tested with 1.9 or newer.
+* RabbitMQ 3.6.x - tested with 3.6.10.
 * For production usage, a web server set up to host Django applications
   (not needed for local-only testing)
 * A database supported by Django (SQLite, MySQL, etc.). Django takes
@@ -41,7 +42,9 @@ Setup instructions:
 1. Edit settings.py to specify a database, EMAIL_HOST, SECRET_KEY and
    other settings specific to your installation. Ensure you set
    LAYER_FETCH_DIR to an absolute path to a location with sufficient
-   space for fetching layer repositories.
+   space for fetching layer repositories. Modify RABBIT_BROKER
+   and RABBIT_BACKEND to reflect the settings used by your RabbitMQ
+   server.
 
 2. Run the following commands within the layerindex-web directory to
    initialise the database:
layerindex/tasks.py new file +31 -0
new file mode 100644
index 0000000..9bb4701
@@ -0,0 +1,31 @@
+from celery import Celery
+from django.core.mail import EmailMessage
+import os
+import time
+
+try:
+  from update import do_update
+except ImportError:
+  from .update import do_update
+
+try:
+  import settings
+except ImportError:
+  # not in a full django env, so settings is inaccessible.
+  # setup django to access settings.
+  from utils import setup_django
+  setup_django()
+  import settings
+
+tasks = Celery('layerindex',
+    broker=settings.RABBIT_BROKER,
+    backend=settings.RABBIT_BACKEND)
+
[email protected]
+def update(**options):
+  return do_update(**options)
+
[email protected]
+def send_email(subject, text_content, from_email=settings.DEFAULT_FROM_EMAIL, to_emails=[]):
+  msg = EmailMessage(subject, text_content, from_email, to_emails)
+  msg.send()
layerindex/update.py +77 -66

221 lines not on this page · raw message

layerindex/views.py +3 -4
index 65a536a..c7c20b8 100644
@@ -19,7 +19,6 @@ from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm
 from django.db import transaction
 from django.contrib.auth.models import User, Permission
 from django.db.models import Q, Count, Sum
-from django.core.mail import EmailMessage
 from django.template.loader import get_template
 from django.template import Context
 from django.utils.decorators import method_decorator
@@ -28,6 +27,7 @@ from django.contrib import messages
 from reversion.models import Revision
 from . import utils
 from . import simplesearch
+from . import tasks
 import settings
 from django.dispatch import receiver
 import reversion
@@ -163,7 +163,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None):
                     # Send email
                     plaintext = get_template('layerindex/submitemail.txt')
                     perm = Permission.objects.get(codename='publish_layer')
-                    users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) ).distinct()
+                    users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) | Q(is_superuser=True) ).distinct()
                     for user in users:
                         if user.first_name:
                             user_name = user.first_name
@@ -178,8 +178,7 @@ def edit_layer_view(request, template_name, branch='master', slug=None):
                         from_email = settings.SUBMIT_EMAIL_FROM
                         to_email = user.email
                         text_content = plaintext.render(d)
-                        msg = EmailMessage(subject, text_content, from_email, [to_email])
-                        msg.send()
+                        tasks.send_email(subject, text_content, from_email, [to_email])
                     return HttpResponseRedirect(reverse('submit_layer_thanks'))
             messages.success(request, 'Layer %s saved successfully.' % layeritem.name)
             if return_url:
requirements.txt +1 -0
index f80e5d4..7cd741b 100644
@@ -1,3 +1,4 @@
+celery==3.1.25
 Django==1.8.17
 django-cors-headers==1.1.0
 django-nvd3==0.9.7
settings.py +4 -0
index 0ecf90b..e98236b 100644
@@ -217,3 +217,7 @@ UPDATE_PURGE_DAYS = 30
 # Settings for layer submission feature
 SUBMIT_EMAIL_FROM = '[email protected]'
 SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'
+
+# RabbitMQ settings
+RABBIT_BROKER = 'amqp://admin:password@localhost:5672/admin_vhost'
+RABBIT_BACKEND = 'rpc://'
Signature · 1 line
2.7.4