Merge pull request #124 from bornhack/emailworker

add emailworker for async emails

src/shop/email.py

@@ -1,9 +1,9 @@
-from utils.email import _send_email
+from utils.email import add_outgoing_email
 import logging
 logger = logging.getLogger("bornhack.%s" % __name__)
 
 
-def send_creditnote_email(creditnote):
+def add_creditnote_email(creditnote):
     # put formatdict together
     formatdict = {
         'creditnote': creditnote,
@@ -11,11 +11,11 @@ def send_creditnote_email(creditnote):
 
     subject = 'BornHack creditnote %s' % creditnote.pk
 
-    # send mail
-    return _send_email(
+    # add email to outgoing email queue
+    return add_outgoing_email(
         text_template='emails/creditnote_email.txt',
         html_template='emails/creditnote_email.html',
-        recipient=creditnote.user.email,
+        to_recipients=creditnote.user.email,
         formatdict=formatdict,
         subject=subject,
         attachment=creditnote.pdf.read(),
@@ -23,7 +23,7 @@ def send_creditnote_email(creditnote):
     )
 
 
-def send_invoice_email(invoice):
+def add_invoice_email(invoice):
     # put formatdict together
     formatdict = {
         'ordernumber': invoice.order.pk,
@@ -33,11 +33,11 @@ def send_invoice_email(invoice):
 
     subject = 'BornHack invoice %s' % invoice.pk
 
-    # send mail
-    return _send_email(
+    # add email to outgoing email queue
+    return add_outgoing_email(
         text_template='emails/invoice_email.txt',
         html_template='emails/invoice_email.html',
-        recipient=invoice.order.user.email,
+        to_recipients=invoice.order.user.email,
         formatdict=formatdict,
         subject=subject,
         attachment=invoice.pdf.read(),
@@ -45,9 +45,9 @@ def send_invoice_email(invoice):
     )
 
 
-def send_test_email(recipient):
-    return _send_email(
+def add_test_email(recipient):
+    return add_outgoing_email(
         text_template='emails/testmail.txt',
-        recipient=recipient,
+        to_recipients=recipient,
         subject='testmail from bornhack website'
     )

src/shop/invoiceworker.py

@@ -1,10 +1,8 @@
 from django.core.files import File
-from django.utils import timezone
 from shop.pdf import generate_pdf_letter
-from shop.email import send_invoice_email, send_creditnote_email
+from shop.email import add_invoice_email, add_creditnote_email
 from shop.models import Order, CustomOrder, Invoice, CreditNote
-from decimal import Decimal
-import logging, importlib
+import logging
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger('bornhack.%s' % __name__)
 
@@ -16,31 +14,26 @@ def do_work():
         that have no PDF. It also emails invoices for shop orders.
     """
 
-    ###############################################################
     # check if we need to generate any invoices for shop orders
     for order in Order.objects.filter(paid=True, invoice__isnull=True):
         # generate invoice for this Order
         Invoice.objects.create(order=order)
         logger.info('Generated Invoice object for %s' % order)
 
-
-    ###############################################################
     # check if we need to generate any invoices for custom orders
     for customorder in CustomOrder.objects.filter(invoice__isnull=True):
         # generate invoice for this CustomOrder
         Invoice.objects.create(customorder=customorder)
         logger.info('Generated Invoice object for %s' % customorder)
 
-
-    ###############################################################
     # check if we need to generate any pdf invoices
     for invoice in Invoice.objects.filter(pdf=''):
         # generate the pdf
         try:
             if invoice.customorder:
-                template='pdf/custominvoice.html'
+                template = 'pdf/custominvoice.html'
             else:
-                template='pdf/invoice.html'
+                template = 'pdf/invoice.html'
             pdffile = generate_pdf_letter(
                 filename=invoice.filename,
                 template=template,
@@ -57,21 +50,23 @@ def do_work():
         invoice.pdf.save(invoice.filename, File(pdffile))
         invoice.save()
 
-
-    ###############################################################
     # check if we need to send out any invoices (only for shop orders, and only where pdf has been generated)
     for invoice in Invoice.objects.filter(order__isnull=False, sent_to_customer=False).exclude(pdf=''):
         logger.info("found unmailed Invoice object: %s" % invoice)
-        # send the email
-        if send_invoice_email(invoice=invoice):
-            invoice.sent_to_customer=True
+        # add email to the outgoing email queue
+        if add_invoice_email(invoice=invoice):
+            invoice.sent_to_customer = True
             invoice.save()
-            logger.info('OK: Invoice email sent to %s' % invoice.order.user.email)
+            logger.info('OK: Invoice email to {} added to queue.'.format(
+                invoice.order.user.email)
+            )
         else:
-            logger.error('Unable to send invoice email for order %s to %s' % (invoice.order.pk, invoice.order.user.email))
+            logger.error('Unable to add email for invoice {} to {}'.format(
+                    invoice.pk,
+                    invoice.order.user.email
+                )
+            )
 
-
-    ###############################################################
     # check if we need to generate any pdf creditnotes?
     for creditnote in CreditNote.objects.filter(pdf=''):
         # generate the pdf
@@ -92,15 +87,13 @@ def do_work():
         creditnote.pdf.save(creditnote.filename, File(pdffile))
         creditnote.save()
 
-
-    ###############################################################
     # check if we need to send out any creditnotes (only where pdf has been generated)
     for creditnote in CreditNote.objects.filter(sent_to_customer=False).exclude(pdf=''):
         # send the email
-        if send_creditnote_email(creditnote=creditnote):
-            logger.info('OK: Creditnote email sent to %s' % creditnote.user.email)
-            creditnote.sent_to_customer=True
+        if add_creditnote_email(creditnote=creditnote):
+            logger.info('OK: Creditnote email to %s added' % creditnote.user.email)
+            creditnote.sent_to_customer = True
             creditnote.save()
         else:
-            logger.error('Unable to send creditnote email for creditnote %s to %s' % (creditnote.pk, creditnote.user.email))
+            logger.error('Unable to add creditnote email for creditnote %s to %s' % (creditnote.pk, creditnote.user.email))
 

src/shop/pdf.py

@@ -3,7 +3,9 @@ from wkhtmltopdf.views import PDFTemplateResponse
 from PyPDF2 import PdfFileWriter, PdfFileReader
 from django.test.client import RequestFactory
 from django.conf import settings
-import io, logging
+import logging
+import io
+import os
 logger = logging.getLogger("bornhack.%s" % __name__)
 
 
@@ -13,10 +15,10 @@ def generate_pdf_letter(filename, template, formatdict):
     request.user = AnonymousUser()
     request.session = {}
 
-    ### produce text-only PDF from template
+    # produce text-only PDF from template
     pdfgenerator = PDFTemplateResponse(
         request=request,
-        template=template, 
+        template=template,
         context=formatdict,
         cmd_options={
             'margin-top': 50,
@@ -26,33 +28,35 @@ def generate_pdf_letter(filename, template, formatdict):
     textonlypdf = io.BytesIO()
     textonlypdf.write(pdfgenerator.rendered_content)
 
-    ### create a blank pdf to work with
+    # create a blank pdf to work with
     finalpdf = PdfFileWriter()
 
-    ### open the text-only pdf
+    # open the text-only pdf
     pdfreader = PdfFileReader(textonlypdf)
 
-    ### get watermark from watermark file
-    watermark = PdfFileReader(open("%s/pdf/%s" % (settings.STATICFILES_DIRS[0], settings.PDF_LETTERHEAD_FILENAME), 'rb'))
+    # get watermark from watermark file
+    watermark = PdfFileReader(
+        open(os.path.join(settings.STATICFILES_DIRS[0], 'pdf', settings.PDF_LETTERHEAD_FILENAME), 'rb')
+    )
 
-    ### add the watermark to all pages
+    # add the watermark to all pages
     for pagenum in range(pdfreader.getNumPages()):
         page = watermark.getPage(0)
         try:
             page.mergePage(pdfreader.getPage(pagenum))
         except ValueError:
-            ### watermark pdf might be broken?
+            # watermark pdf might be broken?
             return False
-        ### add page to output
+        # add page to output
         finalpdf.addPage(page)
 
-    ### save the generated pdf to the archive
-    fullpath = settings.PDF_ARCHIVE_PATH+filename
+    # save the generated pdf to the archive
+    fullpath = os.path.join(settings.PDF_ARCHIVE_PATH, filename)
     with open(fullpath, 'wb') as fh:
         finalpdf.write(fh)
         logger.info('Saved pdf to archive: %s' % fullpath)
 
-    ### return a file object with the data
+    # return a file object with the data
     returnfile = io.BytesIO()
     finalpdf.write(returnfile)
     return returnfile

src/teams/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 from .models import Team, TeamArea, TeamMember
-from .email import send_add_membership_email, send_remove_membership_email
+from .email import add_added_membership_email, add_removed_membership_email
 
 admin.site.register(TeamArea)
 
@@ -41,7 +41,7 @@ class TeamMemberAdmin(admin.ModelAdmin):
             membership.approved = True
             membership.save()
             updated += 1
-            send_add_membership_email(membership)
+            add_added_membership_email(membership)
 
         self.message_user(
             request,
@@ -57,7 +57,7 @@ class TeamMemberAdmin(admin.ModelAdmin):
         updated = 0
 
         for membership in queryset:
-            send_remove_membership_email(membership)
+            add_removed_membership_email(membership)
             membership.delete()
             updated += 1
 
@@ -69,4 +69,3 @@ class TeamMemberAdmin(admin.ModelAdmin):
             )
         )
     remove_member.description = 'Remove a user from the team.'
-

src/teams/email.py

@@ -1,24 +1,24 @@
-from utils.email import _send_email
+from utils.email import add_outgoing_email
 import logging
 logger = logging.getLogger("bornhack.%s" % __name__)
 
 
-def send_add_membership_email(membership):
+def add_added_membership_email(membership):
     formatdict = {
         'team': membership.team.name,
         'camp': membership.team.camp.title
     }
 
-    return _send_email(
+    return add_outgoing_email(
         text_template='emails/add_membership_email.txt',
         html_template='emails/add_membership_email.html',
-        recipient=membership.user.email,
+        to_recipients=membership.user.email,
         formatdict=formatdict,
         subject='Team update from {}'.format(membership.team.camp.title)
     )
 
 
-def send_remove_membership_email(membership):
+def add_removed_membership_email(membership):
     formatdict = {
         'team': membership.team.name,
         'camp': membership.team.camp.title
@@ -31,25 +31,25 @@ def send_remove_membership_email(membership):
         text_template = 'emails/unapproved_membership_email.txt',
         html_template = 'emails/unapproved_membership_email.html'
 
-    return _send_email(
+    return add_outgoing_email(
         text_template=text_template,
         html_template=html_template,
-        recipient=membership.user.email,
+        to_recipients=membership.user.email,
         formatdict=formatdict,
         subject='Team update from {}'.format(membership.team.camp.title)
     )
 
 
-def send_new_membership_email(membership):
+def add_new_membership_email(membership):
     formatdict = {
         'team': membership.team.name,
         'camp': membership.team.camp.title
     }
 
-    return _send_email(
+    return add_outgoing_email(
         text_template='emails/new_membership_email.txt',
         html_template='emails/new_membership_email.html',
-        recipient=[resp.email for resp in membership.team.responsible],
+        to_recipients=[resp.email for resp in membership.team.responsible],
         formatdict=formatdict,
         subject='New membership request for {} at {}'.format(
             membership.team.name,

src/teams/models.py

@@ -3,9 +3,11 @@ from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils.text import slugify
 from utils.models import CampRelatedModel
-from .email import send_new_membership_email
+from .email import add_new_membership_email
 from django.core.exceptions import ValidationError
 from django.contrib.auth.models import User
+import logging
+logger = logging.getLogger("bornhack.%s" % __name__)
 
 
 class TeamArea(CampRelatedModel):
@@ -96,6 +98,7 @@ class TeamMember(models.Model):
 
 
 @receiver(post_save, sender=TeamMember)
-def send_responsible_email(sender, instance, created, **kwargs):
+def add_responsible_email(sender, instance, created, **kwargs):
     if created:
-        send_new_membership_email(instance)
+        if not add_new_membership_email(instance):
+            logger.error('Error adding email to outgoing queue')

src/utils/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+
+from .models import OutgoingEmail
+
+
+@admin.register(OutgoingEmail)
+class OutgoingEmailAdmin(admin.ModelAdmin):
+    pass

src/utils/email.py

@@ -1,49 +1,105 @@
 from django.core.mail import EmailMultiAlternatives
+from django.core.validators import validate_email
+from django.core.exceptions import ValidationError
+from django.core.files.base import ContentFile
 from django.conf import settings
 from django.template.loader import render_to_string
+from .models import OutgoingEmail
 import logging
 logger = logging.getLogger("bornhack.%s" % __name__)
 
 
 def _send_email(
     text_template,
-    recipient,
-    formatdict,
     subject,
-    html_template=None,
+    to_recipients=[],
+    cc_recipients=[],
+    bcc_recipients=[],
+    html_template='',
     sender='BornHack <info@bornhack.dk>',
     attachment=None,
-    attachment_filename=None
+    attachment_filename=''
 ):
-    if not isinstance(recipient, list):
-        recipient = [recipient]
+    if not isinstance(to_recipients, list):
+        to_recipients = [to_recipients]
 
     try:
         # put the basic email together
         msg = EmailMultiAlternatives(
             subject,
-            render_to_string(text_template, formatdict),
+            text_template,
             sender,
-            recipient,
-            [settings.ARCHIVE_EMAIL]
+            to_recipients,
+            bcc_recipients + [settings.ARCHIVE_EMAIL] if bcc_recipients else [settings.ARCHIVE_EMAIL],
+            cc_recipients
         )
 
         # is there a html version of this email?
         if html_template:
             msg.attach_alternative(
-                render_to_string(html_template, formatdict),
+                html_template,
                 'text/html'
             )
 
         # is there a pdf attachment to this mail?
         if attachment:
             msg.attach(attachment_filename, attachment, 'application/pdf')
-
     except Exception as e:
         logger.exception('exception while rendering email: {}'.format(e))
         return False
 
     # send the email
-    msg.send()
+    try:
+        msg.send(fail_silently=False)
+    except Exception as e:
+        logger.exception('exception while sending email: {}'.format(e))
+        return False
+
+    return True
+
+
+def add_outgoing_email(
+    text_template,
+    formatdict,
+    subject,
+    to_recipients=[],
+    cc_recipients=[],
+    bcc_recipients=[],
+    html_template='',
+    sender='BornHack <info@bornhack.dk>',
+    attachment=None,
+    attachment_filename=''
+):
+    """ adds an email to the outgoing queue
+        recipients is a list of to recipients
+    """
+    text_template = render_to_string(text_template, formatdict)
+
+    if html_template:
+        html_template = render_to_string(html_template, formatdict)
+
+    if not isinstance(to_recipients, list):
+        to_recipients = [to_recipients]
+
+    for recipient in to_recipients:
+        try:
+            validate_email(recipient)
+        except ValidationError:
+            return False
+
+    email = OutgoingEmail.objects.create(
+        text_template=text_template,
+        html_template=html_template,
+        subject=subject,
+        sender=sender,
+        to_recipients=to_recipients,
+        cc_recipients=cc_recipients,
+        bcc_recipients=bcc_recipients
+    )
+
+    if attachment:
+        django_file = ContentFile(attachment)
+        django_file.name = attachment_filename
+        email.attachment.save(attachment_filename, django_file, save=True)
 
     return True

src/utils/migrations/0001_initial.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-04-30 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='OutgoingEmail',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('subject', models.CharField(max_length=500)),
+                ('text_template', models.TextField()),
+                ('html_template', models.TextField(blank=True)),
+                ('recipient', models.CharField(max_length=500)),
+                ('sender', models.CharField(max_length=500)),
+                ('attachment', models.FileField(blank=True, upload_to='')),
+                ('processed', models.BooleanField(default=False)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

src/utils/migrations/0002_remove_outgoingemail_recipient.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-05-21 16:08
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('utils', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='outgoingemail',
+            name='recipient',
+        ),
+    ]

src/utils/migrations/0003_auto_20170521_1932.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.5 on 2017-05-21 17:32
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('utils', '0002_remove_outgoingemail_recipient'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='outgoingemail',
+            name='bcc_recipients',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=500), blank=True, null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='outgoingemail',
+            name='cc_recipients',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=500), blank=True, null=True, size=None),
+        ),
+        migrations.AddField(
+            model_name='outgoingemail',
+            name='to_recipients',
+            field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(blank=True, max_length=500), blank=True, null=True, size=None),
+        ),
+    ]

src/utils/models.py

@@ -1,4 +1,5 @@
 import uuid
+from django.contrib.postgres.fields import ArrayField
 from django.core.exceptions import ValidationError
 from django.contrib import messages
 from django.db import models
@@ -68,3 +69,38 @@ class CampRelatedModel(CreatedUpdatedModel):
             raise ValidationError('This camp is in read only mode.')
 
         super().delete(**kwargs)
+
+
+class OutgoingEmail(CreatedUpdatedModel):
+    subject = models.CharField(max_length=500)
+    text_template = models.TextField()
+    html_template = models.TextField(blank=True)
+    sender = models.CharField(max_length=500)
+    to_recipients = ArrayField(
+        models.CharField(max_length=500, blank=True),
+        null=True,
+        blank=True
+    )
+    cc_recipients = ArrayField(
+        models.CharField(max_length=500, blank=True),
+        null=True,
+        blank=True
+    )
+    bcc_recipients = ArrayField(
+        models.CharField(max_length=500, blank=True),
+        null=True,
+        blank=True
+    )
+    attachment = models.FileField(blank=True)
+    processed = models.BooleanField(default=False)
+
+    def __str__(self):
+        return 'OutgoingEmail Object id: {} '.format(self.id)
+
+    def clean(self):
+        if not self.to_recipients \
+           and not self.bcc_recipients \
+           and not self.cc_recipients:
+            raise ValidationError(
+                {'recipient': 'either to_recipient, bcc_recipient or cc_recipient required.'}
+            )

src/utils/outgoingemailworker.py

@@ -0,0 +1,43 @@
+from .models import OutgoingEmail
+from .email import _send_email
+import logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger('bornhack.%s' % __name__)
+
+
+def do_work():
+    """
+        The outgoing email worker sends emails added to the OutgoingEmail
+        queue.
+    """
+    not_processed_email = OutgoingEmail.objects.filter(processed=False)
+
+    if len(not_processed_email) > 0:
+        logger.debug('about to process {} emails'.format(
+            len(not_processed_email))
+        )
+
+    for email in not_processed_email:
+
+        attachment = None
+        attachment_filename = ''
+        if email.attachment:
+            attachment = email.attachment.read()
+            attachment_filename = email.attachment.name
+
+        mail_send_success = _send_email(
+            text_template=email.text_template,
+            to_recipients=email.to_recipients,
+            subject=email.subject,
+            cc_recipients=email.cc_recipients,
+            bcc_recipients=email.bcc_recipients,
+            html_template=email.html_template,
+            attachment=attachment,
+            attachment_filename=attachment_filename
+        )
+        if mail_send_success:
+            email.processed = True
+            email.save()
+            logger.debug('successfully sent {}'.format(email))
+        else:
+            logger.error('unable to sent {}'.format(email))