fixup the shop to work in the new multicamp world, fix pdf generation, add new letterhead templates, rework invoiceworker into a generic worker runner

README.md

@@ -25,6 +25,9 @@ Install system dependencies (method depends on OS):
 - libjpeg (for pdf generation)
   - Debian: libjpeg-dev
   - FreeBSD: graphics/jpeg-turbo
+- wkhtmltopdf (also for pdf generation):
+  - Debian: wkhtmltopdf
+  - FreeBSD: converters/wkhtmltopdf
 
 ### Python packages
 Install pip packages:

src/bornhack/environment_settings.py.dist

@@ -47,6 +47,7 @@ COINIFY_API_SECRET='{{ coinify_api_secret }}'
 COINIFY_IPN_SECRET='{{ coinify_ipn_secret }}'
 
 # shop settings
+PDF_LETTERHEAD_FILENAME='bornhack-2017_test_letterhead.pdf'
 BANKACCOUNT_IBAN='{{ iban }}'
 BANKACCOUNT_SWIFTBIC='{{ swiftbic }}'
 BANKACCOUNT_REG='{{ regno }}'

src/camps/models.py

@@ -11,7 +11,6 @@ import logging
 logger = logging.getLogger("bornhack.%s" % __name__)
 
 
-
 class Camp(CreatedUpdatedModel, UUIDModel):
     class Meta:
         verbose_name = 'Camp'

src/shop/invoiceworker.py

@@ -1,36 +1,40 @@
 from django.core.files import File
-from django.conf import settings
 from django.utils import timezone
 from shop.pdf import generate_pdf_letter
 from shop.email import send_invoice_email, send_creditnote_email
 from shop.models import Order, CustomOrder, Invoice, CreditNote
 from decimal import Decimal
-import logging
+import logging, importlib
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger('bornhack.%s' % __name__)
 
 
-def run_invoice_worker():
-    logging.basicConfig(level=logging.INFO)
-    logger = logging.getLogger(__name__)
+def do_work():
+    """
+        The invoice worker creates Invoice objects for shop orders and
+        for custom orders. It also generates PDF files for Invoice objects
+        that have no PDF. It also emails invoices for shop orders.
+    """
 
+    ###############################################################
     # check if we need to generate any invoices for shop orders
     for order in Order.objects.filter(paid=True, invoice__isnull=True):
         # generate invoice for this Order
         Invoice.objects.create(order=order)
         logger.info('Generated Invoice object for %s' % order)
 
+
+    ###############################################################
     # check if we need to generate any invoices for custom orders
     for customorder in CustomOrder.objects.filter(invoice__isnull=True):
         # generate invoice for this CustomOrder
         Invoice.objects.create(customorder=customorder)
         logger.info('Generated Invoice object for %s' % customorder)
 
+
+    ###############################################################
     # check if we need to generate any pdf invoices
     for invoice in Invoice.objects.filter(pdf=''):
-        # put the dict with data for the pdf together
-        formatdict = {
-            'invoice': invoice,
-        }
-
         # generate the pdf
         try:
             if invoice.customorder:
@@ -40,26 +44,63 @@ def run_invoice_worker():
             pdffile = generate_pdf_letter(
                 filename=invoice.filename,
                 template=template,
-                formatdict=formatdict,
+                formatdict={
+                    'invoice': invoice,
+                },
             )
             logger.info('Generated pdf for invoice %s' % invoice)
         except Exception as E:
-            logger.error('ERROR: Unable to generate PDF file for invoice #%s. Error: %s' % (invoice.pk, E))
-            continue
-
-        # so, do we have a pdf?
-        if not pdffile:
-            logger.error('ERROR: Unable to generate PDF file for invoice #%s' % invoice.pk)
+            logger.exception('Unable to generate PDF file for invoice #%s. Error: %s' % (invoice.pk, E))
             continue
 
         # update invoice object with the file
         invoice.pdf.save(invoice.filename, File(pdffile))
         invoice.save()
 
+
     ###############################################################
     # check if we need to send out any invoices (only for shop orders, and only where pdf has been generated)
     for invoice in Invoice.objects.filter(order__isnull=False, sent_to_customer=False).exclude(pdf=''):
+        logger.info("found unmailed Invoice object: %s" % invoice)
         # send the email
         if send_invoice_email(invoice=invoice):
+            invoice.sent_to_customer=True
+            invoice.save()
             logger.info('OK: Invoice email sent to %s' % invoice.order.user.email)
+        else:
+            logger.error('Unable to send invoice email for order %s to %s' % (invoice.order.pk, invoice.order.user.email))
+
+
+    ###############################################################
+    # check if we need to generate any pdf creditnotes?
+    for creditnote in CreditNote.objects.filter(pdf=''):
+        # generate the pdf
+        try:
+            pdffile = generate_pdf_letter(
+                filename=creditnote.filename,
+                template='pdf/creditnote.html',
+                formatdict={
+                    'creditnote': creditnote,
+                },
+            )
+            logger.info('Generated pdf for creditnote %s' % creditnote)
+        except Exception as E:
+            logger.exception('Unable to generate PDF file for creditnote #%s. Error: %s' % (creditnote.pk, E))
+            continue
+
+        # update creditnote object with the file
+        creditnote.pdf.save(creditnote.filename, File(pdffile))
+        creditnote.save()
+
+
+    ###############################################################
+    # check if we need to send out any creditnotes (only where pdf has been generated)
+    for creditnote in CreditNote.objects.filter(sent_to_customer=False).exclude(pdf=''):
+        # send the email
+        if send_creditnote_email(creditnote=creditnote):
+            logger.info('OK: Creditnote email sent to %s' % creditnote.user.email)
+            creditnote.sent_to_customer=True
+            creditnote.save()
+        else:
+            logger.error('Unable to send creditnote email for creditnote %s to %s' % (creditnote.pk, creditnote.user.email))
 

src/shop/management/commands/invoice-worker.py

@@ -1,46 +0,0 @@
-from django.core.management.base import BaseCommand
-from shop import invoiceworker
-from time import sleep
-import signal, sys
-import logging
-
-
-class Command(BaseCommand):
-    args = 'none'
-    help = 'Generate invoices and credit notes, and email invoices that have not been sent yet'
-    exit = False
-
-    def __init__(self):
-        logging.basicConfig(level=logging.INFO)
-        self.logger = logging.getLogger('bornhack.%s' % __name__)
-
-    def reload_worker_code(self, signum, frame):
-        self.logger.info("Reloading shop.invoiceworker module...")
-        reload(invoiceworker)
-        self.logger.info("Done reloading shop.invoiceworker module")
-
-    def clean_exit(self, signum, frame):
-        self.logger.info("SIGTERM received, exiting gracefully soon...")
-        self.exit = True
-
-    def handle(self, *args, **options):
-        self.logger.info("Connecting signals...")
-        signal.signal(signal.SIGHUP, self.reload_worker_code)
-        signal.signal(signal.SIGTERM, self.clean_exit)
-        signal.signal(signal.SIGINT, self.clean_exit)
-
-        self.logger.info("Entering main loop...")
-        while True:
-            # run invoiceworker
-            invoiceworker.run_invoice_worker()
-
-            # sleep for 60 seconds, but check sys.exit every second
-            i = 0
-            while i < 60:
-                if self.exit:
-                    self.logger.info("Graceful exit requested, calling sys.exit(0) now")
-                    sys.exit(0)
-                else:
-                    i += 1
-                    sleep(1)
-

src/shop/pdf.py

@@ -1,16 +1,21 @@
+from django.contrib.auth.models import AnonymousUser
 from wkhtmltopdf.views import PDFTemplateResponse
 from PyPDF2 import PdfFileWriter, PdfFileReader
 from django.test.client import RequestFactory
 from django.conf import settings
-import io
-import logging
+import io, logging
 logger = logging.getLogger("bornhack.%s" % __name__)
 
 
 def generate_pdf_letter(filename, template, formatdict):
+    # conjure up a fake request for PDFTemplateResponse
+    request = RequestFactory().get('/')
+    request.user = AnonymousUser()
+    request.session = {}
+
     ### produce text-only PDF from template
     pdfgenerator = PDFTemplateResponse(
-        request=RequestFactory().get('/'),
+        request=request,
         template=template, 
         context=formatdict,
         cmd_options={
@@ -18,7 +23,7 @@ def generate_pdf_letter(filename, template, formatdict):
             'margin-bottom': 50,
         },
     )
-    textonlypdf = io.StringIO()
+    textonlypdf = io.BytesIO()
     textonlypdf.write(pdfgenerator.rendered_content)
 
     ### create a blank pdf to work with
@@ -28,7 +33,7 @@ def generate_pdf_letter(filename, template, formatdict):
     pdfreader = PdfFileReader(textonlypdf)
 
     ### get watermark from watermark file
-    watermark = PdfFileReader(open(settings.LETTERHEAD_PDF_PATH, 'rb'))
+    watermark = PdfFileReader(open("%s/pdf/%s" % (settings.STATICFILES_DIRS[0], settings.PDF_LETTERHEAD_FILENAME), 'rb'))
 
     ### add the watermark to all pages
     for pagenum in range(pdfreader.getNumPages()):
@@ -42,12 +47,13 @@ def generate_pdf_letter(filename, template, formatdict):
         finalpdf.addPage(page)
 
     ### save the generated pdf to the archive
-    with open(settings.PDF_ARCHIVE_PATH+filename, 'wb') as fh:
+    fullpath = settings.PDF_ARCHIVE_PATH+filename
+    with open(fullpath, 'wb') as fh:
         finalpdf.write(fh)
-        logger.info('Saved pdf to archive: %s' % settings.PDF_ARCHIVE_PATH+filename)
+        logger.info('Saved pdf to archive: %s' % fullpath)
 
     ### return a file object with the data
-    returnfile = io.StringIO()
+    returnfile = io.BytesIO()
     finalpdf.write(returnfile)
     return returnfile
 

src/shop/templates/pdf/invoice.html

@@ -23,10 +23,13 @@
         <b>Name
       <td>
         <b>Quantity
-      <td>
+      <td align="right">
         <b>Price
-      <td>
+      <td align="right">
         <b>Total
+
+<tr><td style="height: 1px; line-height: 1px; border-top: 1pt solid black;" colspan="4">&nbsp;</td></tr>
+
 {% for order_product in invoice.order.orderproductrelation_set.all %}
   <tr>
       <td>
@@ -39,19 +42,24 @@
       {{ order_product.total|currency }}
 {% endfor %}
 
+<tr><td style="height: 1px; line-height: 1px; border-top: 1pt solid black;" colspan="4">&nbsp;</td></tr>
+
   <tr>
     <td colspan="2">
     <td>
-      <strong>Danish VAT (25%)</strong>
+      <strong>Included Danish VAT (25%)</strong>
     <td align="right">
       {{ invoice.order.vat|currency }}
 
   <tr>
     <td colspan="2">
     <td>
-      <strong>Total</strong>
+      <strong>Invoice Total</strong>
     <td align="right">
       {{ invoice.order.total|currency }}
+
+  <tr><td colspan="2"></td><td style="height: 1px; line-height: 1px; border-top: 1pt solid black; border-bottom: 1pt solid black;" colspan="4">&nbsp;</td></tr>
+
 </table>
 <br>
 <h3>The order has been paid in full.</h3>
@@ -66,6 +74,6 @@
 <p>Note: Danish law grants everyone a 14 days 'cooling-off' period for 
 internet purchases, in which the customer can regret the purchase for any 
 reason (including just changing your mind). In case you regret this purchase 
-please contact us on info@bornhack.dk for a full refund. This is possible until
+please contact us on <b>info@bornhack.dk</b> for a full refund. This is possible until
 <b>{{ invoice.regretdate|date:"b jS, Y" }}</b>. Please see our General 
 Terms &amp; Conditions on our website for more information on order cancellation.</p>

src/shop/templates/product_detail.html

@@ -21,7 +21,7 @@
     <h3>
       <small>Price</small><br />
       {{ product.price|currency }}<br />
-      ~{{ product.price|approxeur }}
+      (~{{ product.price|approxeur }})
     </h3>
 
     <hr />

src/utils/management/commands/runworker.py

@@ -0,0 +1,68 @@
+from django.core.management.base import BaseCommand
+from time import sleep
+import signal, sys
+import logging, importlib
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger('bornhack.%s' % __name__)
+
+
+class Command(BaseCommand):
+    args = 'none'
+    help = 'Run a worker. Takes the worker module as the first positional argument and calls the do_work() function in it. Optional argumends can be seen with -h / --help'
+    exit_now = False
+
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            'workermodule',
+            type=str,
+            help='The dotted path to the module which contains the do_work() function to call periodically.'
+        )
+        parser.add_argument(
+            '--sleep',
+            type=int,
+            default=60,
+            help='The number of seconds to sleep between calls'
+        )
+
+    def reload_worker(self, signum, frame):
+        # we exit when we receive a HUP (expecting uwsgi or another supervisor to restart this worker)
+        # this is more reliable than using importlib.reload to reload the workermodule code, since that
+        # doesn't reload imports inside the worker module.
+        logger.info("Signal %s (SIGHUP) received, exiting gracefully..." % signum)
+        self.exit_now = True
+
+
+    def clean_exit(self, signum, frame):
+        logger.info("Signal %s (INT or TERM) received, exiting gracefully..." % signum)
+        self.exit_now = True
+
+
+    def handle(self, *args, **options):
+        logger.info("Importing worker module...")
+        self.workermodule = importlib.import_module(options['workermodule'])
+        if not hasattr(self.workermodule, 'do_work'):
+            logger.error("module %s must have a do_work() method to call")
+            sys.exit(1)
+
+        logger.info("Connecting signals...")
+        signal.signal(signal.SIGHUP, self.reload_worker)
+        signal.signal(signal.SIGTERM, self.clean_exit)
+        signal.signal(signal.SIGINT, self.clean_exit)
+
+        logger.info("Entering main loop...")
+        while True:
+            # run worker code
+            getattr(self.workermodule, 'do_work')()
+
+            # sleep for N seconds before calling worker code again
+            i = 0
+            while i < options['sleep']:
+                # but check self.exit_now every second
+                if self.exit_now:
+                    sys.exit(0)
+                else:
+                    i += 1
+                    sleep(1)
+