Unicode encode errors (#49)

Fix unicode and encode errors

* Sending queued emails
* Sending non queued emails
* In admin detail view

Thanks to @xusy2k to find an trying to fix the bugs in PR #48.

Author: sastred

django_yubin/engine.py

@@ -9,7 +9,7 @@
 
 """
 from django.core.mail import get_connection
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_bytes, smart_str
 from django.utils.timezone import now
 from django_yubin import constants, models, settings
 from lockfile import FileLock, AlreadyLocked, LockTimeout
@@ -201,18 +201,23 @@ def send_queued_message(queued_message, smtp_connection=None, blacklist=None,
                         (message.to_address.encode("utf-8"),
                          message.subject.encode("utf-8")))
             opened_connection = smtp_connection.open()
-            smtp_connection.connection.sendmail(
-                message.from_address,
-                [message.to_address],
-                smart_str(message.encoded_message).encode('utf-8'))
+            try:
+                smtp_connection.connection.sendmail(
+                    message.from_address,
+                    [message.to_address],
+                    smart_str(message.encoded_message).encode('utf-8'))
+            except UnicodeDecodeError:
+                smtp_connection.connection.sendmail(
+                    message.from_address,
+                    [message.to_address],
+                    force_bytes(message.encoded_message))
             queued_message.message.date_sent = now()
             queued_message.message.save()
             queued_message.delete()
             result = constants.RESULT_SENT
         except (SocketError, smtplib.SMTPSenderRefused,
-                smtplib.SMTPRecipientsRefused,
-                smtplib.SMTPAuthenticationError,
-                UnicodeEncodeError) as err:
+                smtplib.SMTPRecipientsRefused, smtplib.SMTPAuthenticationError,
+                UnicodeDecodeError, UnicodeEncodeError) as err:
             queued_message.defer()
             logger.warning("Message to %s deferred due to failure: %s" %
                            (message.to_address.encode("utf-8"), err))
@@ -257,18 +262,30 @@ def send_message(email_message, smtp_connection=None):
 
     try:
         opened_connection = smtp_connection.open()
-        smtp_connection.connection.sendmail(
-            email_message.from_email,
-            email_message.recipients(),
-            email_message.message().as_string())
+        try:
+            smtp_connection.connection.sendmail(
+                email_message.from_email,
+                email_message.recipients(),
+                smart_str(email_message.message().as_string()).encode('utf-8'))
+        except UnicodeDecodeError:
+            message = email_message.message()
+            charset = message.get_charset()
+            if charset:
+                charset = charset.get_output_charset()
+            else:
+                charset = 'utf-8'
+            smtp_connection.connection.sendmail(
+                email_message.from_email,
+                email_message.recipients(),
+                force_bytes(message.as_string(), charset))
         result = constants.RESULT_SENT
     except (SocketError, smtplib.SMTPSenderRefused,
-            smtplib.SMTPRecipientsRefused,
-            smtplib.SMTPAuthenticationError,
-            UnicodeEncodeError) as err:
+            smtplib.SMTPRecipientsRefused, smtplib.SMTPAuthenticationError,
+            UnicodeDecodeError, UnicodeEncodeError) as err:
         result = constants.RESULT_FAILED
         logger.warning("Message from %s failed due to: %s" %
                        (email_message.from_email, err))
+
     if opened_connection:
         smtp_connection.close()
     return result

django_yubin/models.py

@@ -3,7 +3,7 @@
 # ----------------------------------------------------------------------------
 
 from django.db import models
-from django.utils.encoding import python_2_unicode_compatible
+from django.utils.encoding import force_bytes, python_2_unicode_compatible
 from django.utils.timezone import now
 from django_yubin import constants, managers
 
@@ -52,6 +52,8 @@ def __str__(self):
     def get_pyz_message(self):
         try:
             msg = message_from_string(self.encoded_message)
+        except UnicodeEncodeError:
+            msg = message_from_string(force_bytes(self.encoded_message))
         except (TypeError, AttributeError):
             msg = message_from_bytes(self.encoded_message)
         return msg

django_yubin/smtp_queue.py

@@ -33,6 +33,6 @@ def send_messages(self, email_messages):
 
         num_sent = 0
         for email_message in email_messages:
-            queue_email_message(email_message)
-            num_sent += 1
+            if queue_email_message(email_message):
+                num_sent += 1
         return num_sent

humans.txt

@@ -11,5 +11,6 @@ Dave Peake (@davepeake)
 Grzegorz Bialy (@grzegorzbialy)
 Carlos Salom (@csalom)
 Juan Luis Garcia (@JuanLuisGarcia)
+Xus Zoyo (@xusy2k)
 
 https://github.com/APSL/django-yubin/graphs/contributors

tests/tests/test_backend.py

@@ -6,6 +6,8 @@
 
 from django.conf import settings as django_settings
 from django.core import mail
+from django.core.management import call_command
+from django.utils.encoding import force_text
 
 from django_yubin import models, constants, settings, mail_admins, mail_managers
 
@@ -95,7 +97,7 @@ def testQueuedMessagePriorities(self):
                              constants.PRIORITY_NORMAL)
 
     @skipIf(not RFC_6532_SUPPORT, 'RFC 6532 not supported')
-    def testUnicodeQueuedMessage(self):
+    def testUnicodeErrorQueuedMessage(self):
         """
         Checks that we capture unicode errors on mail
         """
@@ -111,22 +113,54 @@ def testUnicodeQueuedMessage(self):
         self.assertEqual(num_errors, 1)
 
     @skipIf(not RFC_6532_SUPPORT, 'RFC 6532 not supported')
-    def testUnicodePriorityMessage(self):
+    def testUnicodeQueuedMessage(self):
         """
-        Checks that we capture unicode errors on mail on priority.
-        It's hard to check as by definiton priority email does not Logs its
-        contents.
+        Checks that we capture unicode errors on mail
         """
         from django.core.management import call_command
-        msg = mail.EmailMessage(subject=u'á subject', body='body',
-                                from_email=u'juan.ló[email protected]', to=[u'únñ[email protected]'],
-                                headers={'X-Mail-Queue-Priority': 'now'})
+        msg = mail.EmailMessage(subject=u'Chère maman',
+                                body='Je t\'aime très fort',
+                                from_email='[email protected]',
+                                to=['[email protected]'])
         msg.send()
+
         queued_messages = models.QueuedMessage.objects.all()
         self.assertEqual(queued_messages.count(), 1)
+
         call_command('send_mail', verbosity='0')
+
+        queued_messages = models.QueuedMessage.objects.all()
+        self.assertEqual(queued_messages.count(), 0)
+
         num_errors = models.Log.objects.filter(result=constants.RESULT_FAILED).count()
-        self.assertEqual(num_errors, 1)
+        self.assertEqual(num_errors, 0)
+
+        message = msg.message()
+        self.assertEqual(message['subject'], '=?utf-8?q?Ch=C3=A8re_maman?=')
+        self.assertEqual(force_text(message.get_payload()), 'Je t\'aime très fort')
+
+    @skipIf(not RFC_6532_SUPPORT, 'RFC 6532 not supported')
+    def testUnicodePriorityNowNotQueuedMessage(self):
+        """
+        Checks that we capture unicode errors on mail on priority.
+        It's hard to check as by definiton priority email does not Logs its
+        contents.
+        """
+        msg = mail.EmailMessage(subject=u'Chère maman',
+                                body='Je t\'aime très fort',
+                                from_email='[email protected]',
+                                to=['[email protected]'],
+                                headers={'X-Mail-Queue-Priority': 'now-not-queued'})
+        num_sent = mail.get_connection().send_messages([msg])
+        self.assertEqual(num_sent, 1)
+        queued_messages = models.QueuedMessage.objects.all()
+        self.assertEqual(queued_messages.count(), 0)
+        call_command('send_mail', verbosity='0')
+        num_errors = models.Log.objects.filter(result=constants.RESULT_FAILED).count()
+        self.assertEqual(num_errors, 0)
+        message = msg.message()
+        self.assertEqual(message['subject'], '=?utf-8?q?Ch=C3=A8re_maman?=')
+        self.assertEqual(force_text(message.get_payload()), 'Je t\'aime très fort')
 
     def testSendMessageNowPriority(self):
         # NOW priority message