# -*- coding: utf-8 -*- # # Copyright (C) 2016-2023 by the Free Software Foundation, Inc. # # This file is part of Django-Mailman. # # Django-Mailman is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # Django-Mailman is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along with # Django-Mailman. If not, see . # # Author: Aurelien Bompard # import unittest from email import message_from_binary_file, message_from_file, policy from traceback import format_exc from django_mailman3.lib.scrub import Scrubber from django_mailman3.tests.utils import get_test_file class TestScrubber(unittest.TestCase): def _check_html_attachment(self, value, expected): """ Python's mimetype module does not give predictable results: https://mail.python.org/pipermail/python-list/2014-February/678963.html """ self.assertEqual(value[0], expected[0]) self.assertIn(value[1], ["attachment.html", "attachment.htm"]) self.assertEqual(value[2:4], expected[2:4]) def test_attachment_1(self): with open(get_test_file("attachment-1.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 1) self.assertEqual(attachments[0], ( 2, 'puntogil.vcf', 'text/x-vcard', "utf-8", 'begin:vcard\nfn:gil\nn:;gil\nversion:2.1\n' 'end:vcard\n\n')) self.assertEqual( contents, "This is a test message.\n\n" "\n-- \ndevel mailing list\ndevel@lists.fedoraproject.org\n" "https://admin.fedoraproject.org/mailman/listinfo/devel\n" ) def test_attachment_2(self): with open(get_test_file("attachment-2.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 1) self.assertEqual(attachments[0], ( 3, 'signature.asc', 'application/pgp-signature', None, b'-----BEGIN PGP SIGNATURE-----\nVersion: GnuPG v1.4.12 ' b'(GNU/Linux)\nComment: Using GnuPG with Mozilla - ' b'http://www.enigmail.net/\n\niEYEARECAAYFAlBhm3oACgkQhmBj' b'z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\nEeUAn2OqtaXaXaQV7rx+' b'SmOldmSzcFw4\n=OEJv\n-----END PGP SIGNATURE-----\n')) self.assertEqual( contents, "This is a test message\nNon-ascii chars: Hofm\xfchlgasse\n" "\n-- \ndevel mailing list\ndevel@lists.fedoraproject.org\n" "https://admin.fedoraproject.org/mailman/listinfo/devel\n" ) def test_attachment_3(self): with open(get_test_file("attachment-3.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 2) # HTML part self._check_html_attachment( attachments[0], (3, "attachment.html", "text/html", "iso-8859-1")) self.assertEqual(len(attachments[0][4]), 3114) # Image attachment self.assertEqual( attachments[1][0:4], (4, "GeoffreyRoucourt.jpg", "image/jpeg", None)) self.assertEqual(len(attachments[1][4]), 282180) # Scrubbed content self.assertEqual(contents, "This is a test message\n\n\n") def test_html_email_1(self): with open(get_test_file("html-email-1.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 1) # HTML part self._check_html_attachment( attachments[0], (2, "attachment.html", "text/html", "iso-8859-1")) self.assertEqual(len(attachments[0][4]), 2688) # Scrubbed content self.assertEqual( contents, "This is a test message\n" "Non-ASCII chars: r\xe9ponse fran\xe7ais \n\n\n") def test_html_only_email(self): # This email only has an HTML part, thus the scrubbed content will be # empty. It should be an unicode empty string, not str. with open(get_test_file("html-email-2.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents = scrubber.scrub()[0] self.assertTrue( isinstance(contents, str), "Scrubbed content should always be unicode") def _test_non_ascii_payload(self, enc): with open(get_test_file("payload-%s.txt" % enc), 'rb') as email_file: msg = message_from_binary_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents = scrubber.scrub()[0] self.assertTrue(isinstance(contents, str)) self.assertEqual( contents, 'This message contains non-ascii characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n') # noqa: E501 def test_non_ascii_payload_utf8(self): """Scrubber must handle non-ascii messages""" self._test_non_ascii_payload("utf8") def test_non_ascii_payload_iso8859(self): """Scrubber must handle non-ascii messages""" self._test_non_ascii_payload("iso8859") def test_bad_content_type(self): """Scrubber must handle unknown content-types""" with open(get_test_file("payload-unknown.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) try: contents = scrubber.scrub()[0] except LookupError as e: import traceback print(traceback.format_exc()) self.fail(e) # codec not found self.assertTrue(isinstance(contents, str)) def test_bad_charset_html(self): """Scrubber must handle unknown charsets in html parts too.""" with open(get_test_file("bad_charset.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) try: contents = scrubber.scrub()[0] except LookupError as e: import traceback print(traceback.format_exc()) self.fail(e) # codec not found self.assertTrue(isinstance(contents, str)) def test_attachment_4(self): with open(get_test_file("attachment-4.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 2) # HTML part self._check_html_attachment( attachments[0], (3, "attachment.html", "text/html", "iso-8859-1")) self.assertEqual(len(attachments[0][4]), 113) # text attachment self.assertEqual( attachments[1][0:4], (4, "todo-déjeuner.txt", "text/plain", "utf-8")) self.assertEqual(len(attachments[1][4]), 110) # Scrubbed content self.assertEqual( contents, 'This is a test, HTML message with ' 'accented letters : \xe9 \xe8 \xe7 \xe0.\nAnd an ' 'attachment with an accented filename\n\n\n\n\n') def test_attachment_5(self): with open(get_test_file("attachment-5.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 1) # text attachment self.assertEqual( attachments[0][0:4], (2, "todo-déjeuner.txt", "text/plain", "utf-8")) self.assertEqual(len(attachments[0][4]), 110) # Scrubbed content self.assertEqual( contents, 'This is a test, HTML message with ' 'accented letters : \xe9 \xe8 \xe7 \xe0.\nAnd an ' 'attachment with an accented filename\n\n\n\n\n\n') def test_attachedMmessage_rfc822(self): with open(get_test_file("attached_message.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents, attachments = scrubber.scrub() self.assertEqual(len(attachments), 1) self.assertEqual( attachments[0][0:4], (2, "Moderation.eml", "message/rfc822", None)) self.assertEqual(len(attachments[0][4]), 462) self.assertEqual(contents, 'See the attached.\n\n\n') self.assertIn('Message-ID: <1d3c4594-1268', attachments[0][4]) def test_attachment_name_badly_encoded(self): with open(get_test_file("email-bad-filename.txt"), 'rb') as email_file: msg = message_from_binary_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) try: attachments = scrubber.scrub()[1] except UnicodeDecodeError: print(format_exc()) self.fail("Could not decode the filename") # The filename has non-ascii characters without the encoding specified, # Python will try to decode their name with best guess (ascii) and then # replace the characters that don't correspond to an ascii code # point. Then, we scrub the filename to allow only alpahun with dash, # underscore and dot. self.assertEqual( attachments, [(0, 'non-ascii-u3b5.jpg', 'text/plain', None, 'Dummy content\n')]) def test_remove_next_part_from_content(self): with open(get_test_file("pipermail_nextpart.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) contents = scrubber.scrub()[0] self.assertFalse("-------------- next part --------------" in contents) def test_name_unicode(self): for num in range(1, 6): with open(get_test_file("attachment-%d.txt" % num)) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) attachments = scrubber.scrub()[1] for attachment in attachments: name = attachment[1] self.assertTrue(isinstance(name, str), "attachment %r must be unicode" % name) def test_msg_with_null_byte(self): with open(get_test_file("null_byte.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) body = scrubber.scrub()[0] self.assertEqual(body, 'The body with a -><- null byte\n') def test_multipart_msg_with_null_byte(self): with open(get_test_file("null_byte_multipart.txt")) as email_file: msg = message_from_file(email_file, policy=policy.SMTP) scrubber = Scrubber(msg) body, attachments = scrubber.scrub() self.assertEqual(body, """The body with a -><- null byte -- \nA sig with a -><- null byte """) self.assertEqual(attachments[0][4], b'The attachment with a ->\x00<- null byte')