| 1 |
# -*- coding: utf-8 -*- |
| 2 |
# |
| 3 |
# Copyright (C) 2016-2023 by the Free Software Foundation, Inc. |
| 4 |
# |
| 5 |
# This file is part of Django-Mailman. |
| 6 |
# |
| 7 |
# Django-Mailman is free software: you can redistribute it and/or modify it |
| 8 |
# under the terms of the GNU General Public License as published by the Free |
| 9 |
# Software Foundation, either version 3 of the License, or (at your option) |
| 10 |
# any later version. |
| 11 |
# |
| 12 |
# Django-Mailman is distributed in the hope that it will be useful, but WITHOUT |
| 13 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 14 |
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 15 |
# more details. |
| 16 |
# |
| 17 |
# You should have received a copy of the GNU General Public License along with |
| 18 |
# Django-Mailman. If not, see <http://www.gnu.org/licenses/>. |
| 19 |
# |
| 20 |
# Author: Aurelien Bompard <[email protected]> |
| 21 |
# |
| 22 |
|
| 23 |
import unittest |
| 24 |
from email import message_from_binary_file, message_from_file, policy |
| 25 |
from traceback import format_exc |
| 26 |
|
| 27 |
from django_mailman3.lib.scrub import Scrubber |
| 28 |
from django_mailman3.tests.utils import get_test_file |
| 29 |
|
| 30 |
|
| 31 |
class TestScrubber(unittest.TestCase): |
| 32 |
|
| 33 |
def _check_html_attachment(self, value, expected): |
| 34 |
""" |
| 35 |
Python's mimetype module does not give predictable results: |
| 36 |
https://mail.python.org/pipermail/python-list/2014-February/678963.html |
| 37 |
""" |
| 38 |
self.assertEqual(value[0], expected[0]) |
| 39 |
self.assertIn(value[1], ["attachment.html", "attachment.htm"]) |
| 40 |
self.assertEqual(value[2:4], expected[2:4]) |
| 41 |
|
| 42 |
def test_attachment_1(self): |
| 43 |
with open(get_test_file("attachment-1.txt")) as email_file: |
| 44 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 45 |
scrubber = Scrubber(msg) |
| 46 |
contents, attachments = scrubber.scrub() |
| 47 |
self.assertEqual(len(attachments), 1) |
| 48 |
self.assertEqual(attachments[0], ( |
| 49 |
2, 'puntogil.vcf', 'text/x-vcard', "utf-8", |
| 50 |
'begin:vcard\nfn:gil\nn:;gil\nversion:2.1\n' |
| 51 |
'end:vcard\n\n')) |
| 52 |
self.assertEqual( |
| 53 |
contents, |
| 54 |
"This is a test message.\n\n" |
| 55 |
"\n-- \ndevel mailing list\[email protected]\n" |
| 56 |
"https://admin.fedoraproject.org/mailman/listinfo/devel\n" |
| 57 |
) |
| 58 |
|
| 59 |
def test_attachment_2(self): |
| 60 |
with open(get_test_file("attachment-2.txt")) as email_file: |
| 61 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 62 |
scrubber = Scrubber(msg) |
| 63 |
contents, attachments = scrubber.scrub() |
| 64 |
self.assertEqual(len(attachments), 1) |
| 65 |
self.assertEqual(attachments[0], ( |
| 66 |
3, 'signature.asc', 'application/pgp-signature', None, |
| 67 |
b'-----BEGIN PGP SIGNATURE-----\nVersion: GnuPG v1.4.12 ' |
| 68 |
b'(GNU/Linux)\nComment: Using GnuPG with Mozilla - ' |
| 69 |
b'http://www.enigmail.net/\n\niEYEARECAAYFAlBhm3oACgkQhmBj' |
| 70 |
b'z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\nEeUAn2OqtaXaXaQV7rx+' |
| 71 |
b'SmOldmSzcFw4\n=OEJv\n-----END PGP SIGNATURE-----\n')) |
| 72 |
self.assertEqual( |
| 73 |
contents, |
| 74 |
"This is a test message\nNon-ascii chars: Hofm\xfchlgasse\n" |
| 75 |
"\n-- \ndevel mailing list\[email protected]\n" |
| 76 |
"https://admin.fedoraproject.org/mailman/listinfo/devel\n" |
| 77 |
) |
| 78 |
|
| 79 |
def test_attachment_3(self): |
| 80 |
with open(get_test_file("attachment-3.txt")) as email_file: |
| 81 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 82 |
scrubber = Scrubber(msg) |
| 83 |
contents, attachments = scrubber.scrub() |
| 84 |
self.assertEqual(len(attachments), 2) |
| 85 |
# HTML part |
| 86 |
self._check_html_attachment( |
| 87 |
attachments[0], |
| 88 |
(3, "attachment.html", "text/html", "iso-8859-1")) |
| 89 |
self.assertEqual(len(attachments[0][4]), 3114) |
| 90 |
# Image attachment |
| 91 |
self.assertEqual( |
| 92 |
attachments[1][0:4], |
| 93 |
(4, "GeoffreyRoucourt.jpg", "image/jpeg", None)) |
| 94 |
self.assertEqual(len(attachments[1][4]), 282180) |
| 95 |
# Scrubbed content |
| 96 |
self.assertEqual(contents, "This is a test message\n\n\n") |
| 97 |
|
| 98 |
def test_html_email_1(self): |
| 99 |
with open(get_test_file("html-email-1.txt")) as email_file: |
| 100 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 101 |
scrubber = Scrubber(msg) |
| 102 |
contents, attachments = scrubber.scrub() |
| 103 |
self.assertEqual(len(attachments), 1) |
| 104 |
# HTML part |
| 105 |
self._check_html_attachment( |
| 106 |
attachments[0], |
| 107 |
(2, "attachment.html", "text/html", "iso-8859-1")) |
| 108 |
self.assertEqual(len(attachments[0][4]), 2688) |
| 109 |
# Scrubbed content |
| 110 |
self.assertEqual( |
| 111 |
contents, |
| 112 |
"This is a test message\n" |
| 113 |
"Non-ASCII chars: r\xe9ponse fran\xe7ais \n\n\n") |
| 114 |
|
| 115 |
def test_html_only_email(self): |
| 116 |
# This email only has an HTML part, thus the scrubbed content will be |
| 117 |
# empty. It should be an unicode empty string, not str. |
| 118 |
with open(get_test_file("html-email-2.txt")) as email_file: |
| 119 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 120 |
scrubber = Scrubber(msg) |
| 121 |
contents = scrubber.scrub()[0] |
| 122 |
self.assertTrue( |
| 123 |
isinstance(contents, str), |
| 124 |
"Scrubbed content should always be unicode") |
| 125 |
|
| 126 |
def _test_non_ascii_payload(self, enc): |
| 127 |
with open(get_test_file("payload-%s.txt" % enc), 'rb') as email_file: |
| 128 |
msg = message_from_binary_file(email_file, policy=policy.SMTP) |
| 129 |
scrubber = Scrubber(msg) |
| 130 |
contents = scrubber.scrub()[0] |
| 131 |
self.assertTrue(isinstance(contents, str)) |
| 132 |
self.assertEqual( |
| 133 |
contents, |
| 134 |
'This message contains non-ascii characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n') # noqa: E501 |
| 135 |
|
| 136 |
def test_non_ascii_payload_utf8(self): |
| 137 |
"""Scrubber must handle non-ascii messages""" |
| 138 |
self._test_non_ascii_payload("utf8") |
| 139 |
|
| 140 |
def test_non_ascii_payload_iso8859(self): |
| 141 |
"""Scrubber must handle non-ascii messages""" |
| 142 |
self._test_non_ascii_payload("iso8859") |
| 143 |
|
| 144 |
def test_bad_content_type(self): |
| 145 |
"""Scrubber must handle unknown content-types""" |
| 146 |
with open(get_test_file("payload-unknown.txt")) as email_file: |
| 147 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 148 |
scrubber = Scrubber(msg) |
| 149 |
try: |
| 150 |
contents = scrubber.scrub()[0] |
| 151 |
except LookupError as e: |
| 152 |
import traceback |
| 153 |
print(traceback.format_exc()) |
| 154 |
self.fail(e) # codec not found |
| 155 |
self.assertTrue(isinstance(contents, str)) |
| 156 |
|
| 157 |
def test_bad_charset_html(self): |
| 158 |
"""Scrubber must handle unknown charsets in html parts too.""" |
| 159 |
with open(get_test_file("bad_charset.txt")) as email_file: |
| 160 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 161 |
scrubber = Scrubber(msg) |
| 162 |
try: |
| 163 |
contents = scrubber.scrub()[0] |
| 164 |
except LookupError as e: |
| 165 |
import traceback |
| 166 |
print(traceback.format_exc()) |
| 167 |
self.fail(e) # codec not found |
| 168 |
self.assertTrue(isinstance(contents, str)) |
| 169 |
|
| 170 |
def test_attachment_4(self): |
| 171 |
with open(get_test_file("attachment-4.txt")) as email_file: |
| 172 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 173 |
scrubber = Scrubber(msg) |
| 174 |
contents, attachments = scrubber.scrub() |
| 175 |
self.assertEqual(len(attachments), 2) |
| 176 |
# HTML part |
| 177 |
self._check_html_attachment( |
| 178 |
attachments[0], |
| 179 |
(3, "attachment.html", "text/html", "iso-8859-1")) |
| 180 |
self.assertEqual(len(attachments[0][4]), 113) |
| 181 |
# text attachment |
| 182 |
self.assertEqual( |
| 183 |
attachments[1][0:4], |
| 184 |
(4, "todo-déjeuner.txt", "text/plain", "utf-8")) |
| 185 |
self.assertEqual(len(attachments[1][4]), 110) |
| 186 |
# Scrubbed content |
| 187 |
self.assertEqual( |
| 188 |
contents, |
| 189 |
'This is a test, HTML message with ' |
| 190 |
'accented letters : \xe9 \xe8 \xe7 \xe0.\nAnd an ' |
| 191 |
'attachment with an accented filename\n\n\n\n\n') |
| 192 |
|
| 193 |
def test_attachment_5(self): |
| 194 |
with open(get_test_file("attachment-5.txt")) as email_file: |
| 195 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 196 |
scrubber = Scrubber(msg) |
| 197 |
contents, attachments = scrubber.scrub() |
| 198 |
self.assertEqual(len(attachments), 1) |
| 199 |
# text attachment |
| 200 |
self.assertEqual( |
| 201 |
attachments[0][0:4], |
| 202 |
(2, "todo-déjeuner.txt", "text/plain", "utf-8")) |
| 203 |
self.assertEqual(len(attachments[0][4]), 110) |
| 204 |
# Scrubbed content |
| 205 |
self.assertEqual( |
| 206 |
contents, |
| 207 |
'This is a test, HTML message with ' |
| 208 |
'accented letters : \xe9 \xe8 \xe7 \xe0.\nAnd an ' |
| 209 |
'attachment with an accented filename\n\n\n\n\n\n') |
| 210 |
|
| 211 |
def test_attachedMmessage_rfc822(self): |
| 212 |
with open(get_test_file("attached_message.txt")) as email_file: |
| 213 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 214 |
scrubber = Scrubber(msg) |
| 215 |
contents, attachments = scrubber.scrub() |
| 216 |
self.assertEqual(len(attachments), 1) |
| 217 |
self.assertEqual( |
| 218 |
attachments[0][0:4], |
| 219 |
(2, "Moderation.eml", "message/rfc822", None)) |
| 220 |
self.assertEqual(len(attachments[0][4]), 462) |
| 221 |
self.assertEqual(contents, 'See the attached.\n\n\n') |
| 222 |
self.assertIn('Message-ID: <1d3c4594-1268', attachments[0][4]) |
| 223 |
|
| 224 |
def test_attachment_name_badly_encoded(self): |
| 225 |
with open(get_test_file("email-bad-filename.txt"), 'rb') as email_file: |
| 226 |
msg = message_from_binary_file(email_file, policy=policy.SMTP) |
| 227 |
scrubber = Scrubber(msg) |
| 228 |
try: |
| 229 |
attachments = scrubber.scrub()[1] |
| 230 |
except UnicodeDecodeError: |
| 231 |
print(format_exc()) |
| 232 |
self.fail("Could not decode the filename") |
| 233 |
# The filename has non-ascii characters without the encoding specified, |
| 234 |
# Python will try to decode their name with best guess (ascii) and then |
| 235 |
# replace the characters that don't correspond to an ascii code |
| 236 |
# point. Then, we scrub the filename to allow only alpahun with dash, |
| 237 |
# underscore and dot. |
| 238 |
self.assertEqual( |
| 239 |
attachments, |
| 240 |
[(0, 'non-ascii-u3b5.jpg', 'text/plain', None, 'Dummy content\n')]) |
| 241 |
|
| 242 |
def test_remove_next_part_from_content(self): |
| 243 |
with open(get_test_file("pipermail_nextpart.txt")) as email_file: |
| 244 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 245 |
scrubber = Scrubber(msg) |
| 246 |
contents = scrubber.scrub()[0] |
| 247 |
|
| 248 |
self.assertFalse("-------------- next part --------------" in contents) |
| 249 |
|
| 250 |
def test_name_unicode(self): |
| 251 |
for num in range(1, 6): |
| 252 |
with open(get_test_file("attachment-%d.txt" % num)) as email_file: |
| 253 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 254 |
scrubber = Scrubber(msg) |
| 255 |
attachments = scrubber.scrub()[1] |
| 256 |
for attachment in attachments: |
| 257 |
name = attachment[1] |
| 258 |
self.assertTrue(isinstance(name, str), |
| 259 |
"attachment %r must be unicode" % name) |
| 260 |
|
| 261 |
def test_msg_with_null_byte(self): |
| 262 |
with open(get_test_file("null_byte.txt")) as email_file: |
| 263 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 264 |
scrubber = Scrubber(msg) |
| 265 |
body = scrubber.scrub()[0] |
| 266 |
self.assertEqual(body, 'The body with a -><- null byte\n') |
| 267 |
|
| 268 |
def test_multipart_msg_with_null_byte(self): |
| 269 |
with open(get_test_file("null_byte_multipart.txt")) as email_file: |
| 270 |
msg = message_from_file(email_file, policy=policy.SMTP) |
| 271 |
scrubber = Scrubber(msg) |
| 272 |
body, attachments = scrubber.scrub() |
| 273 |
self.assertEqual(body, """The body with a -><- null byte |
| 274 |
|
| 275 |
-- \nA sig with a -><- null byte |
| 276 |
""") |
| 277 |
self.assertEqual(attachments[0][4], |
| 278 |
b'The attachment with a ->\x00<- null byte') |