Debian Codegraph @

django_mailman3/tests/test_scrub.py

raw
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (C) 2016-2023 by the Free Software Foundation, Inc.
4 #
5 # This file is part of Django-Mailman.
6 #
7 # Django-Mailman is free software: you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by the Free
9 # Software Foundation, either version 3 of the License, or (at your option)
10 # any later version.
11 #
12 # Django-Mailman is distributed in the hope that it will be useful, but WITHOUT
13 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 # more details.
16 #
17 # You should have received a copy of the GNU General Public License along with
18 # Django-Mailman. If not, see <http://www.gnu.org/licenses/>.
19 #
20 # Author: Aurelien Bompard <[email protected]>
21 #
22
23 import unittest
24 from email import message_from_binary_file, message_from_file, policy
25 from traceback import format_exc
26
27 from django_mailman3.lib.scrub import Scrubber
28 from django_mailman3.tests.utils import get_test_file
29
30
31 class TestScrubber(unittest.TestCase):
32
33 def _check_html_attachment(self, value, expected):
34 """
35 Python's mimetype module does not give predictable results:
36 https://mail.python.org/pipermail/python-list/2014-February/678963.html
37 """
38 self.assertEqual(value[0], expected[0])
39 self.assertIn(value[1], ["attachment.html", "attachment.htm"])
40 self.assertEqual(value[2:4], expected[2:4])
41
42 def test_attachment_1(self):
43 with open(get_test_file("attachment-1.txt")) as email_file:
44 msg = message_from_file(email_file, policy=policy.SMTP)
45 scrubber = Scrubber(msg)
46 contents, attachments = scrubber.scrub()
47 self.assertEqual(len(attachments), 1)
48 self.assertEqual(attachments[0], (
49 2, 'puntogil.vcf', 'text/x-vcard', "utf-8",
50 'begin:vcard\nfn:gil\nn:;gil\nversion:2.1\n'
51 'end:vcard\n\n'))
52 self.assertEqual(
53 contents,
54 "This is a test message.\n\n"
55 "\n-- \ndevel mailing list\[email protected]\n"
56 "https://admin.fedoraproject.org/mailman/listinfo/devel\n"
57 )
58
59 def test_attachment_2(self):
60 with open(get_test_file("attachment-2.txt")) as email_file:
61 msg = message_from_file(email_file, policy=policy.SMTP)
62 scrubber = Scrubber(msg)
63 contents, attachments = scrubber.scrub()
64 self.assertEqual(len(attachments), 1)
65 self.assertEqual(attachments[0], (
66 3, 'signature.asc', 'application/pgp-signature', None,
67 b'-----BEGIN PGP SIGNATURE-----\nVersion: GnuPG v1.4.12 '
68 b'(GNU/Linux)\nComment: Using GnuPG with Mozilla - '
69 b'http://www.enigmail.net/\n\niEYEARECAAYFAlBhm3oACgkQhmBj'
70 b'z394AnmMnQCcC+6tWcqE1dPQmIdRbLXgKGVp\nEeUAn2OqtaXaXaQV7rx+'
71 b'SmOldmSzcFw4\n=OEJv\n-----END PGP SIGNATURE-----\n'))
72 self.assertEqual(
73 contents,
74 "This is a test message\nNon-ascii chars: Hofm\xfchlgasse\n"
75 "\n-- \ndevel mailing list\[email protected]\n"
76 "https://admin.fedoraproject.org/mailman/listinfo/devel\n"
77 )
78
79 def test_attachment_3(self):
80 with open(get_test_file("attachment-3.txt")) as email_file:
81 msg = message_from_file(email_file, policy=policy.SMTP)
82 scrubber = Scrubber(msg)
83 contents, attachments = scrubber.scrub()
84 self.assertEqual(len(attachments), 2)
85 # HTML part
86 self._check_html_attachment(
87 attachments[0],
88 (3, "attachment.html", "text/html", "iso-8859-1"))
89 self.assertEqual(len(attachments[0][4]), 3114)
90 # Image attachment
91 self.assertEqual(
92 attachments[1][0:4],
93 (4, "GeoffreyRoucourt.jpg", "image/jpeg", None))
94 self.assertEqual(len(attachments[1][4]), 282180)
95 # Scrubbed content
96 self.assertEqual(contents, "This is a test message\n\n\n")
97
98 def test_html_email_1(self):
99 with open(get_test_file("html-email-1.txt")) as email_file:
100 msg = message_from_file(email_file, policy=policy.SMTP)
101 scrubber = Scrubber(msg)
102 contents, attachments = scrubber.scrub()
103 self.assertEqual(len(attachments), 1)
104 # HTML part
105 self._check_html_attachment(
106 attachments[0],
107 (2, "attachment.html", "text/html", "iso-8859-1"))
108 self.assertEqual(len(attachments[0][4]), 2688)
109 # Scrubbed content
110 self.assertEqual(
111 contents,
112 "This is a test message\n"
113 "Non-ASCII chars: r\xe9ponse fran\xe7ais \n\n\n")
114
115 def test_html_only_email(self):
116 # This email only has an HTML part, thus the scrubbed content will be
117 # empty. It should be an unicode empty string, not str.
118 with open(get_test_file("html-email-2.txt")) as email_file:
119 msg = message_from_file(email_file, policy=policy.SMTP)
120 scrubber = Scrubber(msg)
121 contents = scrubber.scrub()[0]
122 self.assertTrue(
123 isinstance(contents, str),
124 "Scrubbed content should always be unicode")
125
126 def _test_non_ascii_payload(self, enc):
127 with open(get_test_file("payload-%s.txt" % enc), 'rb') as email_file:
128 msg = message_from_binary_file(email_file, policy=policy.SMTP)
129 scrubber = Scrubber(msg)
130 contents = scrubber.scrub()[0]
131 self.assertTrue(isinstance(contents, str))
132 self.assertEqual(
133 contents,
134 'This message contains non-ascii characters:\n\xe9 \xe8 \xe7 \xe0 \xee \xef \xeb \u20ac\n') # noqa: E501
135
136 def test_non_ascii_payload_utf8(self):
137 """Scrubber must handle non-ascii messages"""
138 self._test_non_ascii_payload("utf8")
139
140 def test_non_ascii_payload_iso8859(self):
141 """Scrubber must handle non-ascii messages"""
142 self._test_non_ascii_payload("iso8859")
143
144 def test_bad_content_type(self):
145 """Scrubber must handle unknown content-types"""
146 with open(get_test_file("payload-unknown.txt")) as email_file:
147 msg = message_from_file(email_file, policy=policy.SMTP)
148 scrubber = Scrubber(msg)
149 try:
150 contents = scrubber.scrub()[0]
151 except LookupError as e:
152 import traceback
153 print(traceback.format_exc())
154 self.fail(e) # codec not found
155 self.assertTrue(isinstance(contents, str))
156
157 def test_bad_charset_html(self):
158 """Scrubber must handle unknown charsets in html parts too."""
159 with open(get_test_file("bad_charset.txt")) as email_file:
160 msg = message_from_file(email_file, policy=policy.SMTP)
161 scrubber = Scrubber(msg)
162 try:
163 contents = scrubber.scrub()[0]
164 except LookupError as e:
165 import traceback
166 print(traceback.format_exc())
167 self.fail(e) # codec not found
168 self.assertTrue(isinstance(contents, str))
169
170 def test_attachment_4(self):
171 with open(get_test_file("attachment-4.txt")) as email_file:
172 msg = message_from_file(email_file, policy=policy.SMTP)
173 scrubber = Scrubber(msg)
174 contents, attachments = scrubber.scrub()
175 self.assertEqual(len(attachments), 2)
176 # HTML part
177 self._check_html_attachment(
178 attachments[0],
179 (3, "attachment.html", "text/html", "iso-8859-1"))
180 self.assertEqual(len(attachments[0][4]), 113)
181 # text attachment
182 self.assertEqual(
183 attachments[1][0:4],
184 (4, "todo-déjeuner.txt", "text/plain", "utf-8"))
185 self.assertEqual(len(attachments[1][4]), 110)
186 # Scrubbed content
187 self.assertEqual(
188 contents,
189 'This is a test, HTML message with '
190 'accented letters : \xe9 \xe8 \xe7 \xe0.\nAnd an '
191 'attachment with an accented filename\n\n\n\n\n')
192
193 def test_attachment_5(self):
194 with open(get_test_file("attachment-5.txt")) as email_file:
195 msg = message_from_file(email_file, policy=policy.SMTP)
196 scrubber = Scrubber(msg)
197 contents, attachments = scrubber.scrub()
198 self.assertEqual(len(attachments), 1)
199 # text attachment
200 self.assertEqual(
201 attachments[0][0:4],
202 (2, "todo-déjeuner.txt", "text/plain", "utf-8"))
203 self.assertEqual(len(attachments[0][4]), 110)
204 # Scrubbed content
205 self.assertEqual(
206 contents,
207 'This is a test, HTML message with '
208 'accented letters : \xe9 \xe8 \xe7 \xe0.\nAnd an '
209 'attachment with an accented filename\n\n\n\n\n\n')
210
211 def test_attachedMmessage_rfc822(self):
212 with open(get_test_file("attached_message.txt")) as email_file:
213 msg = message_from_file(email_file, policy=policy.SMTP)
214 scrubber = Scrubber(msg)
215 contents, attachments = scrubber.scrub()
216 self.assertEqual(len(attachments), 1)
217 self.assertEqual(
218 attachments[0][0:4],
219 (2, "Moderation.eml", "message/rfc822", None))
220 self.assertEqual(len(attachments[0][4]), 462)
221 self.assertEqual(contents, 'See the attached.\n\n\n')
222 self.assertIn('Message-ID: <1d3c4594-1268', attachments[0][4])
223
224 def test_attachment_name_badly_encoded(self):
225 with open(get_test_file("email-bad-filename.txt"), 'rb') as email_file:
226 msg = message_from_binary_file(email_file, policy=policy.SMTP)
227 scrubber = Scrubber(msg)
228 try:
229 attachments = scrubber.scrub()[1]
230 except UnicodeDecodeError:
231 print(format_exc())
232 self.fail("Could not decode the filename")
233 # The filename has non-ascii characters without the encoding specified,
234 # Python will try to decode their name with best guess (ascii) and then
235 # replace the characters that don't correspond to an ascii code
236 # point. Then, we scrub the filename to allow only alpahun with dash,
237 # underscore and dot.
238 self.assertEqual(
239 attachments,
240 [(0, 'non-ascii-u3b5.jpg', 'text/plain', None, 'Dummy content\n')])
241
242 def test_remove_next_part_from_content(self):
243 with open(get_test_file("pipermail_nextpart.txt")) as email_file:
244 msg = message_from_file(email_file, policy=policy.SMTP)
245 scrubber = Scrubber(msg)
246 contents = scrubber.scrub()[0]
247
248 self.assertFalse("-------------- next part --------------" in contents)
249
250 def test_name_unicode(self):
251 for num in range(1, 6):
252 with open(get_test_file("attachment-%d.txt" % num)) as email_file:
253 msg = message_from_file(email_file, policy=policy.SMTP)
254 scrubber = Scrubber(msg)
255 attachments = scrubber.scrub()[1]
256 for attachment in attachments:
257 name = attachment[1]
258 self.assertTrue(isinstance(name, str),
259 "attachment %r must be unicode" % name)
260
261 def test_msg_with_null_byte(self):
262 with open(get_test_file("null_byte.txt")) as email_file:
263 msg = message_from_file(email_file, policy=policy.SMTP)
264 scrubber = Scrubber(msg)
265 body = scrubber.scrub()[0]
266 self.assertEqual(body, 'The body with a -><- null byte\n')
267
268 def test_multipart_msg_with_null_byte(self):
269 with open(get_test_file("null_byte_multipart.txt")) as email_file:
270 msg = message_from_file(email_file, policy=policy.SMTP)
271 scrubber = Scrubber(msg)
272 body, attachments = scrubber.scrub()
273 self.assertEqual(body, """The body with a -><- null byte
274
275 -- \nA sig with a -><- null byte
276 """)
277 self.assertEqual(attachments[0][4],
278 b'The attachment with a ->\x00<- null byte')