Debian Codegraph @

Lib/test/test_ftplib.py

raw
1 """Test script for ftplib module."""
2
3 # Modified by Giampaolo Rodola' to test FTP class, IPv6 and TLS
4 # environment
5
6 import ftplib
7 import socket
8 import io
9 import errno
10 import os
11 import threading
12 import time
13 import unittest
14 try:
15 import ssl
16 except ImportError:
17 ssl = None
18
19 from unittest import mock, TestCase, skipUnless
20 from test import support
21 from test.support import requires_subprocess
22 from test.support import threading_helper
23 from test.support import socket_helper
24 from test.support import warnings_helper
25 from test.support import asynchat
26 from test.support import asyncore
27 from test.support.socket_helper import HOST, HOSTv6
28
29
30 support.requires_working_socket(module=True)
31
32 TIMEOUT = support.LOOPBACK_TIMEOUT
33 DEFAULT_ENCODING = 'utf-8'
34 # the dummy data returned by server over the data channel when
35 # RETR, LIST, NLST, MLSD commands are issued
36 RETR_DATA = 'abcde\xB9\xB2\xB3\xA4\xA6\r\n' * 1000
37 LIST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n'
38 NLST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n'
39 MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n"
40 "type=pdir;perm=e;unique==keVO1+d?3; ..\r\n"
41 "type=OS.unix=slink:/foobar;perm=;unique==keVO1+4G4; foobar\r\n"
42 "type=OS.unix=chr-13/29;perm=;unique==keVO1+5G4; device\r\n"
43 "type=OS.unix=blk-11/108;perm=;unique==keVO1+6G4; block\r\n"
44 "type=file;perm=awr;unique==keVO1+8G4; writable\r\n"
45 "type=dir;perm=cpmel;unique==keVO1+7G4; promiscuous\r\n"
46 "type=dir;perm=;unique==keVO1+1t2; no-exec\r\n"
47 "type=file;perm=r;unique==keVO1+EG4; two words\r\n"
48 "type=file;perm=r;unique==keVO1+IH4; leading space\r\n"
49 "type=file;perm=r;unique==keVO1+1G4; file1\r\n"
50 "type=dir;perm=cpmel;unique==keVO1+7G4; incoming\r\n"
51 "type=file;perm=r;unique==keVO1+1G4; file2\r\n"
52 "type=file;perm=r;unique==keVO1+1G4; file3\r\n"
53 "type=file;perm=r;unique==keVO1+1G4; file4\r\n"
54 "type=dir;perm=cpmel;unique==SGP1; dir \xAE non-ascii char\r\n"
55 "type=file;perm=r;unique==SGP2; file \xAE non-ascii char\r\n")
56
57
58 def default_error_handler():
59 # bpo-44359: Silently ignore socket errors. Such errors occur when a client
60 # socket is closed, in TestFTPClass.tearDown() and makepasv() tests, and
61 # the server gets an error on its side.
62 pass
63
64
65 class DummyDTPHandler(asynchat.async_chat):
66 dtp_conn_closed = False
67
68 def __init__(self, conn, baseclass):
69 asynchat.async_chat.__init__(self, conn)
70 self.baseclass = baseclass
71 self.baseclass.last_received_data = bytearray()
72 self.encoding = baseclass.encoding
73
74 def handle_read(self):
75 new_data = self.recv(1024)
76 self.baseclass.last_received_data += new_data
77
78 def handle_close(self):
79 # XXX: this method can be called many times in a row for a single
80 # connection, including in clear-text (non-TLS) mode.
81 # (behaviour witnessed with test_data_connection)
82 if not self.dtp_conn_closed:
83 self.baseclass.push('226 transfer complete')
84 self.shutdown()
85 self.dtp_conn_closed = True
86
87 def push(self, what):
88 if self.baseclass.next_data is not None:
89 what = self.baseclass.next_data
90 self.baseclass.next_data = None
91 if not what:
92 return self.close_when_done()
93 super(DummyDTPHandler, self).push(what.encode(self.encoding))
94
95 def handle_error(self):
96 default_error_handler()
97
98 def shutdown(self):
99 self.close()
100
101
102 class DummyFTPHandler(asynchat.async_chat):
103
104 dtp_handler = DummyDTPHandler
105
106 def __init__(self, conn, encoding=DEFAULT_ENCODING):
107 asynchat.async_chat.__init__(self, conn)
108 # tells the socket to handle urgent data inline (ABOR command)
109 self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_OOBINLINE, 1)
110 self.set_terminator(b"\r\n")
111 self.in_buffer = []
112 self.dtp = None
113 self.last_received_cmd = None
114 self.last_received_data = bytearray()
115 self.next_response = ''
116 self.next_data = None
117 self.rest = None
118 self.next_retr_data = RETR_DATA
119 self.push('220 welcome')
120 self.encoding = encoding
121 # We use this as the string IPv4 address to direct the client
122 # to in response to a PASV command. To test security behavior.
123 # https://bugs.python.org/issue43285/.
124 self.fake_pasv_server_ip = '252.253.254.255'
125
126 def collect_incoming_data(self, data):
127 self.in_buffer.append(data)
128
129 def found_terminator(self):
130 line = b''.join(self.in_buffer).decode(self.encoding)
131 self.in_buffer = []
132 if self.next_response:
133 self.push(self.next_response)
134 self.next_response = ''
135 cmd = line.split(' ')[0].lower()
136 self.last_received_cmd = cmd
137 space = line.find(' ')
138 if space != -1:
139 arg = line[space + 1:]
140 else:
141 arg = ""
142 if hasattr(self, 'cmd_' + cmd):
143 method = getattr(self, 'cmd_' + cmd)
144 method(arg)
145 else:
146 self.push('550 command "%s" not understood.' %cmd)
147
148 def handle_error(self):
149 default_error_handler()
150
151 def push(self, data):
152 asynchat.async_chat.push(self, data.encode(self.encoding) + b'\r\n')
153
154 def cmd_port(self, arg):
155 addr = list(map(int, arg.split(',')))
156 ip = '%d.%d.%d.%d' %tuple(addr[:4])
157 port = (addr[4] * 256) + addr[5]
158 s = socket.create_connection((ip, port), timeout=TIMEOUT)
159 self.dtp = self.dtp_handler(s, baseclass=self)
160 self.push('200 active data connection established')
161
162 def cmd_pasv(self, arg):
163 with socket.create_server((self.socket.getsockname()[0], 0)) as sock:
164 sock.settimeout(TIMEOUT)
165 port = sock.getsockname()[1]
166 ip = self.fake_pasv_server_ip
167 ip = ip.replace('.', ','); p1 = port / 256; p2 = port % 256
168 self.push('227 entering passive mode (%s,%d,%d)' %(ip, p1, p2))
169 conn, addr = sock.accept()
170 self.dtp = self.dtp_handler(conn, baseclass=self)
171
172 def cmd_eprt(self, arg):
173 af, ip, port = arg.split(arg[0])[1:-1]
174 port = int(port)
175 s = socket.create_connection((ip, port), timeout=TIMEOUT)
176 self.dtp = self.dtp_handler(s, baseclass=self)
177 self.push('200 active data connection established')
178
179 def cmd_epsv(self, arg):
180 with socket.create_server((self.socket.getsockname()[0], 0),
181 family=socket.AF_INET6) as sock:
182 sock.settimeout(TIMEOUT)
183 port = sock.getsockname()[1]
184 self.push('229 entering extended passive mode (|||%d|)' %port)
185 conn, addr = sock.accept()
186 self.dtp = self.dtp_handler(conn, baseclass=self)
187
188 def cmd_echo(self, arg):
189 # sends back the received string (used by the test suite)
190 self.push(arg)
191
192 def cmd_noop(self, arg):
193 self.push('200 noop ok')
194
195 def cmd_user(self, arg):
196 self.push('331 username ok')
197
198 def cmd_pass(self, arg):
199 self.push('230 password ok')
200
201 def cmd_acct(self, arg):
202 self.push('230 acct ok')
203
204 def cmd_rnfr(self, arg):
205 self.push('350 rnfr ok')
206
207 def cmd_rnto(self, arg):
208 self.push('250 rnto ok')
209
210 def cmd_dele(self, arg):
211 self.push('250 dele ok')
212
213 def cmd_cwd(self, arg):
214 self.push('250 cwd ok')
215
216 def cmd_size(self, arg):
217 self.push('250 1000')
218
219 def cmd_mkd(self, arg):
220 self.push('257 "%s"' %arg)
221
222 def cmd_rmd(self, arg):
223 self.push('250 rmd ok')
224
225 def cmd_pwd(self, arg):
226 self.push('257 "pwd ok"')
227
228 def cmd_type(self, arg):
229 self.push('200 type ok')
230
231 def cmd_quit(self, arg):
232 self.push('221 quit ok')
233 self.shutdown()
234
235 def cmd_abor(self, arg):
236 self.push('226 abor ok')
237
238 def cmd_stor(self, arg):
239 self.push('125 stor ok')
240
241 def cmd_rest(self, arg):
242 self.rest = arg
243 self.push('350 rest ok')
244
245 def cmd_retr(self, arg):
246 self.push('125 retr ok')
247 if self.rest is not None:
248 offset = int(self.rest)
249 else:
250 offset = 0
251 self.dtp.push(self.next_retr_data[offset:])
252 self.dtp.close_when_done()
253 self.rest = None
254
255 def cmd_list(self, arg):
256 self.push('125 list ok')
257 self.dtp.push(LIST_DATA)
258 self.dtp.close_when_done()
259
260 def cmd_nlst(self, arg):
261 self.push('125 nlst ok')
262 self.dtp.push(NLST_DATA)
263 self.dtp.close_when_done()
264
265 def cmd_opts(self, arg):
266 self.push('200 opts ok')
267
268 def cmd_mlsd(self, arg):
269 self.push('125 mlsd ok')
270 self.dtp.push(MLSD_DATA)
271 self.dtp.close_when_done()
272
273 def cmd_setlongretr(self, arg):
274 # For testing. Next RETR will return long line.
275 self.next_retr_data = 'x' * int(arg)
276 self.push('125 setlongretr ok')
277
278
279 class DummyFTPServer(asyncore.dispatcher, threading.Thread):
280
281 handler = DummyFTPHandler
282
283 def __init__(self, address, af=socket.AF_INET, encoding=DEFAULT_ENCODING):
284 threading.Thread.__init__(self)
285 asyncore.dispatcher.__init__(self)
286 self.daemon = True
287 self.create_socket(af, socket.SOCK_STREAM)
288 self.bind(address)
289 self.listen(5)
290 self.active = False
291 self.active_lock = threading.Lock()
292 self.host, self.port = self.socket.getsockname()[:2]
293 self.handler_instance = None
294 self.encoding = encoding
295
296 def start(self):
297 assert not self.active
298 self.__flag = threading.Event()
299 threading.Thread.start(self)
300 self.__flag.wait()
301
302 def run(self):
303 self.active = True
304 self.__flag.set()
305 while self.active and asyncore.socket_map:
306 self.active_lock.acquire()
307 asyncore.loop(timeout=0.1, count=1)
308 self.active_lock.release()
309 asyncore.close_all(ignore_all=True)
310
311 def stop(self):
312 assert self.active
313 self.active = False
314 self.join()
315
316 def handle_accepted(self, conn, addr):
317 self.handler_instance = self.handler(conn, encoding=self.encoding)
318
319 def handle_connect(self):
320 self.shutdown()
321 handle_read = handle_connect
322
323 def writable(self):
324 return 0
325
326 def handle_error(self):
327 default_error_handler()
328
329
330 if ssl is not None:
331
332 CERTFILE = os.path.join(os.path.dirname(__file__), "certdata", "keycert3.pem")
333 CAFILE = os.path.join(os.path.dirname(__file__), "certdata", "pycacert.pem")
334
335 class SSLConnection(asyncore.dispatcher):
336 """An asyncore.dispatcher subclass supporting TLS/SSL."""
337
338 _ssl_accepting = False
339 _ssl_closing = False
340
341 def secure_connection(self):
342 context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
343 context.load_cert_chain(CERTFILE)
344 socket = context.wrap_socket(self.socket,
345 suppress_ragged_eofs=False,
346 server_side=True,
347 do_handshake_on_connect=False)
348 self.del_channel()
349 self.set_socket(socket)
350 self._ssl_accepting = True
351
352 def _do_ssl_handshake(self):
353 try:
354 self.socket.do_handshake()
355 except ssl.SSLError as err:
356 if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
357 ssl.SSL_ERROR_WANT_WRITE):
358 return
359 elif err.args[0] == ssl.SSL_ERROR_EOF:
360 return self.handle_close()
361 # TODO: SSLError does not expose alert information
362 elif "SSLV3_ALERT_BAD_CERTIFICATE" in err.args[1]:
363 return self.handle_close()
364 raise
365 except OSError as err:
366 if err.args[0] == errno.ECONNABORTED:
367 return self.handle_close()
368 else:
369 self._ssl_accepting = False
370
371 def _do_ssl_shutdown(self):
372 self._ssl_closing = True
373 try:
374 self.socket = self.socket.unwrap()
375 except ssl.SSLError as err:
376 if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
377 ssl.SSL_ERROR_WANT_WRITE):
378 return
379 except OSError:
380 # Any "socket error" corresponds to a SSL_ERROR_SYSCALL return
381 # from OpenSSL's SSL_shutdown(), corresponding to a
382 # closed socket condition. See also:
383 # http://www.mail-archive.com/[email protected]/msg60710.html
384 pass
385 self._ssl_closing = False
386 if getattr(self, '_ccc', False) is False:
387 super(SSLConnection, self).close()
388 else:
389 pass
390
391 def handle_read_event(self):
392 if self._ssl_accepting:
393 self._do_ssl_handshake()
394 elif self._ssl_closing:
395 self._do_ssl_shutdown()
396 else:
397 super(SSLConnection, self).handle_read_event()
398
399 def handle_write_event(self):
400 if self._ssl_accepting:
401 self._do_ssl_handshake()
402 elif self._ssl_closing:
403 self._do_ssl_shutdown()
404 else:
405 super(SSLConnection, self).handle_write_event()
406
407 def send(self, data):
408 try:
409 return super(SSLConnection, self).send(data)
410 except ssl.SSLError as err:
411 if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN,
412 ssl.SSL_ERROR_WANT_READ,
413 ssl.SSL_ERROR_WANT_WRITE):
414 return 0
415 raise
416
417 def recv(self, buffer_size):
418 try:
419 return super(SSLConnection, self).recv(buffer_size)
420 except ssl.SSLError as err:
421 if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
422 ssl.SSL_ERROR_WANT_WRITE):
423 return b''
424 if err.args[0] in (ssl.SSL_ERROR_EOF, ssl.SSL_ERROR_ZERO_RETURN):
425 self.handle_close()
426 return b''
427 raise
428
429 def handle_error(self):
430 default_error_handler()
431
432 def shutdown(self):
433 if (isinstance(self.socket, ssl.SSLSocket) and
434 self.socket._sslobj is not None):
435 self._do_ssl_shutdown()
436 else:
437 self.close()
438
439
440 class DummyTLS_DTPHandler(SSLConnection, DummyDTPHandler):
441 """A DummyDTPHandler subclass supporting TLS/SSL."""
442
443 def __init__(self, conn, baseclass):
444 DummyDTPHandler.__init__(self, conn, baseclass)
445 if self.baseclass.secure_data_channel:
446 self.secure_connection()
447
448
449 class DummyTLS_FTPHandler(SSLConnection, DummyFTPHandler):
450 """A DummyFTPHandler subclass supporting TLS/SSL."""
451
452 dtp_handler = DummyTLS_DTPHandler
453
454 def __init__(self, conn, encoding=DEFAULT_ENCODING):
455 DummyFTPHandler.__init__(self, conn, encoding=encoding)
456 self.secure_data_channel = False
457 self._ccc = False
458
459 def cmd_auth(self, line):
460 """Set up secure control channel."""
461 self.push('234 AUTH TLS successful')
462 self.secure_connection()
463
464 def cmd_ccc(self, line):
465 self.push('220 Reverting back to clear-text')
466 self._ccc = True
467 self._do_ssl_shutdown()
468
469 def cmd_pbsz(self, line):
470 """Negotiate size of buffer for secure data transfer.
471 For TLS/SSL the only valid value for the parameter is '0'.
472 Any other value is accepted but ignored.
473 """
474 self.push('200 PBSZ=0 successful.')
475
476 def cmd_prot(self, line):
477 """Setup un/secure data channel."""
478 arg = line.upper()
479 if arg == 'C':
480 self.push('200 Protection set to Clear')
481 self.secure_data_channel = False
482 elif arg == 'P':
483 self.push('200 Protection set to Private')
484 self.secure_data_channel = True
485 else:
486 self.push("502 Unrecognized PROT type (use C or P).")
487
488
489 class DummyTLS_FTPServer(DummyFTPServer):
490 handler = DummyTLS_FTPHandler
491
492
493 class TestFTPClass(TestCase):
494
495 def setUp(self, encoding=DEFAULT_ENCODING):
496 self.server = DummyFTPServer((HOST, 0), encoding=encoding)
497 self.server.start()
498 self.client = ftplib.FTP(timeout=TIMEOUT, encoding=encoding)
499 self.client.connect(self.server.host, self.server.port)
500
501 def tearDown(self):
502 self.client.close()
503 self.server.stop()
504 # Explicitly clear the attribute to prevent dangling thread
505 self.server = None
506 asyncore.close_all(ignore_all=True)
507
508 def check_data(self, received, expected):
509 self.assertEqual(len(received), len(expected))
510 self.assertEqual(received, expected)
511
512 def test_getwelcome(self):
513 self.assertEqual(self.client.getwelcome(), '220 welcome')
514
515 def test_sanitize(self):
516 self.assertEqual(self.client.sanitize('foo'), repr('foo'))
517 self.assertEqual(self.client.sanitize('pass 12345'), repr('pass *****'))
518 self.assertEqual(self.client.sanitize('PASS 12345'), repr('PASS *****'))
519
520 def test_exceptions(self):
521 self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r\n0')
522 self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\n0')
523 self.assertRaises(ValueError, self.client.sendcmd, 'echo 40\r0')
524 self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 400')
525 self.assertRaises(ftplib.error_temp, self.client.sendcmd, 'echo 499')
526 self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 500')
527 self.assertRaises(ftplib.error_perm, self.client.sendcmd, 'echo 599')
528 self.assertRaises(ftplib.error_proto, self.client.sendcmd, 'echo 999')
529
530 def test_all_errors(self):
531 exceptions = (ftplib.error_reply, ftplib.error_temp, ftplib.error_perm,
532 ftplib.error_proto, ftplib.Error, OSError,
533 EOFError)
534 for x in exceptions:
535 try:
536 raise x('exception not included in all_errors set')
537 except ftplib.all_errors:
538 pass
539
540 def test_set_pasv(self):
541 # passive mode is supposed to be enabled by default
542 self.assertTrue(self.client.passiveserver)
543 self.client.set_pasv(True)
544 self.assertTrue(self.client.passiveserver)
545 self.client.set_pasv(False)
546 self.assertFalse(self.client.passiveserver)
547
548 def test_voidcmd(self):
549 self.assertEqual(self.client.voidcmd('echo 200'), '200')
550 self.assertEqual(self.client.voidcmd('echo 299'), '299')
551 self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 199')
552 self.assertRaises(ftplib.error_reply, self.client.voidcmd, 'echo 300')
553
554 def test_login(self):
555 self.client.login()
556
557 def test_acct(self):
558 self.client.acct('passwd')
559
560 def test_rename(self):
561 self.client.rename('a', 'b')
562 self.server.handler_instance.next_response = '200'
563 self.assertRaises(ftplib.error_reply, self.client.rename, 'a', 'b')
564
565 def test_delete(self):
566 self.client.delete('foo')
567 self.server.handler_instance.next_response = '199'
568 self.assertRaises(ftplib.error_reply, self.client.delete, 'foo')
569
570 def test_size(self):
571 self.client.size('foo')
572
573 def test_mkd(self):
574 dir = self.client.mkd('/foo')
575 self.assertEqual(dir, '/foo')
576
577 def test_rmd(self):
578 self.client.rmd('foo')
579
580 def test_cwd(self):
581 dir = self.client.cwd('/foo')
582 self.assertEqual(dir, '250 cwd ok')
583
584 def test_pwd(self):
585 dir = self.client.pwd()
586 self.assertEqual(dir, 'pwd ok')
587
588 def test_quit(self):
589 self.assertEqual(self.client.quit(), '221 quit ok')
590 # Ensure the connection gets closed; sock attribute should be None
591 self.assertEqual(self.client.sock, None)
592
593 def test_abort(self):
594 self.client.abort()
595
596 def test_retrbinary(self):
597 received = []
598 self.client.retrbinary('retr', received.append)
599 self.check_data(b''.join(received),
600 RETR_DATA.encode(self.client.encoding))
601
602 def test_retrbinary_rest(self):
603 for rest in (0, 10, 20):
604 received = []
605 self.client.retrbinary('retr', received.append, rest=rest)
606 self.check_data(b''.join(received),
607 RETR_DATA[rest:].encode(self.client.encoding))
608
609 def test_retrlines(self):
610 received = []
611 self.client.retrlines('retr', received.append)
612 self.check_data(''.join(received), RETR_DATA.replace('\r\n', ''))
613
614 def test_storbinary(self):
615 f = io.BytesIO(RETR_DATA.encode(self.client.encoding))
616 self.client.storbinary('stor', f)
617 self.check_data(self.server.handler_instance.last_received_data,
618 RETR_DATA.encode(self.server.encoding))
619 # test new callback arg
620 flag = []
621 f.seek(0)
622 self.client.storbinary('stor', f, callback=lambda x: flag.append(None))
623 self.assertTrue(flag)
624
625 def test_storbinary_rest(self):
626 data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding)
627 f = io.BytesIO(data)
628 for r in (30, '30'):
629 f.seek(0)
630 self.client.storbinary('stor', f, rest=r)
631 self.assertEqual(self.server.handler_instance.rest, str(r))
632
633 def test_storlines(self):
634 data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding)
635 f = io.BytesIO(data)
636 self.client.storlines('stor', f)
637 self.check_data(self.server.handler_instance.last_received_data,
638 RETR_DATA.encode(self.server.encoding))
639 # test new callback arg
640 flag = []
641 f.seek(0)
642 self.client.storlines('stor foo', f, callback=lambda x: flag.append(None))
643 self.assertTrue(flag)
644
645 f = io.StringIO(RETR_DATA.replace('\r\n', '\n'))
646 # storlines() expects a binary file, not a text file
647 with warnings_helper.check_warnings(('', BytesWarning), quiet=True):
648 self.assertRaises(TypeError, self.client.storlines, 'stor foo', f)
649
650 def test_nlst(self):
651 self.client.nlst()
652 self.assertEqual(self.client.nlst(), NLST_DATA.split('\r\n')[:-1])
653
654 def test_dir(self):
655 l = []
656 self.client.dir(l.append)
657 self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', ''))
658
659 def test_mlsd(self):
660 list(self.client.mlsd())
661 list(self.client.mlsd(path='/'))
662 list(self.client.mlsd(path='/', facts=['size', 'type']))
663
664 ls = list(self.client.mlsd())
665 for name, facts in ls:
666 self.assertIsInstance(name, str)
667 self.assertIsInstance(facts, dict)
668 self.assertTrue(name)
669 self.assertIn('type', facts)
670 self.assertIn('perm', facts)
671 self.assertIn('unique', facts)
672
673 def set_data(data):
674 self.server.handler_instance.next_data = data
675
676 def test_entry(line, type=None, perm=None, unique=None, name=None):
677 type = 'type' if type is None else type
678 perm = 'perm' if perm is None else perm
679 unique = 'unique' if unique is None else unique
680 name = 'name' if name is None else name
681 set_data(line)
682 _name, facts = next(self.client.mlsd())
683 self.assertEqual(_name, name)
684 self.assertEqual(facts['type'], type)
685 self.assertEqual(facts['perm'], perm)
686 self.assertEqual(facts['unique'], unique)
687
688 # plain
689 test_entry('type=type;perm=perm;unique=unique; name\r\n')
690 # "=" in fact value
691 test_entry('type=ty=pe;perm=perm;unique=unique; name\r\n', type="ty=pe")
692 test_entry('type==type;perm=perm;unique=unique; name\r\n', type="=type")
693 test_entry('type=t=y=pe;perm=perm;unique=unique; name\r\n', type="t=y=pe")
694 test_entry('type=====;perm=perm;unique=unique; name\r\n', type="====")
695 # spaces in name
696 test_entry('type=type;perm=perm;unique=unique; na me\r\n', name="na me")
697 test_entry('type=type;perm=perm;unique=unique; name \r\n', name="name ")
698 test_entry('type=type;perm=perm;unique=unique; name\r\n', name=" name")
699 test_entry('type=type;perm=perm;unique=unique; n am e\r\n', name="n am e")
700 # ";" in name
701 test_entry('type=type;perm=perm;unique=unique; na;me\r\n', name="na;me")
702 test_entry('type=type;perm=perm;unique=unique; ;name\r\n', name=";name")
703 test_entry('type=type;perm=perm;unique=unique; ;name;\r\n', name=";name;")
704 test_entry('type=type;perm=perm;unique=unique; ;;;;\r\n', name=";;;;")
705 # case sensitiveness
706 set_data('Type=type;TyPe=perm;UNIQUE=unique; name\r\n')
707 _name, facts = next(self.client.mlsd())
708 for x in facts:
709 self.assertTrue(x.islower())
710 # no data (directory empty)
711 set_data('')
712 self.assertRaises(StopIteration, next, self.client.mlsd())
713 set_data('')
714 for x in self.client.mlsd():
715 self.fail("unexpected data %s" % x)
716
717 def test_makeport(self):
718 with self.client.makeport():
719 # IPv4 is in use, just make sure send_eprt has not been used
720 self.assertEqual(self.server.handler_instance.last_received_cmd,
721 'port')
722
723 def test_makepasv(self):
724 host, port = self.client.makepasv()
725 conn = socket.create_connection((host, port), timeout=TIMEOUT)
726 conn.close()
727 # IPv4 is in use, just make sure send_epsv has not been used
728 self.assertEqual(self.server.handler_instance.last_received_cmd, 'pasv')
729
730 def test_makepasv_issue43285_security_disabled(self):
731 """Test the opt-in to the old vulnerable behavior."""
732 self.client.trust_server_pasv_ipv4_address = True
733 bad_host, port = self.client.makepasv()
734 self.assertEqual(
735 bad_host, self.server.handler_instance.fake_pasv_server_ip)
736 # Opening and closing a connection keeps the dummy server happy
737 # instead of timing out on accept.
738 socket.create_connection((self.client.sock.getpeername()[0], port),
739 timeout=TIMEOUT).close()
740
741 def test_makepasv_issue43285_security_enabled_default(self):
742 self.assertFalse(self.client.trust_server_pasv_ipv4_address)
743 trusted_host, port = self.client.makepasv()
744 self.assertNotEqual(
745 trusted_host, self.server.handler_instance.fake_pasv_server_ip)
746 # Opening and closing a connection keeps the dummy server happy
747 # instead of timing out on accept.
748 socket.create_connection((trusted_host, port), timeout=TIMEOUT).close()
749
750 def test_with_statement(self):
751 self.client.quit()
752
753 def is_client_connected():
754 if self.client.sock is None:
755 return False
756 try:
757 self.client.sendcmd('noop')
758 except (OSError, EOFError):
759 return False
760 return True
761
762 # base test
763 with ftplib.FTP(timeout=TIMEOUT) as self.client:
764 self.client.connect(self.server.host, self.server.port)
765 self.client.sendcmd('noop')
766 self.assertTrue(is_client_connected())
767 self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit')
768 self.assertFalse(is_client_connected())
769
770 # QUIT sent inside the with block
771 with ftplib.FTP(timeout=TIMEOUT) as self.client:
772 self.client.connect(self.server.host, self.server.port)
773 self.client.sendcmd('noop')
774 self.client.quit()
775 self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit')
776 self.assertFalse(is_client_connected())
777
778 # force a wrong response code to be sent on QUIT: error_perm
779 # is expected and the connection is supposed to be closed
780 try:
781 with ftplib.FTP(timeout=TIMEOUT) as self.client:
782 self.client.connect(self.server.host, self.server.port)
783 self.client.sendcmd('noop')
784 self.server.handler_instance.next_response = '550 error on quit'
785 except ftplib.error_perm as err:
786 self.assertEqual(str(err), '550 error on quit')
787 else:
788 self.fail('Exception not raised')
789 # needed to give the threaded server some time to set the attribute
790 # which otherwise would still be == 'noop'
791 time.sleep(0.1)
792 self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit')
793 self.assertFalse(is_client_connected())
794
795 def test_source_address(self):
796 self.client.quit()
797 port = socket_helper.find_unused_port()
798 try:
799 self.client.connect(self.server.host, self.server.port,
800 source_address=(HOST, port))
801 self.assertEqual(self.client.sock.getsockname()[1], port)
802 self.client.quit()
803 except OSError as e:
804 if e.errno == errno.EADDRINUSE:
805 self.skipTest("couldn't bind to port %d" % port)
806 raise
807
808 def test_source_address_passive_connection(self):
809 port = socket_helper.find_unused_port()
810 self.client.source_address = (HOST, port)
811 try:
812 with self.client.transfercmd('list') as sock:
813 self.assertEqual(sock.getsockname()[1], port)
814 except OSError as e:
815 if e.errno == errno.EADDRINUSE:
816 self.skipTest("couldn't bind to port %d" % port)
817 raise
818
819 def test_parse257(self):
820 self.assertEqual(ftplib.parse257('257 "/foo/bar"'), '/foo/bar')
821 self.assertEqual(ftplib.parse257('257 "/foo/bar" created'), '/foo/bar')
822 self.assertEqual(ftplib.parse257('257 ""'), '')
823 self.assertEqual(ftplib.parse257('257 "" created'), '')
824 self.assertRaises(ftplib.error_reply, ftplib.parse257, '250 "/foo/bar"')
825 # The 257 response is supposed to include the directory
826 # name and in case it contains embedded double-quotes
827 # they must be doubled (see RFC-959, chapter 7, appendix 2).
828 self.assertEqual(ftplib.parse257('257 "/foo/b""ar"'), '/foo/b"ar')
829 self.assertEqual(ftplib.parse257('257 "/foo/b""ar" created'), '/foo/b"ar')
830
831 def test_line_too_long(self):
832 self.assertRaises(ftplib.Error, self.client.sendcmd,
833 'x' * self.client.maxline * 2)
834
835 def test_retrlines_too_long(self):
836 self.client.sendcmd('SETLONGRETR %d' % (self.client.maxline * 2))
837 received = []
838 self.assertRaises(ftplib.Error,
839 self.client.retrlines, 'retr', received.append)
840
841 def test_storlines_too_long(self):
842 f = io.BytesIO(b'x' * self.client.maxline * 2)
843 self.assertRaises(ftplib.Error, self.client.storlines, 'stor', f)
844
845 def test_encoding_param(self):
846 encodings = ['latin-1', 'utf-8']
847 for encoding in encodings:
848 with self.subTest(encoding=encoding):
849 self.tearDown()
850 self.setUp(encoding=encoding)
851 self.assertEqual(encoding, self.client.encoding)
852 self.test_retrbinary()
853 self.test_storbinary()
854 self.test_retrlines()
855 new_dir = self.client.mkd('/non-ascii dir \xAE')
856 self.check_data(new_dir, '/non-ascii dir \xAE')
857 # Check default encoding
858 client = ftplib.FTP(timeout=TIMEOUT)
859 self.assertEqual(DEFAULT_ENCODING, client.encoding)
860
861
862 @skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled")
863 class TestIPv6Environment(TestCase):
864
865 def setUp(self):
866 self.server = DummyFTPServer((HOSTv6, 0),
867 af=socket.AF_INET6,
868 encoding=DEFAULT_ENCODING)
869 self.server.start()
870 self.client = ftplib.FTP(timeout=TIMEOUT, encoding=DEFAULT_ENCODING)
871 self.client.connect(self.server.host, self.server.port)
872
873 def tearDown(self):
874 self.client.close()
875 self.server.stop()
876 # Explicitly clear the attribute to prevent dangling thread
877 self.server = None
878 asyncore.close_all(ignore_all=True)
879
880 def test_af(self):
881 self.assertEqual(self.client.af, socket.AF_INET6)
882
883 def test_makeport(self):
884 with self.client.makeport():
885 self.assertEqual(self.server.handler_instance.last_received_cmd,
886 'eprt')
887
888 def test_makepasv(self):
889 host, port = self.client.makepasv()
890 conn = socket.create_connection((host, port), timeout=TIMEOUT)
891 conn.close()
892 self.assertEqual(self.server.handler_instance.last_received_cmd, 'epsv')
893
894 def test_transfer(self):
895 def retr():
896 received = []
897 self.client.retrbinary('retr', received.append)
898 self.assertEqual(b''.join(received),
899 RETR_DATA.encode(self.client.encoding))
900 self.client.set_pasv(True)
901 retr()
902 self.client.set_pasv(False)
903 retr()
904
905
906 @skipUnless(ssl, "SSL not available")
907 @requires_subprocess()
908 class TestTLS_FTPClassMixin(TestFTPClass):
909 """Repeat TestFTPClass tests starting the TLS layer for both control
910 and data connections first.
911 """
912
913 def setUp(self, encoding=DEFAULT_ENCODING):
914 self.server = DummyTLS_FTPServer((HOST, 0), encoding=encoding)
915 self.server.start()
916 self.client = ftplib.FTP_TLS(timeout=TIMEOUT, encoding=encoding)
917 self.client.connect(self.server.host, self.server.port)
918 # enable TLS
919 self.client.auth()
920 self.client.prot_p()
921
922
923 @skipUnless(ssl, "SSL not available")
924 @requires_subprocess()
925 class TestTLS_FTPClass(TestCase):
926 """Specific TLS_FTP class tests."""
927
928 def setUp(self, encoding=DEFAULT_ENCODING):
929 self.server = DummyTLS_FTPServer((HOST, 0), encoding=encoding)
930 self.server.start()
931 self.client = ftplib.FTP_TLS(timeout=TIMEOUT)
932 self.client.connect(self.server.host, self.server.port)
933
934 def tearDown(self):
935 self.client.close()
936 self.server.stop()
937 # Explicitly clear the attribute to prevent dangling thread
938 self.server = None
939 asyncore.close_all(ignore_all=True)
940
941 def test_control_connection(self):
942 self.assertNotIsInstance(self.client.sock, ssl.SSLSocket)
943 self.client.auth()
944 self.assertIsInstance(self.client.sock, ssl.SSLSocket)
945
946 def test_data_connection(self):
947 # clear text
948 with self.client.transfercmd('list') as sock:
949 self.assertNotIsInstance(sock, ssl.SSLSocket)
950 self.assertEqual(sock.recv(1024),
951 LIST_DATA.encode(self.client.encoding))
952 self.assertEqual(self.client.voidresp(), "226 transfer complete")
953
954 # secured, after PROT P
955 self.client.prot_p()
956 with self.client.transfercmd('list') as sock:
957 self.assertIsInstance(sock, ssl.SSLSocket)
958 # consume from SSL socket to finalize handshake and avoid
959 # "SSLError [SSL] shutdown while in init"
960 self.assertEqual(sock.recv(1024),
961 LIST_DATA.encode(self.client.encoding))
962 self.assertEqual(self.client.voidresp(), "226 transfer complete")
963
964 # PROT C is issued, the connection must be in cleartext again
965 self.client.prot_c()
966 with self.client.transfercmd('list') as sock:
967 self.assertNotIsInstance(sock, ssl.SSLSocket)
968 self.assertEqual(sock.recv(1024),
969 LIST_DATA.encode(self.client.encoding))
970 self.assertEqual(self.client.voidresp(), "226 transfer complete")
971
972 def test_login(self):
973 # login() is supposed to implicitly secure the control connection
974 self.assertNotIsInstance(self.client.sock, ssl.SSLSocket)
975 self.client.login()
976 self.assertIsInstance(self.client.sock, ssl.SSLSocket)
977 # make sure that AUTH TLS doesn't get issued again
978 self.client.login()
979
980 def test_auth_issued_twice(self):
981 self.client.auth()
982 self.assertRaises(ValueError, self.client.auth)
983
984 def test_context(self):
985 self.client.quit()
986 ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
987 ctx.check_hostname = False
988 ctx.verify_mode = ssl.CERT_NONE
989 self.assertRaises(TypeError, ftplib.FTP_TLS, keyfile=CERTFILE,
990 context=ctx)
991 self.assertRaises(TypeError, ftplib.FTP_TLS, certfile=CERTFILE,
992 context=ctx)
993 self.assertRaises(TypeError, ftplib.FTP_TLS, certfile=CERTFILE,
994 keyfile=CERTFILE, context=ctx)
995
996 self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT)
997 self.client.connect(self.server.host, self.server.port)
998 self.assertNotIsInstance(self.client.sock, ssl.SSLSocket)
999 self.client.auth()
1000 self.assertIs(self.client.sock.context, ctx)
1001 self.assertIsInstance(self.client.sock, ssl.SSLSocket)
1002
1003 self.client.prot_p()
1004 with self.client.transfercmd('list') as sock:
1005 self.assertIs(sock.context, ctx)
1006 self.assertIsInstance(sock, ssl.SSLSocket)
1007
1008 def test_ccc(self):
1009 self.assertRaises(ValueError, self.client.ccc)
1010 self.client.login(secure=True)
1011 self.assertIsInstance(self.client.sock, ssl.SSLSocket)
1012 self.client.ccc()
1013 self.assertRaises(ValueError, self.client.sock.unwrap)
1014
1015 @skipUnless(False, "FIXME: bpo-32706")
1016 def test_check_hostname(self):
1017 self.client.quit()
1018 ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
1019 self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
1020 self.assertEqual(ctx.check_hostname, True)
1021 ctx.load_verify_locations(CAFILE)
1022 self.client = ftplib.FTP_TLS(context=ctx, timeout=TIMEOUT)
1023
1024 # 127.0.0.1 doesn't match SAN
1025 self.client.connect(self.server.host, self.server.port)
1026 with self.assertRaises(ssl.CertificateError):
1027 self.client.auth()
1028 # exception quits connection
1029
1030 self.client.connect(self.server.host, self.server.port)
1031 self.client.prot_p()
1032 with self.assertRaises(ssl.CertificateError):
1033 with self.client.transfercmd("list") as sock:
1034 pass
1035 self.client.quit()
1036
1037 self.client.connect("localhost", self.server.port)
1038 self.client.auth()
1039 self.client.quit()
1040
1041 self.client.connect("localhost", self.server.port)
1042 self.client.prot_p()
1043 with self.client.transfercmd("list") as sock:
1044 pass
1045
1046
1047 class TestTimeouts(TestCase):
1048
1049 def setUp(self):
1050 self.evt = threading.Event()
1051 self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1052 self.sock.settimeout(20)
1053 self.port = socket_helper.bind_port(self.sock)
1054 self.server_thread = threading.Thread(target=self.server)
1055 self.server_thread.daemon = True
1056 self.server_thread.start()
1057 # Wait for the server to be ready.
1058 self.evt.wait()
1059 self.evt.clear()
1060 self.old_port = ftplib.FTP.port
1061 ftplib.FTP.port = self.port
1062
1063 def tearDown(self):
1064 ftplib.FTP.port = self.old_port
1065 self.server_thread.join()
1066 # Explicitly clear the attribute to prevent dangling thread
1067 self.server_thread = None
1068
1069 def server(self):
1070 # This method sets the evt 3 times:
1071 # 1) when the connection is ready to be accepted.
1072 # 2) when it is safe for the caller to close the connection
1073 # 3) when we have closed the socket
1074 self.sock.listen()
1075 # (1) Signal the caller that we are ready to accept the connection.
1076 self.evt.set()
1077 try:
1078 conn, addr = self.sock.accept()
1079 except TimeoutError:
1080 pass
1081 else:
1082 conn.sendall(b"1 Hola mundo\n")
1083 conn.shutdown(socket.SHUT_WR)
1084 # (2) Signal the caller that it is safe to close the socket.
1085 self.evt.set()
1086 conn.close()
1087 finally:
1088 self.sock.close()
1089
1090 def testTimeoutDefault(self):
1091 # default -- use global socket timeout
1092 self.assertIsNone(socket.getdefaulttimeout())
1093 socket.setdefaulttimeout(30)
1094 try:
1095 ftp = ftplib.FTP(HOST)
1096 finally:
1097 socket.setdefaulttimeout(None)
1098 self.assertEqual(ftp.sock.gettimeout(), 30)
1099 self.evt.wait()
1100 ftp.close()
1101
1102 def testTimeoutNone(self):
1103 # no timeout -- do not use global socket timeout
1104 self.assertIsNone(socket.getdefaulttimeout())
1105 socket.setdefaulttimeout(30)
1106 try:
1107 ftp = ftplib.FTP(HOST, timeout=None)
1108 finally:
1109 socket.setdefaulttimeout(None)
1110 self.assertIsNone(ftp.sock.gettimeout())
1111 self.evt.wait()
1112 ftp.close()
1113
1114 def testTimeoutValue(self):
1115 # a value
1116 ftp = ftplib.FTP(HOST, timeout=30)
1117 self.assertEqual(ftp.sock.gettimeout(), 30)
1118 self.evt.wait()
1119 ftp.close()
1120
1121 # bpo-39259
1122 with self.assertRaises(ValueError):
1123 ftplib.FTP(HOST, timeout=0)
1124
1125 def testTimeoutConnect(self):
1126 ftp = ftplib.FTP()
1127 ftp.connect(HOST, timeout=30)
1128 self.assertEqual(ftp.sock.gettimeout(), 30)
1129 self.evt.wait()
1130 ftp.close()
1131
1132 def testTimeoutDifferentOrder(self):
1133 ftp = ftplib.FTP(timeout=30)
1134 ftp.connect(HOST)
1135 self.assertEqual(ftp.sock.gettimeout(), 30)
1136 self.evt.wait()
1137 ftp.close()
1138
1139 def testTimeoutDirectAccess(self):
1140 ftp = ftplib.FTP()
1141 ftp.timeout = 30
1142 ftp.connect(HOST)
1143 self.assertEqual(ftp.sock.gettimeout(), 30)
1144 self.evt.wait()
1145 ftp.close()
1146
1147
1148 class TestFtpcpSecurity(TestCase):
1149 """ftpcp() must not trust the host a source server advertises in PASV.
1150
1151 A malicious source server can otherwise redirect the target server's
1152 data connection to an arbitrary host:port (SSRF), so ftpcp() uses the
1153 source server's actual peer address instead, the same as FTP.makepasv().
1154 """
1155
1156 def _make_pair(self, *, advertised_host, real_host, trust=False):
1157 source = mock.Mock(spec=ftplib.FTP)
1158 source.trust_server_pasv_ipv4_address = trust
1159 source.sock.getpeername.return_value = (real_host, 21)
1160 # PASV replies give the host as comma-separated octets, not dotted.
1161 advertised = advertised_host.replace('.', ',')
1162 source.sendcmd.side_effect = lambda cmd: (
1163 f'227 Entering Passive Mode ({advertised},1,2).'
1164 if cmd == 'PASV' else '150 ok')
1165 target = mock.Mock(spec=ftplib.FTP)
1166 target.sendcmd.return_value = '150 ok'
1167 return source, target
1168
1169 def test_ftpcp_ignores_untrusted_pasv_host(self):
1170 source, target = self._make_pair(advertised_host='10.0.0.5',
1171 real_host='198.51.100.7')
1172 ftplib.ftpcp(source, 'a', target, 'b')
1173 target.sendport.assert_called_once_with('198.51.100.7', 258)
1174
1175 def test_ftpcp_trust_server_pasv_ipv4_address(self):
1176 source, target = self._make_pair(advertised_host='10.0.0.5',
1177 real_host='198.51.100.7', trust=True)
1178 ftplib.ftpcp(source, 'a', target, 'b')
1179 target.sendport.assert_called_once_with('10.0.0.5', 258)
1180
1181
1182 class MiscTestCase(TestCase):
1183 def test__all__(self):
1184 not_exported = {
1185 'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', 'Error',
1186 'parse150', 'parse227', 'parse229', 'parse257', 'print_line',
1187 'ftpcp', 'test'}
1188 support.check__all__(self, ftplib, not_exported=not_exported)
1189
1190
1191 def setUpModule():
1192 thread_info = threading_helper.threading_setup()
1193 unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)
1194
1195
1196 if __name__ == '__main__':
1197 unittest.main()