Debian Codegraph @

Lib/test/test_android.py

raw
1 import io
2 import queue
3 import re
4 import subprocess
5 import sys
6 import unittest
7 from _android_support import TextLogStream
8 from array import array
9 from contextlib import ExitStack, contextmanager
10 from threading import Thread
11 from test.support import LOOPBACK_TIMEOUT
12 from time import time
13 from unittest.mock import patch
14
15
16 if sys.platform != "android":
17 raise unittest.SkipTest("Android-specific")
18
19 # (name, level, fileno)
20 STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)]
21
22
23 # Test redirection of stdout and stderr to the Android log.
24 class TestAndroidOutput(unittest.TestCase):
25 maxDiff = None
26
27 def setUp(self):
28 self.logcat_process = subprocess.Popen(
29 ["logcat", "-v", "tag"], stdout=subprocess.PIPE,
30 errors="backslashreplace"
31 )
32 self.logcat_queue = queue.Queue()
33
34 def logcat_thread():
35 for line in self.logcat_process.stdout:
36 self.logcat_queue.put(line.rstrip("\n"))
37 self.logcat_process.stdout.close()
38
39 self.logcat_thread = Thread(target=logcat_thread)
40 self.logcat_thread.start()
41
42 try:
43 from ctypes import CDLL, c_char_p, c_int
44 android_log_write = getattr(CDLL("liblog.so"), "__android_log_write")
45 android_log_write.argtypes = (c_int, c_char_p, c_char_p)
46 ANDROID_LOG_INFO = 4
47
48 # Separate tests using a marker line with a different tag.
49 tag, message = "python.test", f"{self.id()} {time()}"
50 android_log_write(
51 ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8"))
52 self.assert_log("I", tag, message, skip=True)
53 except:
54 # If setUp throws an exception, tearDown is not automatically
55 # called. Avoid leaving a dangling thread which would keep the
56 # Python process alive indefinitely.
57 self.tearDown()
58 raise
59
60 def assert_logs(self, level, tag, expected, **kwargs):
61 for line in expected:
62 self.assert_log(level, tag, line, **kwargs)
63
64 def assert_log(self, level, tag, expected, *, skip=False):
65 deadline = time() + LOOPBACK_TIMEOUT
66 while True:
67 try:
68 line = self.logcat_queue.get(timeout=(deadline - time()))
69 except queue.Empty:
70 raise self.failureException(
71 f"line not found: {expected!r}"
72 ) from None
73 if match := re.fullmatch(fr"(.)/{tag}: (.*)", line):
74 try:
75 self.assertEqual(level, match[1])
76 self.assertEqual(expected, match[2])
77 break
78 except AssertionError:
79 if not skip:
80 raise
81
82 def tearDown(self):
83 self.logcat_process.terminate()
84 self.logcat_process.wait(LOOPBACK_TIMEOUT)
85 self.logcat_thread.join(LOOPBACK_TIMEOUT)
86
87 # Avoid an irrelevant warning about threading._dangling.
88 self.logcat_thread = None
89
90 @contextmanager
91 def reconfigure(self, stream, **settings):
92 original_settings = {key: getattr(stream, key, None) for key in settings.keys()}
93 stream.reconfigure(**settings)
94 try:
95 yield
96 finally:
97 stream.reconfigure(**original_settings)
98
99 def stream_context(self, stream_name, level):
100 stack = ExitStack()
101 stack.enter_context(self.subTest(stream_name))
102
103 # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't
104 # test them directly. Detect this mode and use some temporary streams with
105 # the same properties.
106 stream = getattr(sys, stream_name)
107 native_stream = getattr(sys, f"__{stream_name}__")
108 if isinstance(stream, io.StringIO):
109 # https://developer.android.com/ndk/reference/group/logging
110 prio = {"I": 4, "W": 5}[level]
111 stack.enter_context(
112 patch(
113 f"sys.{stream_name}",
114 stream := TextLogStream(
115 prio, f"python.{stream_name}", native_stream,
116 ),
117 )
118 )
119
120 # The tests assume the stream is initially buffered.
121 stack.enter_context(self.reconfigure(stream, write_through=False))
122
123 return stack
124
125 def test_str(self):
126 for stream_name, level, fileno in STREAM_INFO:
127 with self.stream_context(stream_name, level):
128 stream = getattr(sys, stream_name)
129 tag = f"python.{stream_name}"
130 self.assertEqual(f"<TextLogStream '{tag}'>", repr(stream))
131
132 self.assertIs(stream.writable(), True)
133 self.assertIs(stream.readable(), False)
134 self.assertEqual(stream.fileno(), fileno)
135 self.assertEqual("UTF-8", stream.encoding)
136 self.assertEqual("backslashreplace", stream.errors)
137 self.assertIs(stream.line_buffering, True)
138 self.assertIs(stream.write_through, False)
139
140 def write(s, lines=None, *, write_len=None):
141 if write_len is None:
142 write_len = len(s)
143 self.assertEqual(write_len, stream.write(s))
144 if lines is None:
145 lines = [s]
146 self.assert_logs(level, tag, lines)
147
148 # Single-line messages,
149 with self.reconfigure(stream, write_through=True):
150 write("", [])
151
152 write("a")
153 write("Hello")
154 write("Hello world")
155 write(" ")
156 write(" ")
157
158 # Non-ASCII text
159 write("ol\u00e9") # Spanish
160 write("\u4e2d\u6587") # Chinese
161
162 # Non-BMP emoji
163 write("\U0001f600")
164
165 # Non-encodable surrogates
166 write("\ud800\udc00", [r"\ud800\udc00"])
167
168 # Code used by surrogateescape (which isn't enabled here)
169 write("\udc80", [r"\udc80"])
170
171 # Null characters are logged using "modified UTF-8".
172 write("\u0000", [r"\xc0\x80"])
173 write("a\u0000", [r"a\xc0\x80"])
174 write("\u0000b", [r"\xc0\x80b"])
175 write("a\u0000b", [r"a\xc0\x80b"])
176
177 # Multi-line messages. Avoid identical consecutive lines, as
178 # they may activate "chatty" filtering and break the tests.
179 #
180 # Additional spaces will appear in the output where necessary to
181 # protect leading newlines.
182 write("\nx", [" "])
183 write("\na\n", ["x", "a"])
184 write("\n", [" "])
185 write("\n\n", [" ", " "])
186 write("b\n", ["b"])
187 write("c\n\n", ["c", " "])
188 write("d\ne", ["d"])
189 write("xx", [])
190 write("f\n\ng", ["exxf", " "])
191 write("\n", ["g"])
192
193 # Since this is a line-based logging system, line buffering
194 # cannot be turned off, i.e. a newline always causes a flush.
195 stream.reconfigure(line_buffering=False)
196 self.assertIs(stream.line_buffering, True)
197
198 # However, buffering can be turned off completely if you want a
199 # flush after every write.
200 with self.reconfigure(stream, write_through=True):
201 write("\nx", [" ", "x"])
202 write("\na\n", [" ", "a"])
203 write("\n", [" "])
204 write("\n\n", [" ", " "])
205 write("b\n", ["b"])
206 write("c\n\n", ["c", " "])
207 write("d\ne", ["d", "e"])
208 write("xx", ["xx"])
209 write("f\n\ng", ["f", " ", "g"])
210 write("\n", [" "])
211
212 # "\r\n" should be translated into "\n".
213 write("hello\r\n", ["hello"])
214 write("hello\r\nworld\r\n", ["hello", "world"])
215 write("\r\n", [""])
216
217 # Non-standard line separators should be preserved.
218 write("before form feed\x0cafter form feed\n",
219 ["before form feed\x0cafter form feed"])
220 write("before line separator\u2028after line separator\n",
221 ["before line separator\u2028after line separator"])
222
223 # String subclasses are accepted, but they should be converted
224 # to a standard str without calling any of their methods.
225 class CustomStr(str):
226 def splitlines(self, *args, **kwargs):
227 raise AssertionError()
228
229 def __len__(self):
230 raise AssertionError()
231
232 def __str__(self):
233 raise AssertionError()
234
235 write(CustomStr("custom\n"), ["custom"], write_len=7)
236
237 # Non-string classes are not accepted.
238 for obj in [b"", b"hello", None, 42]:
239 with self.subTest(obj=obj):
240 with self.assertRaisesRegex(
241 TypeError,
242 fr"write\(\) argument must be str, not "
243 fr"{type(obj).__name__}"
244 ):
245 stream.write(obj)
246
247 # Manual flushing is supported.
248 write("hello", [])
249 stream.flush()
250 self.assert_log(level, tag, "hello")
251 write("hello", [])
252 write("world", [])
253 stream.flush()
254 self.assert_log(level, tag, "helloworld")
255
256 # Long lines are split into blocks of 1000 characters
257 # (MAX_CHARS_PER_WRITE in _android_support.py), but
258 # TextIOWrapper should then join them back together as much as
259 # possible without exceeding 4000 UTF-8 bytes
260 # (MAX_BYTES_PER_WRITE).
261 #
262 # ASCII (1 byte per character)
263 write(("foobar" * 700) + "\n", # 4200 bytes in
264 [("foobar" * 666) + "foob", # 4000 bytes out
265 "ar" + ("foobar" * 33)]) # 200 bytes out
266
267 # "Full-width" digits 0-9 (3 bytes per character)
268 s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19"
269 write((s * 150) + "\n", # 4500 bytes in
270 [s * 100, # 3000 bytes out
271 s * 50]) # 1500 bytes out
272
273 s = "0123456789"
274 write(s * 200, []) # 2000 bytes in
275 write(s * 150, []) # 1500 bytes in
276 write(s * 51, [s * 350]) # 510 bytes in, 3500 bytes out
277 write("\n", [s * 51]) # 0 bytes in, 510 bytes out
278
279 def test_bytes(self):
280 for stream_name, level, fileno in STREAM_INFO:
281 with self.stream_context(stream_name, level):
282 stream = getattr(sys, stream_name).buffer
283 tag = f"python.{stream_name}"
284 self.assertEqual(f"<BinaryLogStream '{tag}'>", repr(stream))
285 self.assertIs(stream.writable(), True)
286 self.assertIs(stream.readable(), False)
287 self.assertEqual(stream.fileno(), fileno)
288
289 def write(b, lines=None, *, write_len=None):
290 if write_len is None:
291 write_len = len(b)
292 self.assertEqual(write_len, stream.write(b))
293 if lines is None:
294 lines = [b.decode()]
295 self.assert_logs(level, tag, lines)
296
297 # Single-line messages,
298 write(b"", [])
299
300 write(b"a")
301 write(b"Hello")
302 write(b"Hello world")
303 write(b" ")
304 write(b" ")
305
306 # Non-ASCII text
307 write(b"ol\xc3\xa9") # Spanish
308 write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese
309
310 # Non-BMP emoji
311 write(b"\xf0\x9f\x98\x80")
312
313 # Null bytes are logged using "modified UTF-8".
314 write(b"\x00", [r"\xc0\x80"])
315 write(b"a\x00", [r"a\xc0\x80"])
316 write(b"\x00b", [r"\xc0\x80b"])
317 write(b"a\x00b", [r"a\xc0\x80b"])
318
319 # Invalid UTF-8
320 write(b"\xff", [r"\xff"])
321 write(b"a\xff", [r"a\xff"])
322 write(b"\xffb", [r"\xffb"])
323 write(b"a\xffb", [r"a\xffb"])
324
325 # Log entries containing newlines are shown differently by
326 # `logcat -v tag`, `logcat -v long`, and Android Studio. We
327 # currently use `logcat -v tag`, which shows each line as if it
328 # was a separate log entry, but strips a single trailing
329 # newline.
330 write(b"\nx", [" ", "x"])
331 write(b"\na\n", [" ", "a"])
332 write(b"\n", [" "])
333 write(b"\n\n", [" ", ""])
334 write(b"b\n", ["b"])
335 write(b"c\n\n", ["c", ""])
336 write(b"d\ne", ["d", "e"])
337 write(b"xx", ["xx"])
338 write(b"f\n\ng", ["f", "", "g"])
339 write(b"\n", [" "])
340
341 # "\r\n" should be translated into "\n".
342 write(b"hello\r\n", ["hello"])
343 write(b"hello\r\nworld\r\n", ["hello", "world"])
344 write(b"\r\n", [""])
345
346 # Other bytes-like objects are accepted.
347 write(bytearray(b"bytearray"))
348
349 mv = memoryview(b"memoryview")
350 write(mv, ["memoryview"]) # Continuous
351 write(mv[::2], ["mmrve"]) # Discontinuous
352
353 write(
354 # Android only supports little-endian architectures, so the
355 # bytes representation is as follows:
356 array("H", [
357 0, # 00 00
358 1, # 01 00
359 65534, # FE FF
360 65535, # FF FF
361 ]),
362
363 # After encoding null bytes with modified UTF-8, the only
364 # valid UTF-8 sequence is \x01. All other bytes are handled
365 # by backslashreplace.
366 ["\\xc0\\x80\\xc0\\x80"
367 "\x01\\xc0\\x80"
368 "\\xfe\\xff"
369 "\\xff\\xff"],
370 write_len=8,
371 )
372
373 # Non-bytes-like classes are not accepted.
374 for obj in ["", "hello", None, 42]:
375 with self.subTest(obj=obj):
376 with self.assertRaisesRegex(
377 TypeError,
378 fr"write\(\) argument must be bytes-like, not "
379 fr"{type(obj).__name__}"
380 ):
381 stream.write(obj)
382
383
384 class TestAndroidRateLimit(unittest.TestCase):
385 def test_rate_limit(self):
386 # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
387 PER_MESSAGE_OVERHEAD = 28
388
389 # https://developer.android.com/ndk/reference/group/logging
390 ANDROID_LOG_DEBUG = 3
391
392 # To avoid flooding the test script output, use a different tag rather
393 # than stdout or stderr.
394 tag = "python.rate_limit"
395 stream = TextLogStream(ANDROID_LOG_DEBUG, tag)
396
397 # Make a test message which consumes 1 KB of the logcat buffer.
398 message = "Line {:03d} "
399 message += "." * (
400 1024 - PER_MESSAGE_OVERHEAD - len(tag) - len(message.format(0))
401 ) + "\n"
402
403 # To avoid depending on the performance of the test device, we mock the
404 # passage of time.
405 mock_now = time()
406
407 def mock_time():
408 # Avoid division by zero by simulating a small delay.
409 mock_sleep(0.0001)
410 return mock_now
411
412 def mock_sleep(duration):
413 nonlocal mock_now
414 mock_now += duration
415
416 # See _android_support.py. The default values of these parameters work
417 # well across a wide range of devices, but we'll use smaller values to
418 # ensure a quick and reliable test that doesn't flood the log too much.
419 MAX_KB_PER_SECOND = 100
420 BUCKET_KB = 10
421 with (
422 patch("_android_support.MAX_BYTES_PER_SECOND", MAX_KB_PER_SECOND * 1024),
423 patch("_android_support.BUCKET_SIZE", BUCKET_KB * 1024),
424 patch("_android_support.sleep", mock_sleep),
425 patch("_android_support.time", mock_time),
426 ):
427 # Make sure the token bucket is full.
428 stream.write("Initial message to reset _prev_write_time")
429 mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND)
430 line_num = 0
431
432 # Write BUCKET_KB messages, and return the rate at which they were
433 # accepted in KB per second.
434 def write_bucketful():
435 nonlocal line_num
436 start = mock_time()
437 max_line_num = line_num + BUCKET_KB
438 while line_num < max_line_num:
439 stream.write(message.format(line_num))
440 line_num += 1
441 return BUCKET_KB / (mock_time() - start)
442
443 # The first bucketful should be written with minimal delay. The
444 # factor of 2 here is not arbitrary: it verifies that the system can
445 # write fast enough to empty the bucket within two bucketfuls, which
446 # the next part of the test depends on.
447 self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2)
448
449 # Write another bucketful to empty the token bucket completely.
450 write_bucketful()
451
452 # The next bucketful should be written at the rate limit.
453 self.assertAlmostEqual(
454 write_bucketful(), MAX_KB_PER_SECOND,
455 delta=MAX_KB_PER_SECOND * 0.1
456 )
457
458 # Once the token bucket refills, we should go back to full speed.
459 mock_sleep(BUCKET_KB / MAX_KB_PER_SECOND)
460 self.assertGreater(write_bucketful(), MAX_KB_PER_SECOND * 2)