| 1 |
""" Standard "encodings" Package |
| 2 |
|
| 3 |
Standard Python encoding modules are stored in this package |
| 4 |
directory. |
| 5 |
|
| 6 |
Codec modules must have names corresponding to normalized encoding |
| 7 |
names as defined in the normalize_encoding() function below, e.g. |
| 8 |
'utf-8' must be implemented by the module 'utf_8.py'. |
| 9 |
|
| 10 |
Each codec module must export the following interface: |
| 11 |
|
| 12 |
* getregentry() -> codecs.CodecInfo object |
| 13 |
The getregentry() API must return a CodecInfo object with encoder, decoder, |
| 14 |
incrementalencoder, incrementaldecoder, streamwriter and streamreader |
| 15 |
attributes which adhere to the Python Codec Interface Standard. |
| 16 |
|
| 17 |
In addition, a module may optionally also define the following |
| 18 |
APIs which are then used by the package's codec search function: |
| 19 |
|
| 20 |
* getaliases() -> sequence of encoding name strings to use as aliases |
| 21 |
|
| 22 |
Alias names returned by getaliases() must be normalized encoding |
| 23 |
names as defined by normalize_encoding(). |
| 24 |
|
| 25 |
Written by Marc-Andre Lemburg ([email protected]). |
| 26 |
|
| 27 |
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. |
| 28 |
|
| 29 |
"""#" |
| 30 |
|
| 31 |
import codecs |
| 32 |
import sys |
| 33 |
from . import aliases |
| 34 |
|
| 35 |
_cache = {} |
| 36 |
_MAXCACHE = 500 |
| 37 |
_unknown = '--unknown--' |
| 38 |
_import_tail = ['*'] |
| 39 |
_aliases = aliases.aliases |
| 40 |
|
| 41 |
class CodecRegistryError(LookupError, SystemError): |
| 42 |
pass |
| 43 |
|
| 44 |
def normalize_encoding(encoding): |
| 45 |
|
| 46 |
""" Normalize an encoding name. |
| 47 |
|
| 48 |
Normalization works as follows: all non-alphanumeric |
| 49 |
characters except the dot used for Python package names are |
| 50 |
collapsed and replaced with a single underscore, e.g. ' -;#' |
| 51 |
becomes '_'. Leading and trailing underscores are removed. |
| 52 |
|
| 53 |
Note that encoding names should be ASCII only. |
| 54 |
|
| 55 |
""" |
| 56 |
if isinstance(encoding, bytes): |
| 57 |
encoding = str(encoding, "ascii") |
| 58 |
|
| 59 |
chars = [] |
| 60 |
punct = False |
| 61 |
for c in encoding: |
| 62 |
if c.isalnum() or c == '.': |
| 63 |
if punct and chars: |
| 64 |
chars.append('_') |
| 65 |
if c.isascii(): |
| 66 |
chars.append(c) |
| 67 |
punct = False |
| 68 |
else: |
| 69 |
punct = True |
| 70 |
return ''.join(chars) |
| 71 |
|
| 72 |
def search_function(encoding): |
| 73 |
|
| 74 |
# Cache lookup |
| 75 |
entry = _cache.get(encoding, _unknown) |
| 76 |
if entry is not _unknown: |
| 77 |
return entry |
| 78 |
|
| 79 |
# Import the module: |
| 80 |
# |
| 81 |
# First try to find an alias for the normalized encoding |
| 82 |
# name and lookup the module using the aliased name, then try to |
| 83 |
# lookup the module using the standard import scheme, i.e. first |
| 84 |
# try in the encodings package, then at top-level. |
| 85 |
# |
| 86 |
norm_encoding = normalize_encoding(encoding) |
| 87 |
aliased_encoding = _aliases.get(norm_encoding) or \ |
| 88 |
_aliases.get(norm_encoding.replace('.', '_')) |
| 89 |
if aliased_encoding is not None: |
| 90 |
modnames = [aliased_encoding, |
| 91 |
norm_encoding] |
| 92 |
else: |
| 93 |
modnames = [norm_encoding] |
| 94 |
for modname in modnames: |
| 95 |
if not modname or '.' in modname: |
| 96 |
continue |
| 97 |
try: |
| 98 |
# Import is absolute to prevent the possibly malicious import of a |
| 99 |
# module with side-effects that is not in the 'encodings' package. |
| 100 |
mod = __import__('encodings.' + modname, fromlist=_import_tail, |
| 101 |
level=0) |
| 102 |
except ImportError: |
| 103 |
# ImportError may occur because 'encodings.(modname)' does not exist, |
| 104 |
# or because it imports a name that does not exist (see mbcs and oem) |
| 105 |
pass |
| 106 |
else: |
| 107 |
break |
| 108 |
else: |
| 109 |
mod = None |
| 110 |
|
| 111 |
try: |
| 112 |
getregentry = mod.getregentry |
| 113 |
except AttributeError: |
| 114 |
# Not a codec module |
| 115 |
mod = None |
| 116 |
|
| 117 |
if mod is None: |
| 118 |
# Cache misses |
| 119 |
if len(_cache) >= _MAXCACHE: |
| 120 |
_cache.clear() |
| 121 |
_cache[encoding] = None |
| 122 |
return None |
| 123 |
|
| 124 |
# Now ask the module for the registry entry |
| 125 |
entry = getregentry() |
| 126 |
if not isinstance(entry, codecs.CodecInfo): |
| 127 |
if not 4 <= len(entry) <= 7: |
| 128 |
raise CodecRegistryError('module "%s" (%s) failed to register' |
| 129 |
% (mod.__name__, mod.__file__)) |
| 130 |
if not callable(entry[0]) or not callable(entry[1]) or \ |
| 131 |
(entry[2] is not None and not callable(entry[2])) or \ |
| 132 |
(entry[3] is not None and not callable(entry[3])) or \ |
| 133 |
(len(entry) > 4 and entry[4] is not None and not callable(entry[4])) or \ |
| 134 |
(len(entry) > 5 and entry[5] is not None and not callable(entry[5])): |
| 135 |
raise CodecRegistryError('incompatible codecs in module "%s" (%s)' |
| 136 |
% (mod.__name__, mod.__file__)) |
| 137 |
if len(entry)<7 or entry[6] is None: |
| 138 |
entry += (None,)*(6-len(entry)) + (mod.__name__.split(".", 1)[1],) |
| 139 |
entry = codecs.CodecInfo(*entry) |
| 140 |
|
| 141 |
# Cache the codec registry entry |
| 142 |
if len(_cache) >= _MAXCACHE: |
| 143 |
_cache.clear() |
| 144 |
_cache[encoding] = entry |
| 145 |
|
| 146 |
# Register its aliases (without overwriting previously registered |
| 147 |
# aliases) |
| 148 |
try: |
| 149 |
codecaliases = mod.getaliases() |
| 150 |
except AttributeError: |
| 151 |
pass |
| 152 |
else: |
| 153 |
for alias in codecaliases: |
| 154 |
if alias not in _aliases: |
| 155 |
_aliases[alias] = modname |
| 156 |
|
| 157 |
# Return the registry entry |
| 158 |
return entry |
| 159 |
|
| 160 |
# Register the search_function in the Python codec registry |
| 161 |
codecs.register(search_function) |
| 162 |
|
| 163 |
if sys.platform == 'win32': |
| 164 |
from ._win_cp_codecs import create_win32_code_page_codec |
| 165 |
|
| 166 |
def win32_code_page_search_function(encoding): |
| 167 |
encoding = encoding.lower() |
| 168 |
if not encoding.startswith('cp'): |
| 169 |
return None |
| 170 |
try: |
| 171 |
cp = int(encoding[2:]) |
| 172 |
except ValueError: |
| 173 |
return None |
| 174 |
# Test if the code page is supported |
| 175 |
try: |
| 176 |
codecs.code_page_encode(cp, 'x') |
| 177 |
except (OverflowError, OSError): |
| 178 |
return None |
| 179 |
|
| 180 |
return create_win32_code_page_codec(cp) |
| 181 |
|
| 182 |
codecs.register(win32_code_page_search_function) |