Debian Codegraph @

.cargo/vendor/encoding_rs/src/lib.rs

raw
1 // Copyright Mozilla Foundation. See the COPYRIGHT
2 // file at the top-level directory of this distribution.
3 //
4 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5 // https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7 // option. This file may not be copied, modified, or distributed
8 // except according to those terms.
9
10 #![cfg_attr(
11 feature = "cargo-clippy",
12 allow(doc_markdown, inline_always, new_ret_no_self)
13 )]
14
15 //! encoding_rs is a Gecko-oriented Free Software / Open Source implementation
16 //! of the [Encoding Standard](https://encoding.spec.whatwg.org/) in Rust.
17 //! Gecko-oriented means that converting to and from UTF-16 is supported in
18 //! addition to converting to and from UTF-8, that the performance and
19 //! streamability goals are browser-oriented, and that FFI-friendliness is a
20 //! goal.
21 //!
22 //! Additionally, the `mem` module provides functions that are useful for
23 //! applications that need to be able to deal with legacy in-memory
24 //! representations of Unicode.
25 //!
26 //! For expectation setting, please be sure to read the sections
27 //! [_UTF-16LE, UTF-16BE and Unicode Encoding Schemes_](#utf-16le-utf-16be-and-unicode-encoding-schemes),
28 //! [_ISO-8859-1_](#iso-8859-1) and [_Web / Browser Focus_](#web--browser-focus) below.
29 //!
30 //! There is a [long-form write-up](https://hsivonen.fi/encoding_rs/) about the
31 //! design and internals of the crate.
32 //!
33 //! # Availability
34 //!
35 //! The code is available under the
36 //! [Apache license, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
37 //! or the [MIT license](https://opensource.org/licenses/MIT), at your option.
38 //! See the
39 //! [`COPYRIGHT`](https://github.com/hsivonen/encoding_rs/blob/master/COPYRIGHT)
40 //! file for details.
41 //! The [repository is on GitHub](https://github.com/hsivonen/encoding_rs). The
42 //! [crate is available on crates.io](https://crates.io/crates/encoding_rs).
43 //!
44 //! # Integration with `std::io`
45 //!
46 //! This crate doesn't implement traits from `std::io`. However, for the case of
47 //! wrapping a `std::io::Read` in a decoder that implements `std::io::Read` and
48 //! presents the data from the wrapped `std::io::Read` as UTF-8 is addressed by
49 //! the [`encoding_rs_io`](https://docs.rs/encoding_rs_io/) crate.
50 //!
51 //! # Examples
52 //!
53 //! Example programs:
54 //!
55 //! * [Rust](https://github.com/hsivonen/recode_rs)
56 //! * [C](https://github.com/hsivonen/recode_c)
57 //! * [C++](https://github.com/hsivonen/recode_cpp)
58 //!
59 //! Decode using the non-streaming API:
60 //!
61 //! ```
62 //! #[cfg(feature = "alloc")] {
63 //! use encoding_rs::*;
64 //!
65 //! let expectation = "\u{30CF}\u{30ED}\u{30FC}\u{30FB}\u{30EF}\u{30FC}\u{30EB}\u{30C9}";
66 //! let bytes = b"\x83n\x83\x8D\x81[\x81E\x83\x8F\x81[\x83\x8B\x83h";
67 //!
68 //! let (cow, encoding_used, had_errors) = SHIFT_JIS.decode(bytes);
69 //! assert_eq!(&cow[..], expectation);
70 //! assert_eq!(encoding_used, SHIFT_JIS);
71 //! assert!(!had_errors);
72 //! }
73 //! ```
74 //!
75 //! Decode using the streaming API with minimal `unsafe`:
76 //!
77 //! ```
78 //! use encoding_rs::*;
79 //!
80 //! let expectation = "\u{30CF}\u{30ED}\u{30FC}\u{30FB}\u{30EF}\u{30FC}\u{30EB}\u{30C9}";
81 //!
82 //! // Use an array of byte slices to demonstrate content arriving piece by
83 //! // piece from the network.
84 //! let bytes: [&'static [u8]; 4] = [b"\x83",
85 //! b"n\x83\x8D\x81",
86 //! b"[\x81E\x83\x8F\x81[\x83",
87 //! b"\x8B\x83h"];
88 //!
89 //! // Very short output buffer to demonstrate the output buffer getting full.
90 //! // Normally, you'd use something like `[0u8; 2048]`.
91 //! let mut buffer_bytes = [0u8; 8];
92 //! let mut buffer: &mut str = std::str::from_utf8_mut(&mut buffer_bytes[..]).unwrap();
93 //!
94 //! // How many bytes in the buffer currently hold significant data.
95 //! let mut bytes_in_buffer = 0usize;
96 //!
97 //! // Collect the output to a string for demonstration purposes.
98 //! let mut output = String::new();
99 //!
100 //! // The `Decoder`
101 //! let mut decoder = SHIFT_JIS.new_decoder();
102 //!
103 //! // Track whether we see errors.
104 //! let mut total_had_errors = false;
105 //!
106 //! // Decode using a fixed-size intermediate buffer (for demonstrating the
107 //! // use of a fixed-size buffer; normally when the output of an incremental
108 //! // decode goes to a `String` one would use `Decoder.decode_to_string()` to
109 //! // avoid the intermediate buffer).
110 //! for input in &bytes[..] {
111 //! // The number of bytes already read from current `input` in total.
112 //! let mut total_read_from_current_input = 0usize;
113 //!
114 //! loop {
115 //! let (result, read, written, had_errors) =
116 //! decoder.decode_to_str(&input[total_read_from_current_input..],
117 //! &mut buffer[bytes_in_buffer..],
118 //! false);
119 //! total_read_from_current_input += read;
120 //! bytes_in_buffer += written;
121 //! total_had_errors |= had_errors;
122 //! match result {
123 //! CoderResult::InputEmpty => {
124 //! // We have consumed the current input buffer. Break out of
125 //! // the inner loop to get the next input buffer from the
126 //! // outer loop.
127 //! break;
128 //! },
129 //! CoderResult::OutputFull => {
130 //! // Write the current buffer out and consider the buffer
131 //! // empty.
132 //! output.push_str(&buffer[..bytes_in_buffer]);
133 //! bytes_in_buffer = 0usize;
134 //! continue;
135 //! }
136 //! }
137 //! }
138 //! }
139 //!
140 //! // Process EOF
141 //! loop {
142 //! let (result, _, written, had_errors) =
143 //! decoder.decode_to_str(b"",
144 //! &mut buffer[bytes_in_buffer..],
145 //! true);
146 //! bytes_in_buffer += written;
147 //! total_had_errors |= had_errors;
148 //! // Write the current buffer out and consider the buffer empty.
149 //! // Need to do this here for both `match` arms, because we exit the
150 //! // loop on `CoderResult::InputEmpty`.
151 //! output.push_str(&buffer[..bytes_in_buffer]);
152 //! bytes_in_buffer = 0usize;
153 //! match result {
154 //! CoderResult::InputEmpty => {
155 //! // Done!
156 //! break;
157 //! },
158 //! CoderResult::OutputFull => {
159 //! continue;
160 //! }
161 //! }
162 //! }
163 //!
164 //! assert_eq!(&output[..], expectation);
165 //! assert!(!total_had_errors);
166 //! ```
167 //!
168 //! ## UTF-16LE, UTF-16BE and Unicode Encoding Schemes
169 //!
170 //! The Encoding Standard doesn't specify encoders for UTF-16LE and UTF-16BE,
171 //! __so this crate does not provide encoders for those encodings__!
172 //! Along with the replacement encoding, their _output encoding_ (i.e. the
173 //! encoding used for form submission and error handling in the query string
174 //! of URLs) is UTF-8, so you get an UTF-8 encoder if you request an encoder
175 //! for them.
176 //!
177 //! Additionally, the Encoding Standard factors BOM handling into wrapper
178 //! algorithms so that BOM handling isn't part of the definition of the
179 //! encodings themselves. The Unicode _encoding schemes_ in the Unicode
180 //! Standard define BOM handling or lack thereof as part of the encoding
181 //! scheme.
182 //!
183 //! When used with the `_without_bom_handling` entry points, the UTF-16LE
184 //! and UTF-16BE _encodings_ match the same-named _encoding schemes_ from
185 //! the Unicode Standard.
186 //!
187 //! When used with the `_with_bom_removal` entry points, the UTF-8
188 //! _encoding_ matches the UTF-8 _encoding scheme_ from the Unicode
189 //! Standard.
190 //!
191 //! This crate does not provide a mode that matches the UTF-16 _encoding
192 //! scheme_ from the Unicode Stardard. The UTF-16BE encoding used with
193 //! the entry points without `_bom_` qualifiers is the closest match,
194 //! but in that case, the UTF-8 BOM triggers UTF-8 decoding, which is
195 //! not part of the behavior of the UTF-16 _encoding scheme_ per the
196 //! Unicode Standard.
197 //!
198 //! The UTF-32 family of Unicode encoding schemes is not supported
199 //! by this crate. The Encoding Standard doesn't define any UTF-32
200 //! family encodings, since they aren't necessary for consuming Web
201 //! content.
202 //!
203 //! While gb18030 is capable of representing U+FEFF, the Encoding
204 //! Standard does not treat the gb18030 byte representation of U+FEFF
205 //! as a BOM, so neither does this crate.
206 //!
207 //! ## ISO-8859-1
208 //!
209 //! ISO-8859-1 does not exist as a distinct encoding from windows-1252 in
210 //! the Encoding Standard. Therefore, an encoding that maps the unsigned
211 //! byte value to the same Unicode scalar value is not available via
212 //! `Encoding` in this crate.
213 //!
214 //! However, the functions whose name starts with `convert` and contains
215 //! `latin1` in the `mem` module support such conversions, which are known as
216 //! [_isomorphic decode_](https://infra.spec.whatwg.org/#isomorphic-decode)
217 //! and [_isomorphic encode_](https://infra.spec.whatwg.org/#isomorphic-encode)
218 //! in the [Infra Standard](https://infra.spec.whatwg.org/).
219 //!
220 //! ## Web / Browser Focus
221 //!
222 //! Both in terms of scope and performance, the focus is on the Web. For scope,
223 //! this means that encoding_rs implements the Encoding Standard fully and
224 //! doesn't implement encodings that are not specified in the Encoding
225 //! Standard. For performance, this means that decoding performance is
226 //! important as well as performance for encoding into UTF-8 or encoding the
227 //! Basic Latin range (ASCII) into legacy encodings. Non-Basic Latin needs to
228 //! be encoded into legacy encodings in only two places in the Web platform: in
229 //! the query part of URLs, in which case it's a matter of relatively rare
230 //! error handling, and in form submission, in which case the user action and
231 //! networking tend to hide the performance of the encoder.
232 //!
233 //! Deemphasizing performance of encoding non-Basic Latin text into legacy
234 //! encodings enables smaller code size thanks to the encoder side using the
235 //! decode-optimized data tables without having encode-optimized data tables at
236 //! all. Even in decoders, smaller lookup table size is preferred over avoiding
237 //! multiplication operations.
238 //!
239 //! Additionally, performance is a non-goal for the ASCII-incompatible
240 //! ISO-2022-JP encoding, which are rarely used on the Web. Instead of
241 //! performance, the decoder for ISO-2022-JP optimizes for ease/clarity
242 //! of implementation.
243 //!
244 //! Despite the browser focus, the hope is that non-browser applications
245 //! that wish to consume Web content or submit Web forms in a Web-compatible
246 //! way will find encoding_rs useful. While encoding_rs does not try to match
247 //! Windows behavior, many of the encodings are close enough to legacy
248 //! encodings implemented by Windows that applications that need to consume
249 //! data in legacy Windows encodins may find encoding_rs useful. The
250 //! [codepage](https://crates.io/crates/codepage) crate maps from Windows
251 //! code page identifiers onto encoding_rs `Encoding`s and vice versa.
252 //!
253 //! For decoding email, UTF-7 support is needed (unfortunately) in additition
254 //! to the encodings defined in the Encoding Standard. The
255 //! [charset](https://crates.io/crates/charset) wraps encoding_rs and adds
256 //! UTF-7 decoding for email purposes.
257 //!
258 //! For single-byte DOS encodings beyond the ones supported by the Encoding
259 //! Standard, there is the [`oem_cp`](https://crates.io/crates/oem_cp) crate.
260 //!
261 //! # Preparing Text for the Encoders
262 //!
263 //! Normalizing text into Unicode Normalization Form C prior to encoding text
264 //! into a legacy encoding minimizes unmappable characters. Text can be
265 //! normalized to Unicode Normalization Form C using the
266 //! [`icu_normalizer`](https://crates.io/crates/icu_normalizer) crate, which
267 //! is part of [ICU4X](https://icu4x.unicode.org/).
268 //!
269 //! The exception is windows-1258, which after normalizing to Unicode
270 //! Normalization Form C requires tone marks to be decomposed in order to
271 //! minimize unmappable characters. Vietnamese tone marks can be decomposed
272 //! using the [`detone`](https://crates.io/crates/detone) crate.
273 //!
274 //! # Streaming & Non-Streaming; Rust & C/C++
275 //!
276 //! The API in Rust has two modes of operation: streaming and non-streaming.
277 //! The streaming API is the foundation of the implementation and should be
278 //! used when processing data that arrives piecemeal from an i/o stream. The
279 //! streaming API has an FFI wrapper (as a [separate crate][1]) that exposes it
280 //! to C callers. The non-streaming part of the API is for Rust callers only and
281 //! is smart about borrowing instead of copying when possible. When
282 //! streamability is not needed, the non-streaming API should be preferrer in
283 //! order to avoid copying data when a borrow suffices.
284 //!
285 //! There is no analogous C API exposed via FFI, mainly because C doesn't have
286 //! standard types for growable byte buffers and Unicode strings that know
287 //! their length.
288 //!
289 //! The C API (header file generated at `target/include/encoding_rs.h` when
290 //! building encoding_rs) can, in turn, be wrapped for use from C++. Such a
291 //! C++ wrapper can re-create the non-streaming API in C++ for C++ callers.
292 //! The C binding comes with a [C++17 wrapper][2] that uses standard library +
293 //! [GSL][3] types and that recreates the non-streaming API in C++ on top of
294 //! the streaming API. A C++ wrapper with XPCOM/MFBT types is available as
295 //! [`mozilla::Encoding`][4].
296 //!
297 //! The `Encoding` type is common to both the streaming and non-streaming
298 //! modes. In the streaming mode, decoding operations are performed with a
299 //! `Decoder` and encoding operations with an `Encoder` object obtained via
300 //! `Encoding`. In the non-streaming mode, decoding and encoding operations are
301 //! performed using methods on `Encoding` objects themselves, so the `Decoder`
302 //! and `Encoder` objects are not used at all.
303 //!
304 //! [1]: https://github.com/hsivonen/encoding_c
305 //! [2]: https://github.com/hsivonen/encoding_c/blob/master/include/encoding_rs_cpp.h
306 //! [3]: https://github.com/Microsoft/GSL/
307 //! [4]: https://searchfox.org/mozilla-central/source/intl/Encoding.h
308 //!
309 //! # Memory management
310 //!
311 //! The non-streaming mode never performs heap allocations (even the methods
312 //! that write into a `Vec<u8>` or a `String` by taking them as arguments do
313 //! not reallocate the backing buffer of the `Vec<u8>` or the `String`). That
314 //! is, the non-streaming mode uses caller-allocated buffers exclusively.
315 //!
316 //! The methods of the streaming mode that return a `Vec<u8>` or a `String`
317 //! perform heap allocations but only to allocate the backing buffer of the
318 //! `Vec<u8>` or the `String`.
319 //!
320 //! `Encoding` is always statically allocated. `Decoder` and `Encoder` need no
321 //! `Drop` cleanup.
322 //!
323 //! # Buffer reading and writing behavior
324 //!
325 //! Based on experience gained with the `java.nio.charset` encoding converter
326 //! API and with the Gecko uconv encoding converter API, the buffer reading
327 //! and writing behaviors of encoding_rs are asymmetric: input buffers are
328 //! fully drained but output buffers are not always fully filled.
329 //!
330 //! When reading from an input buffer, encoding_rs always consumes all input
331 //! up to the next error or to the end of the buffer. In particular, when
332 //! decoding, even if the input buffer ends in the middle of a byte sequence
333 //! for a character, the decoder consumes all input. This has the benefit that
334 //! the caller of the API can always fill the next buffer from the start from
335 //! whatever source the bytes come from and never has to first copy the last
336 //! bytes of the previous buffer to the start of the next buffer. However, when
337 //! encoding, the UTF-8 input buffers have to end at a character boundary, which
338 //! is a requirement for the Rust `str` type anyway, and UTF-16 input buffer
339 //! boundaries falling in the middle of a surrogate pair result in both
340 //! suggorates being treated individually as unpaired surrogates.
341 //!
342 //! Additionally, decoders guarantee that they can be fed even one byte at a
343 //! time and encoders guarantee that they can be fed even one code point at a
344 //! time. This has the benefit of not placing restrictions on the size of
345 //! chunks the content arrives e.g. from network.
346 //!
347 //! When writing into an output buffer, encoding_rs makes sure that the code
348 //! unit sequence for a character is never split across output buffer
349 //! boundaries. This may result in wasted space at the end of an output buffer,
350 //! but the advantages are that the output side of both decoders and encoders
351 //! is greatly simplified compared to designs that attempt to fill output
352 //! buffers exactly even when that entails splitting a code unit sequence and
353 //! when encoding_rs methods return to the caller, the output produces thus
354 //! far is always valid taken as whole. (In the case of encoding to ISO-2022-JP,
355 //! the output needs to be considered as a whole, because the latest output
356 //! buffer taken alone might not be valid taken alone if the transition away
357 //! from the ASCII state occurred in an earlier output buffer. However, since
358 //! the ISO-2022-JP decoder doesn't treat streams that don't end in the ASCII
359 //! state as being in error despite the encoder generating a transition to the
360 //! ASCII state at the end, the claim about the partial output taken as a whole
361 //! being valid is true even for ISO-2022-JP.)
362 //!
363 //! # Error Reporting
364 //!
365 //! Based on experience gained with the `java.nio.charset` encoding converter
366 //! API and with the Gecko uconv encoding converter API, the error reporting
367 //! behaviors of encoding_rs are asymmetric: decoder errors include offsets
368 //! that leave it up to the caller to extract the erroneous bytes from the
369 //! input stream if the caller wishes to do so but encoder errors provide the
370 //! code point associated with the error without requiring the caller to
371 //! extract it from the input on its own.
372 //!
373 //! On the encoder side, an error is always triggered by the most recently
374 //! pushed Unicode scalar, which makes it simple to pass the `char` to the
375 //! caller. Also, it's very typical for the caller to wish to do something with
376 //! this data: generate a numeric escape for the character. Additionally, the
377 //! ISO-2022-JP encoder reports U+FFFD instead of the actual input character in
378 //! certain cases, so requiring the caller to extract the character from the
379 //! input buffer would require the caller to handle ISO-2022-JP details.
380 //! Furthermore, requiring the caller to extract the character from the input
381 //! buffer would require the caller to implement UTF-8 or UTF-16 math, which is
382 //! the job of an encoding conversion library.
383 //!
384 //! On the decoder side, errors are triggered in more complex ways. For
385 //! example, when decoding the sequence ESC, '$', _buffer boundary_, 'A' as
386 //! ISO-2022-JP, the ESC byte is in error, but this is discovered only after
387 //! the buffer boundary when processing 'A'. Thus, the bytes in error might not
388 //! be the ones most recently pushed to the decoder and the error might not even
389 //! be in the current buffer.
390 //!
391 //! Some encoding conversion APIs address the problem by not acknowledging
392 //! trailing bytes of an input buffer as consumed if it's still possible for
393 //! future bytes to cause the trailing bytes to be in error. This way, error
394 //! reporting can always refer to the most recently pushed buffer. This has the
395 //! problem that the caller of the API has to copy the unconsumed trailing
396 //! bytes to the start of the next buffer before being able to fill the rest
397 //! of the next buffer. This is annoying, error-prone and inefficient.
398 //!
399 //! A possible solution would be making the decoder remember recently consumed
400 //! bytes in order to be able to include a copy of the erroneous bytes when
401 //! reporting an error. This has two problem: First, callers a rarely
402 //! interested in the erroneous bytes, so attempts to identify them are most
403 //! often just overhead anyway. Second, the rare applications that are
404 //! interested typically care about the location of the error in the input
405 //! stream.
406 //!
407 //! To keep the API convenient for common uses and the overhead low while making
408 //! it possible to develop applications, such as HTML validators, that care
409 //! about which bytes were in error, encoding_rs reports the length of the
410 //! erroneous sequence and the number of bytes consumed after the erroneous
411 //! sequence. As long as the caller doesn't discard the 6 most recent bytes,
412 //! this makes it possible for callers that care about the erroneous bytes to
413 //! locate them.
414 //!
415 //! # No Convenience API for Custom Replacements
416 //!
417 //! The Web Platform and, therefore, the Encoding Standard supports only one
418 //! error recovery mode for decoders and only one error recovery mode for
419 //! encoders. The supported error recovery mode for decoders is emitting the
420 //! REPLACEMENT CHARACTER on error. The supported error recovery mode for
421 //! encoders is emitting an HTML decimal numeric character reference for
422 //! unmappable characters.
423 //!
424 //! Since encoding_rs is Web-focused, these are the only error recovery modes
425 //! for which convenient support is provided. Moreover, on the decoder side,
426 //! there aren't really good alternatives for emitting the REPLACEMENT CHARACTER
427 //! on error (other than treating errors as fatal). In particular, simply
428 //! ignoring errors is a
429 //! [security problem](http://www.unicode.org/reports/tr36/#Substituting_for_Ill_Formed_Subsequences),
430 //! so it would be a bad idea for encoding_rs to provide a mode that encouraged
431 //! callers to ignore errors.
432 //!
433 //! On the encoder side, there are plausible alternatives for HTML decimal
434 //! numeric character references. For example, when outputting CSS, CSS-style
435 //! escapes would seem to make sense. However, instead of facilitating the
436 //! output of CSS, JS, etc. in non-UTF-8 encodings, encoding_rs takes the design
437 //! position that you shouldn't generate output in encodings other than UTF-8,
438 //! except where backward compatibility with interacting with the legacy Web
439 //! requires it. The legacy Web requires it only when parsing the query strings
440 //! of URLs and when submitting forms, and those two both use HTML decimal
441 //! numeric character references.
442 //!
443 //! While encoding_rs doesn't make encoder replacements other than HTML decimal
444 //! numeric character references easy, it does make them _possible_.
445 //! `encode_from_utf8()`, which emits HTML decimal numeric character references
446 //! for unmappable characters, is implemented on top of
447 //! `encode_from_utf8_without_replacement()`. Applications that really, really
448 //! want other replacement schemes for unmappable characters can likewise
449 //! implement them on top of `encode_from_utf8_without_replacement()`.
450 //!
451 //! # No Extensibility by Design
452 //!
453 //! The set of encodings supported by encoding_rs is not extensible by design.
454 //! That is, `Encoding`, `Decoder` and `Encoder` are intentionally `struct`s
455 //! rather than `trait`s. encoding_rs takes the design position that all future
456 //! text interchange should be done using UTF-8, which can represent all of
457 //! Unicode. (It is, in fact, the only encoding supported by the Encoding
458 //! Standard and encoding_rs that can represent all of Unicode and that has
459 //! encoder support. UTF-16LE and UTF-16BE don't have encoder support, and
460 //! gb18030 cannot encode U+E5E5.) The other encodings are supported merely for
461 //! legacy compatibility and not due to non-UTF-8 encodings having benefits
462 //! other than being able to consume legacy content.
463 //!
464 //! Considering that UTF-8 can represent all of Unicode and is already supported
465 //! by all Web browsers, introducing a new encoding wouldn't add to the
466 //! expressiveness but would add to compatibility problems. In that sense,
467 //! adding new encodings to the Web Platform doesn't make sense, and, in fact,
468 //! post-UTF-8 attempts at encodings, such as BOCU-1, have been rejected from
469 //! the Web Platform. On the other hand, the set of legacy encodings that must
470 //! be supported for a Web browser to be able to be successful is not going to
471 //! expand. Empirically, the set of encodings specified in the Encoding Standard
472 //! is already sufficient and the set of legacy encodings won't grow
473 //! retroactively.
474 //!
475 //! Since extensibility doesn't make sense considering the Web focus of
476 //! encoding_rs and adding encodings to Web clients would be actively harmful,
477 //! it makes sense to make the set of encodings that encoding_rs supports
478 //! non-extensible and to take the (admittedly small) benefits arising from
479 //! that, such as the size of `Decoder` and `Encoder` objects being known ahead
480 //! of time, which enables stack allocation thereof.
481 //!
482 //! This does have downsides for applications that might want to put encoding_rs
483 //! to non-Web uses if those non-Web uses involve legacy encodings that aren't
484 //! needed for Web uses. The needs of such applications should not complicate
485 //! encoding_rs itself, though. It is up to those applications to provide a
486 //! framework that delegates the operations with encodings that encoding_rs
487 //! supports to encoding_rs and operations with other encodings to something
488 //! else (as opposed to encoding_rs itself providing an extensibility
489 //! framework).
490 //!
491 //! # Panics
492 //!
493 //! Methods in encoding_rs can panic if the API is used against the requirements
494 //! stated in the documentation, if a state that's supposed to be impossible
495 //! is reached due to an internal bug or on integer overflow. When used
496 //! according to documentation with buffer sizes that stay below integer
497 //! overflow, in the absence of internal bugs, encoding_rs does not panic.
498 //!
499 //! Panics arising from API misuse aren't documented beyond this on individual
500 //! methods.
501 //!
502 //! # At-Risk Parts of the API
503 //!
504 //! The foreseeable source of partially backward-incompatible API change is the
505 //! way the instances of `Encoding` are made available.
506 //!
507 //! If Rust changes to allow the entries of `[&'static Encoding; N]` to be
508 //! initialized with `static`s of type `&'static Encoding`, the non-reference
509 //! `FOO_INIT` public `Encoding` instances will be removed from the public API.
510 //!
511 //! If Rust changes to make the referent of `pub const FOO: &'static Encoding`
512 //! unique when the constant is used in different crates, the reference-typed
513 //! `static`s for the encoding instances will be changed from `static` to
514 //! `const` and the non-reference-typed `_INIT` instances will be removed.
515 //!
516 //! # Mapping Spec Concepts onto the API
517 //!
518 //! <table>
519 //! <thead>
520 //! <tr><th>Spec Concept</th><th>Streaming</th><th>Non-Streaming</th></tr>
521 //! </thead>
522 //! <tbody>
523 //! <tr><td><a href="https://encoding.spec.whatwg.org/#encoding">encoding</a></td><td><code>&amp;'static Encoding</code></td><td><code>&amp;'static Encoding</code></td></tr>
524 //! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8">UTF-8 encoding</a></td><td><code>UTF_8</code></td><td><code>UTF_8</code></td></tr>
525 //! <tr><td><a href="https://encoding.spec.whatwg.org/#concept-encoding-get">get an encoding</a></td><td><code>Encoding::for_label(<var>label</var>)</code></td><td><code>Encoding::for_label(<var>label</var>)</code></td></tr>
526 //! <tr><td><a href="https://encoding.spec.whatwg.org/#name">name</a></td><td><code><var>encoding</var>.name()</code></td><td><code><var>encoding</var>.name()</code></td></tr>
527 //! <tr><td><a href="https://encoding.spec.whatwg.org/#get-an-output-encoding">get an output encoding</a></td><td><code><var>encoding</var>.output_encoding()</code></td><td><code><var>encoding</var>.output_encoding()</code></td></tr>
528 //! <tr><td><a href="https://encoding.spec.whatwg.org/#decode">decode</a></td><td><code>let d = <var>encoding</var>.new_decoder();<br>let res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code><var>encoding</var>.decode(<var>src</var>)</code></td></tr>
529 //! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-decode">UTF-8 decode</a></td><td><code>let d = UTF_8.new_decoder_with_bom_removal();<br>let res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code>UTF_8.decode_with_bom_removal(<var>src</var>)</code></td></tr>
530 //! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-decode-without-bom">UTF-8 decode without BOM</a></td><td><code>let d = UTF_8.new_decoder_without_bom_handling();<br>let res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = d.decode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code>UTF_8.decode_without_bom_handling(<var>src</var>)</code></td></tr>
531 //! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-decode-without-bom-or-fail">UTF-8 decode without BOM or fail</a></td><td><code>let d = UTF_8.new_decoder_without_bom_handling();<br>let res = d.decode_to_<var>*</var>_without_replacement(<var>src</var>, <var>dst</var>, false);<br>// &hellip; (fail if malformed)</br>let last_res = d.decode_to_<var>*</var>_without_replacement(<var>src</var>, <var>dst</var>, true);<br>// (fail if malformed)</code></td><td><code>UTF_8.decode_without_bom_handling_and_without_replacement(<var>src</var>)</code></td></tr>
532 //! <tr><td><a href="https://encoding.spec.whatwg.org/#encode">encode</a></td><td><code>let e = <var>encoding</var>.new_encoder();<br>let res = e.encode_to_<var>*</var>(<var>src</var>, <var>dst</var>, false);<br>// &hellip;</br>let last_res = e.encode_to_<var>*</var>(<var>src</var>, <var>dst</var>, true);</code></td><td><code><var>encoding</var>.encode(<var>src</var>)</code></td></tr>
533 //! <tr><td><a href="https://encoding.spec.whatwg.org/#utf-8-encode">UTF-8 encode</a></td><td>Use the UTF-8 nature of Rust strings directly:<br><code><var>write</var>(<var>src</var>.as_bytes());<br>// refill src<br><var>write</var>(<var>src</var>.as_bytes());<br>// refill src<br><var>write</var>(<var>src</var>.as_bytes());<br>// &hellip;</code></td><td>Use the UTF-8 nature of Rust strings directly:<br><code><var>src</var>.as_bytes()</code></td></tr>
534 //! </tbody>
535 //! </table>
536 //!
537 //! # Compatibility with the rust-encoding API
538 //!
539 //! The crate
540 //! [encoding_rs_compat](https://github.com/hsivonen/encoding_rs_compat/)
541 //! is a drop-in replacement for rust-encoding 0.2.32 that implements (most of)
542 //! the API of rust-encoding 0.2.32 on top of encoding_rs.
543 //!
544 //! # Mapping rust-encoding concepts to encoding_rs concepts
545 //!
546 //! The following table provides a mapping from rust-encoding constructs to
547 //! encoding_rs ones.
548 //!
549 //! <table>
550 //! <thead>
551 //! <tr><th>rust-encoding</th><th>encoding_rs</th></tr>
552 //! </thead>
553 //! <tbody>
554 //! <tr><td><code>encoding::EncodingRef</code></td><td><code>&amp;'static encoding_rs::Encoding</code></td></tr>
555 //! <tr><td><code>encoding::all::<var>WINDOWS_31J</var></code> (not based on the WHATWG name for some encodings)</td><td><code>encoding_rs::<var>SHIFT_JIS</var></code> (always the WHATWG name uppercased and hyphens replaced with underscores)</td></tr>
556 //! <tr><td><code>encoding::all::ERROR</code></td><td>Not available because not in the Encoding Standard</td></tr>
557 //! <tr><td><code>encoding::all::ASCII</code></td><td>Not available because not in the Encoding Standard</td></tr>
558 //! <tr><td><code>encoding::all::ISO_8859_1</code></td><td>Not available because not in the Encoding Standard</td></tr>
559 //! <tr><td><code>encoding::all::HZ</code></td><td>Not available because not in the Encoding Standard</td></tr>
560 //! <tr><td><code>encoding::label::encoding_from_whatwg_label(<var>string</var>)</code></td><td><code>encoding_rs::Encoding::for_label(<var>string</var>)</code></td></tr>
561 //! <tr><td><code><var>enc</var>.whatwg_name()</code> (always lower case)</td><td><code><var>enc</var>.name()</code> (potentially mixed case)</td></tr>
562 //! <tr><td><code><var>enc</var>.name()</code></td><td>Not available because not in the Encoding Standard</td></tr>
563 //! <tr><td><code>encoding::decode(<var>bytes</var>, encoding::DecoderTrap::Replace, <var>enc</var>)</code></td><td><code><var>enc</var>.decode(<var>bytes</var>)</code></td></tr>
564 //! <tr><td><code><var>enc</var>.decode(<var>bytes</var>, encoding::DecoderTrap::Replace)</code></td><td><code><var>enc</var>.decode_without_bom_handling(<var>bytes</var>)</code></td></tr>
565 //! <tr><td><code><var>enc</var>.encode(<var>string</var>, encoding::EncoderTrap::NcrEscape)</code></td><td><code><var>enc</var>.encode(<var>string</var>)</code></td></tr>
566 //! <tr><td><code><var>enc</var>.raw_decoder()</code></td><td><code><var>enc</var>.new_decoder_without_bom_handling()</code></td></tr>
567 //! <tr><td><code><var>enc</var>.raw_encoder()</code></td><td><code><var>enc</var>.new_encoder()</code></td></tr>
568 //! <tr><td><code>encoding::RawDecoder</code></td><td><code>encoding_rs::Decoder</code></td></tr>
569 //! <tr><td><code>encoding::RawEncoder</code></td><td><code>encoding_rs::Encoder</code></td></tr>
570 //! <tr><td><code><var>raw_decoder</var>.raw_feed(<var>src</var>, <var>dst_string</var>)</code></td><td><code><var>dst_string</var>.reserve(<var>decoder</var>.max_utf8_buffer_length_without_replacement(<var>src</var>.len()));<br><var>decoder</var>.decode_to_string_without_replacement(<var>src</var>, <var>dst_string</var>, false)</code></td></tr>
571 //! <tr><td><code><var>raw_encoder</var>.raw_feed(<var>src</var>, <var>dst_vec</var>)</code></td><td><code><var>dst_vec</var>.reserve(<var>encoder</var>.max_buffer_length_from_utf8_without_replacement(<var>src</var>.len()));<br><var>encoder</var>.encode_from_utf8_to_vec_without_replacement(<var>src</var>, <var>dst_vec</var>, false)</code></td></tr>
572 //! <tr><td><code><var>raw_decoder</var>.raw_finish(<var>dst</var>)</code></td><td><code><var>dst_string</var>.reserve(<var>decoder</var>.max_utf8_buffer_length_without_replacement(0));<br><var>decoder</var>.decode_to_string_without_replacement(b"", <var>dst</var>, true)</code></td></tr>
573 //! <tr><td><code><var>raw_encoder</var>.raw_finish(<var>dst</var>)</code></td><td><code><var>dst_vec</var>.reserve(<var>encoder</var>.max_buffer_length_from_utf8_without_replacement(0));<br><var>encoder</var>.encode_from_utf8_to_vec_without_replacement("", <var>dst</var>, true)</code></td></tr>
574 //! <tr><td><code>encoding::DecoderTrap::Strict</code></td><td><code>decode*</code> methods that have <code>_without_replacement</code> in their name (and treating the `Malformed` result as fatal).</td></tr>
575 //! <tr><td><code>encoding::DecoderTrap::Replace</code></td><td><code>decode*</code> methods that <i>do not</i> have <code>_without_replacement</code> in their name.</td></tr>
576 //! <tr><td><code>encoding::DecoderTrap::Ignore</code></td><td>It is a bad idea to ignore errors due to security issues, but this could be implemented using <code>decode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
577 //! <tr><td><code>encoding::DecoderTrap::Call(DecoderTrapFunc)</code></td><td>Can be implemented using <code>decode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
578 //! <tr><td><code>encoding::EncoderTrap::Strict</code></td><td><code>encode*</code> methods that have <code>_without_replacement</code> in their name (and treating the `Unmappable` result as fatal).</td></tr>
579 //! <tr><td><code>encoding::EncoderTrap::Replace</code></td><td>Can be implemented using <code>encode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
580 //! <tr><td><code>encoding::EncoderTrap::Ignore</code></td><td>It is a bad idea to ignore errors due to security issues, but this could be implemented using <code>encode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
581 //! <tr><td><code>encoding::EncoderTrap::NcrEscape</code></td><td><code>encode*</code> methods that <i>do not</i> have <code>_without_replacement</code> in their name.</td></tr>
582 //! <tr><td><code>encoding::EncoderTrap::Call(EncoderTrapFunc)</code></td><td>Can be implemented using <code>encode*</code> methods that have <code>_without_replacement</code> in their name.</td></tr>
583 //! </tbody>
584 //! </table>
585 //!
586 //! # Relationship with Windows Code Pages
587 //!
588 //! Despite the Web and browser focus, the encodings defined by the Encoding
589 //! Standard and implemented by this crate may be useful for decoding legacy
590 //! data that uses Windows code pages. The following table names the single-byte
591 //! encodings
592 //! that have a closely related Windows code page, the number of the closest
593 //! code page, a column indicating whether Windows maps unassigned code points
594 //! to the Unicode Private Use Area instead of U+FFFD and a remark number
595 //! indicating remarks in the list after the table.
596 //!
597 //! <table>
598 //! <thead>
599 //! <tr><th>Encoding</th><th>Code Page</th><th>PUA</th><th>Remarks</th></tr>
600 //! </thead>
601 //! <tbody>
602 //! <tr><td>Shift_JIS</td><td>932</td><td></td><td></td></tr>
603 //! <tr><td>GBK</td><td>936</td><td></td><td></td></tr>
604 //! <tr><td>EUC-KR</td><td>949</td><td></td><td></td></tr>
605 //! <tr><td>Big5</td><td>950</td><td></td><td></td></tr>
606 //! <tr><td>IBM866</td><td>866</td><td></td><td></td></tr>
607 //! <tr><td>windows-874</td><td>874</td><td>&bullet;</td><td></td></tr>
608 //! <tr><td>UTF-16LE</td><td>1200</td><td></td><td></td></tr>
609 //! <tr><td>UTF-16BE</td><td>1201</td><td></td><td></td></tr>
610 //! <tr><td>windows-1250</td><td>1250</td><td></td><td></td></tr>
611 //! <tr><td>windows-1251</td><td>1251</td><td></td><td></td></tr>
612 //! <tr><td>windows-1252</td><td>1252</td><td></td><td></td></tr>
613 //! <tr><td>windows-1253</td><td>1253</td><td>&bullet;</td><td></td></tr>
614 //! <tr><td>windows-1254</td><td>1254</td><td></td><td></td></tr>
615 //! <tr><td>windows-1255</td><td>1255</td><td>&bullet;</td><td></td></tr>
616 //! <tr><td>windows-1256</td><td>1256</td><td></td><td></td></tr>
617 //! <tr><td>windows-1257</td><td>1257</td><td>&bullet;</td><td></td></tr>
618 //! <tr><td>windows-1258</td><td>1258</td><td></td><td></td></tr>
619 //! <tr><td>macintosh</td><td>10000</td><td></td><td>1</td></tr>
620 //! <tr><td>x-mac-cyrillic</td><td>10017</td><td></td><td>2</td></tr>
621 //! <tr><td>KOI8-R</td><td>20866</td><td></td><td></td></tr>
622 //! <tr><td>EUC-JP</td><td>20932</td><td></td><td></td></tr>
623 //! <tr><td>KOI8-U</td><td>21866</td><td></td><td></td></tr>
624 //! <tr><td>ISO-8859-2</td><td>28592</td><td></td><td></td></tr>
625 //! <tr><td>ISO-8859-3</td><td>28593</td><td></td><td></td></tr>
626 //! <tr><td>ISO-8859-4</td><td>28594</td><td></td><td></td></tr>
627 //! <tr><td>ISO-8859-5</td><td>28595</td><td></td><td></td></tr>
628 //! <tr><td>ISO-8859-6</td><td>28596</td><td>&bullet;</td><td></td></tr>
629 //! <tr><td>ISO-8859-7</td><td>28597</td><td>&bullet;</td><td>3</td></tr>
630 //! <tr><td>ISO-8859-8</td><td>28598</td><td>&bullet;</td><td>4</td></tr>
631 //! <tr><td>ISO-8859-13</td><td>28603</td><td>&bullet;</td><td></td></tr>
632 //! <tr><td>ISO-8859-15</td><td>28605</td><td></td><td></td></tr>
633 //! <tr><td>ISO-8859-8-I</td><td>38598</td><td></td><td>5</td></tr>
634 //! <tr><td>ISO-2022-JP</td><td>50220</td><td></td><td></td></tr>
635 //! <tr><td>gb18030</td><td>54936</td><td></td><td></td></tr>
636 //! <tr><td>UTF-8</td><td>65001</td><td></td><td></td></tr>
637 //! </tbody>
638 //! </table>
639 //!
640 //! 1. Windows decodes 0xBD to U+2126 OHM SIGN instead of U+03A9 GREEK CAPITAL LETTER OMEGA.
641 //! 2. Windows decodes 0xFF to U+00A4 CURRENCY SIGN instead of U+20AC EURO SIGN.
642 //! 3. Windows decodes the currency signs at 0xA4 and 0xA5 as well as 0xAA,
643 //! which should be U+037A GREEK YPOGEGRAMMENI, to PUA code points. Windows
644 //! decodes 0xA1 to U+02BD MODIFIER LETTER REVERSED COMMA instead of U+2018
645 //! LEFT SINGLE QUOTATION MARK and 0xA2 to U+02BC MODIFIER LETTER APOSTROPHE
646 //! instead of U+2019 RIGHT SINGLE QUOTATION MARK.
647 //! 4. Windows decodes 0xAF to OVERLINE instead of MACRON and 0xFE and 0xFD to PUA instead
648 //! of LRM and RLM.
649 //! 5. Remarks from the previous item apply.
650 //!
651 //! The differences between this crate and Windows in the case of multibyte encodings
652 //! are not yet fully documented here. The lack of remarks above should not be taken
653 //! as indication of lack of differences.
654 //!
655 //! # Notable Differences from IANA Naming
656 //!
657 //! In some cases, the Encoding Standard specifies the popular unextended encoding
658 //! name where in IANA terms one of the other labels would be more precise considering
659 //! the extensions that the Encoding Standard has unified into the encoding.
660 //!
661 //! <table>
662 //! <thead>
663 //! <tr><th>Encoding</th><th>IANA</th></tr>
664 //! </thead>
665 //! <tbody>
666 //! <tr><td>Big5</td><td>Big5-HKSCS</td></tr>
667 //! <tr><td>EUC-KR</td><td>windows-949</td></tr>
668 //! <tr><td>Shift_JIS</td><td>windows-31j</td></tr>
669 //! <tr><td>x-mac-cyrillic</td><td>x-mac-ukrainian</td></tr>
670 //! </tbody>
671 //! </table>
672 //!
673 //! In other cases where the Encoding Standard unifies unextended and extended
674 //! variants of an encoding, the encoding gets the name of the extended
675 //! variant.
676 //!
677 //! <table>
678 //! <thead>
679 //! <tr><th>IANA</th><th>Unified into Encoding</th></tr>
680 //! </thead>
681 //! <tbody>
682 //! <tr><td>ISO-8859-1</td><td>windows-1252</td></tr>
683 //! <tr><td>ISO-8859-9</td><td>windows-1254</td></tr>
684 //! <tr><td>TIS-620</td><td>windows-874</td></tr>
685 //! </tbody>
686 //! </table>
687 //!
688 //! See the section [_UTF-16LE, UTF-16BE and Unicode Encoding Schemes_](#utf-16le-utf-16be-and-unicode-encoding-schemes)
689 //! for discussion about the UTF-16 family.
690
691 #![no_std]
692 #![cfg_attr(feature = "simd-accel", feature(core_intrinsics, portable_simd))]
693
694 #[cfg(feature = "alloc")]
695 #[cfg_attr(test, macro_use)]
696 extern crate alloc;
697
698 extern crate core;
699 #[macro_use]
700 extern crate cfg_if;
701
702 #[cfg(feature = "serde")]
703 extern crate serde;
704
705 #[cfg(all(test, feature = "serde"))]
706 extern crate bincode;
707 #[cfg(all(test, feature = "serde"))]
708 #[macro_use]
709 extern crate serde_derive;
710 #[cfg(all(test, feature = "serde"))]
711 extern crate serde_json;
712
713 #[macro_use]
714 mod macros;
715
716 #[cfg(all(
717 feature = "simd-accel",
718 any(
719 target_feature = "sse2",
720 all(target_endian = "little", target_arch = "aarch64"),
721 all(target_endian = "little", target_feature = "neon")
722 )
723 ))]
724 mod simd_funcs;
725
726 #[cfg(all(test, feature = "alloc"))]
727 mod testing;
728
729 mod big5;
730 mod euc_jp;
731 mod euc_kr;
732 mod gb18030;
733 mod iso_2022_jp;
734 mod replacement;
735 mod shift_jis;
736 mod single_byte;
737 mod utf_16;
738 mod utf_8;
739 mod x_user_defined;
740
741 mod ascii;
742 mod data;
743 mod handles;
744 mod variant;
745
746 pub mod mem;
747
748 use crate::ascii::ascii_valid_up_to;
749 use crate::ascii::iso_2022_jp_ascii_valid_up_to;
750 use crate::utf_8::utf8_valid_up_to;
751 use crate::variant::*;
752
753 #[cfg(feature = "alloc")]
754 use alloc::borrow::Cow;
755 #[cfg(feature = "alloc")]
756 use alloc::string::String;
757 #[cfg(feature = "alloc")]
758 use alloc::vec::Vec;
759 use core::cmp::Ordering;
760 use core::hash::Hash;
761 use core::hash::Hasher;
762
763 #[cfg(feature = "serde")]
764 use serde::de::Visitor;
765 #[cfg(feature = "serde")]
766 use serde::{Deserialize, Deserializer, Serialize, Serializer};
767
768 /// This has to be the max length of an NCR instead of max
769 /// minus one, because we can't rely on getting the minus
770 /// one from the space reserved for the current unmappable,
771 /// because the ISO-2022-JP encoder can fill up that space
772 /// with a state transition escape.
773 const NCR_EXTRA: usize = 10; // &#1114111;
774
775 // BEGIN GENERATED CODE. PLEASE DO NOT EDIT.
776 // Instead, please regenerate using generate-encoding-data.py
777
778 const LONGEST_LABEL_LENGTH: usize = 19; // cseucpkdfmtjapanese
779
780 /// The initializer for the [Big5](static.BIG5.html) encoding.
781 ///
782 /// For use only for taking the address of this form when
783 /// Rust prohibits the use of the non-`_INIT` form directly,
784 /// such as in initializers of other `static`s. If in doubt,
785 /// use the corresponding non-`_INIT` reference-typed `static`.
786 ///
787 /// This part of the public API will go away if Rust changes
788 /// to make the referent of `pub const FOO: &'static Encoding`
789 /// unique cross-crate or if Rust starts allowing static arrays
790 /// to be initialized with `pub static FOO: &'static Encoding`
791 /// items.
792 pub static BIG5_INIT: Encoding = Encoding {
793 name: "Big5",
794 variant: VariantEncoding::Big5,
795 };
796
797 /// The Big5 encoding.
798 ///
799 /// This is Big5 with HKSCS with mappings to more recent Unicode assignments
800 /// instead of the Private Use Area code points that have been used historically.
801 /// It is believed to be able to decode existing Web content in a way that makes
802 /// sense.
803 ///
804 /// To avoid form submissions generating data that Web servers don't understand,
805 /// the encoder doesn't use the HKSCS byte sequences that precede the unextended
806 /// Big5 in the lexical order.
807 ///
808 /// [Index visualization](https://encoding.spec.whatwg.org/big5.html),
809 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/big5-bmp.html)
810 ///
811 /// This encoding is designed to be suited for decoding the Windows code page 950
812 /// and its HKSCS patched "951" variant such that the text makes sense, given
813 /// assignments that Unicode has made after those encodings used Private Use
814 /// Area characters.
815 ///
816 /// This will change from `static` to `const` if Rust changes
817 /// to make the referent of `pub const FOO: &'static Encoding`
818 /// unique cross-crate, so don't take the address of this
819 /// `static`.
820 pub static BIG5: &'static Encoding = &BIG5_INIT;
821
822 /// The initializer for the [EUC-JP](static.EUC_JP.html) encoding.
823 ///
824 /// For use only for taking the address of this form when
825 /// Rust prohibits the use of the non-`_INIT` form directly,
826 /// such as in initializers of other `static`s. If in doubt,
827 /// use the corresponding non-`_INIT` reference-typed `static`.
828 ///
829 /// This part of the public API will go away if Rust changes
830 /// to make the referent of `pub const FOO: &'static Encoding`
831 /// unique cross-crate or if Rust starts allowing static arrays
832 /// to be initialized with `pub static FOO: &'static Encoding`
833 /// items.
834 pub static EUC_JP_INIT: Encoding = Encoding {
835 name: "EUC-JP",
836 variant: VariantEncoding::EucJp,
837 };
838
839 /// The EUC-JP encoding.
840 ///
841 /// This is the legacy Unix encoding for Japanese.
842 ///
843 /// For compatibility with Web servers that don't expect three-byte sequences
844 /// in form submissions, the encoder doesn't generate three-byte sequences.
845 /// That is, the JIS X 0212 support is decode-only.
846 ///
847 /// [Index visualization](https://encoding.spec.whatwg.org/euc-jp.html),
848 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/euc-jp-bmp.html)
849 ///
850 /// This encoding roughly matches the Windows code page 20932. There are error
851 /// handling differences and a handful of 2-byte sequences that decode differently.
852 /// Additionall, Windows doesn't support 3-byte sequences.
853 ///
854 /// This will change from `static` to `const` if Rust changes
855 /// to make the referent of `pub const FOO: &'static Encoding`
856 /// unique cross-crate, so don't take the address of this
857 /// `static`.
858 pub static EUC_JP: &'static Encoding = &EUC_JP_INIT;
859
860 /// The initializer for the [EUC-KR](static.EUC_KR.html) encoding.
861 ///
862 /// For use only for taking the address of this form when
863 /// Rust prohibits the use of the non-`_INIT` form directly,
864 /// such as in initializers of other `static`s. If in doubt,
865 /// use the corresponding non-`_INIT` reference-typed `static`.
866 ///
867 /// This part of the public API will go away if Rust changes
868 /// to make the referent of `pub const FOO: &'static Encoding`
869 /// unique cross-crate or if Rust starts allowing static arrays
870 /// to be initialized with `pub static FOO: &'static Encoding`
871 /// items.
872 pub static EUC_KR_INIT: Encoding = Encoding {
873 name: "EUC-KR",
874 variant: VariantEncoding::EucKr,
875 };
876
877 /// The EUC-KR encoding.
878 ///
879 /// This is the Korean encoding for Windows. It extends the Unix legacy encoding
880 /// for Korean, based on KS X 1001 (which also formed the base of MacKorean on Mac OS
881 /// Classic), with all the characters from the Hangul Syllables block of Unicode.
882 ///
883 /// [Index visualization](https://encoding.spec.whatwg.org/euc-kr.html),
884 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/euc-kr-bmp.html)
885 ///
886 /// This encoding matches the Windows code page 949, except Windows decodes byte 0x80
887 /// to U+0080 and some byte sequences that are error per the Encoding Standard to
888 /// the question mark or the Private Use Area.
889 ///
890 /// This will change from `static` to `const` if Rust changes
891 /// to make the referent of `pub const FOO: &'static Encoding`
892 /// unique cross-crate, so don't take the address of this
893 /// `static`.
894 pub static EUC_KR: &'static Encoding = &EUC_KR_INIT;
895
896 /// The initializer for the [GBK](static.GBK.html) encoding.
897 ///
898 /// For use only for taking the address of this form when
899 /// Rust prohibits the use of the non-`_INIT` form directly,
900 /// such as in initializers of other `static`s. If in doubt,
901 /// use the corresponding non-`_INIT` reference-typed `static`.
902 ///
903 /// This part of the public API will go away if Rust changes
904 /// to make the referent of `pub const FOO: &'static Encoding`
905 /// unique cross-crate or if Rust starts allowing static arrays
906 /// to be initialized with `pub static FOO: &'static Encoding`
907 /// items.
908 pub static GBK_INIT: Encoding = Encoding {
909 name: "GBK",
910 variant: VariantEncoding::Gbk,
911 };
912
913 /// The GBK encoding.
914 ///
915 /// The decoder for this encoding is the same as the decoder for gb18030.
916 /// The encoder side of this encoding is GBK with Windows code page 936 euro
917 /// sign behavior. GBK extends GB2312-80 to cover the CJK Unified Ideographs
918 /// Unicode block as well as a handful of ideographs from the CJK Unified
919 /// Ideographs Extension A and CJK Compatibility Ideographs blocks.
920 ///
921 /// Unlike e.g. in the case of ISO-8859-1 and windows-1252, GBK encoder wasn't
922 /// unified with the gb18030 encoder in the Encoding Standard out of concern
923 /// that servers that expect GBK form submissions might not be able to handle
924 /// the four-byte sequences.
925 ///
926 /// [Index visualization for the two-byte sequences](https://encoding.spec.whatwg.org/gb18030.html),
927 /// [Visualization of BMP coverage of the two-byte index](https://encoding.spec.whatwg.org/gb18030-bmp.html)
928 ///
929 /// The encoder of this encoding roughly matches the Windows code page 936.
930 /// The decoder side is a superset.
931 ///
932 /// This will change from `static` to `const` if Rust changes
933 /// to make the referent of `pub const FOO: &'static Encoding`
934 /// unique cross-crate, so don't take the address of this
935 /// `static`.
936 pub static GBK: &'static Encoding = &GBK_INIT;
937
938 /// The initializer for the [IBM866](static.IBM866.html) encoding.
939 ///
940 /// For use only for taking the address of this form when
941 /// Rust prohibits the use of the non-`_INIT` form directly,
942 /// such as in initializers of other `static`s. If in doubt,
943 /// use the corresponding non-`_INIT` reference-typed `static`.
944 ///
945 /// This part of the public API will go away if Rust changes
946 /// to make the referent of `pub const FOO: &'static Encoding`
947 /// unique cross-crate or if Rust starts allowing static arrays
948 /// to be initialized with `pub static FOO: &'static Encoding`
949 /// items.
950 pub static IBM866_INIT: Encoding = Encoding {
951 name: "IBM866",
952 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.ibm866, 0x0440, 96, 16),
953 };
954
955 /// The IBM866 encoding.
956 ///
957 /// This the most notable one of the DOS Cyrillic code pages. It has the same
958 /// box drawing characters as code page 437, so it can be used for decoding
959 /// DOS-era ASCII + box drawing data.
960 ///
961 /// [Index visualization](https://encoding.spec.whatwg.org/ibm866.html),
962 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/ibm866-bmp.html)
963 ///
964 /// This encoding matches the Windows code page 866.
965 ///
966 /// This will change from `static` to `const` if Rust changes
967 /// to make the referent of `pub const FOO: &'static Encoding`
968 /// unique cross-crate, so don't take the address of this
969 /// `static`.
970 pub static IBM866: &'static Encoding = &IBM866_INIT;
971
972 /// The initializer for the [ISO-2022-JP](static.ISO_2022_JP.html) encoding.
973 ///
974 /// For use only for taking the address of this form when
975 /// Rust prohibits the use of the non-`_INIT` form directly,
976 /// such as in initializers of other `static`s. If in doubt,
977 /// use the corresponding non-`_INIT` reference-typed `static`.
978 ///
979 /// This part of the public API will go away if Rust changes
980 /// to make the referent of `pub const FOO: &'static Encoding`
981 /// unique cross-crate or if Rust starts allowing static arrays
982 /// to be initialized with `pub static FOO: &'static Encoding`
983 /// items.
984 pub static ISO_2022_JP_INIT: Encoding = Encoding {
985 name: "ISO-2022-JP",
986 variant: VariantEncoding::Iso2022Jp,
987 };
988
989 /// The ISO-2022-JP encoding.
990 ///
991 /// This the primary pre-UTF-8 encoding for Japanese email. It uses the ASCII
992 /// byte range to encode non-Basic Latin characters. It's the only encoding
993 /// supported by this crate whose encoder is stateful.
994 ///
995 /// [Index visualization](https://encoding.spec.whatwg.org/jis0208.html),
996 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/jis0208-bmp.html)
997 ///
998 /// This encoding roughly matches the Windows code page 50220. Notably, Windows
999 /// uses U+30FB in place of the REPLACEMENT CHARACTER and otherwise differs in
1000 /// error handling.
1001 ///
1002 /// This will change from `static` to `const` if Rust changes
1003 /// to make the referent of `pub const FOO: &'static Encoding`
1004 /// unique cross-crate, so don't take the address of this
1005 /// `static`.
1006 pub static ISO_2022_JP: &'static Encoding = &ISO_2022_JP_INIT;
1007
1008 /// The initializer for the [ISO-8859-10](static.ISO_8859_10.html) encoding.
1009 ///
1010 /// For use only for taking the address of this form when
1011 /// Rust prohibits the use of the non-`_INIT` form directly,
1012 /// such as in initializers of other `static`s. If in doubt,
1013 /// use the corresponding non-`_INIT` reference-typed `static`.
1014 ///
1015 /// This part of the public API will go away if Rust changes
1016 /// to make the referent of `pub const FOO: &'static Encoding`
1017 /// unique cross-crate or if Rust starts allowing static arrays
1018 /// to be initialized with `pub static FOO: &'static Encoding`
1019 /// items.
1020 pub static ISO_8859_10_INIT: Encoding = Encoding {
1021 name: "ISO-8859-10",
1022 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_10, 0x00DA, 90, 6),
1023 };
1024
1025 /// The ISO-8859-10 encoding.
1026 ///
1027 /// This is the Nordic part of the ISO/IEC 8859 encoding family. This encoding
1028 /// is also known as Latin 6.
1029 ///
1030 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-10.html),
1031 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-10-bmp.html)
1032 ///
1033 /// The Windows code page number for this encoding is 28600, but kernel32.dll
1034 /// does not support this encoding.
1035 ///
1036 /// This will change from `static` to `const` if Rust changes
1037 /// to make the referent of `pub const FOO: &'static Encoding`
1038 /// unique cross-crate, so don't take the address of this
1039 /// `static`.
1040 pub static ISO_8859_10: &'static Encoding = &ISO_8859_10_INIT;
1041
1042 /// The initializer for the [ISO-8859-13](static.ISO_8859_13.html) encoding.
1043 ///
1044 /// For use only for taking the address of this form when
1045 /// Rust prohibits the use of the non-`_INIT` form directly,
1046 /// such as in initializers of other `static`s. If in doubt,
1047 /// use the corresponding non-`_INIT` reference-typed `static`.
1048 ///
1049 /// This part of the public API will go away if Rust changes
1050 /// to make the referent of `pub const FOO: &'static Encoding`
1051 /// unique cross-crate or if Rust starts allowing static arrays
1052 /// to be initialized with `pub static FOO: &'static Encoding`
1053 /// items.
1054 pub static ISO_8859_13_INIT: Encoding = Encoding {
1055 name: "ISO-8859-13",
1056 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_13, 0x00DF, 95, 1),
1057 };
1058
1059 /// The ISO-8859-13 encoding.
1060 ///
1061 /// This is the Baltic part of the ISO/IEC 8859 encoding family. This encoding
1062 /// is also known as Latin 7.
1063 ///
1064 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-13.html),
1065 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-13-bmp.html)
1066 ///
1067 /// This encoding matches the Windows code page 28603, except Windows decodes
1068 /// unassigned code points to the Private Use Area of Unicode.
1069 ///
1070 /// This will change from `static` to `const` if Rust changes
1071 /// to make the referent of `pub const FOO: &'static Encoding`
1072 /// unique cross-crate, so don't take the address of this
1073 /// `static`.
1074 pub static ISO_8859_13: &'static Encoding = &ISO_8859_13_INIT;
1075
1076 /// The initializer for the [ISO-8859-14](static.ISO_8859_14.html) encoding.
1077 ///
1078 /// For use only for taking the address of this form when
1079 /// Rust prohibits the use of the non-`_INIT` form directly,
1080 /// such as in initializers of other `static`s. If in doubt,
1081 /// use the corresponding non-`_INIT` reference-typed `static`.
1082 ///
1083 /// This part of the public API will go away if Rust changes
1084 /// to make the referent of `pub const FOO: &'static Encoding`
1085 /// unique cross-crate or if Rust starts allowing static arrays
1086 /// to be initialized with `pub static FOO: &'static Encoding`
1087 /// items.
1088 pub static ISO_8859_14_INIT: Encoding = Encoding {
1089 name: "ISO-8859-14",
1090 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_14, 0x00DF, 95, 17),
1091 };
1092
1093 /// The ISO-8859-14 encoding.
1094 ///
1095 /// This is the Celtic part of the ISO/IEC 8859 encoding family. This encoding
1096 /// is also known as Latin 8.
1097 ///
1098 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-14.html),
1099 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-14-bmp.html)
1100 ///
1101 /// The Windows code page number for this encoding is 28604, but kernel32.dll
1102 /// does not support this encoding.
1103 ///
1104 /// This will change from `static` to `const` if Rust changes
1105 /// to make the referent of `pub const FOO: &'static Encoding`
1106 /// unique cross-crate, so don't take the address of this
1107 /// `static`.
1108 pub static ISO_8859_14: &'static Encoding = &ISO_8859_14_INIT;
1109
1110 /// The initializer for the [ISO-8859-15](static.ISO_8859_15.html) encoding.
1111 ///
1112 /// For use only for taking the address of this form when
1113 /// Rust prohibits the use of the non-`_INIT` form directly,
1114 /// such as in initializers of other `static`s. If in doubt,
1115 /// use the corresponding non-`_INIT` reference-typed `static`.
1116 ///
1117 /// This part of the public API will go away if Rust changes
1118 /// to make the referent of `pub const FOO: &'static Encoding`
1119 /// unique cross-crate or if Rust starts allowing static arrays
1120 /// to be initialized with `pub static FOO: &'static Encoding`
1121 /// items.
1122 pub static ISO_8859_15_INIT: Encoding = Encoding {
1123 name: "ISO-8859-15",
1124 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_15, 0x00BF, 63, 65),
1125 };
1126
1127 /// The ISO-8859-15 encoding.
1128 ///
1129 /// This is the revised Western European part of the ISO/IEC 8859 encoding
1130 /// family. This encoding is also known as Latin 9.
1131 ///
1132 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-15.html),
1133 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-15-bmp.html)
1134 ///
1135 /// This encoding matches the Windows code page 28605.
1136 ///
1137 /// This will change from `static` to `const` if Rust changes
1138 /// to make the referent of `pub const FOO: &'static Encoding`
1139 /// unique cross-crate, so don't take the address of this
1140 /// `static`.
1141 pub static ISO_8859_15: &'static Encoding = &ISO_8859_15_INIT;
1142
1143 /// The initializer for the [ISO-8859-16](static.ISO_8859_16.html) encoding.
1144 ///
1145 /// For use only for taking the address of this form when
1146 /// Rust prohibits the use of the non-`_INIT` form directly,
1147 /// such as in initializers of other `static`s. If in doubt,
1148 /// use the corresponding non-`_INIT` reference-typed `static`.
1149 ///
1150 /// This part of the public API will go away if Rust changes
1151 /// to make the referent of `pub const FOO: &'static Encoding`
1152 /// unique cross-crate or if Rust starts allowing static arrays
1153 /// to be initialized with `pub static FOO: &'static Encoding`
1154 /// items.
1155 pub static ISO_8859_16_INIT: Encoding = Encoding {
1156 name: "ISO-8859-16",
1157 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_16, 0x00DF, 95, 4),
1158 };
1159
1160 /// The ISO-8859-16 encoding.
1161 ///
1162 /// This is the South-Eastern European part of the ISO/IEC 8859 encoding
1163 /// family. This encoding is also known as Latin 10.
1164 ///
1165 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-16.html),
1166 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-16-bmp.html)
1167 ///
1168 /// The Windows code page number for this encoding is 28606, but kernel32.dll
1169 /// does not support this encoding.
1170 ///
1171 /// This will change from `static` to `const` if Rust changes
1172 /// to make the referent of `pub const FOO: &'static Encoding`
1173 /// unique cross-crate, so don't take the address of this
1174 /// `static`.
1175 pub static ISO_8859_16: &'static Encoding = &ISO_8859_16_INIT;
1176
1177 /// The initializer for the [ISO-8859-2](static.ISO_8859_2.html) encoding.
1178 ///
1179 /// For use only for taking the address of this form when
1180 /// Rust prohibits the use of the non-`_INIT` form directly,
1181 /// such as in initializers of other `static`s. If in doubt,
1182 /// use the corresponding non-`_INIT` reference-typed `static`.
1183 ///
1184 /// This part of the public API will go away if Rust changes
1185 /// to make the referent of `pub const FOO: &'static Encoding`
1186 /// unique cross-crate or if Rust starts allowing static arrays
1187 /// to be initialized with `pub static FOO: &'static Encoding`
1188 /// items.
1189 pub static ISO_8859_2_INIT: Encoding = Encoding {
1190 name: "ISO-8859-2",
1191 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_2, 0x00DF, 95, 1),
1192 };
1193
1194 /// The ISO-8859-2 encoding.
1195 ///
1196 /// This is the Central European part of the ISO/IEC 8859 encoding family. This encoding is also known as Latin 2.
1197 ///
1198 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-2.html),
1199 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-2-bmp.html)
1200 ///
1201 /// This encoding matches the Windows code page 28592.
1202 ///
1203 /// This will change from `static` to `const` if Rust changes
1204 /// to make the referent of `pub const FOO: &'static Encoding`
1205 /// unique cross-crate, so don't take the address of this
1206 /// `static`.
1207 pub static ISO_8859_2: &'static Encoding = &ISO_8859_2_INIT;
1208
1209 /// The initializer for the [ISO-8859-3](static.ISO_8859_3.html) encoding.
1210 ///
1211 /// For use only for taking the address of this form when
1212 /// Rust prohibits the use of the non-`_INIT` form directly,
1213 /// such as in initializers of other `static`s. If in doubt,
1214 /// use the corresponding non-`_INIT` reference-typed `static`.
1215 ///
1216 /// This part of the public API will go away if Rust changes
1217 /// to make the referent of `pub const FOO: &'static Encoding`
1218 /// unique cross-crate or if Rust starts allowing static arrays
1219 /// to be initialized with `pub static FOO: &'static Encoding`
1220 /// items.
1221 pub static ISO_8859_3_INIT: Encoding = Encoding {
1222 name: "ISO-8859-3",
1223 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_3, 0x00DF, 95, 4),
1224 };
1225
1226 /// The ISO-8859-3 encoding.
1227 ///
1228 /// This is the South European part of the ISO/IEC 8859 encoding family. This encoding is also known as Latin 3.
1229 ///
1230 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-3.html),
1231 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-3-bmp.html)
1232 ///
1233 /// This encoding matches the Windows code page 28593.
1234 ///
1235 /// This will change from `static` to `const` if Rust changes
1236 /// to make the referent of `pub const FOO: &'static Encoding`
1237 /// unique cross-crate, so don't take the address of this
1238 /// `static`.
1239 pub static ISO_8859_3: &'static Encoding = &ISO_8859_3_INIT;
1240
1241 /// The initializer for the [ISO-8859-4](static.ISO_8859_4.html) encoding.
1242 ///
1243 /// For use only for taking the address of this form when
1244 /// Rust prohibits the use of the non-`_INIT` form directly,
1245 /// such as in initializers of other `static`s. If in doubt,
1246 /// use the corresponding non-`_INIT` reference-typed `static`.
1247 ///
1248 /// This part of the public API will go away if Rust changes
1249 /// to make the referent of `pub const FOO: &'static Encoding`
1250 /// unique cross-crate or if Rust starts allowing static arrays
1251 /// to be initialized with `pub static FOO: &'static Encoding`
1252 /// items.
1253 pub static ISO_8859_4_INIT: Encoding = Encoding {
1254 name: "ISO-8859-4",
1255 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_4, 0x00DF, 95, 1),
1256 };
1257
1258 /// The ISO-8859-4 encoding.
1259 ///
1260 /// This is the North European part of the ISO/IEC 8859 encoding family. This encoding is also known as Latin 4.
1261 ///
1262 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-4.html),
1263 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-4-bmp.html)
1264 ///
1265 /// This encoding matches the Windows code page 28594.
1266 ///
1267 /// This will change from `static` to `const` if Rust changes
1268 /// to make the referent of `pub const FOO: &'static Encoding`
1269 /// unique cross-crate, so don't take the address of this
1270 /// `static`.
1271 pub static ISO_8859_4: &'static Encoding = &ISO_8859_4_INIT;
1272
1273 /// The initializer for the [ISO-8859-5](static.ISO_8859_5.html) encoding.
1274 ///
1275 /// For use only for taking the address of this form when
1276 /// Rust prohibits the use of the non-`_INIT` form directly,
1277 /// such as in initializers of other `static`s. If in doubt,
1278 /// use the corresponding non-`_INIT` reference-typed `static`.
1279 ///
1280 /// This part of the public API will go away if Rust changes
1281 /// to make the referent of `pub const FOO: &'static Encoding`
1282 /// unique cross-crate or if Rust starts allowing static arrays
1283 /// to be initialized with `pub static FOO: &'static Encoding`
1284 /// items.
1285 pub static ISO_8859_5_INIT: Encoding = Encoding {
1286 name: "ISO-8859-5",
1287 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_5, 0x040E, 46, 66),
1288 };
1289
1290 /// The ISO-8859-5 encoding.
1291 ///
1292 /// This is the Cyrillic part of the ISO/IEC 8859 encoding family.
1293 ///
1294 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-5.html),
1295 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-5-bmp.html)
1296 ///
1297 /// This encoding matches the Windows code page 28595.
1298 ///
1299 /// This will change from `static` to `const` if Rust changes
1300 /// to make the referent of `pub const FOO: &'static Encoding`
1301 /// unique cross-crate, so don't take the address of this
1302 /// `static`.
1303 pub static ISO_8859_5: &'static Encoding = &ISO_8859_5_INIT;
1304
1305 /// The initializer for the [ISO-8859-6](static.ISO_8859_6.html) encoding.
1306 ///
1307 /// For use only for taking the address of this form when
1308 /// Rust prohibits the use of the non-`_INIT` form directly,
1309 /// such as in initializers of other `static`s. If in doubt,
1310 /// use the corresponding non-`_INIT` reference-typed `static`.
1311 ///
1312 /// This part of the public API will go away if Rust changes
1313 /// to make the referent of `pub const FOO: &'static Encoding`
1314 /// unique cross-crate or if Rust starts allowing static arrays
1315 /// to be initialized with `pub static FOO: &'static Encoding`
1316 /// items.
1317 pub static ISO_8859_6_INIT: Encoding = Encoding {
1318 name: "ISO-8859-6",
1319 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_6, 0x0621, 65, 26),
1320 };
1321
1322 /// The ISO-8859-6 encoding.
1323 ///
1324 /// This is the Arabic part of the ISO/IEC 8859 encoding family.
1325 ///
1326 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-6.html),
1327 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-6-bmp.html)
1328 ///
1329 /// This encoding matches the Windows code page 28596, except Windows decodes
1330 /// unassigned code points to the Private Use Area of Unicode.
1331 ///
1332 /// This will change from `static` to `const` if Rust changes
1333 /// to make the referent of `pub const FOO: &'static Encoding`
1334 /// unique cross-crate, so don't take the address of this
1335 /// `static`.
1336 pub static ISO_8859_6: &'static Encoding = &ISO_8859_6_INIT;
1337
1338 /// The initializer for the [ISO-8859-7](static.ISO_8859_7.html) encoding.
1339 ///
1340 /// For use only for taking the address of this form when
1341 /// Rust prohibits the use of the non-`_INIT` form directly,
1342 /// such as in initializers of other `static`s. If in doubt,
1343 /// use the corresponding non-`_INIT` reference-typed `static`.
1344 ///
1345 /// This part of the public API will go away if Rust changes
1346 /// to make the referent of `pub const FOO: &'static Encoding`
1347 /// unique cross-crate or if Rust starts allowing static arrays
1348 /// to be initialized with `pub static FOO: &'static Encoding`
1349 /// items.
1350 pub static ISO_8859_7_INIT: Encoding = Encoding {
1351 name: "ISO-8859-7",
1352 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_7, 0x03A3, 83, 44),
1353 };
1354
1355 /// The ISO-8859-7 encoding.
1356 ///
1357 /// This is the Greek part of the ISO/IEC 8859 encoding family.
1358 ///
1359 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-7.html),
1360 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-7-bmp.html)
1361 ///
1362 /// This encoding roughly matches the Windows code page 28597. Windows decodes
1363 /// unassigned code points, the currency signs at 0xA4 and 0xA5 as well as
1364 /// 0xAA, which should be U+037A GREEK YPOGEGRAMMENI, to the Private Use Area
1365 /// of Unicode. Windows decodes 0xA1 to U+02BD MODIFIER LETTER REVERSED COMMA
1366 /// instead of U+2018 LEFT SINGLE QUOTATION MARK and 0xA2 to U+02BC MODIFIER
1367 /// LETTER APOSTROPHE instead of U+2019 RIGHT SINGLE QUOTATION MARK.
1368 ///
1369 /// This will change from `static` to `const` if Rust changes
1370 /// to make the referent of `pub const FOO: &'static Encoding`
1371 /// unique cross-crate, so don't take the address of this
1372 /// `static`.
1373 pub static ISO_8859_7: &'static Encoding = &ISO_8859_7_INIT;
1374
1375 /// The initializer for the [ISO-8859-8](static.ISO_8859_8.html) encoding.
1376 ///
1377 /// For use only for taking the address of this form when
1378 /// Rust prohibits the use of the non-`_INIT` form directly,
1379 /// such as in initializers of other `static`s. If in doubt,
1380 /// use the corresponding non-`_INIT` reference-typed `static`.
1381 ///
1382 /// This part of the public API will go away if Rust changes
1383 /// to make the referent of `pub const FOO: &'static Encoding`
1384 /// unique cross-crate or if Rust starts allowing static arrays
1385 /// to be initialized with `pub static FOO: &'static Encoding`
1386 /// items.
1387 pub static ISO_8859_8_INIT: Encoding = Encoding {
1388 name: "ISO-8859-8",
1389 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_8, 0x05D0, 96, 27),
1390 };
1391
1392 /// The ISO-8859-8 encoding.
1393 ///
1394 /// This is the Hebrew part of the ISO/IEC 8859 encoding family in visual order.
1395 ///
1396 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-8.html),
1397 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-8-bmp.html)
1398 ///
1399 /// This encoding roughly matches the Windows code page 28598. Windows decodes
1400 /// 0xAF to OVERLINE instead of MACRON and 0xFE and 0xFD to the Private Use
1401 /// Area instead of LRM and RLM. Windows decodes unassigned code points to
1402 /// the private use area.
1403 ///
1404 /// This will change from `static` to `const` if Rust changes
1405 /// to make the referent of `pub const FOO: &'static Encoding`
1406 /// unique cross-crate, so don't take the address of this
1407 /// `static`.
1408 pub static ISO_8859_8: &'static Encoding = &ISO_8859_8_INIT;
1409
1410 /// The initializer for the [ISO-8859-8-I](static.ISO_8859_8_I.html) encoding.
1411 ///
1412 /// For use only for taking the address of this form when
1413 /// Rust prohibits the use of the non-`_INIT` form directly,
1414 /// such as in initializers of other `static`s. If in doubt,
1415 /// use the corresponding non-`_INIT` reference-typed `static`.
1416 ///
1417 /// This part of the public API will go away if Rust changes
1418 /// to make the referent of `pub const FOO: &'static Encoding`
1419 /// unique cross-crate or if Rust starts allowing static arrays
1420 /// to be initialized with `pub static FOO: &'static Encoding`
1421 /// items.
1422 pub static ISO_8859_8_I_INIT: Encoding = Encoding {
1423 name: "ISO-8859-8-I",
1424 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.iso_8859_8, 0x05D0, 96, 27),
1425 };
1426
1427 /// The ISO-8859-8-I encoding.
1428 ///
1429 /// This is the Hebrew part of the ISO/IEC 8859 encoding family in logical order.
1430 ///
1431 /// [Index visualization](https://encoding.spec.whatwg.org/iso-8859-8.html),
1432 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/iso-8859-8-bmp.html)
1433 ///
1434 /// This encoding roughly matches the Windows code page 38598. Windows decodes
1435 /// 0xAF to OVERLINE instead of MACRON and 0xFE and 0xFD to the Private Use
1436 /// Area instead of LRM and RLM. Windows decodes unassigned code points to
1437 /// the private use area.
1438 ///
1439 /// This will change from `static` to `const` if Rust changes
1440 /// to make the referent of `pub const FOO: &'static Encoding`
1441 /// unique cross-crate, so don't take the address of this
1442 /// `static`.
1443 pub static ISO_8859_8_I: &'static Encoding = &ISO_8859_8_I_INIT;
1444
1445 /// The initializer for the [KOI8-R](static.KOI8_R.html) encoding.
1446 ///
1447 /// For use only for taking the address of this form when
1448 /// Rust prohibits the use of the non-`_INIT` form directly,
1449 /// such as in initializers of other `static`s. If in doubt,
1450 /// use the corresponding non-`_INIT` reference-typed `static`.
1451 ///
1452 /// This part of the public API will go away if Rust changes
1453 /// to make the referent of `pub const FOO: &'static Encoding`
1454 /// unique cross-crate or if Rust starts allowing static arrays
1455 /// to be initialized with `pub static FOO: &'static Encoding`
1456 /// items.
1457 pub static KOI8_R_INIT: Encoding = Encoding {
1458 name: "KOI8-R",
1459 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.koi8_r, 0x044E, 64, 1),
1460 };
1461
1462 /// The KOI8-R encoding.
1463 ///
1464 /// This is an encoding for Russian from [RFC 1489](https://tools.ietf.org/html/rfc1489).
1465 ///
1466 /// [Index visualization](https://encoding.spec.whatwg.org/koi8-r.html),
1467 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/koi8-r-bmp.html)
1468 ///
1469 /// This encoding matches the Windows code page 20866.
1470 ///
1471 /// This will change from `static` to `const` if Rust changes
1472 /// to make the referent of `pub const FOO: &'static Encoding`
1473 /// unique cross-crate, so don't take the address of this
1474 /// `static`.
1475 pub static KOI8_R: &'static Encoding = &KOI8_R_INIT;
1476
1477 /// The initializer for the [KOI8-U](static.KOI8_U.html) encoding.
1478 ///
1479 /// For use only for taking the address of this form when
1480 /// Rust prohibits the use of the non-`_INIT` form directly,
1481 /// such as in initializers of other `static`s. If in doubt,
1482 /// use the corresponding non-`_INIT` reference-typed `static`.
1483 ///
1484 /// This part of the public API will go away if Rust changes
1485 /// to make the referent of `pub const FOO: &'static Encoding`
1486 /// unique cross-crate or if Rust starts allowing static arrays
1487 /// to be initialized with `pub static FOO: &'static Encoding`
1488 /// items.
1489 pub static KOI8_U_INIT: Encoding = Encoding {
1490 name: "KOI8-U",
1491 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.koi8_u, 0x044E, 64, 1),
1492 };
1493
1494 /// The KOI8-U encoding.
1495 ///
1496 /// This is an encoding for Ukrainian adapted from KOI8-R.
1497 ///
1498 /// [Index visualization](https://encoding.spec.whatwg.org/koi8-u.html),
1499 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/koi8-u-bmp.html)
1500 ///
1501 /// This encoding matches the Windows code page 21866.
1502 ///
1503 /// This will change from `static` to `const` if Rust changes
1504 /// to make the referent of `pub const FOO: &'static Encoding`
1505 /// unique cross-crate, so don't take the address of this
1506 /// `static`.
1507 pub static KOI8_U: &'static Encoding = &KOI8_U_INIT;
1508
1509 /// The initializer for the [Shift_JIS](static.SHIFT_JIS.html) encoding.
1510 ///
1511 /// For use only for taking the address of this form when
1512 /// Rust prohibits the use of the non-`_INIT` form directly,
1513 /// such as in initializers of other `static`s. If in doubt,
1514 /// use the corresponding non-`_INIT` reference-typed `static`.
1515 ///
1516 /// This part of the public API will go away if Rust changes
1517 /// to make the referent of `pub const FOO: &'static Encoding`
1518 /// unique cross-crate or if Rust starts allowing static arrays
1519 /// to be initialized with `pub static FOO: &'static Encoding`
1520 /// items.
1521 pub static SHIFT_JIS_INIT: Encoding = Encoding {
1522 name: "Shift_JIS",
1523 variant: VariantEncoding::ShiftJis,
1524 };
1525
1526 /// The Shift_JIS encoding.
1527 ///
1528 /// This is the Japanese encoding for Windows.
1529 ///
1530 /// [Index visualization](https://encoding.spec.whatwg.org/shift_jis.html),
1531 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/shift_jis-bmp.html)
1532 ///
1533 /// This encoding matches the Windows code page 932, except Windows decodes some byte
1534 /// sequences that are error per the Encoding Standard to the question mark or the
1535 /// Private Use Area and generally uses U+30FB in place of the REPLACEMENT CHARACTER.
1536 ///
1537 /// This will change from `static` to `const` if Rust changes
1538 /// to make the referent of `pub const FOO: &'static Encoding`
1539 /// unique cross-crate, so don't take the address of this
1540 /// `static`.
1541 pub static SHIFT_JIS: &'static Encoding = &SHIFT_JIS_INIT;
1542
1543 /// The initializer for the [UTF-16BE](static.UTF_16BE.html) encoding.
1544 ///
1545 /// For use only for taking the address of this form when
1546 /// Rust prohibits the use of the non-`_INIT` form directly,
1547 /// such as in initializers of other `static`s. If in doubt,
1548 /// use the corresponding non-`_INIT` reference-typed `static`.
1549 ///
1550 /// This part of the public API will go away if Rust changes
1551 /// to make the referent of `pub const FOO: &'static Encoding`
1552 /// unique cross-crate or if Rust starts allowing static arrays
1553 /// to be initialized with `pub static FOO: &'static Encoding`
1554 /// items.
1555 pub static UTF_16BE_INIT: Encoding = Encoding {
1556 name: "UTF-16BE",
1557 variant: VariantEncoding::Utf16Be,
1558 };
1559
1560 /// The UTF-16BE encoding.
1561 ///
1562 /// This decode-only encoding uses 16-bit code units due to Unicode originally
1563 /// having been designed as a 16-bit reportoire. In the absence of a byte order
1564 /// mark the big endian byte order is assumed.
1565 ///
1566 /// There is no corresponding encoder in this crate or in the Encoding
1567 /// Standard. The output encoding of this encoding is UTF-8.
1568 ///
1569 /// This encoding matches the Windows code page 1201.
1570 ///
1571 /// This will change from `static` to `const` if Rust changes
1572 /// to make the referent of `pub const FOO: &'static Encoding`
1573 /// unique cross-crate, so don't take the address of this
1574 /// `static`.
1575 pub static UTF_16BE: &'static Encoding = &UTF_16BE_INIT;
1576
1577 /// The initializer for the [UTF-16LE](static.UTF_16LE.html) encoding.
1578 ///
1579 /// For use only for taking the address of this form when
1580 /// Rust prohibits the use of the non-`_INIT` form directly,
1581 /// such as in initializers of other `static`s. If in doubt,
1582 /// use the corresponding non-`_INIT` reference-typed `static`.
1583 ///
1584 /// This part of the public API will go away if Rust changes
1585 /// to make the referent of `pub const FOO: &'static Encoding`
1586 /// unique cross-crate or if Rust starts allowing static arrays
1587 /// to be initialized with `pub static FOO: &'static Encoding`
1588 /// items.
1589 pub static UTF_16LE_INIT: Encoding = Encoding {
1590 name: "UTF-16LE",
1591 variant: VariantEncoding::Utf16Le,
1592 };
1593
1594 /// The UTF-16LE encoding.
1595 ///
1596 /// This decode-only encoding uses 16-bit code units due to Unicode originally
1597 /// having been designed as a 16-bit reportoire. In the absence of a byte order
1598 /// mark the little endian byte order is assumed.
1599 ///
1600 /// There is no corresponding encoder in this crate or in the Encoding
1601 /// Standard. The output encoding of this encoding is UTF-8.
1602 ///
1603 /// This encoding matches the Windows code page 1200.
1604 ///
1605 /// This will change from `static` to `const` if Rust changes
1606 /// to make the referent of `pub const FOO: &'static Encoding`
1607 /// unique cross-crate, so don't take the address of this
1608 /// `static`.
1609 pub static UTF_16LE: &'static Encoding = &UTF_16LE_INIT;
1610
1611 /// The initializer for the [UTF-8](static.UTF_8.html) encoding.
1612 ///
1613 /// For use only for taking the address of this form when
1614 /// Rust prohibits the use of the non-`_INIT` form directly,
1615 /// such as in initializers of other `static`s. If in doubt,
1616 /// use the corresponding non-`_INIT` reference-typed `static`.
1617 ///
1618 /// This part of the public API will go away if Rust changes
1619 /// to make the referent of `pub const FOO: &'static Encoding`
1620 /// unique cross-crate or if Rust starts allowing static arrays
1621 /// to be initialized with `pub static FOO: &'static Encoding`
1622 /// items.
1623 pub static UTF_8_INIT: Encoding = Encoding {
1624 name: "UTF-8",
1625 variant: VariantEncoding::Utf8,
1626 };
1627
1628 /// The UTF-8 encoding.
1629 ///
1630 /// This is the encoding that should be used for all new development it can
1631 /// represent all of Unicode.
1632 ///
1633 /// This encoding matches the Windows code page 65001, except Windows differs
1634 /// in the number of errors generated for some erroneous byte sequences.
1635 ///
1636 /// This will change from `static` to `const` if Rust changes
1637 /// to make the referent of `pub const FOO: &'static Encoding`
1638 /// unique cross-crate, so don't take the address of this
1639 /// `static`.
1640 pub static UTF_8: &'static Encoding = &UTF_8_INIT;
1641
1642 /// The initializer for the [gb18030](static.GB18030.html) encoding.
1643 ///
1644 /// For use only for taking the address of this form when
1645 /// Rust prohibits the use of the non-`_INIT` form directly,
1646 /// such as in initializers of other `static`s. If in doubt,
1647 /// use the corresponding non-`_INIT` reference-typed `static`.
1648 ///
1649 /// This part of the public API will go away if Rust changes
1650 /// to make the referent of `pub const FOO: &'static Encoding`
1651 /// unique cross-crate or if Rust starts allowing static arrays
1652 /// to be initialized with `pub static FOO: &'static Encoding`
1653 /// items.
1654 pub static GB18030_INIT: Encoding = Encoding {
1655 name: "gb18030",
1656 variant: VariantEncoding::Gb18030,
1657 };
1658
1659 /// The gb18030 encoding.
1660 ///
1661 /// This encoding matches GB18030-2005 except the two-byte sequence 0xA3 0xA0
1662 /// maps to U+3000 for compatibility with existing Web content. As a result,
1663 /// this encoding can represent all of Unicode except for the private-use
1664 /// character U+E5E5.
1665 ///
1666 /// [Index visualization for the two-byte sequences](https://encoding.spec.whatwg.org/gb18030.html),
1667 /// [Visualization of BMP coverage of the two-byte index](https://encoding.spec.whatwg.org/gb18030-bmp.html)
1668 ///
1669 /// This encoding matches the Windows code page 54936.
1670 ///
1671 /// This will change from `static` to `const` if Rust changes
1672 /// to make the referent of `pub const FOO: &'static Encoding`
1673 /// unique cross-crate, so don't take the address of this
1674 /// `static`.
1675 pub static GB18030: &'static Encoding = &GB18030_INIT;
1676
1677 /// The initializer for the [macintosh](static.MACINTOSH.html) encoding.
1678 ///
1679 /// For use only for taking the address of this form when
1680 /// Rust prohibits the use of the non-`_INIT` form directly,
1681 /// such as in initializers of other `static`s. If in doubt,
1682 /// use the corresponding non-`_INIT` reference-typed `static`.
1683 ///
1684 /// This part of the public API will go away if Rust changes
1685 /// to make the referent of `pub const FOO: &'static Encoding`
1686 /// unique cross-crate or if Rust starts allowing static arrays
1687 /// to be initialized with `pub static FOO: &'static Encoding`
1688 /// items.
1689 pub static MACINTOSH_INIT: Encoding = Encoding {
1690 name: "macintosh",
1691 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.macintosh, 0x00CD, 106, 3),
1692 };
1693
1694 /// The macintosh encoding.
1695 ///
1696 /// This is the MacRoman encoding from Mac OS Classic.
1697 ///
1698 /// [Index visualization](https://encoding.spec.whatwg.org/macintosh.html),
1699 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/macintosh-bmp.html)
1700 ///
1701 /// This encoding matches the Windows code page 10000, except Windows decodes
1702 /// 0xBD to U+2126 OHM SIGN instead of U+03A9 GREEK CAPITAL LETTER OMEGA.
1703 ///
1704 /// This will change from `static` to `const` if Rust changes
1705 /// to make the referent of `pub const FOO: &'static Encoding`
1706 /// unique cross-crate, so don't take the address of this
1707 /// `static`.
1708 pub static MACINTOSH: &'static Encoding = &MACINTOSH_INIT;
1709
1710 /// The initializer for the [replacement](static.REPLACEMENT.html) encoding.
1711 ///
1712 /// For use only for taking the address of this form when
1713 /// Rust prohibits the use of the non-`_INIT` form directly,
1714 /// such as in initializers of other `static`s. If in doubt,
1715 /// use the corresponding non-`_INIT` reference-typed `static`.
1716 ///
1717 /// This part of the public API will go away if Rust changes
1718 /// to make the referent of `pub const FOO: &'static Encoding`
1719 /// unique cross-crate or if Rust starts allowing static arrays
1720 /// to be initialized with `pub static FOO: &'static Encoding`
1721 /// items.
1722 pub static REPLACEMENT_INIT: Encoding = Encoding {
1723 name: "replacement",
1724 variant: VariantEncoding::Replacement,
1725 };
1726
1727 /// The replacement encoding.
1728 ///
1729 /// This decode-only encoding decodes all non-zero-length streams to a single
1730 /// REPLACEMENT CHARACTER. Its purpose is to avoid the use of an
1731 /// ASCII-compatible fallback encoding (typically windows-1252) for some
1732 /// encodings that are no longer supported by the Web Platform and that
1733 /// would be dangerous to treat as ASCII-compatible.
1734 ///
1735 /// There is no corresponding encoder. The output encoding of this encoding
1736 /// is UTF-8.
1737 ///
1738 /// This encoding does not have a Windows code page number.
1739 ///
1740 /// This will change from `static` to `const` if Rust changes
1741 /// to make the referent of `pub const FOO: &'static Encoding`
1742 /// unique cross-crate, so don't take the address of this
1743 /// `static`.
1744 pub static REPLACEMENT: &'static Encoding = &REPLACEMENT_INIT;
1745
1746 /// The initializer for the [windows-1250](static.WINDOWS_1250.html) encoding.
1747 ///
1748 /// For use only for taking the address of this form when
1749 /// Rust prohibits the use of the non-`_INIT` form directly,
1750 /// such as in initializers of other `static`s. If in doubt,
1751 /// use the corresponding non-`_INIT` reference-typed `static`.
1752 ///
1753 /// This part of the public API will go away if Rust changes
1754 /// to make the referent of `pub const FOO: &'static Encoding`
1755 /// unique cross-crate or if Rust starts allowing static arrays
1756 /// to be initialized with `pub static FOO: &'static Encoding`
1757 /// items.
1758 pub static WINDOWS_1250_INIT: Encoding = Encoding {
1759 name: "windows-1250",
1760 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1250, 0x00DC, 92, 2),
1761 };
1762
1763 /// The windows-1250 encoding.
1764 ///
1765 /// This is the Central European encoding for Windows.
1766 ///
1767 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1250.html),
1768 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1250-bmp.html)
1769 ///
1770 /// This encoding matches the Windows code page 1250.
1771 ///
1772 /// This will change from `static` to `const` if Rust changes
1773 /// to make the referent of `pub const FOO: &'static Encoding`
1774 /// unique cross-crate, so don't take the address of this
1775 /// `static`.
1776 pub static WINDOWS_1250: &'static Encoding = &WINDOWS_1250_INIT;
1777
1778 /// The initializer for the [windows-1251](static.WINDOWS_1251.html) encoding.
1779 ///
1780 /// For use only for taking the address of this form when
1781 /// Rust prohibits the use of the non-`_INIT` form directly,
1782 /// such as in initializers of other `static`s. If in doubt,
1783 /// use the corresponding non-`_INIT` reference-typed `static`.
1784 ///
1785 /// This part of the public API will go away if Rust changes
1786 /// to make the referent of `pub const FOO: &'static Encoding`
1787 /// unique cross-crate or if Rust starts allowing static arrays
1788 /// to be initialized with `pub static FOO: &'static Encoding`
1789 /// items.
1790 pub static WINDOWS_1251_INIT: Encoding = Encoding {
1791 name: "windows-1251",
1792 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1251, 0x0410, 64, 64),
1793 };
1794
1795 /// The windows-1251 encoding.
1796 ///
1797 /// This is the Cyrillic encoding for Windows.
1798 ///
1799 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1251.html),
1800 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1251-bmp.html)
1801 ///
1802 /// This encoding matches the Windows code page 1251.
1803 ///
1804 /// This will change from `static` to `const` if Rust changes
1805 /// to make the referent of `pub const FOO: &'static Encoding`
1806 /// unique cross-crate, so don't take the address of this
1807 /// `static`.
1808 pub static WINDOWS_1251: &'static Encoding = &WINDOWS_1251_INIT;
1809
1810 /// The initializer for the [windows-1252](static.WINDOWS_1252.html) encoding.
1811 ///
1812 /// For use only for taking the address of this form when
1813 /// Rust prohibits the use of the non-`_INIT` form directly,
1814 /// such as in initializers of other `static`s. If in doubt,
1815 /// use the corresponding non-`_INIT` reference-typed `static`.
1816 ///
1817 /// This part of the public API will go away if Rust changes
1818 /// to make the referent of `pub const FOO: &'static Encoding`
1819 /// unique cross-crate or if Rust starts allowing static arrays
1820 /// to be initialized with `pub static FOO: &'static Encoding`
1821 /// items.
1822 pub static WINDOWS_1252_INIT: Encoding = Encoding {
1823 name: "windows-1252",
1824 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1252, 0x00A0, 32, 96),
1825 };
1826
1827 /// The windows-1252 encoding.
1828 ///
1829 /// This is the Western encoding for Windows. It is an extension of ISO-8859-1,
1830 /// which is known as Latin 1.
1831 ///
1832 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1252.html),
1833 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1252-bmp.html)
1834 ///
1835 /// This encoding matches the Windows code page 1252.
1836 ///
1837 /// This will change from `static` to `const` if Rust changes
1838 /// to make the referent of `pub const FOO: &'static Encoding`
1839 /// unique cross-crate, so don't take the address of this
1840 /// `static`.
1841 pub static WINDOWS_1252: &'static Encoding = &WINDOWS_1252_INIT;
1842
1843 /// The initializer for the [windows-1253](static.WINDOWS_1253.html) encoding.
1844 ///
1845 /// For use only for taking the address of this form when
1846 /// Rust prohibits the use of the non-`_INIT` form directly,
1847 /// such as in initializers of other `static`s. If in doubt,
1848 /// use the corresponding non-`_INIT` reference-typed `static`.
1849 ///
1850 /// This part of the public API will go away if Rust changes
1851 /// to make the referent of `pub const FOO: &'static Encoding`
1852 /// unique cross-crate or if Rust starts allowing static arrays
1853 /// to be initialized with `pub static FOO: &'static Encoding`
1854 /// items.
1855 pub static WINDOWS_1253_INIT: Encoding = Encoding {
1856 name: "windows-1253",
1857 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1253, 0x03A3, 83, 44),
1858 };
1859
1860 /// The windows-1253 encoding.
1861 ///
1862 /// This is the Greek encoding for Windows. It is mostly an extension of
1863 /// ISO-8859-7, but U+0386 is mapped to a different byte.
1864 ///
1865 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1253.html),
1866 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1253-bmp.html)
1867 ///
1868 /// This encoding matches the Windows code page 1253, except Windows decodes
1869 /// unassigned code points to the Private Use Area of Unicode.
1870 ///
1871 /// This will change from `static` to `const` if Rust changes
1872 /// to make the referent of `pub const FOO: &'static Encoding`
1873 /// unique cross-crate, so don't take the address of this
1874 /// `static`.
1875 pub static WINDOWS_1253: &'static Encoding = &WINDOWS_1253_INIT;
1876
1877 /// The initializer for the [windows-1254](static.WINDOWS_1254.html) encoding.
1878 ///
1879 /// For use only for taking the address of this form when
1880 /// Rust prohibits the use of the non-`_INIT` form directly,
1881 /// such as in initializers of other `static`s. If in doubt,
1882 /// use the corresponding non-`_INIT` reference-typed `static`.
1883 ///
1884 /// This part of the public API will go away if Rust changes
1885 /// to make the referent of `pub const FOO: &'static Encoding`
1886 /// unique cross-crate or if Rust starts allowing static arrays
1887 /// to be initialized with `pub static FOO: &'static Encoding`
1888 /// items.
1889 pub static WINDOWS_1254_INIT: Encoding = Encoding {
1890 name: "windows-1254",
1891 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1254, 0x00DF, 95, 17),
1892 };
1893
1894 /// The windows-1254 encoding.
1895 ///
1896 /// This is the Turkish encoding for Windows. It is an extension of ISO-8859-9,
1897 /// which is known as Latin 5.
1898 ///
1899 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1254.html),
1900 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1254-bmp.html)
1901 ///
1902 /// This encoding matches the Windows code page 1254.
1903 ///
1904 /// This will change from `static` to `const` if Rust changes
1905 /// to make the referent of `pub const FOO: &'static Encoding`
1906 /// unique cross-crate, so don't take the address of this
1907 /// `static`.
1908 pub static WINDOWS_1254: &'static Encoding = &WINDOWS_1254_INIT;
1909
1910 /// The initializer for the [windows-1255](static.WINDOWS_1255.html) encoding.
1911 ///
1912 /// For use only for taking the address of this form when
1913 /// Rust prohibits the use of the non-`_INIT` form directly,
1914 /// such as in initializers of other `static`s. If in doubt,
1915 /// use the corresponding non-`_INIT` reference-typed `static`.
1916 ///
1917 /// This part of the public API will go away if Rust changes
1918 /// to make the referent of `pub const FOO: &'static Encoding`
1919 /// unique cross-crate or if Rust starts allowing static arrays
1920 /// to be initialized with `pub static FOO: &'static Encoding`
1921 /// items.
1922 pub static WINDOWS_1255_INIT: Encoding = Encoding {
1923 name: "windows-1255",
1924 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1255, 0x05D0, 96, 27),
1925 };
1926
1927 /// The windows-1255 encoding.
1928 ///
1929 /// This is the Hebrew encoding for Windows. It is an extension of ISO-8859-8-I,
1930 /// except for a currency sign swap.
1931 ///
1932 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1255.html),
1933 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1255-bmp.html)
1934 ///
1935 /// This encoding matches the Windows code page 1255, except Windows decodes
1936 /// unassigned code points to the Private Use Area of Unicode.
1937 ///
1938 /// This will change from `static` to `const` if Rust changes
1939 /// to make the referent of `pub const FOO: &'static Encoding`
1940 /// unique cross-crate, so don't take the address of this
1941 /// `static`.
1942 pub static WINDOWS_1255: &'static Encoding = &WINDOWS_1255_INIT;
1943
1944 /// The initializer for the [windows-1256](static.WINDOWS_1256.html) encoding.
1945 ///
1946 /// For use only for taking the address of this form when
1947 /// Rust prohibits the use of the non-`_INIT` form directly,
1948 /// such as in initializers of other `static`s. If in doubt,
1949 /// use the corresponding non-`_INIT` reference-typed `static`.
1950 ///
1951 /// This part of the public API will go away if Rust changes
1952 /// to make the referent of `pub const FOO: &'static Encoding`
1953 /// unique cross-crate or if Rust starts allowing static arrays
1954 /// to be initialized with `pub static FOO: &'static Encoding`
1955 /// items.
1956 pub static WINDOWS_1256_INIT: Encoding = Encoding {
1957 name: "windows-1256",
1958 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1256, 0x0621, 65, 22),
1959 };
1960
1961 /// The windows-1256 encoding.
1962 ///
1963 /// This is the Arabic encoding for Windows.
1964 ///
1965 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1256.html),
1966 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1256-bmp.html)
1967 ///
1968 /// This encoding matches the Windows code page 1256.
1969 ///
1970 /// This will change from `static` to `const` if Rust changes
1971 /// to make the referent of `pub const FOO: &'static Encoding`
1972 /// unique cross-crate, so don't take the address of this
1973 /// `static`.
1974 pub static WINDOWS_1256: &'static Encoding = &WINDOWS_1256_INIT;
1975
1976 /// The initializer for the [windows-1257](static.WINDOWS_1257.html) encoding.
1977 ///
1978 /// For use only for taking the address of this form when
1979 /// Rust prohibits the use of the non-`_INIT` form directly,
1980 /// such as in initializers of other `static`s. If in doubt,
1981 /// use the corresponding non-`_INIT` reference-typed `static`.
1982 ///
1983 /// This part of the public API will go away if Rust changes
1984 /// to make the referent of `pub const FOO: &'static Encoding`
1985 /// unique cross-crate or if Rust starts allowing static arrays
1986 /// to be initialized with `pub static FOO: &'static Encoding`
1987 /// items.
1988 pub static WINDOWS_1257_INIT: Encoding = Encoding {
1989 name: "windows-1257",
1990 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1257, 0x00DF, 95, 1),
1991 };
1992
1993 /// The windows-1257 encoding.
1994 ///
1995 /// This is the Baltic encoding for Windows.
1996 ///
1997 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1257.html),
1998 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1257-bmp.html)
1999 ///
2000 /// This encoding matches the Windows code page 1257, except Windows decodes
2001 /// unassigned code points to the Private Use Area of Unicode.
2002 ///
2003 /// This will change from `static` to `const` if Rust changes
2004 /// to make the referent of `pub const FOO: &'static Encoding`
2005 /// unique cross-crate, so don't take the address of this
2006 /// `static`.
2007 pub static WINDOWS_1257: &'static Encoding = &WINDOWS_1257_INIT;
2008
2009 /// The initializer for the [windows-1258](static.WINDOWS_1258.html) encoding.
2010 ///
2011 /// For use only for taking the address of this form when
2012 /// Rust prohibits the use of the non-`_INIT` form directly,
2013 /// such as in initializers of other `static`s. If in doubt,
2014 /// use the corresponding non-`_INIT` reference-typed `static`.
2015 ///
2016 /// This part of the public API will go away if Rust changes
2017 /// to make the referent of `pub const FOO: &'static Encoding`
2018 /// unique cross-crate or if Rust starts allowing static arrays
2019 /// to be initialized with `pub static FOO: &'static Encoding`
2020 /// items.
2021 pub static WINDOWS_1258_INIT: Encoding = Encoding {
2022 name: "windows-1258",
2023 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_1258, 0x00DF, 95, 4),
2024 };
2025
2026 /// The windows-1258 encoding.
2027 ///
2028 /// This is the Vietnamese encoding for Windows.
2029 ///
2030 /// [Index visualization](https://encoding.spec.whatwg.org/windows-1258.html),
2031 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-1258-bmp.html)
2032 ///
2033 /// This encoding matches the Windows code page 1258 when used in the
2034 /// non-normalizing mode. Unlike with the other single-byte encodings, the
2035 /// result of decoding is not necessarily in Normalization Form C. On the
2036 /// other hand, input in the Normalization Form C is not encoded without
2037 /// replacement. In general, it's a bad idea to encode to encodings other
2038 /// than UTF-8, but this encoding is especially hazardous to encode to.
2039 ///
2040 /// This will change from `static` to `const` if Rust changes
2041 /// to make the referent of `pub const FOO: &'static Encoding`
2042 /// unique cross-crate, so don't take the address of this
2043 /// `static`.
2044 pub static WINDOWS_1258: &'static Encoding = &WINDOWS_1258_INIT;
2045
2046 /// The initializer for the [windows-874](static.WINDOWS_874.html) encoding.
2047 ///
2048 /// For use only for taking the address of this form when
2049 /// Rust prohibits the use of the non-`_INIT` form directly,
2050 /// such as in initializers of other `static`s. If in doubt,
2051 /// use the corresponding non-`_INIT` reference-typed `static`.
2052 ///
2053 /// This part of the public API will go away if Rust changes
2054 /// to make the referent of `pub const FOO: &'static Encoding`
2055 /// unique cross-crate or if Rust starts allowing static arrays
2056 /// to be initialized with `pub static FOO: &'static Encoding`
2057 /// items.
2058 pub static WINDOWS_874_INIT: Encoding = Encoding {
2059 name: "windows-874",
2060 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.windows_874, 0x0E01, 33, 58),
2061 };
2062
2063 /// The windows-874 encoding.
2064 ///
2065 /// This is the Thai encoding for Windows. It is an extension of TIS-620 / ISO-8859-11.
2066 ///
2067 /// [Index visualization](https://encoding.spec.whatwg.org/windows-874.html),
2068 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/windows-874-bmp.html)
2069 ///
2070 /// This encoding matches the Windows code page 874, except Windows decodes
2071 /// unassigned code points to the Private Use Area of Unicode.
2072 ///
2073 /// This will change from `static` to `const` if Rust changes
2074 /// to make the referent of `pub const FOO: &'static Encoding`
2075 /// unique cross-crate, so don't take the address of this
2076 /// `static`.
2077 pub static WINDOWS_874: &'static Encoding = &WINDOWS_874_INIT;
2078
2079 /// The initializer for the [x-mac-cyrillic](static.X_MAC_CYRILLIC.html) encoding.
2080 ///
2081 /// For use only for taking the address of this form when
2082 /// Rust prohibits the use of the non-`_INIT` form directly,
2083 /// such as in initializers of other `static`s. If in doubt,
2084 /// use the corresponding non-`_INIT` reference-typed `static`.
2085 ///
2086 /// This part of the public API will go away if Rust changes
2087 /// to make the referent of `pub const FOO: &'static Encoding`
2088 /// unique cross-crate or if Rust starts allowing static arrays
2089 /// to be initialized with `pub static FOO: &'static Encoding`
2090 /// items.
2091 pub static X_MAC_CYRILLIC_INIT: Encoding = Encoding {
2092 name: "x-mac-cyrillic",
2093 variant: VariantEncoding::SingleByte(&data::SINGLE_BYTE_DATA.x_mac_cyrillic, 0x0430, 96, 31),
2094 };
2095
2096 /// The x-mac-cyrillic encoding.
2097 ///
2098 /// This is the MacUkrainian encoding from Mac OS Classic.
2099 ///
2100 /// [Index visualization](https://encoding.spec.whatwg.org/x-mac-cyrillic.html),
2101 /// [Visualization of BMP coverage](https://encoding.spec.whatwg.org/x-mac-cyrillic-bmp.html)
2102 ///
2103 /// This encoding matches the Windows code page 10017.
2104 ///
2105 /// This will change from `static` to `const` if Rust changes
2106 /// to make the referent of `pub const FOO: &'static Encoding`
2107 /// unique cross-crate, so don't take the address of this
2108 /// `static`.
2109 pub static X_MAC_CYRILLIC: &'static Encoding = &X_MAC_CYRILLIC_INIT;
2110
2111 /// The initializer for the [x-user-defined](static.X_USER_DEFINED.html) encoding.
2112 ///
2113 /// For use only for taking the address of this form when
2114 /// Rust prohibits the use of the non-`_INIT` form directly,
2115 /// such as in initializers of other `static`s. If in doubt,
2116 /// use the corresponding non-`_INIT` reference-typed `static`.
2117 ///
2118 /// This part of the public API will go away if Rust changes
2119 /// to make the referent of `pub const FOO: &'static Encoding`
2120 /// unique cross-crate or if Rust starts allowing static arrays
2121 /// to be initialized with `pub static FOO: &'static Encoding`
2122 /// items.
2123 pub static X_USER_DEFINED_INIT: Encoding = Encoding {
2124 name: "x-user-defined",
2125 variant: VariantEncoding::UserDefined,
2126 };
2127
2128 /// The x-user-defined encoding.
2129 ///
2130 /// This encoding offsets the non-ASCII bytes by `0xF700` thereby decoding
2131 /// them to the Private Use Area of Unicode. It was used for loading binary
2132 /// data into a JavaScript string using `XMLHttpRequest` before XHR supported
2133 /// the `"arraybuffer"` response type.
2134 ///
2135 /// This encoding does not have a Windows code page number.
2136 ///
2137 /// This will change from `static` to `const` if Rust changes
2138 /// to make the referent of `pub const FOO: &'static Encoding`
2139 /// unique cross-crate, so don't take the address of this
2140 /// `static`.
2141 pub static X_USER_DEFINED: &'static Encoding = &X_USER_DEFINED_INIT;
2142
2143 static LABELS_SORTED: [&'static str; 228] = [
2144 "l1",
2145 "l2",
2146 "l3",
2147 "l4",
2148 "l5",
2149 "l6",
2150 "l9",
2151 "866",
2152 "mac",
2153 "koi",
2154 "gbk",
2155 "big5",
2156 "utf8",
2157 "koi8",
2158 "sjis",
2159 "ucs-2",
2160 "ms932",
2161 "cp866",
2162 "utf-8",
2163 "cp819",
2164 "ascii",
2165 "x-gbk",
2166 "greek",
2167 "cp1250",
2168 "cp1251",
2169 "latin1",
2170 "gb2312",
2171 "cp1252",
2172 "latin2",
2173 "cp1253",
2174 "latin3",
2175 "cp1254",
2176 "latin4",
2177 "cp1255",
2178 "csbig5",
2179 "latin5",
2180 "utf-16",
2181 "cp1256",
2182 "ibm866",
2183 "latin6",
2184 "cp1257",
2185 "cp1258",
2186 "greek8",
2187 "ibm819",
2188 "arabic",
2189 "visual",
2190 "korean",
2191 "euc-jp",
2192 "koi8-r",
2193 "koi8_r",
2194 "euc-kr",
2195 "x-sjis",
2196 "koi8-u",
2197 "hebrew",
2198 "tis-620",
2199 "gb18030",
2200 "ksc5601",
2201 "gb_2312",
2202 "dos-874",
2203 "cn-big5",
2204 "unicode",
2205 "chinese",
2206 "logical",
2207 "cskoi8r",
2208 "cseuckr",
2209 "koi8-ru",
2210 "x-cp1250",
2211 "ksc_5601",
2212 "x-cp1251",
2213 "iso88591",
2214 "csgb2312",
2215 "x-cp1252",
2216 "iso88592",
2217 "x-cp1253",
2218 "iso88593",
2219 "ecma-114",
2220 "x-cp1254",
2221 "iso88594",
2222 "x-cp1255",
2223 "iso88595",
2224 "x-x-big5",
2225 "x-cp1256",
2226 "csibm866",
2227 "iso88596",
2228 "x-cp1257",
2229 "iso88597",
2230 "asmo-708",
2231 "ecma-118",
2232 "elot_928",
2233 "x-cp1258",
2234 "iso88598",
2235 "iso88599",
2236 "cyrillic",
2237 "utf-16be",
2238 "utf-16le",
2239 "us-ascii",
2240 "ms_kanji",
2241 "x-euc-jp",
2242 "iso885910",
2243 "iso8859-1",
2244 "iso885911",
2245 "iso8859-2",
2246 "iso8859-3",
2247 "iso885913",
2248 "iso8859-4",
2249 "iso885914",
2250 "iso8859-5",
2251 "iso885915",
2252 "iso8859-6",
2253 "iso8859-7",
2254 "iso8859-8",
2255 "iso-ir-58",
2256 "iso8859-9",
2257 "csunicode",
2258 "macintosh",
2259 "shift-jis",
2260 "shift_jis",
2261 "iso-ir-100",
2262 "iso8859-10",
2263 "iso-ir-110",
2264 "gb_2312-80",
2265 "iso-8859-1",
2266 "iso_8859-1",
2267 "iso-ir-101",
2268 "iso8859-11",
2269 "iso-8859-2",
2270 "iso_8859-2",
2271 "hz-gb-2312",
2272 "iso-8859-3",
2273 "iso_8859-3",
2274 "iso8859-13",
2275 "iso-8859-4",
2276 "iso_8859-4",
2277 "iso8859-14",
2278 "iso-ir-144",
2279 "iso-8859-5",
2280 "iso_8859-5",
2281 "iso8859-15",
2282 "iso-8859-6",
2283 "iso_8859-6",
2284 "iso-ir-126",
2285 "iso-8859-7",
2286 "iso_8859-7",
2287 "iso-ir-127",
2288 "iso-ir-157",
2289 "iso-8859-8",
2290 "iso_8859-8",
2291 "iso-ir-138",
2292 "iso-ir-148",
2293 "iso-8859-9",
2294 "iso_8859-9",
2295 "iso-ir-109",
2296 "iso-ir-149",
2297 "big5-hkscs",
2298 "csshiftjis",
2299 "iso-8859-10",
2300 "iso-8859-11",
2301 "csisolatin1",
2302 "csisolatin2",
2303 "iso-8859-13",
2304 "csisolatin3",
2305 "iso-8859-14",
2306 "windows-874",
2307 "csisolatin4",
2308 "iso-8859-15",
2309 "iso_8859-15",
2310 "csisolatin5",
2311 "iso-8859-16",
2312 "csisolatin6",
2313 "windows-949",
2314 "csisolatin9",
2315 "csiso88596e",
2316 "csiso88598e",
2317 "unicodefffe",
2318 "unicodefeff",
2319 "csmacintosh",
2320 "csiso88596i",
2321 "csiso88598i",
2322 "windows-31j",
2323 "x-mac-roman",
2324 "iso-2022-cn",
2325 "iso-2022-jp",
2326 "csiso2022jp",
2327 "iso-2022-kr",
2328 "csiso2022kr",
2329 "replacement",
2330 "windows-1250",
2331 "windows-1251",
2332 "windows-1252",
2333 "windows-1253",
2334 "windows-1254",
2335 "windows-1255",
2336 "windows-1256",
2337 "windows-1257",
2338 "windows-1258",
2339 "iso-8859-6-e",
2340 "iso-8859-8-e",
2341 "iso-8859-6-i",
2342 "iso-8859-8-i",
2343 "sun_eu_greek",
2344 "csksc56011987",
2345 "unicode20utf8",
2346 "unicode11utf8",
2347 "ks_c_5601-1987",
2348 "ansi_x3.4-1968",
2349 "ks_c_5601-1989",
2350 "x-mac-cyrillic",
2351 "x-user-defined",
2352 "csiso58gb231280",
2353 "iso-10646-ucs-2",
2354 "iso_8859-1:1987",
2355 "iso_8859-2:1987",
2356 "iso_8859-6:1987",
2357 "iso_8859-7:1987",
2358 "iso_8859-3:1988",
2359 "iso_8859-4:1988",
2360 "iso_8859-5:1988",
2361 "iso_8859-8:1988",
2362 "x-unicode20utf8",
2363 "iso_8859-9:1989",
2364 "csisolatingreek",
2365 "x-mac-ukrainian",
2366 "iso-2022-cn-ext",
2367 "csisolatinarabic",
2368 "csisolatinhebrew",
2369 "unicode-1-1-utf-8",
2370 "csisolatincyrillic",
2371 "cseucpkdfmtjapanese",
2372 ];
2373
2374 static ENCODINGS_IN_LABEL_SORT: [&'static Encoding; 228] = [
2375 &WINDOWS_1252_INIT,
2376 &ISO_8859_2_INIT,
2377 &ISO_8859_3_INIT,
2378 &ISO_8859_4_INIT,
2379 &WINDOWS_1254_INIT,
2380 &ISO_8859_10_INIT,
2381 &ISO_8859_15_INIT,
2382 &IBM866_INIT,
2383 &MACINTOSH_INIT,
2384 &KOI8_R_INIT,
2385 &GBK_INIT,
2386 &BIG5_INIT,
2387 &UTF_8_INIT,
2388 &KOI8_R_INIT,
2389 &SHIFT_JIS_INIT,
2390 &UTF_16LE_INIT,
2391 &SHIFT_JIS_INIT,
2392 &IBM866_INIT,
2393 &UTF_8_INIT,
2394 &WINDOWS_1252_INIT,
2395 &WINDOWS_1252_INIT,
2396 &GBK_INIT,
2397 &ISO_8859_7_INIT,
2398 &WINDOWS_1250_INIT,
2399 &WINDOWS_1251_INIT,
2400 &WINDOWS_1252_INIT,
2401 &GBK_INIT,
2402 &WINDOWS_1252_INIT,
2403 &ISO_8859_2_INIT,
2404 &WINDOWS_1253_INIT,
2405 &ISO_8859_3_INIT,
2406 &WINDOWS_1254_INIT,
2407 &ISO_8859_4_INIT,
2408 &WINDOWS_1255_INIT,
2409 &BIG5_INIT,
2410 &WINDOWS_1254_INIT,
2411 &UTF_16LE_INIT,
2412 &WINDOWS_1256_INIT,
2413 &IBM866_INIT,
2414 &ISO_8859_10_INIT,
2415 &WINDOWS_1257_INIT,
2416 &WINDOWS_1258_INIT,
2417 &ISO_8859_7_INIT,
2418 &WINDOWS_1252_INIT,
2419 &ISO_8859_6_INIT,
2420 &ISO_8859_8_INIT,
2421 &EUC_KR_INIT,
2422 &EUC_JP_INIT,
2423 &KOI8_R_INIT,
2424 &KOI8_R_INIT,
2425 &EUC_KR_INIT,
2426 &SHIFT_JIS_INIT,
2427 &KOI8_U_INIT,
2428 &ISO_8859_8_INIT,
2429 &WINDOWS_874_INIT,
2430 &GB18030_INIT,
2431 &EUC_KR_INIT,
2432 &GBK_INIT,
2433 &WINDOWS_874_INIT,
2434 &BIG5_INIT,
2435 &UTF_16LE_INIT,
2436 &GBK_INIT,
2437 &ISO_8859_8_I_INIT,
2438 &KOI8_R_INIT,
2439 &EUC_KR_INIT,
2440 &KOI8_U_INIT,
2441 &WINDOWS_1250_INIT,
2442 &EUC_KR_INIT,
2443 &WINDOWS_1251_INIT,
2444 &WINDOWS_1252_INIT,
2445 &GBK_INIT,
2446 &WINDOWS_1252_INIT,
2447 &ISO_8859_2_INIT,
2448 &WINDOWS_1253_INIT,
2449 &ISO_8859_3_INIT,
2450 &ISO_8859_6_INIT,
2451 &WINDOWS_1254_INIT,
2452 &ISO_8859_4_INIT,
2453 &WINDOWS_1255_INIT,
2454 &ISO_8859_5_INIT,
2455 &BIG5_INIT,
2456 &WINDOWS_1256_INIT,
2457 &IBM866_INIT,
2458 &ISO_8859_6_INIT,
2459 &WINDOWS_1257_INIT,
2460 &ISO_8859_7_INIT,
2461 &ISO_8859_6_INIT,
2462 &ISO_8859_7_INIT,
2463 &ISO_8859_7_INIT,
2464 &WINDOWS_1258_INIT,
2465 &ISO_8859_8_INIT,
2466 &WINDOWS_1254_INIT,
2467 &ISO_8859_5_INIT,
2468 &UTF_16BE_INIT,
2469 &UTF_16LE_INIT,
2470 &WINDOWS_1252_INIT,
2471 &SHIFT_JIS_INIT,
2472 &EUC_JP_INIT,
2473 &ISO_8859_10_INIT,
2474 &WINDOWS_1252_INIT,
2475 &WINDOWS_874_INIT,
2476 &ISO_8859_2_INIT,
2477 &ISO_8859_3_INIT,
2478 &ISO_8859_13_INIT,
2479 &ISO_8859_4_INIT,
2480 &ISO_8859_14_INIT,
2481 &ISO_8859_5_INIT,
2482 &ISO_8859_15_INIT,
2483 &ISO_8859_6_INIT,
2484 &ISO_8859_7_INIT,
2485 &ISO_8859_8_INIT,
2486 &GBK_INIT,
2487 &WINDOWS_1254_INIT,
2488 &UTF_16LE_INIT,
2489 &MACINTOSH_INIT,
2490 &SHIFT_JIS_INIT,
2491 &SHIFT_JIS_INIT,
2492 &WINDOWS_1252_INIT,
2493 &ISO_8859_10_INIT,
2494 &ISO_8859_4_INIT,
2495 &GBK_INIT,
2496 &WINDOWS_1252_INIT,
2497 &WINDOWS_1252_INIT,
2498 &ISO_8859_2_INIT,
2499 &WINDOWS_874_INIT,
2500 &ISO_8859_2_INIT,
2501 &ISO_8859_2_INIT,
2502 &REPLACEMENT_INIT,
2503 &ISO_8859_3_INIT,
2504 &ISO_8859_3_INIT,
2505 &ISO_8859_13_INIT,
2506 &ISO_8859_4_INIT,
2507 &ISO_8859_4_INIT,
2508 &ISO_8859_14_INIT,
2509 &ISO_8859_5_INIT,
2510 &ISO_8859_5_INIT,
2511 &ISO_8859_5_INIT,
2512 &ISO_8859_15_INIT,
2513 &ISO_8859_6_INIT,
2514 &ISO_8859_6_INIT,
2515 &ISO_8859_7_INIT,
2516 &ISO_8859_7_INIT,
2517 &ISO_8859_7_INIT,
2518 &ISO_8859_6_INIT,
2519 &ISO_8859_10_INIT,
2520 &ISO_8859_8_INIT,
2521 &ISO_8859_8_INIT,
2522 &ISO_8859_8_INIT,
2523 &WINDOWS_1254_INIT,
2524 &WINDOWS_1254_INIT,
2525 &WINDOWS_1254_INIT,
2526 &ISO_8859_3_INIT,
2527 &EUC_KR_INIT,
2528 &BIG5_INIT,
2529 &SHIFT_JIS_INIT,
2530 &ISO_8859_10_INIT,
2531 &WINDOWS_874_INIT,
2532 &WINDOWS_1252_INIT,
2533 &ISO_8859_2_INIT,
2534 &ISO_8859_13_INIT,
2535 &ISO_8859_3_INIT,
2536 &ISO_8859_14_INIT,
2537 &WINDOWS_874_INIT,
2538 &ISO_8859_4_INIT,
2539 &ISO_8859_15_INIT,
2540 &ISO_8859_15_INIT,
2541 &WINDOWS_1254_INIT,
2542 &ISO_8859_16_INIT,
2543 &ISO_8859_10_INIT,
2544 &EUC_KR_INIT,
2545 &ISO_8859_15_INIT,
2546 &ISO_8859_6_INIT,
2547 &ISO_8859_8_INIT,
2548 &UTF_16BE_INIT,
2549 &UTF_16LE_INIT,
2550 &MACINTOSH_INIT,
2551 &ISO_8859_6_INIT,
2552 &ISO_8859_8_I_INIT,
2553 &SHIFT_JIS_INIT,
2554 &MACINTOSH_INIT,
2555 &REPLACEMENT_INIT,
2556 &ISO_2022_JP_INIT,
2557 &ISO_2022_JP_INIT,
2558 &REPLACEMENT_INIT,
2559 &REPLACEMENT_INIT,
2560 &REPLACEMENT_INIT,
2561 &WINDOWS_1250_INIT,
2562 &WINDOWS_1251_INIT,
2563 &WINDOWS_1252_INIT,
2564 &WINDOWS_1253_INIT,
2565 &WINDOWS_1254_INIT,
2566 &WINDOWS_1255_INIT,
2567 &WINDOWS_1256_INIT,
2568 &WINDOWS_1257_INIT,
2569 &WINDOWS_1258_INIT,
2570 &ISO_8859_6_INIT,
2571 &ISO_8859_8_INIT,
2572 &ISO_8859_6_INIT,
2573 &ISO_8859_8_I_INIT,
2574 &ISO_8859_7_INIT,
2575 &EUC_KR_INIT,
2576 &UTF_8_INIT,
2577 &UTF_8_INIT,
2578 &EUC_KR_INIT,
2579 &WINDOWS_1252_INIT,
2580 &EUC_KR_INIT,
2581 &X_MAC_CYRILLIC_INIT,
2582 &X_USER_DEFINED_INIT,
2583 &GBK_INIT,
2584 &UTF_16LE_INIT,
2585 &WINDOWS_1252_INIT,
2586 &ISO_8859_2_INIT,
2587 &ISO_8859_6_INIT,
2588 &ISO_8859_7_INIT,
2589 &ISO_8859_3_INIT,
2590 &ISO_8859_4_INIT,
2591 &ISO_8859_5_INIT,
2592 &ISO_8859_8_INIT,
2593 &UTF_8_INIT,
2594 &WINDOWS_1254_INIT,
2595 &ISO_8859_7_INIT,
2596 &X_MAC_CYRILLIC_INIT,
2597 &REPLACEMENT_INIT,
2598 &ISO_8859_6_INIT,
2599 &ISO_8859_8_INIT,
2600 &UTF_8_INIT,
2601 &ISO_8859_5_INIT,
2602 &EUC_JP_INIT,
2603 ];
2604
2605 // END GENERATED CODE
2606
2607 /// An encoding as defined in the [Encoding Standard][1].
2608 ///
2609 /// An _encoding_ defines a mapping from a `u8` sequence to a `char` sequence
2610 /// and, in most cases, vice versa. Each encoding has a name, an output
2611 /// encoding, and one or more labels.
2612 ///
2613 /// _Labels_ are ASCII-case-insensitive strings that are used to identify an
2614 /// encoding in formats and protocols. The _name_ of the encoding is the
2615 /// preferred label in the case appropriate for returning from the
2616 /// [`characterSet`][2] property of the `Document` DOM interface.
2617 ///
2618 /// The _output encoding_ is the encoding used for form submission and URL
2619 /// parsing on Web pages in the encoding. This is UTF-8 for the replacement,
2620 /// UTF-16LE and UTF-16BE encodings and the encoding itself for other
2621 /// encodings.
2622 ///
2623 /// [1]: https://encoding.spec.whatwg.org/
2624 /// [2]: https://dom.spec.whatwg.org/#dom-document-characterset
2625 ///
2626 /// # Streaming vs. Non-Streaming
2627 ///
2628 /// When you have the entire input in a single buffer, you can use the
2629 /// methods [`decode()`][3], [`decode_with_bom_removal()`][3],
2630 /// [`decode_without_bom_handling()`][5],
2631 /// [`decode_without_bom_handling_and_without_replacement()`][6] and
2632 /// [`encode()`][7]. (These methods are available to Rust callers only and are
2633 /// not available in the C API.) Unlike the rest of the API available to Rust,
2634 /// these methods perform heap allocations. You should the `Decoder` and
2635 /// `Encoder` objects when your input is split into multiple buffers or when
2636 /// you want to control the allocation of the output buffers.
2637 ///
2638 /// [3]: #method.decode
2639 /// [4]: #method.decode_with_bom_removal
2640 /// [5]: #method.decode_without_bom_handling
2641 /// [6]: #method.decode_without_bom_handling_and_without_replacement
2642 /// [7]: #method.encode
2643 ///
2644 /// # Instances
2645 ///
2646 /// All instances of `Encoding` are statically allocated and have the `'static`
2647 /// lifetime. There is precisely one unique `Encoding` instance for each
2648 /// encoding defined in the Encoding Standard.
2649 ///
2650 /// To obtain a reference to a particular encoding whose identity you know at
2651 /// compile time, use a `static` that refers to encoding. There is a `static`
2652 /// for each encoding. The `static`s are named in all caps with hyphens
2653 /// replaced with underscores (and in C/C++ have `_ENCODING` appended to the
2654 /// name). For example, if you know at compile time that you will want to
2655 /// decode using the UTF-8 encoding, use the `UTF_8` `static` (`UTF_8_ENCODING`
2656 /// in C/C++).
2657 ///
2658 /// Additionally, there are non-reference-typed forms ending with `_INIT` to
2659 /// work around the problem that `static`s of the type `&'static Encoding`
2660 /// cannot be used to initialize items of an array whose type is
2661 /// `[&'static Encoding; N]`.
2662 ///
2663 /// If you don't know what encoding you need at compile time and need to
2664 /// dynamically get an encoding by label, use
2665 /// <code>Encoding::<a href="#method.for_label">for_label</a>(<var>label</var>)</code>.
2666 ///
2667 /// Instances of `Encoding` can be compared with `==` (in both Rust and in
2668 /// C/C++).
2669 pub struct Encoding {
2670 name: &'static str,
2671 variant: VariantEncoding,
2672 }
2673
2674 impl Encoding {
2675 /// Implements the
2676 /// [_get an encoding_](https://encoding.spec.whatwg.org/#concept-encoding-get)
2677 /// algorithm.
2678 ///
2679 /// If, after ASCII-lowercasing and removing leading and trailing
2680 /// whitespace, the argument matches a label defined in the Encoding
2681 /// Standard, `Some(&'static Encoding)` representing the corresponding
2682 /// encoding is returned. If there is no match, `None` is returned.
2683 ///
2684 /// This is the right method to use if the action upon the method returning
2685 /// `None` is to use a fallback encoding (e.g. `WINDOWS_1252`) instead.
2686 /// When the action upon the method returning `None` is not to proceed with
2687 /// a fallback but to refuse processing, `for_label_no_replacement()` is more
2688 /// appropriate.
2689 ///
2690 /// The argument is of type `&[u8]` instead of `&str` to save callers
2691 /// that are extracting the label from a non-UTF-8 protocol the trouble
2692 /// of conversion to UTF-8. (If you have a `&str`, just call `.as_bytes()`
2693 /// on it.)
2694 ///
2695 /// Available via the C wrapper.
2696 ///
2697 /// # Example
2698 /// ```
2699 /// use encoding_rs::Encoding;
2700 ///
2701 /// assert_eq!(Some(encoding_rs::UTF_8), Encoding::for_label(b"utf-8"));
2702 /// assert_eq!(Some(encoding_rs::UTF_8), Encoding::for_label(b"unicode11utf8"));
2703 ///
2704 /// assert_eq!(Some(encoding_rs::ISO_8859_2), Encoding::for_label(b"latin2"));
2705 ///
2706 /// assert_eq!(Some(encoding_rs::UTF_16BE), Encoding::for_label(b"utf-16be"));
2707 ///
2708 /// assert_eq!(None, Encoding::for_label(b"unrecognized label"));
2709 /// ```
2710 pub fn for_label(label: &[u8]) -> Option<&'static Encoding> {
2711 let mut trimmed = [0u8; LONGEST_LABEL_LENGTH];
2712 let mut trimmed_pos = 0usize;
2713 let mut iter = label.into_iter();
2714 // before
2715 loop {
2716 match iter.next() {
2717 None => {
2718 return None;
2719 }
2720 Some(byte) => {
2721 // The characters used in labels are:
2722 // a-z (except q, but excluding it below seems excessive)
2723 // 0-9
2724 // . _ - :
2725 match *byte {
2726 0x09u8 | 0x0Au8 | 0x0Cu8 | 0x0Du8 | 0x20u8 => {
2727 continue;
2728 }
2729 b'A'..=b'Z' => {
2730 trimmed[trimmed_pos] = *byte + 0x20u8;
2731 trimmed_pos = 1usize;
2732 break;
2733 }
2734 b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b':' | b'.' => {
2735 trimmed[trimmed_pos] = *byte;
2736 trimmed_pos = 1usize;
2737 break;
2738 }
2739 _ => {
2740 return None;
2741 }
2742 }
2743 }
2744 }
2745 }
2746 // inside
2747 loop {
2748 match iter.next() {
2749 None => {
2750 break;
2751 }
2752 Some(byte) => {
2753 match *byte {
2754 0x09u8 | 0x0Au8 | 0x0Cu8 | 0x0Du8 | 0x20u8 => {
2755 break;
2756 }
2757 b'A'..=b'Z' => {
2758 if trimmed_pos == LONGEST_LABEL_LENGTH {
2759 // There's no encoding with a label this long
2760 return None;
2761 }
2762 trimmed[trimmed_pos] = *byte + 0x20u8;
2763 trimmed_pos += 1usize;
2764 continue;
2765 }
2766 b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b':' | b'.' => {
2767 if trimmed_pos == LONGEST_LABEL_LENGTH {
2768 // There's no encoding with a label this long
2769 return None;
2770 }
2771 trimmed[trimmed_pos] = *byte;
2772 trimmed_pos += 1usize;
2773 continue;
2774 }
2775 _ => {
2776 return None;
2777 }
2778 }
2779 }
2780 }
2781 }
2782 // after
2783 loop {
2784 match iter.next() {
2785 None => {
2786 break;
2787 }
2788 Some(byte) => {
2789 match *byte {
2790 0x09u8 | 0x0Au8 | 0x0Cu8 | 0x0Du8 | 0x20u8 => {
2791 continue;
2792 }
2793 _ => {
2794 // There's no label with space in the middle
2795 return None;
2796 }
2797 }
2798 }
2799 }
2800 }
2801 let candidate = &trimmed[..trimmed_pos];
2802 match LABELS_SORTED.binary_search_by(|probe| {
2803 let bytes = probe.as_bytes();
2804 let c = bytes.len().cmp(&candidate.len());
2805 if c != Ordering::Equal {
2806 return c;
2807 }
2808 let probe_iter = bytes.iter().rev();
2809 let candidate_iter = candidate.iter().rev();
2810 probe_iter.cmp(candidate_iter)
2811 }) {
2812 Ok(i) => Some(ENCODINGS_IN_LABEL_SORT[i]),
2813 Err(_) => None,
2814 }
2815 }
2816
2817 /// This method behaves the same as `for_label()`, except when `for_label()`
2818 /// would return `Some(REPLACEMENT)`, this method returns `None` instead.
2819 ///
2820 /// This method is useful in scenarios where a fatal error is required
2821 /// upon invalid label, because in those cases the caller typically wishes
2822 /// to treat the labels that map to the replacement encoding as fatal
2823 /// errors, too.
2824 ///
2825 /// It is not OK to use this method when the action upon the method returning
2826 /// `None` is to use a fallback encoding (e.g. `WINDOWS_1252`). In such a
2827 /// case, the `for_label()` method should be used instead in order to avoid
2828 /// unsafe fallback for labels that `for_label()` maps to `Some(REPLACEMENT)`.
2829 ///
2830 /// Available via the C wrapper.
2831 #[inline]
2832 pub fn for_label_no_replacement(label: &[u8]) -> Option<&'static Encoding> {
2833 match Encoding::for_label(label) {
2834 None => None,
2835 Some(encoding) => {
2836 if encoding == REPLACEMENT {
2837 None
2838 } else {
2839 Some(encoding)
2840 }
2841 }
2842 }
2843 }
2844
2845 /// Performs non-incremental BOM sniffing.
2846 ///
2847 /// The argument must either be a buffer representing the entire input
2848 /// stream (non-streaming case) or a buffer representing at least the first
2849 /// three bytes of the input stream (streaming case).
2850 ///
2851 /// Returns `Some((UTF_8, 3))`, `Some((UTF_16LE, 2))` or
2852 /// `Some((UTF_16BE, 2))` if the argument starts with the UTF-8, UTF-16LE
2853 /// or UTF-16BE BOM or `None` otherwise.
2854 ///
2855 /// Available via the C wrapper.
2856 #[inline]
2857 pub fn for_bom(buffer: &[u8]) -> Option<(&'static Encoding, usize)> {
2858 if buffer.starts_with(b"\xEF\xBB\xBF") {
2859 Some((UTF_8, 3))
2860 } else if buffer.starts_with(b"\xFF\xFE") {
2861 Some((UTF_16LE, 2))
2862 } else if buffer.starts_with(b"\xFE\xFF") {
2863 Some((UTF_16BE, 2))
2864 } else {
2865 None
2866 }
2867 }
2868
2869 /// Returns the name of this encoding.
2870 ///
2871 /// This name is appropriate to return as-is from the DOM
2872 /// `document.characterSet` property.
2873 ///
2874 /// Available via the C wrapper.
2875 #[inline]
2876 pub fn name(&'static self) -> &'static str {
2877 self.name
2878 }
2879
2880 /// Checks whether the _output encoding_ of this encoding can encode every
2881 /// `char`. (Only true if the output encoding is UTF-8.)
2882 ///
2883 /// Available via the C wrapper.
2884 #[inline]
2885 pub fn can_encode_everything(&'static self) -> bool {
2886 self.output_encoding() == UTF_8
2887 }
2888
2889 /// Checks whether the bytes 0x00...0x7F map exclusively to the characters
2890 /// U+0000...U+007F and vice versa.
2891 ///
2892 /// Available via the C wrapper.
2893 #[inline]
2894 pub fn is_ascii_compatible(&'static self) -> bool {
2895 !(self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE || self == ISO_2022_JP)
2896 }
2897
2898 /// Checks whether this encoding maps one byte to one Basic Multilingual
2899 /// Plane code point (i.e. byte length equals decoded UTF-16 length) and
2900 /// vice versa (for mappable characters).
2901 ///
2902 /// `true` iff this encoding is on the list of [Legacy single-byte
2903 /// encodings](https://encoding.spec.whatwg.org/#legacy-single-byte-encodings)
2904 /// in the spec or x-user-defined.
2905 ///
2906 /// Available via the C wrapper.
2907 #[inline]
2908 pub fn is_single_byte(&'static self) -> bool {
2909 self.variant.is_single_byte()
2910 }
2911
2912 /// Checks whether the bytes 0x00...0x7F map mostly to the characters
2913 /// U+0000...U+007F and vice versa.
2914 #[cfg(feature = "alloc")]
2915 #[inline]
2916 fn is_potentially_borrowable(&'static self) -> bool {
2917 !(self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE)
2918 }
2919
2920 /// Returns the _output encoding_ of this encoding. This is UTF-8 for
2921 /// UTF-16BE, UTF-16LE, and replacement and the encoding itself otherwise.
2922 ///
2923 /// _Note:_ The _output encoding_ concept is needed for form submission and
2924 /// error handling in the query strings of URLs in the Web Platform.
2925 ///
2926 /// Available via the C wrapper.
2927 #[inline]
2928 pub fn output_encoding(&'static self) -> &'static Encoding {
2929 if self == REPLACEMENT || self == UTF_16BE || self == UTF_16LE {
2930 UTF_8
2931 } else {
2932 self
2933 }
2934 }
2935
2936 /// Decode complete input to `Cow<'a, str>` _with BOM sniffing_ and with
2937 /// malformed sequences replaced with the REPLACEMENT CHARACTER when the
2938 /// entire input is available as a single buffer (i.e. the end of the
2939 /// buffer marks the end of the stream).
2940 ///
2941 /// The BOM, if any, does not appear in the output.
2942 ///
2943 /// This method implements the (non-streaming version of) the
2944 /// [_decode_](https://encoding.spec.whatwg.org/#decode) spec concept.
2945 ///
2946 /// The second item in the returned tuple is the encoding that was actually
2947 /// used (which may differ from this encoding thanks to BOM sniffing).
2948 ///
2949 /// The third item in the returned tuple indicates whether there were
2950 /// malformed sequences (that were replaced with the REPLACEMENT CHARACTER).
2951 ///
2952 /// _Note:_ It is wrong to use this when the input buffer represents only
2953 /// a segment of the input instead of the whole input. Use `new_decoder()`
2954 /// when decoding segmented input.
2955 ///
2956 /// This method performs a one or two heap allocations for the backing
2957 /// buffer of the `String` when unable to borrow. (One allocation if not
2958 /// errors and potentially another one in the presence of errors.) The
2959 /// first allocation assumes jemalloc and may not be optimal with
2960 /// allocators that do not use power-of-two buckets. A borrow is performed
2961 /// if decoding UTF-8 and the input is valid UTF-8, if decoding an
2962 /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
2963 /// ISO-2022-JP and the input is entirely in the ASCII state without state
2964 /// transitions.
2965 ///
2966 /// # Panics
2967 ///
2968 /// If the size calculation for a heap-allocated backing buffer overflows
2969 /// `usize`.
2970 ///
2971 /// Available to Rust only and only with the `alloc` feature enabled (enabled
2972 /// by default).
2973 #[cfg(feature = "alloc")]
2974 #[inline]
2975 pub fn decode<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, &'static Encoding, bool) {
2976 let (encoding, without_bom) = match Encoding::for_bom(bytes) {
2977 Some((encoding, bom_length)) => (encoding, &bytes[bom_length..]),
2978 None => (self, bytes),
2979 };
2980 let (cow, had_errors) = encoding.decode_without_bom_handling(without_bom);
2981 (cow, encoding, had_errors)
2982 }
2983
2984 /// Decode complete input to `Cow<'a, str>` _with BOM removal_ and with
2985 /// malformed sequences replaced with the REPLACEMENT CHARACTER when the
2986 /// entire input is available as a single buffer (i.e. the end of the
2987 /// buffer marks the end of the stream).
2988 ///
2989 /// Only an initial byte sequence that is a BOM for this encoding is removed.
2990 ///
2991 /// When invoked on `UTF_8`, this method implements the (non-streaming
2992 /// version of) the
2993 /// [_UTF-8 decode_](https://encoding.spec.whatwg.org/#utf-8-decode) spec
2994 /// concept.
2995 ///
2996 /// The second item in the returned pair indicates whether there were
2997 /// malformed sequences (that were replaced with the REPLACEMENT CHARACTER).
2998 ///
2999 /// _Note:_ It is wrong to use this when the input buffer represents only
3000 /// a segment of the input instead of the whole input. Use
3001 /// `new_decoder_with_bom_removal()` when decoding segmented input.
3002 ///
3003 /// This method performs a one or two heap allocations for the backing
3004 /// buffer of the `String` when unable to borrow. (One allocation if not
3005 /// errors and potentially another one in the presence of errors.) The
3006 /// first allocation assumes jemalloc and may not be optimal with
3007 /// allocators that do not use power-of-two buckets. A borrow is performed
3008 /// if decoding UTF-8 and the input is valid UTF-8, if decoding an
3009 /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
3010 /// ISO-2022-JP and the input is entirely in the ASCII state without state
3011 /// transitions.
3012 ///
3013 /// # Panics
3014 ///
3015 /// If the size calculation for a heap-allocated backing buffer overflows
3016 /// `usize`.
3017 ///
3018 /// Available to Rust only and only with the `alloc` feature enabled (enabled
3019 /// by default).
3020 #[cfg(feature = "alloc")]
3021 #[inline]
3022 pub fn decode_with_bom_removal<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, bool) {
3023 let without_bom = if self == UTF_8 && bytes.starts_with(b"\xEF\xBB\xBF") {
3024 &bytes[3..]
3025 } else if (self == UTF_16LE && bytes.starts_with(b"\xFF\xFE"))
3026 || (self == UTF_16BE && bytes.starts_with(b"\xFE\xFF"))
3027 {
3028 &bytes[2..]
3029 } else {
3030 bytes
3031 };
3032 self.decode_without_bom_handling(without_bom)
3033 }
3034
3035 /// Decode complete input to `Cow<'a, str>` _without BOM handling_ and
3036 /// with malformed sequences replaced with the REPLACEMENT CHARACTER when
3037 /// the entire input is available as a single buffer (i.e. the end of the
3038 /// buffer marks the end of the stream).
3039 ///
3040 /// When invoked on `UTF_8`, this method implements the (non-streaming
3041 /// version of) the
3042 /// [_UTF-8 decode without BOM_](https://encoding.spec.whatwg.org/#utf-8-decode-without-bom)
3043 /// spec concept.
3044 ///
3045 /// The second item in the returned pair indicates whether there were
3046 /// malformed sequences (that were replaced with the REPLACEMENT CHARACTER).
3047 ///
3048 /// _Note:_ It is wrong to use this when the input buffer represents only
3049 /// a segment of the input instead of the whole input. Use
3050 /// `new_decoder_without_bom_handling()` when decoding segmented input.
3051 ///
3052 /// This method performs a one or two heap allocations for the backing
3053 /// buffer of the `String` when unable to borrow. (One allocation if not
3054 /// errors and potentially another one in the presence of errors.) The
3055 /// first allocation assumes jemalloc and may not be optimal with
3056 /// allocators that do not use power-of-two buckets. A borrow is performed
3057 /// if decoding UTF-8 and the input is valid UTF-8, if decoding an
3058 /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
3059 /// ISO-2022-JP and the input is entirely in the ASCII state without state
3060 /// transitions.
3061 ///
3062 /// # Panics
3063 ///
3064 /// If the size calculation for a heap-allocated backing buffer overflows
3065 /// `usize`.
3066 ///
3067 /// Available to Rust only and only with the `alloc` feature enabled (enabled
3068 /// by default).
3069 #[cfg(feature = "alloc")]
3070 pub fn decode_without_bom_handling<'a>(&'static self, bytes: &'a [u8]) -> (Cow<'a, str>, bool) {
3071 let (mut decoder, mut string, mut total_read) = if self.is_potentially_borrowable() {
3072 let valid_up_to = if self == UTF_8 {
3073 utf8_valid_up_to(bytes)
3074 } else if self == ISO_2022_JP {
3075 iso_2022_jp_ascii_valid_up_to(bytes)
3076 } else {
3077 ascii_valid_up_to(bytes)
3078 };
3079 if valid_up_to == bytes.len() {
3080 let str: &str = unsafe { core::str::from_utf8_unchecked(bytes) };
3081 return (Cow::Borrowed(str), false);
3082 }
3083 let decoder = self.new_decoder_without_bom_handling();
3084
3085 let rounded_without_replacement = checked_next_power_of_two(checked_add(
3086 valid_up_to,
3087 decoder.max_utf8_buffer_length_without_replacement(bytes.len() - valid_up_to),
3088 ));
3089 let with_replacement = checked_add(
3090 valid_up_to,
3091 decoder.max_utf8_buffer_length(bytes.len() - valid_up_to),
3092 );
3093 let mut string = String::with_capacity(
3094 checked_min(rounded_without_replacement, with_replacement).unwrap(),
3095 );
3096 unsafe {
3097 let vec = string.as_mut_vec();
3098 vec.set_len(valid_up_to);
3099 core::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
3100 }
3101 (decoder, string, valid_up_to)
3102 } else {
3103 let decoder = self.new_decoder_without_bom_handling();
3104 let rounded_without_replacement = checked_next_power_of_two(
3105 decoder.max_utf8_buffer_length_without_replacement(bytes.len()),
3106 );
3107 let with_replacement = decoder.max_utf8_buffer_length(bytes.len());
3108 let string = String::with_capacity(
3109 checked_min(rounded_without_replacement, with_replacement).unwrap(),
3110 );
3111 (decoder, string, 0)
3112 };
3113
3114 let mut total_had_errors = false;
3115 loop {
3116 let (result, read, had_errors) =
3117 decoder.decode_to_string(&bytes[total_read..], &mut string, true);
3118 total_read += read;
3119 total_had_errors |= had_errors;
3120 match result {
3121 CoderResult::InputEmpty => {
3122 debug_assert_eq!(total_read, bytes.len());
3123 return (Cow::Owned(string), total_had_errors);
3124 }
3125 CoderResult::OutputFull => {
3126 // Allocate for the worst case. That is, we should come
3127 // here at most once per invocation of this method.
3128 let needed = decoder.max_utf8_buffer_length(bytes.len() - total_read);
3129 string.reserve(needed.unwrap());
3130 }
3131 }
3132 }
3133 }
3134
3135 /// Decode complete input to `Cow<'a, str>` _without BOM handling_ and
3136 /// _with malformed sequences treated as fatal_ when the entire input is
3137 /// available as a single buffer (i.e. the end of the buffer marks the end
3138 /// of the stream).
3139 ///
3140 /// When invoked on `UTF_8`, this method implements the (non-streaming
3141 /// version of) the
3142 /// [_UTF-8 decode without BOM or fail_](https://encoding.spec.whatwg.org/#utf-8-decode-without-bom-or-fail)
3143 /// spec concept.
3144 ///
3145 /// Returns `None` if a malformed sequence was encountered and the result
3146 /// of the decode as `Some(String)` otherwise.
3147 ///
3148 /// _Note:_ It is wrong to use this when the input buffer represents only
3149 /// a segment of the input instead of the whole input. Use
3150 /// `new_decoder_without_bom_handling()` when decoding segmented input.
3151 ///
3152 /// This method performs a single heap allocation for the backing
3153 /// buffer of the `String` when unable to borrow. A borrow is performed if
3154 /// decoding UTF-8 and the input is valid UTF-8, if decoding an
3155 /// ASCII-compatible encoding and the input is ASCII-only, or when decoding
3156 /// ISO-2022-JP and the input is entirely in the ASCII state without state
3157 /// transitions.
3158 ///
3159 /// # Panics
3160 ///
3161 /// If the size calculation for a heap-allocated backing buffer overflows
3162 /// `usize`.
3163 ///
3164 /// Available to Rust only and only with the `alloc` feature enabled (enabled
3165 /// by default).
3166 #[cfg(feature = "alloc")]
3167 pub fn decode_without_bom_handling_and_without_replacement<'a>(
3168 &'static self,
3169 bytes: &'a [u8],
3170 ) -> Option<Cow<'a, str>> {
3171 if self == UTF_8 {
3172 let valid_up_to = utf8_valid_up_to(bytes);
3173 if valid_up_to == bytes.len() {
3174 let str: &str = unsafe { core::str::from_utf8_unchecked(bytes) };
3175 return Some(Cow::Borrowed(str));
3176 }
3177 return None;
3178 }
3179 let (mut decoder, mut string, input) = if self.is_potentially_borrowable() {
3180 let valid_up_to = if self == ISO_2022_JP {
3181 iso_2022_jp_ascii_valid_up_to(bytes)
3182 } else {
3183 ascii_valid_up_to(bytes)
3184 };
3185 if valid_up_to == bytes.len() {
3186 let str: &str = unsafe { core::str::from_utf8_unchecked(bytes) };
3187 return Some(Cow::Borrowed(str));
3188 }
3189 let decoder = self.new_decoder_without_bom_handling();
3190 let mut string = String::with_capacity(
3191 checked_add(
3192 valid_up_to,
3193 decoder.max_utf8_buffer_length_without_replacement(bytes.len() - valid_up_to),
3194 )
3195 .unwrap(),
3196 );
3197 unsafe {
3198 let vec = string.as_mut_vec();
3199 vec.set_len(valid_up_to);
3200 core::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
3201 }
3202 (decoder, string, &bytes[valid_up_to..])
3203 } else {
3204 let decoder = self.new_decoder_without_bom_handling();
3205 let string = String::with_capacity(
3206 decoder
3207 .max_utf8_buffer_length_without_replacement(bytes.len())
3208 .unwrap(),
3209 );
3210 (decoder, string, bytes)
3211 };
3212 let (result, read) = decoder.decode_to_string_without_replacement(input, &mut string, true);
3213 match result {
3214 DecoderResult::InputEmpty => {
3215 debug_assert_eq!(read, input.len());
3216 Some(Cow::Owned(string))
3217 }
3218 DecoderResult::Malformed(_, _) => None,
3219 DecoderResult::OutputFull => unreachable!(),
3220 }
3221 }
3222
3223 /// Encode complete input to `Cow<'a, [u8]>` using the
3224 /// [_output encoding_](Encoding::output_encoding) of this encoding with
3225 /// unmappable characters replaced with decimal numeric character references
3226 /// when the entire input is available as a single buffer (i.e. the end of
3227 /// the buffer marks the end of the stream).
3228 ///
3229 /// This method implements the (non-streaming version of) the
3230 /// [_encode_](https://encoding.spec.whatwg.org/#encode) spec concept. For
3231 /// the [_UTF-8 encode_](https://encoding.spec.whatwg.org/#utf-8-encode)
3232 /// spec concept, it is slightly more efficient to use
3233 /// <code><var>string</var>.as_bytes()</code> instead of invoking this
3234 /// method on `UTF_8`.
3235 ///
3236 /// The second item in the returned tuple is the encoding that was actually
3237 /// used (*which may differ from this encoding thanks to some encodings
3238 /// having UTF-8 as their output encoding*).
3239 ///
3240 /// The third item in the returned tuple indicates whether there were
3241 /// unmappable characters (that were replaced with HTML numeric character
3242 /// references).
3243 ///
3244 /// _Note:_ It is wrong to use this when the input buffer represents only
3245 /// a segment of the input instead of the whole input. Use `new_encoder()`
3246 /// when encoding segmented output.
3247 ///
3248 /// When encoding to UTF-8 or when encoding an ASCII-only input to a
3249 /// ASCII-compatible encoding, this method returns a borrow of the input
3250 /// without a heap allocation. Otherwise, this method performs a single
3251 /// heap allocation for the backing buffer of the `Vec<u8>` if there are no
3252 /// unmappable characters and potentially multiple heap allocations if
3253 /// there are. These allocations are tuned for jemalloc and may not be
3254 /// optimal when using a different allocator that doesn't use power-of-two
3255 /// buckets.
3256 ///
3257 /// # Panics
3258 ///
3259 /// If the size calculation for a heap-allocated backing buffer overflows
3260 /// `usize`.
3261 ///
3262 /// Available to Rust only and only with the `alloc` feature enabled (enabled
3263 /// by default).
3264 #[cfg(feature = "alloc")]
3265 pub fn encode<'a>(&'static self, string: &'a str) -> (Cow<'a, [u8]>, &'static Encoding, bool) {
3266 let output_encoding = self.output_encoding();
3267 if output_encoding == UTF_8 {
3268 return (Cow::Borrowed(string.as_bytes()), output_encoding, false);
3269 }
3270 debug_assert!(output_encoding.is_potentially_borrowable());
3271 let bytes = string.as_bytes();
3272 let valid_up_to = if output_encoding == ISO_2022_JP {
3273 iso_2022_jp_ascii_valid_up_to(bytes)
3274 } else {
3275 ascii_valid_up_to(bytes)
3276 };
3277 if valid_up_to == bytes.len() {
3278 return (Cow::Borrowed(bytes), output_encoding, false);
3279 }
3280 let mut encoder = output_encoding.new_encoder();
3281 let mut vec: Vec<u8> = Vec::with_capacity(
3282 (checked_add(
3283 valid_up_to,
3284 encoder.max_buffer_length_from_utf8_if_no_unmappables(string.len() - valid_up_to),
3285 ))
3286 .unwrap()
3287 .next_power_of_two(),
3288 );
3289 unsafe {
3290 vec.set_len(valid_up_to);
3291 core::ptr::copy_nonoverlapping(bytes.as_ptr(), vec.as_mut_ptr(), valid_up_to);
3292 }
3293 let mut total_read = valid_up_to;
3294 let mut total_had_errors = false;
3295 loop {
3296 let (result, read, had_errors) =
3297 encoder.encode_from_utf8_to_vec(&string[total_read..], &mut vec, true);
3298 total_read += read;
3299 total_had_errors |= had_errors;
3300 match result {
3301 CoderResult::InputEmpty => {
3302 debug_assert_eq!(total_read, string.len());
3303 return (Cow::Owned(vec), output_encoding, total_had_errors);
3304 }
3305 CoderResult::OutputFull => {
3306 // reserve_exact wants to know how much more on top of current
3307 // length--not current capacity.
3308 let needed = encoder
3309 .max_buffer_length_from_utf8_if_no_unmappables(string.len() - total_read);
3310 let rounded = (checked_add(vec.capacity(), needed))
3311 .unwrap()
3312 .next_power_of_two();
3313 let additional = rounded - vec.len();
3314 vec.reserve_exact(additional);
3315 }
3316 }
3317 }
3318 }
3319
3320 fn new_variant_decoder(&'static self) -> VariantDecoder {
3321 self.variant.new_variant_decoder()
3322 }
3323
3324 /// Instantiates a new decoder for this encoding with BOM sniffing enabled.
3325 ///
3326 /// BOM sniffing may cause the returned decoder to morph into a decoder
3327 /// for UTF-8, UTF-16LE or UTF-16BE instead of this encoding. The BOM
3328 /// does not appear in the output.
3329 ///
3330 /// Available via the C wrapper.
3331 #[inline]
3332 pub fn new_decoder(&'static self) -> Decoder {
3333 Decoder::new(self, self.new_variant_decoder(), BomHandling::Sniff)
3334 }
3335
3336 /// Instantiates a new decoder for this encoding with BOM removal.
3337 ///
3338 /// If the input starts with bytes that are the BOM for this encoding,
3339 /// those bytes are removed. However, the decoder never morphs into a
3340 /// decoder for another encoding: A BOM for another encoding is treated as
3341 /// (potentially malformed) input to the decoding algorithm for this
3342 /// encoding.
3343 ///
3344 /// Available via the C wrapper.
3345 #[inline]
3346 pub fn new_decoder_with_bom_removal(&'static self) -> Decoder {
3347 Decoder::new(self, self.new_variant_decoder(), BomHandling::Remove)
3348 }
3349
3350 /// Instantiates a new decoder for this encoding with BOM handling disabled.
3351 ///
3352 /// If the input starts with bytes that look like a BOM, those bytes are
3353 /// not treated as a BOM. (Hence, the decoder never morphs into a decoder
3354 /// for another encoding.)
3355 ///
3356 /// _Note:_ If the caller has performed BOM sniffing on its own but has not
3357 /// removed the BOM, the caller should use `new_decoder_with_bom_removal()`
3358 /// instead of this method to cause the BOM to be removed.
3359 ///
3360 /// Available via the C wrapper.
3361 #[inline]
3362 pub fn new_decoder_without_bom_handling(&'static self) -> Decoder {
3363 Decoder::new(self, self.new_variant_decoder(), BomHandling::Off)
3364 }
3365
3366 /// Instantiates a new encoder for the [_output encoding_](Encoding::output_encoding)
3367 /// of this encoding.
3368 ///
3369 /// _Note:_ The output encoding of UTF-16BE, UTF-16LE, and replacement is UTF-8. There
3370 /// is no encoder for UTF-16BE, UTF-16LE, and replacement themselves.
3371 ///
3372 /// Available via the C wrapper.
3373 #[inline]
3374 pub fn new_encoder(&'static self) -> Encoder {
3375 let enc = self.output_encoding();
3376 enc.variant.new_encoder(enc)
3377 }
3378
3379 /// Validates UTF-8.
3380 ///
3381 /// Returns the index of the first byte that makes the input malformed as
3382 /// UTF-8 or the length of the slice if the slice is entirely valid.
3383 ///
3384 /// This is currently faster than the corresponding standard library
3385 /// functionality. If this implementation gets upstreamed to the standard
3386 /// library, this method may be removed in the future.
3387 ///
3388 /// Available via the C wrapper.
3389 pub fn utf8_valid_up_to(bytes: &[u8]) -> usize {
3390 utf8_valid_up_to(bytes)
3391 }
3392
3393 /// Validates ASCII.
3394 ///
3395 /// Returns the index of the first byte that makes the input malformed as
3396 /// ASCII or the length of the slice if the slice is entirely valid.
3397 ///
3398 /// Available via the C wrapper.
3399 pub fn ascii_valid_up_to(bytes: &[u8]) -> usize {
3400 ascii_valid_up_to(bytes)
3401 }
3402
3403 /// Validates ISO-2022-JP ASCII-state data.
3404 ///
3405 /// Returns the index of the first byte that makes the input not
3406 /// representable in the ASCII state of ISO-2022-JP or the length of the
3407 /// slice if the slice is entirely representable in the ASCII state of
3408 /// ISO-2022-JP.
3409 ///
3410 /// Available via the C wrapper.
3411 pub fn iso_2022_jp_ascii_valid_up_to(bytes: &[u8]) -> usize {
3412 iso_2022_jp_ascii_valid_up_to(bytes)
3413 }
3414 }
3415
3416 impl PartialEq for Encoding {
3417 #[inline]
3418 fn eq(&self, other: &Encoding) -> bool {
3419 (self as *const Encoding) == (other as *const Encoding)
3420 }
3421 }
3422
3423 impl Eq for Encoding {}
3424
3425 #[cfg(test)]
3426 impl PartialOrd for Encoding {
3427 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
3428 (self as *const Encoding as usize).partial_cmp(&(other as *const Encoding as usize))
3429 }
3430 }
3431
3432 #[cfg(test)]
3433 impl Ord for Encoding {
3434 fn cmp(&self, other: &Self) -> Ordering {
3435 (self as *const Encoding as usize).cmp(&(other as *const Encoding as usize))
3436 }
3437 }
3438
3439 impl Hash for Encoding {
3440 #[inline]
3441 fn hash<H: Hasher>(&self, state: &mut H) {
3442 (self as *const Encoding).hash(state);
3443 }
3444 }
3445
3446 impl core::fmt::Debug for Encoding {
3447 #[inline]
3448 fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
3449 write!(f, "Encoding {{ {} }}", self.name)
3450 }
3451 }
3452
3453 #[cfg(feature = "serde")]
3454 impl Serialize for Encoding {
3455 #[inline]
3456 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
3457 where
3458 S: Serializer,
3459 {
3460 serializer.serialize_str(self.name)
3461 }
3462 }
3463
3464 #[cfg(feature = "serde")]
3465 struct EncodingVisitor;
3466
3467 #[cfg(feature = "serde")]
3468 impl<'de> Visitor<'de> for EncodingVisitor {
3469 type Value = &'static Encoding;
3470
3471 fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result {
3472 formatter.write_str("a valid encoding label")
3473 }
3474
3475 fn visit_str<E>(self, value: &str) -> Result<&'static Encoding, E>
3476 where
3477 E: serde::de::Error,
3478 {
3479 if let Some(enc) = Encoding::for_label(value.as_bytes()) {
3480 Ok(enc)
3481 } else {
3482 Err(E::custom(alloc::format!(
3483 "invalid encoding label: {}",
3484 value
3485 )))
3486 }
3487 }
3488 }
3489
3490 #[cfg(feature = "serde")]
3491 impl<'de> Deserialize<'de> for &'static Encoding {
3492 fn deserialize<D>(deserializer: D) -> Result<&'static Encoding, D::Error>
3493 where
3494 D: Deserializer<'de>,
3495 {
3496 deserializer.deserialize_str(EncodingVisitor)
3497 }
3498 }
3499
3500 /// Tracks the life cycle of a decoder from BOM sniffing to conversion to end.
3501 #[derive(PartialEq, Debug, Copy, Clone)]
3502 enum DecoderLifeCycle {
3503 /// The decoder has seen no input yet.
3504 AtStart,
3505 /// The decoder has seen no input yet but expects UTF-8.
3506 AtUtf8Start,
3507 /// The decoder has seen no input yet but expects UTF-16BE.
3508 AtUtf16BeStart,
3509 /// The decoder has seen no input yet but expects UTF-16LE.
3510 AtUtf16LeStart,
3511 /// The decoder has seen EF.
3512 SeenUtf8First,
3513 /// The decoder has seen EF, BB.
3514 SeenUtf8Second,
3515 /// The decoder has seen FE.
3516 SeenUtf16BeFirst,
3517 /// The decoder has seen FF.
3518 SeenUtf16LeFirst,
3519 /// Saw EF, BB but not BF, there was a buffer boundary after BB and the
3520 /// underlying decoder reported EF as an error, so we need to remember to
3521 /// push BB before the next buffer.
3522 ConvertingWithPendingBB,
3523 /// No longer looking for a BOM and EOF not yet seen.
3524 Converting,
3525 /// EOF has been seen.
3526 Finished,
3527 }
3528
3529 /// Communicate the BOM handling mode.
3530 #[derive(Debug, Copy, Clone)]
3531 enum BomHandling {
3532 /// Don't handle the BOM
3533 Off,
3534 /// Sniff for UTF-8, UTF-16BE or UTF-16LE BOM
3535 Sniff,
3536 /// Remove the BOM only if it's the BOM for this encoding
3537 Remove,
3538 }
3539
3540 /// Result of a (potentially partial) decode or encode operation with
3541 /// replacement.
3542 #[must_use]
3543 #[derive(Debug, PartialEq, Eq)]
3544 pub enum CoderResult {
3545 /// The input was exhausted.
3546 ///
3547 /// If this result was returned from a call where `last` was `true`, the
3548 /// conversion process has completed. Otherwise, the caller should call a
3549 /// decode or encode method again with more input.
3550 InputEmpty,
3551
3552 /// The converter cannot produce another unit of output, because the output
3553 /// buffer does not have enough space left.
3554 ///
3555 /// The caller must provide more output space upon the next call and re-push
3556 /// the remaining input to the converter.
3557 OutputFull,
3558 }
3559
3560 /// Result of a (potentially partial) decode operation without replacement.
3561 #[must_use]
3562 #[derive(Debug, PartialEq, Eq)]
3563 pub enum DecoderResult {
3564 /// The input was exhausted.
3565 ///
3566 /// If this result was returned from a call where `last` was `true`, the
3567 /// decoding process has completed. Otherwise, the caller should call a
3568 /// decode method again with more input.
3569 InputEmpty,
3570
3571 /// The decoder cannot produce another unit of output, because the output
3572 /// buffer does not have enough space left.
3573 ///
3574 /// The caller must provide more output space upon the next call and re-push
3575 /// the remaining input to the decoder.
3576 OutputFull,
3577
3578 /// The decoder encountered a malformed byte sequence.
3579 ///
3580 /// The caller must either treat this as a fatal error or must append one
3581 /// REPLACEMENT CHARACTER (U+FFFD) to the output and then re-push the
3582 /// the remaining input to the decoder.
3583 ///
3584 /// The first wrapped integer indicates the length of the malformed byte
3585 /// sequence. The second wrapped integer indicates the number of bytes
3586 /// that were consumed after the malformed sequence. If the second
3587 /// integer is zero, the last byte that was consumed is the last byte of
3588 /// the malformed sequence. Note that the malformed bytes may have been part
3589 /// of an earlier input buffer.
3590 ///
3591 /// The first wrapped integer can have values 1, 2, 3 or 4. The second
3592 /// wrapped integer can have values 0, 1, 2 or 3. The worst-case sum
3593 /// of the two is 6, which happens with ISO-2022-JP.
3594 Malformed(u8, u8), // u8 instead of usize to avoid useless bloat
3595 }
3596
3597 /// A converter that decodes a byte stream into Unicode according to a
3598 /// character encoding in a streaming (incremental) manner.
3599 ///
3600 /// The various `decode_*` methods take an input buffer (`src`) and an output
3601 /// buffer `dst` both of which are caller-allocated. There are variants for
3602 /// both UTF-8 and UTF-16 output buffers.
3603 ///
3604 /// A `decode_*` method decodes bytes from `src` into Unicode characters stored
3605 /// into `dst` until one of the following three things happens:
3606 ///
3607 /// 1. A malformed byte sequence is encountered (`*_without_replacement`
3608 /// variants only).
3609 ///
3610 /// 2. The output buffer has been filled so near capacity that the decoder
3611 /// cannot be sure that processing an additional byte of input wouldn't
3612 /// cause so much output that the output buffer would overflow.
3613 ///
3614 /// 3. All the input bytes have been processed.
3615 ///
3616 /// The `decode_*` method then returns tuple of a status indicating which one
3617 /// of the three reasons to return happened, how many input bytes were read,
3618 /// how many output code units (`u8` when decoding into UTF-8 and `u16`
3619 /// when decoding to UTF-16) were written (except when decoding into `String`,
3620 /// whose length change indicates this), and in the case of the
3621 /// variants performing replacement, a boolean indicating whether an error was
3622 /// replaced with the REPLACEMENT CHARACTER during the call.
3623 ///
3624 /// The number of bytes "written" is what's logically written. Garbage may be
3625 /// written in the output buffer beyond the point logically written to.
3626 /// Therefore, if you wish to decode into an `&mut str`, you should use the
3627 /// methods that take an `&mut str` argument instead of the ones that take an
3628 /// `&mut [u8]` argument. The former take care of overwriting the trailing
3629 /// garbage to ensure the UTF-8 validity of the `&mut str` as a whole, but the
3630 /// latter don't.
3631 ///
3632 /// In the case of the `*_without_replacement` variants, the status is a
3633 /// [`DecoderResult`][1] enumeration (possibilities `Malformed`, `OutputFull` and
3634 /// `InputEmpty` corresponding to the three cases listed above).
3635 ///
3636 /// In the case of methods whose name does not end with
3637 /// `*_without_replacement`, malformed sequences are automatically replaced
3638 /// with the REPLACEMENT CHARACTER and errors do not cause the methods to
3639 /// return early.
3640 ///
3641 /// When decoding to UTF-8, the output buffer must have at least 4 bytes of
3642 /// space. When decoding to UTF-16, the output buffer must have at least two
3643 /// UTF-16 code units (`u16`) of space.
3644 ///
3645 /// When decoding to UTF-8 without replacement, the methods are guaranteed
3646 /// not to return indicating that more output space is needed if the length
3647 /// of the output buffer is at least the length returned by
3648 /// [`max_utf8_buffer_length_without_replacement()`][2]. When decoding to UTF-8
3649 /// with replacement, the length of the output buffer that guarantees the
3650 /// methods not to return indicating that more output space is needed is given
3651 /// by [`max_utf8_buffer_length()`][3]. When decoding to UTF-16 with
3652 /// or without replacement, the length of the output buffer that guarantees
3653 /// the methods not to return indicating that more output space is needed is
3654 /// given by [`max_utf16_buffer_length()`][4].
3655 ///
3656 /// The output written into `dst` is guaranteed to be valid UTF-8 or UTF-16,
3657 /// and the output after each `decode_*` call is guaranteed to consist of
3658 /// complete characters. (I.e. the code unit sequence for the last character is
3659 /// guaranteed not to be split across output buffers.)
3660 ///
3661 /// The boolean argument `last` indicates that the end of the stream is reached
3662 /// when all the bytes in `src` have been consumed.
3663 ///
3664 /// A `Decoder` object can be used to incrementally decode a byte stream.
3665 ///
3666 /// During the processing of a single stream, the caller must call `decode_*`
3667 /// zero or more times with `last` set to `false` and then call `decode_*` at
3668 /// least once with `last` set to `true`. If `decode_*` returns `InputEmpty`,
3669 /// the processing of the stream has ended. Otherwise, the caller must call
3670 /// `decode_*` again with `last` set to `true` (or treat a `Malformed` result as
3671 /// a fatal error).
3672 ///
3673 /// Once the stream has ended, the `Decoder` object must not be used anymore.
3674 /// That is, you need to create another one to process another stream.
3675 ///
3676 /// When the decoder returns `OutputFull` or the decoder returns `Malformed` and
3677 /// the caller does not wish to treat it as a fatal error, the input buffer
3678 /// `src` may not have been completely consumed. In that case, the caller must
3679 /// pass the unconsumed contents of `src` to `decode_*` again upon the next
3680 /// call.
3681 ///
3682 /// [1]: enum.DecoderResult.html
3683 /// [2]: #method.max_utf8_buffer_length_without_replacement
3684 /// [3]: #method.max_utf8_buffer_length
3685 /// [4]: #method.max_utf16_buffer_length
3686 ///
3687 /// # Infinite loops
3688 ///
3689 /// When converting with a fixed-size output buffer whose size is too small to
3690 /// accommodate one character or (when applicable) one numeric character
3691 /// reference of output, an infinite loop ensues. When converting with a
3692 /// fixed-size output buffer, it generally makes sense to make the buffer
3693 /// fairly large (e.g. couple of kilobytes).
3694 pub struct Decoder {
3695 encoding: &'static Encoding,
3696 variant: VariantDecoder,
3697 life_cycle: DecoderLifeCycle,
3698 }
3699
3700 impl Decoder {
3701 fn new(enc: &'static Encoding, decoder: VariantDecoder, sniffing: BomHandling) -> Decoder {
3702 Decoder {
3703 encoding: enc,
3704 variant: decoder,
3705 life_cycle: match sniffing {
3706 BomHandling::Off => DecoderLifeCycle::Converting,
3707 BomHandling::Sniff => DecoderLifeCycle::AtStart,
3708 BomHandling::Remove => {
3709 if enc == UTF_8 {
3710 DecoderLifeCycle::AtUtf8Start
3711 } else if enc == UTF_16BE {
3712 DecoderLifeCycle::AtUtf16BeStart
3713 } else if enc == UTF_16LE {
3714 DecoderLifeCycle::AtUtf16LeStart
3715 } else {
3716 DecoderLifeCycle::Converting
3717 }
3718 }
3719 },
3720 }
3721 }
3722
3723 /// The `Encoding` this `Decoder` is for.
3724 ///
3725 /// BOM sniffing can change the return value of this method during the life
3726 /// of the decoder.
3727 ///
3728 /// Available via the C wrapper.
3729 #[inline]
3730 pub fn encoding(&self) -> &'static Encoding {
3731 self.encoding
3732 }
3733
3734 /// Query the worst-case UTF-8 output size _with replacement_.
3735 ///
3736 /// Returns the size of the output buffer in UTF-8 code units (`u8`)
3737 /// that will not overflow given the current state of the decoder and
3738 /// `byte_length` number of additional input bytes when decoding with
3739 /// errors handled by outputting a REPLACEMENT CHARACTER for each malformed
3740 /// sequence or `None` if `usize` would overflow.
3741 ///
3742 /// Available via the C wrapper.
3743 pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> {
3744 // Need to consider a) the decoder morphing due to the BOM and b) a partial
3745 // BOM getting pushed to the underlying decoder.
3746 match self.life_cycle {
3747 DecoderLifeCycle::Converting
3748 | DecoderLifeCycle::AtUtf8Start
3749 | DecoderLifeCycle::AtUtf16LeStart
3750 | DecoderLifeCycle::AtUtf16BeStart => {
3751 return self.variant.max_utf8_buffer_length(byte_length);
3752 }
3753 DecoderLifeCycle::AtStart => {
3754 if let Some(utf8_bom) = checked_add(3, byte_length.checked_mul(3)) {
3755 if let Some(utf16_bom) = checked_add(
3756 1,
3757 checked_mul(3, checked_div(byte_length.checked_add(1), 2)),
3758 ) {
3759 let utf_bom = core::cmp::max(utf8_bom, utf16_bom);
3760 let encoding = self.encoding();
3761 if encoding == UTF_8 || encoding == UTF_16LE || encoding == UTF_16BE {
3762 // No need to consider the internal state of the underlying decoder,
3763 // because it is at start, because no data has reached it yet.
3764 return Some(utf_bom);
3765 } else if let Some(non_bom) =
3766 self.variant.max_utf8_buffer_length(byte_length)
3767 {
3768 return Some(core::cmp::max(utf_bom, non_bom));
3769 }
3770 }
3771 }
3772 }
3773 DecoderLifeCycle::SeenUtf8First | DecoderLifeCycle::SeenUtf8Second => {
3774 // Add two bytes even when only one byte has been seen,
3775 // because the one byte can become a lead byte in multibyte
3776 // decoders, but only after the decoder has been queried
3777 // for max length, so the decoder's own logic for adding
3778 // one for a pending lead cannot work.
3779 if let Some(sum) = byte_length.checked_add(2) {
3780 if let Some(utf8_bom) = checked_add(3, sum.checked_mul(3)) {
3781 if self.encoding() == UTF_8 {
3782 // No need to consider the internal state of the underlying decoder,
3783 // because it is at start, because no data has reached it yet.
3784 return Some(utf8_bom);
3785 } else if let Some(non_bom) = self.variant.max_utf8_buffer_length(sum) {
3786 return Some(core::cmp::max(utf8_bom, non_bom));
3787 }
3788 }
3789 }
3790 }
3791 DecoderLifeCycle::ConvertingWithPendingBB => {
3792 if let Some(sum) = byte_length.checked_add(2) {
3793 return self.variant.max_utf8_buffer_length(sum);
3794 }
3795 }
3796 DecoderLifeCycle::SeenUtf16LeFirst | DecoderLifeCycle::SeenUtf16BeFirst => {
3797 // Add two bytes even when only one byte has been seen,
3798 // because the one byte can become a lead byte in multibyte
3799 // decoders, but only after the decoder has been queried
3800 // for max length, so the decoder's own logic for adding
3801 // one for a pending lead cannot work.
3802 if let Some(sum) = byte_length.checked_add(2) {
3803 if let Some(utf16_bom) =
3804 checked_add(1, checked_mul(3, checked_div(sum.checked_add(1), 2)))
3805 {
3806 let encoding = self.encoding();
3807 if encoding == UTF_16LE || encoding == UTF_16BE {
3808 // No need to consider the internal state of the underlying decoder,
3809 // because it is at start, because no data has reached it yet.
3810 return Some(utf16_bom);
3811 } else if let Some(non_bom) = self.variant.max_utf8_buffer_length(sum) {
3812 return Some(core::cmp::max(utf16_bom, non_bom));
3813 }
3814 }
3815 }
3816 }
3817 DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
3818 }
3819 None
3820 }
3821
3822 /// Query the worst-case UTF-8 output size _without replacement_.
3823 ///
3824 /// Returns the size of the output buffer in UTF-8 code units (`u8`)
3825 /// that will not overflow given the current state of the decoder and
3826 /// `byte_length` number of additional input bytes when decoding without
3827 /// replacement error handling or `None` if `usize` would overflow.
3828 ///
3829 /// Note that this value may be too small for the `_with_replacement` case.
3830 /// Use `max_utf8_buffer_length()` for that case.
3831 ///
3832 /// Available via the C wrapper.
3833 pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> {
3834 // Need to consider a) the decoder morphing due to the BOM and b) a partial
3835 // BOM getting pushed to the underlying decoder.
3836 match self.life_cycle {
3837 DecoderLifeCycle::Converting
3838 | DecoderLifeCycle::AtUtf8Start
3839 | DecoderLifeCycle::AtUtf16LeStart
3840 | DecoderLifeCycle::AtUtf16BeStart => {
3841 return self
3842 .variant
3843 .max_utf8_buffer_length_without_replacement(byte_length);
3844 }
3845 DecoderLifeCycle::AtStart => {
3846 if let Some(utf8_bom) = byte_length.checked_add(3) {
3847 if let Some(utf16_bom) = checked_add(
3848 1,
3849 checked_mul(3, checked_div(byte_length.checked_add(1), 2)),
3850 ) {
3851 let utf_bom = core::cmp::max(utf8_bom, utf16_bom);
3852 let encoding = self.encoding();
3853 if encoding == UTF_8 || encoding == UTF_16LE || encoding == UTF_16BE {
3854 // No need to consider the internal state of the underlying decoder,
3855 // because it is at start, because no data has reached it yet.
3856 return Some(utf_bom);
3857 } else if let Some(non_bom) = self
3858 .variant
3859 .max_utf8_buffer_length_without_replacement(byte_length)
3860 {
3861 return Some(core::cmp::max(utf_bom, non_bom));
3862 }
3863 }
3864 }
3865 }
3866 DecoderLifeCycle::SeenUtf8First | DecoderLifeCycle::SeenUtf8Second => {
3867 // Add two bytes even when only one byte has been seen,
3868 // because the one byte can become a lead byte in multibyte
3869 // decoders, but only after the decoder has been queried
3870 // for max length, so the decoder's own logic for adding
3871 // one for a pending lead cannot work.
3872 if let Some(sum) = byte_length.checked_add(2) {
3873 if let Some(utf8_bom) = sum.checked_add(3) {
3874 if self.encoding() == UTF_8 {
3875 // No need to consider the internal state of the underlying decoder,
3876 // because it is at start, because no data has reached it yet.
3877 return Some(utf8_bom);
3878 } else if let Some(non_bom) =
3879 self.variant.max_utf8_buffer_length_without_replacement(sum)
3880 {
3881 return Some(core::cmp::max(utf8_bom, non_bom));
3882 }
3883 }
3884 }
3885 }
3886 DecoderLifeCycle::ConvertingWithPendingBB => {
3887 if let Some(sum) = byte_length.checked_add(2) {
3888 return self.variant.max_utf8_buffer_length_without_replacement(sum);
3889 }
3890 }
3891 DecoderLifeCycle::SeenUtf16LeFirst | DecoderLifeCycle::SeenUtf16BeFirst => {
3892 // Add two bytes even when only one byte has been seen,
3893 // because the one byte can become a lead byte in multibyte
3894 // decoders, but only after the decoder has been queried
3895 // for max length, so the decoder's own logic for adding
3896 // one for a pending lead cannot work.
3897 if let Some(sum) = byte_length.checked_add(2) {
3898 if let Some(utf16_bom) =
3899 checked_add(1, checked_mul(3, checked_div(sum.checked_add(1), 2)))
3900 {
3901 let encoding = self.encoding();
3902 if encoding == UTF_16LE || encoding == UTF_16BE {
3903 // No need to consider the internal state of the underlying decoder,
3904 // because it is at start, because no data has reached it yet.
3905 return Some(utf16_bom);
3906 } else if let Some(non_bom) =
3907 self.variant.max_utf8_buffer_length_without_replacement(sum)
3908 {
3909 return Some(core::cmp::max(utf16_bom, non_bom));
3910 }
3911 }
3912 }
3913 }
3914 DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
3915 }
3916 None
3917 }
3918
3919 /// Incrementally decode a byte stream into UTF-8 with malformed sequences
3920 /// replaced with the REPLACEMENT CHARACTER.
3921 ///
3922 /// See the documentation of the struct for documentation for `decode_*`
3923 /// methods collectively.
3924 ///
3925 /// Available via the C wrapper.
3926 pub fn decode_to_utf8(
3927 &mut self,
3928 src: &[u8],
3929 dst: &mut [u8],
3930 last: bool,
3931 ) -> (CoderResult, usize, usize, bool) {
3932 let mut had_errors = false;
3933 let mut total_read = 0usize;
3934 let mut total_written = 0usize;
3935 loop {
3936 let (result, read, written) = self.decode_to_utf8_without_replacement(
3937 &src[total_read..],
3938 &mut dst[total_written..],
3939 last,
3940 );
3941 total_read += read;
3942 total_written += written;
3943 match result {
3944 DecoderResult::InputEmpty => {
3945 return (
3946 CoderResult::InputEmpty,
3947 total_read,
3948 total_written,
3949 had_errors,
3950 );
3951 }
3952 DecoderResult::OutputFull => {
3953 return (
3954 CoderResult::OutputFull,
3955 total_read,
3956 total_written,
3957 had_errors,
3958 );
3959 }
3960 DecoderResult::Malformed(_, _) => {
3961 had_errors = true;
3962 // There should always be space for the U+FFFD, because
3963 // otherwise we'd have gotten OutputFull already.
3964 // XXX: is the above comment actually true for UTF-8 itself?
3965 // TODO: Consider having fewer bound checks here.
3966 dst[total_written] = 0xEFu8;
3967 total_written += 1;
3968 dst[total_written] = 0xBFu8;
3969 total_written += 1;
3970 dst[total_written] = 0xBDu8;
3971 total_written += 1;
3972 }
3973 }
3974 }
3975 }
3976
3977 /// Incrementally decode a byte stream into UTF-8 with malformed sequences
3978 /// replaced with the REPLACEMENT CHARACTER with type system signaling
3979 /// of UTF-8 validity.
3980 ///
3981 /// This methods calls `decode_to_utf8` and then zeroes
3982 /// out up to three bytes that aren't logically part of the write in order
3983 /// to retain the UTF-8 validity even for the unwritten part of the buffer.
3984 ///
3985 /// See the documentation of the struct for documentation for `decode_*`
3986 /// methods collectively.
3987 ///
3988 /// Available to Rust only.
3989 pub fn decode_to_str(
3990 &mut self,
3991 src: &[u8],
3992 dst: &mut str,
3993 last: bool,
3994 ) -> (CoderResult, usize, usize, bool) {
3995 let bytes: &mut [u8] = unsafe { dst.as_bytes_mut() };
3996 let (result, read, written, replaced) = self.decode_to_utf8(src, bytes, last);
3997 let len = bytes.len();
3998 let mut trail = written;
3999 // Non-UTF-8 ASCII-compatible decoders may write up to `MAX_STRIDE_SIZE`
4000 // bytes of trailing garbage. No need to optimize non-ASCII-compatible
4001 // encodings to avoid overwriting here.
4002 if self.encoding != UTF_8 {
4003 let max = core::cmp::min(len, trail + ascii::MAX_STRIDE_SIZE);
4004 while trail < max {
4005 bytes[trail] = 0;
4006 trail += 1;
4007 }
4008 }
4009 while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
4010 bytes[trail] = 0;
4011 trail += 1;
4012 }
4013 (result, read, written, replaced)
4014 }
4015
4016 /// Incrementally decode a byte stream into UTF-8 with malformed sequences
4017 /// replaced with the REPLACEMENT CHARACTER using a `String` receiver.
4018 ///
4019 /// Like the others, this method follows the logic that the output buffer is
4020 /// caller-allocated. This method treats the capacity of the `String` as
4021 /// the output limit. That is, this method guarantees not to cause a
4022 /// reallocation of the backing buffer of `String`.
4023 ///
4024 /// The return value is a tuple that contains the `DecoderResult`, the
4025 /// number of bytes read and a boolean indicating whether replacements
4026 /// were done. The number of bytes written is signaled via the length of
4027 /// the `String` changing.
4028 ///
4029 /// See the documentation of the struct for documentation for `decode_*`
4030 /// methods collectively.
4031 ///
4032 /// Available to Rust only and only with the `alloc` feature enabled (enabled
4033 /// by default).
4034 #[cfg(feature = "alloc")]
4035 pub fn decode_to_string(
4036 &mut self,
4037 src: &[u8],
4038 dst: &mut String,
4039 last: bool,
4040 ) -> (CoderResult, usize, bool) {
4041 unsafe {
4042 let vec = dst.as_mut_vec();
4043 let old_len = vec.len();
4044 let capacity = vec.capacity();
4045 vec.set_len(capacity);
4046 let (result, read, written, replaced) =
4047 self.decode_to_utf8(src, &mut vec[old_len..], last);
4048 vec.set_len(old_len + written);
4049 (result, read, replaced)
4050 }
4051 }
4052
4053 public_decode_function!(/// Incrementally decode a byte stream into UTF-8
4054 /// _without replacement_.
4055 ///
4056 /// See the documentation of the struct for
4057 /// documentation for `decode_*` methods
4058 /// collectively.
4059 ///
4060 /// Available via the C wrapper.
4061 ,
4062 decode_to_utf8_without_replacement,
4063 decode_to_utf8_raw,
4064 decode_to_utf8_checking_end,
4065 decode_to_utf8_after_one_potential_bom_byte,
4066 decode_to_utf8_after_two_potential_bom_bytes,
4067 decode_to_utf8_checking_end_with_offset,
4068 u8);
4069
4070 /// Incrementally decode a byte stream into UTF-8 with type system signaling
4071 /// of UTF-8 validity.
4072 ///
4073 /// This methods calls `decode_to_utf8` and then zeroes out up to three
4074 /// bytes that aren't logically part of the write in order to retain the
4075 /// UTF-8 validity even for the unwritten part of the buffer.
4076 ///
4077 /// See the documentation of the struct for documentation for `decode_*`
4078 /// methods collectively.
4079 ///
4080 /// Available to Rust only.
4081 pub fn decode_to_str_without_replacement(
4082 &mut self,
4083 src: &[u8],
4084 dst: &mut str,
4085 last: bool,
4086 ) -> (DecoderResult, usize, usize) {
4087 let bytes: &mut [u8] = unsafe { dst.as_bytes_mut() };
4088 let (result, read, written) = self.decode_to_utf8_without_replacement(src, bytes, last);
4089 let len = bytes.len();
4090 let mut trail = written;
4091 // Non-UTF-8 ASCII-compatible decoders may write up to `MAX_STRIDE_SIZE`
4092 // bytes of trailing garbage. No need to optimize non-ASCII-compatible
4093 // encodings to avoid overwriting here.
4094 if self.encoding != UTF_8 {
4095 let max = core::cmp::min(len, trail + ascii::MAX_STRIDE_SIZE);
4096 while trail < max {
4097 bytes[trail] = 0;
4098 trail += 1;
4099 }
4100 }
4101 while trail < len && ((bytes[trail] & 0xC0) == 0x80) {
4102 bytes[trail] = 0;
4103 trail += 1;
4104 }
4105 (result, read, written)
4106 }
4107
4108 /// Incrementally decode a byte stream into UTF-8 using a `String` receiver.
4109 ///
4110 /// Like the others, this method follows the logic that the output buffer is
4111 /// caller-allocated. This method treats the capacity of the `String` as
4112 /// the output limit. That is, this method guarantees not to cause a
4113 /// reallocation of the backing buffer of `String`.
4114 ///
4115 /// The return value is a pair that contains the `DecoderResult` and the
4116 /// number of bytes read. The number of bytes written is signaled via
4117 /// the length of the `String` changing.
4118 ///
4119 /// See the documentation of the struct for documentation for `decode_*`
4120 /// methods collectively.
4121 ///
4122 /// Available to Rust only and only with the `alloc` feature enabled (enabled
4123 /// by default).
4124 #[cfg(feature = "alloc")]
4125 pub fn decode_to_string_without_replacement(
4126 &mut self,
4127 src: &[u8],
4128 dst: &mut String,
4129 last: bool,
4130 ) -> (DecoderResult, usize) {
4131 unsafe {
4132 let vec = dst.as_mut_vec();
4133 let old_len = vec.len();
4134 let capacity = vec.capacity();
4135 vec.set_len(capacity);
4136 let (result, read, written) =
4137 self.decode_to_utf8_without_replacement(src, &mut vec[old_len..], last);
4138 vec.set_len(old_len + written);
4139 (result, read)
4140 }
4141 }
4142
4143 /// Query the worst-case UTF-16 output size (with or without replacement).
4144 ///
4145 /// Returns the size of the output buffer in UTF-16 code units (`u16`)
4146 /// that will not overflow given the current state of the decoder and
4147 /// `byte_length` number of additional input bytes or `None` if `usize`
4148 /// would overflow.
4149 ///
4150 /// Since the REPLACEMENT CHARACTER fits into one UTF-16 code unit, the
4151 /// return value of this method applies also in the
4152 /// `_without_replacement` case.
4153 ///
4154 /// Available via the C wrapper.
4155 pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> {
4156 // Need to consider a) the decoder morphing due to the BOM and b) a partial
4157 // BOM getting pushed to the underlying decoder.
4158 match self.life_cycle {
4159 DecoderLifeCycle::Converting
4160 | DecoderLifeCycle::AtUtf8Start
4161 | DecoderLifeCycle::AtUtf16LeStart
4162 | DecoderLifeCycle::AtUtf16BeStart => {
4163 return self.variant.max_utf16_buffer_length(byte_length);
4164 }
4165 DecoderLifeCycle::AtStart => {
4166 if let Some(utf8_bom) = byte_length.checked_add(1) {
4167 if let Some(utf16_bom) =
4168 checked_add(1, checked_div(byte_length.checked_add(1), 2))
4169 {
4170 let utf_bom = core::cmp::max(utf8_bom, utf16_bom);
4171 let encoding = self.encoding();
4172 if encoding == UTF_8 || encoding == UTF_16LE || encoding == UTF_16BE {
4173 // No need to consider the internal state of the underlying decoder,
4174 // because it is at start, because no data has reached it yet.
4175 return Some(utf_bom);
4176 } else if let Some(non_bom) =
4177 self.variant.max_utf16_buffer_length(byte_length)
4178 {
4179 return Some(core::cmp::max(utf_bom, non_bom));
4180 }
4181 }
4182 }
4183 }
4184 DecoderLifeCycle::SeenUtf8First | DecoderLifeCycle::SeenUtf8Second => {
4185 // Add two bytes even when only one byte has been seen,
4186 // because the one byte can become a lead byte in multibyte
4187 // decoders, but only after the decoder has been queried
4188 // for max length, so the decoder's own logic for adding
4189 // one for a pending lead cannot work.
4190 if let Some(sum) = byte_length.checked_add(2) {
4191 if let Some(utf8_bom) = sum.checked_add(1) {
4192 if self.encoding() == UTF_8 {
4193 // No need to consider the internal state of the underlying decoder,
4194 // because it is at start, because no data has reached it yet.
4195 return Some(utf8_bom);
4196 } else if let Some(non_bom) = self.variant.max_utf16_buffer_length(sum) {
4197 return Some(core::cmp::max(utf8_bom, non_bom));
4198 }
4199 }
4200 }
4201 }
4202 DecoderLifeCycle::ConvertingWithPendingBB => {
4203 if let Some(sum) = byte_length.checked_add(2) {
4204 return self.variant.max_utf16_buffer_length(sum);
4205 }
4206 }
4207 DecoderLifeCycle::SeenUtf16LeFirst | DecoderLifeCycle::SeenUtf16BeFirst => {
4208 // Add two bytes even when only one byte has been seen,
4209 // because the one byte can become a lead byte in multibyte
4210 // decoders, but only after the decoder has been queried
4211 // for max length, so the decoder's own logic for adding
4212 // one for a pending lead cannot work.
4213 if let Some(sum) = byte_length.checked_add(2) {
4214 if let Some(utf16_bom) = checked_add(1, checked_div(sum.checked_add(1), 2)) {
4215 let encoding = self.encoding();
4216 if encoding == UTF_16LE || encoding == UTF_16BE {
4217 // No need to consider the internal state of the underlying decoder,
4218 // because it is at start, because no data has reached it yet.
4219 return Some(utf16_bom);
4220 } else if let Some(non_bom) = self.variant.max_utf16_buffer_length(sum) {
4221 return Some(core::cmp::max(utf16_bom, non_bom));
4222 }
4223 }
4224 }
4225 }
4226 DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
4227 }
4228 None
4229 }
4230
4231 /// Incrementally decode a byte stream into UTF-16 with malformed sequences
4232 /// replaced with the REPLACEMENT CHARACTER.
4233 ///
4234 /// See the documentation of the struct for documentation for `decode_*`
4235 /// methods collectively.
4236 ///
4237 /// Available via the C wrapper.
4238 pub fn decode_to_utf16(
4239 &mut self,
4240 src: &[u8],
4241 dst: &mut [u16],
4242 last: bool,
4243 ) -> (CoderResult, usize, usize, bool) {
4244 let mut had_errors = false;
4245 let mut total_read = 0usize;
4246 let mut total_written = 0usize;
4247 loop {
4248 let (result, read, written) = self.decode_to_utf16_without_replacement(
4249 &src[total_read..],
4250 &mut dst[total_written..],
4251 last,
4252 );
4253 total_read += read;
4254 total_written += written;
4255 match result {
4256 DecoderResult::InputEmpty => {
4257 return (
4258 CoderResult::InputEmpty,
4259 total_read,
4260 total_written,
4261 had_errors,
4262 );
4263 }
4264 DecoderResult::OutputFull => {
4265 return (
4266 CoderResult::OutputFull,
4267 total_read,
4268 total_written,
4269 had_errors,
4270 );
4271 }
4272 DecoderResult::Malformed(_, _) => {
4273 had_errors = true;
4274 // There should always be space for the U+FFFD, because
4275 // otherwise we'd have gotten OutputFull already.
4276 dst[total_written] = 0xFFFD;
4277 total_written += 1;
4278 }
4279 }
4280 }
4281 }
4282
4283 public_decode_function!(/// Incrementally decode a byte stream into UTF-16
4284 /// _without replacement_.
4285 ///
4286 /// See the documentation of the struct for
4287 /// documentation for `decode_*` methods
4288 /// collectively.
4289 ///
4290 /// Available via the C wrapper.
4291 ,
4292 decode_to_utf16_without_replacement,
4293 decode_to_utf16_raw,
4294 decode_to_utf16_checking_end,
4295 decode_to_utf16_after_one_potential_bom_byte,
4296 decode_to_utf16_after_two_potential_bom_bytes,
4297 decode_to_utf16_checking_end_with_offset,
4298 u16);
4299
4300 /// Checks for compatibility with storing Unicode scalar values as unsigned
4301 /// bytes taking into account the state of the decoder.
4302 ///
4303 /// Returns `None` if the decoder is not in a neutral state, including waiting
4304 /// for the BOM, or if the encoding is never Latin1-byte-compatible.
4305 ///
4306 /// Otherwise returns the index of the first byte whose unsigned value doesn't
4307 /// directly correspond to the decoded Unicode scalar value, or the length
4308 /// of the input if all bytes in the input decode directly to scalar values
4309 /// corresponding to the unsigned byte values.
4310 ///
4311 /// Does not change the state of the decoder.
4312 ///
4313 /// Do not use this unless you are supporting SpiderMonkey/V8-style string
4314 /// storage optimizations.
4315 ///
4316 /// Available via the C wrapper.
4317 pub fn latin1_byte_compatible_up_to(&self, bytes: &[u8]) -> Option<usize> {
4318 match self.life_cycle {
4319 DecoderLifeCycle::Converting => {
4320 return self.variant.latin1_byte_compatible_up_to(bytes);
4321 }
4322 DecoderLifeCycle::Finished => panic!("Must not use a decoder that has finished."),
4323 _ => None,
4324 }
4325 }
4326 }
4327
4328 /// Result of a (potentially partial) encode operation without replacement.
4329 #[must_use]
4330 #[derive(Debug, PartialEq, Eq)]
4331 pub enum EncoderResult {
4332 /// The input was exhausted.
4333 ///
4334 /// If this result was returned from a call where `last` was `true`, the
4335 /// decoding process has completed. Otherwise, the caller should call a
4336 /// decode method again with more input.
4337 InputEmpty,
4338
4339 /// The encoder cannot produce another unit of output, because the output
4340 /// buffer does not have enough space left.
4341 ///
4342 /// The caller must provide more output space upon the next call and re-push
4343 /// the remaining input to the decoder.
4344 OutputFull,
4345
4346 /// The encoder encountered an unmappable character.
4347 ///
4348 /// The caller must either treat this as a fatal error or must append
4349 /// a placeholder to the output and then re-push the remaining input to the
4350 /// encoder.
4351 Unmappable(char),
4352 }
4353
4354 impl EncoderResult {
4355 fn unmappable_from_bmp(bmp: u16) -> EncoderResult {
4356 EncoderResult::Unmappable(::core::char::from_u32(u32::from(bmp)).unwrap())
4357 }
4358 }
4359
4360 /// A converter that encodes a Unicode stream into bytes according to a
4361 /// character encoding in a streaming (incremental) manner.
4362 ///
4363 /// The various `encode_*` methods take an input buffer (`src`) and an output
4364 /// buffer `dst` both of which are caller-allocated. There are variants for
4365 /// both UTF-8 and UTF-16 input buffers.
4366 ///
4367 /// An `encode_*` method encode characters from `src` into bytes characters
4368 /// stored into `dst` until one of the following three things happens:
4369 ///
4370 /// 1. An unmappable character is encountered (`*_without_replacement` variants
4371 /// only).
4372 ///
4373 /// 2. The output buffer has been filled so near capacity that the decoder
4374 /// cannot be sure that processing an additional character of input wouldn't
4375 /// cause so much output that the output buffer would overflow.
4376 ///
4377 /// 3. All the input characters have been processed.
4378 ///
4379 /// The `encode_*` method then returns tuple of a status indicating which one
4380 /// of the three reasons to return happened, how many input code units (`u8`
4381 /// when encoding from UTF-8 and `u16` when encoding from UTF-16) were read,
4382 /// how many output bytes were written (except when encoding into `Vec<u8>`,
4383 /// whose length change indicates this), and in the case of the variants that
4384 /// perform replacement, a boolean indicating whether an unmappable
4385 /// character was replaced with a numeric character reference during the call.
4386 ///
4387 /// The number of bytes "written" is what's logically written. Garbage may be
4388 /// written in the output buffer beyond the point logically written to.
4389 ///
4390 /// In the case of the methods whose name ends with
4391 /// `*_without_replacement`, the status is an [`EncoderResult`][1] enumeration
4392 /// (possibilities `Unmappable`, `OutputFull` and `InputEmpty` corresponding to
4393 /// the three cases listed above).
4394 ///
4395 /// In the case of methods whose name does not end with
4396 /// `*_without_replacement`, unmappable characters are automatically replaced
4397 /// with the corresponding numeric character references and unmappable
4398 /// characters do not cause the methods to return early.
4399 ///
4400 /// When encoding from UTF-8 without replacement, the methods are guaranteed
4401 /// not to return indicating that more output space is needed if the length
4402 /// of the output buffer is at least the length returned by
4403 /// [`max_buffer_length_from_utf8_without_replacement()`][2]. When encoding from
4404 /// UTF-8 with replacement, the length of the output buffer that guarantees the
4405 /// methods not to return indicating that more output space is needed in the
4406 /// absence of unmappable characters is given by
4407 /// [`max_buffer_length_from_utf8_if_no_unmappables()`][3]. When encoding from
4408 /// UTF-16 without replacement, the methods are guaranteed not to return
4409 /// indicating that more output space is needed if the length of the output
4410 /// buffer is at least the length returned by
4411 /// [`max_buffer_length_from_utf16_without_replacement()`][4]. When encoding
4412 /// from UTF-16 with replacement, the the length of the output buffer that
4413 /// guarantees the methods not to return indicating that more output space is
4414 /// needed in the absence of unmappable characters is given by
4415 /// [`max_buffer_length_from_utf16_if_no_unmappables()`][5].
4416 /// When encoding with replacement, applications are not expected to size the
4417 /// buffer for the worst case ahead of time but to resize the buffer if there
4418 /// are unmappable characters. This is why max length queries are only available
4419 /// for the case where there are no unmappable characters.
4420 ///
4421 /// When encoding from UTF-8, each `src` buffer _must_ be valid UTF-8. (When
4422 /// calling from Rust, the type system takes care of this.) When encoding from
4423 /// UTF-16, unpaired surrogates in the input are treated as U+FFFD REPLACEMENT
4424 /// CHARACTERS. Therefore, in order for astral characters not to turn into a
4425 /// pair of REPLACEMENT CHARACTERS, the caller must ensure that surrogate pairs
4426 /// are not split across input buffer boundaries.
4427 ///
4428 /// After an `encode_*` call returns, the output produced so far, taken as a
4429 /// whole from the start of the stream, is guaranteed to consist of a valid
4430 /// byte sequence in the target encoding. (I.e. the code unit sequence for a
4431 /// character is guaranteed not to be split across output buffers. However, due
4432 /// to the stateful nature of ISO-2022-JP, the stream needs to be considered
4433 /// from the start for it to be valid. For other encodings, the validity holds
4434 /// on a per-output buffer basis.)
4435 ///
4436 /// The boolean argument `last` indicates that the end of the stream is reached
4437 /// when all the characters in `src` have been consumed. This argument is needed
4438 /// for ISO-2022-JP and is ignored for other encodings.
4439 ///
4440 /// An `Encoder` object can be used to incrementally encode a byte stream.
4441 ///
4442 /// During the processing of a single stream, the caller must call `encode_*`
4443 /// zero or more times with `last` set to `false` and then call `encode_*` at
4444 /// least once with `last` set to `true`. If `encode_*` returns `InputEmpty`,
4445 /// the processing of the stream has ended. Otherwise, the caller must call
4446 /// `encode_*` again with `last` set to `true` (or treat an `Unmappable` result
4447 /// as a fatal error).
4448 ///
4449 /// Once the stream has ended, the `Encoder` object must not be used anymore.
4450 /// That is, you need to create another one to process another stream.
4451 ///
4452 /// When the encoder returns `OutputFull` or the encoder returns `Unmappable`
4453 /// and the caller does not wish to treat it as a fatal error, the input buffer
4454 /// `src` may not have been completely consumed. In that case, the caller must
4455 /// pass the unconsumed contents of `src` to `encode_*` again upon the next
4456 /// call.
4457 ///
4458 /// [1]: enum.EncoderResult.html
4459 /// [2]: #method.max_buffer_length_from_utf8_without_replacement
4460 /// [3]: #method.max_buffer_length_from_utf8_if_no_unmappables
4461 /// [4]: #method.max_buffer_length_from_utf16_without_replacement
4462 /// [5]: #method.max_buffer_length_from_utf16_if_no_unmappables
4463 ///
4464 /// # Infinite loops
4465 ///
4466 /// When converting with a fixed-size output buffer whose size is too small to
4467 /// accommodate one character of output, an infinite loop ensues. When
4468 /// converting with a fixed-size output buffer, it generally makes sense to
4469 /// make the buffer fairly large (e.g. couple of kilobytes).
4470 pub struct Encoder {
4471 encoding: &'static Encoding,
4472 variant: VariantEncoder,
4473 }
4474
4475 impl Encoder {
4476 fn new(enc: &'static Encoding, encoder: VariantEncoder) -> Encoder {
4477 Encoder {
4478 encoding: enc,
4479 variant: encoder,
4480 }
4481 }
4482
4483 /// The `Encoding` this `Encoder` is for.
4484 #[inline]
4485 pub fn encoding(&self) -> &'static Encoding {
4486 self.encoding
4487 }
4488
4489 /// Returns `true` if this is an ISO-2022-JP encoder that's not in the
4490 /// ASCII state and `false` otherwise.
4491 #[inline]
4492 pub fn has_pending_state(&self) -> bool {
4493 self.variant.has_pending_state()
4494 }
4495
4496 /// Query the worst-case output size when encoding from UTF-8 with
4497 /// replacement.
4498 ///
4499 /// Returns the size of the output buffer in bytes that will not overflow
4500 /// given the current state of the encoder and `byte_length` number of
4501 /// additional input code units if there are no unmappable characters in
4502 /// the input or `None` if `usize` would overflow.
4503 ///
4504 /// Available via the C wrapper.
4505 pub fn max_buffer_length_from_utf8_if_no_unmappables(
4506 &self,
4507 byte_length: usize,
4508 ) -> Option<usize> {
4509 checked_add(
4510 if self.encoding().can_encode_everything() {
4511 0
4512 } else {
4513 NCR_EXTRA
4514 },
4515 self.max_buffer_length_from_utf8_without_replacement(byte_length),
4516 )
4517 }
4518
4519 /// Query the worst-case output size when encoding from UTF-8 without
4520 /// replacement.
4521 ///
4522 /// Returns the size of the output buffer in bytes that will not overflow
4523 /// given the current state of the encoder and `byte_length` number of
4524 /// additional input code units or `None` if `usize` would overflow.
4525 ///
4526 /// Available via the C wrapper.
4527 pub fn max_buffer_length_from_utf8_without_replacement(
4528 &self,
4529 byte_length: usize,
4530 ) -> Option<usize> {
4531 self.variant
4532 .max_buffer_length_from_utf8_without_replacement(byte_length)
4533 }
4534
4535 /// Incrementally encode into byte stream from UTF-8 with unmappable
4536 /// characters replaced with HTML (decimal) numeric character references.
4537 ///
4538 /// See the documentation of the struct for documentation for `encode_*`
4539 /// methods collectively.
4540 ///
4541 /// Available via the C wrapper.
4542 pub fn encode_from_utf8(
4543 &mut self,
4544 src: &str,
4545 dst: &mut [u8],
4546 last: bool,
4547 ) -> (CoderResult, usize, usize, bool) {
4548 let dst_len = dst.len();
4549 let effective_dst_len = if self.encoding().can_encode_everything() {
4550 dst_len
4551 } else {
4552 if dst_len < NCR_EXTRA {
4553 if src.is_empty() && !(last && self.has_pending_state()) {
4554 return (CoderResult::InputEmpty, 0, 0, false);
4555 }
4556 return (CoderResult::OutputFull, 0, 0, false);
4557 }
4558 dst_len - NCR_EXTRA
4559 };
4560 let mut had_unmappables = false;
4561 let mut total_read = 0usize;
4562 let mut total_written = 0usize;
4563 loop {
4564 let (result, read, written) = self.encode_from_utf8_without_replacement(
4565 &src[total_read..],
4566 &mut dst[total_written..effective_dst_len],
4567 last,
4568 );
4569 total_read += read;
4570 total_written += written;
4571 match result {
4572 EncoderResult::InputEmpty => {
4573 return (
4574 CoderResult::InputEmpty,
4575 total_read,
4576 total_written,
4577 had_unmappables,
4578 );
4579 }
4580 EncoderResult::OutputFull => {
4581 return (
4582 CoderResult::OutputFull,
4583 total_read,
4584 total_written,
4585 had_unmappables,
4586 );
4587 }
4588 EncoderResult::Unmappable(unmappable) => {
4589 had_unmappables = true;
4590 debug_assert!(dst.len() - total_written >= NCR_EXTRA);
4591 debug_assert_ne!(self.encoding(), UTF_16BE);
4592 debug_assert_ne!(self.encoding(), UTF_16LE);
4593 // Additionally, Iso2022JpEncoder is responsible for
4594 // transitioning to ASCII when returning with Unmappable.
4595 total_written += write_ncr(unmappable, &mut dst[total_written..]);
4596 if total_written >= effective_dst_len {
4597 if total_read == src.len() && !(last && self.has_pending_state()) {
4598 return (
4599 CoderResult::InputEmpty,
4600 total_read,
4601 total_written,
4602 had_unmappables,
4603 );
4604 }
4605 return (
4606 CoderResult::OutputFull,
4607 total_read,
4608 total_written,
4609 had_unmappables,
4610 );
4611 }
4612 }
4613 }
4614 }
4615 }
4616
4617 /// Incrementally encode into byte stream from UTF-8 with unmappable
4618 /// characters replaced with HTML (decimal) numeric character references.
4619 ///
4620 /// See the documentation of the struct for documentation for `encode_*`
4621 /// methods collectively.
4622 ///
4623 /// Available to Rust only and only with the `alloc` feature enabled (enabled
4624 /// by default).
4625 #[cfg(feature = "alloc")]
4626 pub fn encode_from_utf8_to_vec(
4627 &mut self,
4628 src: &str,
4629 dst: &mut Vec<u8>,
4630 last: bool,
4631 ) -> (CoderResult, usize, bool) {
4632 unsafe {
4633 let old_len = dst.len();
4634 let capacity = dst.capacity();
4635 dst.set_len(capacity);
4636 let (result, read, written, replaced) =
4637 self.encode_from_utf8(src, &mut dst[old_len..], last);
4638 dst.set_len(old_len + written);
4639 (result, read, replaced)
4640 }
4641 }
4642
4643 /// Incrementally encode into byte stream from UTF-8 _without replacement_.
4644 ///
4645 /// See the documentation of the struct for documentation for `encode_*`
4646 /// methods collectively.
4647 ///
4648 /// Available via the C wrapper.
4649 pub fn encode_from_utf8_without_replacement(
4650 &mut self,
4651 src: &str,
4652 dst: &mut [u8],
4653 last: bool,
4654 ) -> (EncoderResult, usize, usize) {
4655 self.variant.encode_from_utf8_raw(src, dst, last)
4656 }
4657
4658 /// Incrementally encode into byte stream from UTF-8 _without replacement_.
4659 ///
4660 /// See the documentation of the struct for documentation for `encode_*`
4661 /// methods collectively.
4662 ///
4663 /// Available to Rust only and only with the `alloc` feature enabled (enabled
4664 /// by default).
4665 #[cfg(feature = "alloc")]
4666 pub fn encode_from_utf8_to_vec_without_replacement(
4667 &mut self,
4668 src: &str,
4669 dst: &mut Vec<u8>,
4670 last: bool,
4671 ) -> (EncoderResult, usize) {
4672 unsafe {
4673 let old_len = dst.len();
4674 let capacity = dst.capacity();
4675 dst.set_len(capacity);
4676 let (result, read, written) =
4677 self.encode_from_utf8_without_replacement(src, &mut dst[old_len..], last);
4678 dst.set_len(old_len + written);
4679 (result, read)
4680 }
4681 }
4682
4683 /// Query the worst-case output size when encoding from UTF-16 with
4684 /// replacement.
4685 ///
4686 /// Returns the size of the output buffer in bytes that will not overflow
4687 /// given the current state of the encoder and `u16_length` number of
4688 /// additional input code units if there are no unmappable characters in
4689 /// the input or `None` if `usize` would overflow.
4690 ///
4691 /// Available via the C wrapper.
4692 pub fn max_buffer_length_from_utf16_if_no_unmappables(
4693 &self,
4694 u16_length: usize,
4695 ) -> Option<usize> {
4696 checked_add(
4697 if self.encoding().can_encode_everything() {
4698 0
4699 } else {
4700 NCR_EXTRA
4701 },
4702 self.max_buffer_length_from_utf16_without_replacement(u16_length),
4703 )
4704 }
4705
4706 /// Query the worst-case output size when encoding from UTF-16 without
4707 /// replacement.
4708 ///
4709 /// Returns the size of the output buffer in bytes that will not overflow
4710 /// given the current state of the encoder and `u16_length` number of
4711 /// additional input code units or `None` if `usize` would overflow.
4712 ///
4713 /// Available via the C wrapper.
4714 pub fn max_buffer_length_from_utf16_without_replacement(
4715 &self,
4716 u16_length: usize,
4717 ) -> Option<usize> {
4718 self.variant
4719 .max_buffer_length_from_utf16_without_replacement(u16_length)
4720 }
4721
4722 /// Incrementally encode into byte stream from UTF-16 with unmappable
4723 /// characters replaced with HTML (decimal) numeric character references.
4724 ///
4725 /// See the documentation of the struct for documentation for `encode_*`
4726 /// methods collectively.
4727 ///
4728 /// Available via the C wrapper.
4729 pub fn encode_from_utf16(
4730 &mut self,
4731 src: &[u16],
4732 dst: &mut [u8],
4733 last: bool,
4734 ) -> (CoderResult, usize, usize, bool) {
4735 let dst_len = dst.len();
4736 let effective_dst_len = if self.encoding().can_encode_everything() {
4737 dst_len
4738 } else {
4739 if dst_len < NCR_EXTRA {
4740 if src.is_empty() && !(last && self.has_pending_state()) {
4741 return (CoderResult::InputEmpty, 0, 0, false);
4742 }
4743 return (CoderResult::OutputFull, 0, 0, false);
4744 }
4745 dst_len - NCR_EXTRA
4746 };
4747 let mut had_unmappables = false;
4748 let mut total_read = 0usize;
4749 let mut total_written = 0usize;
4750 loop {
4751 let (result, read, written) = self.encode_from_utf16_without_replacement(
4752 &src[total_read..],
4753 &mut dst[total_written..effective_dst_len],
4754 last,
4755 );
4756 total_read += read;
4757 total_written += written;
4758 match result {
4759 EncoderResult::InputEmpty => {
4760 return (
4761 CoderResult::InputEmpty,
4762 total_read,
4763 total_written,
4764 had_unmappables,
4765 );
4766 }
4767 EncoderResult::OutputFull => {
4768 return (
4769 CoderResult::OutputFull,
4770 total_read,
4771 total_written,
4772 had_unmappables,
4773 );
4774 }
4775 EncoderResult::Unmappable(unmappable) => {
4776 had_unmappables = true;
4777 debug_assert!(dst.len() - total_written >= NCR_EXTRA);
4778 // There are no UTF-16 encoders and even if there were,
4779 // they'd never have unmappables.
4780 debug_assert_ne!(self.encoding(), UTF_16BE);
4781 debug_assert_ne!(self.encoding(), UTF_16LE);
4782 // Additionally, Iso2022JpEncoder is responsible for
4783 // transitioning to ASCII when returning with Unmappable
4784 // from the jis0208 state. That is, when we encode
4785 // ISO-2022-JP and come here, the encoder is in either the
4786 // ASCII or the Roman state. We are allowed to generate any
4787 // printable ASCII excluding \ and ~.
4788 total_written += write_ncr(unmappable, &mut dst[total_written..]);
4789 if total_written >= effective_dst_len {
4790 if total_read == src.len() && !(last && self.has_pending_state()) {
4791 return (
4792 CoderResult::InputEmpty,
4793 total_read,
4794 total_written,
4795 had_unmappables,
4796 );
4797 }
4798 return (
4799 CoderResult::OutputFull,
4800 total_read,
4801 total_written,
4802 had_unmappables,
4803 );
4804 }
4805 }
4806 }
4807 }
4808 }
4809
4810 /// Incrementally encode into byte stream from UTF-16 _without replacement_.
4811 ///
4812 /// See the documentation of the struct for documentation for `encode_*`
4813 /// methods collectively.
4814 ///
4815 /// Available via the C wrapper.
4816 pub fn encode_from_utf16_without_replacement(
4817 &mut self,
4818 src: &[u16],
4819 dst: &mut [u8],
4820 last: bool,
4821 ) -> (EncoderResult, usize, usize) {
4822 self.variant.encode_from_utf16_raw(src, dst, last)
4823 }
4824 }
4825
4826 /// Format an unmappable as NCR without heap allocation.
4827 fn write_ncr(unmappable: char, dst: &mut [u8]) -> usize {
4828 // len is the number of decimal digits needed to represent unmappable plus
4829 // 3 (the length of "&#" and ";").
4830 let mut number = unmappable as u32;
4831 let len = if number >= 1_000_000u32 {
4832 10usize
4833 } else if number >= 100_000u32 {
4834 9usize
4835 } else if number >= 10_000u32 {
4836 8usize
4837 } else if number >= 1_000u32 {
4838 7usize
4839 } else if number >= 100u32 {
4840 6usize
4841 } else {
4842 // Review the outcome of https://github.com/whatwg/encoding/issues/15
4843 // to see if this case is possible
4844 5usize
4845 };
4846 debug_assert!(number >= 10u32);
4847 debug_assert!(len <= dst.len());
4848 let mut pos = len - 1;
4849 dst[pos] = b';';
4850 pos -= 1;
4851 loop {
4852 let rightmost = number % 10;
4853 dst[pos] = rightmost as u8 + b'0';
4854 pos -= 1;
4855 if number < 10 {
4856 break;
4857 }
4858 number /= 10;
4859 }
4860 dst[1] = b'#';
4861 dst[0] = b'&';
4862 len
4863 }
4864
4865 #[inline(always)]
4866 fn in_range16(i: u16, start: u16, end: u16) -> bool {
4867 i.wrapping_sub(start) < (end - start)
4868 }
4869
4870 #[inline(always)]
4871 fn in_range32(i: u32, start: u32, end: u32) -> bool {
4872 i.wrapping_sub(start) < (end - start)
4873 }
4874
4875 #[inline(always)]
4876 fn in_inclusive_range8(i: u8, start: u8, end: u8) -> bool {
4877 i.wrapping_sub(start) <= (end - start)
4878 }
4879
4880 #[inline(always)]
4881 fn in_inclusive_range16(i: u16, start: u16, end: u16) -> bool {
4882 i.wrapping_sub(start) <= (end - start)
4883 }
4884
4885 #[inline(always)]
4886 fn in_inclusive_range32(i: u32, start: u32, end: u32) -> bool {
4887 i.wrapping_sub(start) <= (end - start)
4888 }
4889
4890 #[inline(always)]
4891 fn in_inclusive_range(i: usize, start: usize, end: usize) -> bool {
4892 i.wrapping_sub(start) <= (end - start)
4893 }
4894
4895 #[inline(always)]
4896 fn checked_add(num: usize, opt: Option<usize>) -> Option<usize> {
4897 if let Some(n) = opt {
4898 n.checked_add(num)
4899 } else {
4900 None
4901 }
4902 }
4903
4904 #[inline(always)]
4905 fn checked_add_opt(one: Option<usize>, other: Option<usize>) -> Option<usize> {
4906 if let Some(n) = one {
4907 checked_add(n, other)
4908 } else {
4909 None
4910 }
4911 }
4912
4913 #[inline(always)]
4914 fn checked_mul(num: usize, opt: Option<usize>) -> Option<usize> {
4915 if let Some(n) = opt {
4916 n.checked_mul(num)
4917 } else {
4918 None
4919 }
4920 }
4921
4922 #[inline(always)]
4923 fn checked_div(opt: Option<usize>, num: usize) -> Option<usize> {
4924 if let Some(n) = opt {
4925 n.checked_div(num)
4926 } else {
4927 None
4928 }
4929 }
4930
4931 #[cfg(feature = "alloc")]
4932 #[inline(always)]
4933 fn checked_next_power_of_two(opt: Option<usize>) -> Option<usize> {
4934 opt.map(|n| n.next_power_of_two())
4935 }
4936
4937 #[cfg(feature = "alloc")]
4938 #[inline(always)]
4939 fn checked_min(one: Option<usize>, other: Option<usize>) -> Option<usize> {
4940 if let Some(a) = one {
4941 if let Some(b) = other {
4942 Some(::core::cmp::min(a, b))
4943 } else {
4944 Some(a)
4945 }
4946 } else {
4947 other
4948 }
4949 }
4950
4951 // ############## TESTS ###############
4952
4953 #[cfg(all(test, feature = "serde"))]
4954 #[derive(Serialize, Deserialize, Debug, PartialEq)]
4955 struct Demo {
4956 num: u32,
4957 name: String,
4958 enc: &'static Encoding,
4959 }
4960
4961 #[cfg(test)]
4962 mod test_labels_names;
4963
4964 #[cfg(all(test, feature = "alloc"))]
4965 mod tests {
4966 use super::*;
4967 use alloc::borrow::Cow;
4968
4969 fn sniff_to_utf16(
4970 initial_encoding: &'static Encoding,
4971 expected_encoding: &'static Encoding,
4972 bytes: &[u8],
4973 expect: &[u16],
4974 breaks: &[usize],
4975 ) {
4976 let mut decoder = initial_encoding.new_decoder();
4977
4978 let mut dest: Vec<u16> =
4979 Vec::with_capacity(decoder.max_utf16_buffer_length(bytes.len()).unwrap());
4980 let capacity = dest.capacity();
4981 dest.resize(capacity, 0u16);
4982
4983 let mut total_written = 0usize;
4984 let mut start = 0usize;
4985 for br in breaks {
4986 let (result, read, written, _) =
4987 decoder.decode_to_utf16(&bytes[start..*br], &mut dest[total_written..], false);
4988 total_written += written;
4989 assert_eq!(read, *br - start);
4990 match result {
4991 CoderResult::InputEmpty => {}
4992 CoderResult::OutputFull => {
4993 unreachable!();
4994 }
4995 }
4996 start = *br;
4997 }
4998 let (result, read, written, _) =
4999 decoder.decode_to_utf16(&bytes[start..], &mut dest[total_written..], true);
5000 total_written += written;
5001 match result {
5002 CoderResult::InputEmpty => {}
5003 CoderResult::OutputFull => {
5004 unreachable!();
5005 }
5006 }
5007 assert_eq!(read, bytes.len() - start);
5008 assert_eq!(total_written, expect.len());
5009 assert_eq!(&dest[..total_written], expect);
5010 assert_eq!(decoder.encoding(), expected_encoding);
5011 }
5012
5013 // Any copyright to the test code below this comment is dedicated to the
5014 // Public Domain. http://creativecommons.org/publicdomain/zero/1.0/
5015
5016 #[test]
5017 fn test_bom_sniffing() {
5018 // ASCII
5019 sniff_to_utf16(
5020 WINDOWS_1252,
5021 WINDOWS_1252,
5022 b"\x61\x62",
5023 &[0x0061u16, 0x0062u16],
5024 &[],
5025 );
5026 // UTF-8
5027 sniff_to_utf16(
5028 WINDOWS_1252,
5029 UTF_8,
5030 b"\xEF\xBB\xBF\x61\x62",
5031 &[0x0061u16, 0x0062u16],
5032 &[],
5033 );
5034 sniff_to_utf16(
5035 WINDOWS_1252,
5036 UTF_8,
5037 b"\xEF\xBB\xBF\x61\x62",
5038 &[0x0061u16, 0x0062u16],
5039 &[1],
5040 );
5041 sniff_to_utf16(
5042 WINDOWS_1252,
5043 UTF_8,
5044 b"\xEF\xBB\xBF\x61\x62",
5045 &[0x0061u16, 0x0062u16],
5046 &[2],
5047 );
5048 sniff_to_utf16(
5049 WINDOWS_1252,
5050 UTF_8,
5051 b"\xEF\xBB\xBF\x61\x62",
5052 &[0x0061u16, 0x0062u16],
5053 &[3],
5054 );
5055 sniff_to_utf16(
5056 WINDOWS_1252,
5057 UTF_8,
5058 b"\xEF\xBB\xBF\x61\x62",
5059 &[0x0061u16, 0x0062u16],
5060 &[4],
5061 );
5062 sniff_to_utf16(
5063 WINDOWS_1252,
5064 UTF_8,
5065 b"\xEF\xBB\xBF\x61\x62",
5066 &[0x0061u16, 0x0062u16],
5067 &[2, 3],
5068 );
5069 sniff_to_utf16(
5070 WINDOWS_1252,
5071 UTF_8,
5072 b"\xEF\xBB\xBF\x61\x62",
5073 &[0x0061u16, 0x0062u16],
5074 &[1, 2],
5075 );
5076 sniff_to_utf16(
5077 WINDOWS_1252,
5078 UTF_8,
5079 b"\xEF\xBB\xBF\x61\x62",
5080 &[0x0061u16, 0x0062u16],
5081 &[1, 3],
5082 );
5083 sniff_to_utf16(
5084 WINDOWS_1252,
5085 UTF_8,
5086 b"\xEF\xBB\xBF\x61\x62",
5087 &[0x0061u16, 0x0062u16],
5088 &[1, 2, 3, 4],
5089 );
5090 sniff_to_utf16(WINDOWS_1252, UTF_8, b"\xEF\xBB\xBF", &[], &[]);
5091 // Not UTF-8
5092 sniff_to_utf16(
5093 WINDOWS_1252,
5094 WINDOWS_1252,
5095 b"\xEF\xBB\x61\x62",
5096 &[0x00EFu16, 0x00BBu16, 0x0061u16, 0x0062u16],
5097 &[],
5098 );
5099 sniff_to_utf16(
5100 WINDOWS_1252,
5101 WINDOWS_1252,
5102 b"\xEF\xBB\x61\x62",
5103 &[0x00EFu16, 0x00BBu16, 0x0061u16, 0x0062u16],
5104 &[1],
5105 );
5106 sniff_to_utf16(
5107 WINDOWS_1252,
5108 WINDOWS_1252,
5109 b"\xEF\x61\x62",
5110 &[0x00EFu16, 0x0061u16, 0x0062u16],
5111 &[],
5112 );
5113 sniff_to_utf16(
5114 WINDOWS_1252,
5115 WINDOWS_1252,
5116 b"\xEF\x61\x62",
5117 &[0x00EFu16, 0x0061u16, 0x0062u16],
5118 &[1],
5119 );
5120 sniff_to_utf16(
5121 WINDOWS_1252,
5122 WINDOWS_1252,
5123 b"\xEF\xBB",
5124 &[0x00EFu16, 0x00BBu16],
5125 &[],
5126 );
5127 sniff_to_utf16(
5128 WINDOWS_1252,
5129 WINDOWS_1252,
5130 b"\xEF\xBB",
5131 &[0x00EFu16, 0x00BBu16],
5132 &[1],
5133 );
5134 sniff_to_utf16(WINDOWS_1252, WINDOWS_1252, b"\xEF", &[0x00EFu16], &[]);
5135 // Not UTF-16
5136 sniff_to_utf16(
5137 WINDOWS_1252,
5138 WINDOWS_1252,
5139 b"\xFE\x61\x62",
5140 &[0x00FEu16, 0x0061u16, 0x0062u16],
5141 &[],
5142 );
5143 sniff_to_utf16(
5144 WINDOWS_1252,
5145 WINDOWS_1252,
5146 b"\xFE\x61\x62",
5147 &[0x00FEu16, 0x0061u16, 0x0062u16],
5148 &[1],
5149 );
5150 sniff_to_utf16(WINDOWS_1252, WINDOWS_1252, b"\xFE", &[0x00FEu16], &[]);
5151 sniff_to_utf16(
5152 WINDOWS_1252,
5153 WINDOWS_1252,
5154 b"\xFF\x61\x62",
5155 &[0x00FFu16, 0x0061u16, 0x0062u16],
5156 &[],
5157 );
5158 sniff_to_utf16(
5159 WINDOWS_1252,
5160 WINDOWS_1252,
5161 b"\xFF\x61\x62",
5162 &[0x00FFu16, 0x0061u16, 0x0062u16],
5163 &[1],
5164 );
5165 sniff_to_utf16(WINDOWS_1252, WINDOWS_1252, b"\xFF", &[0x00FFu16], &[]);
5166 // UTF-16
5167 sniff_to_utf16(WINDOWS_1252, UTF_16BE, b"\xFE\xFF", &[], &[]);
5168 sniff_to_utf16(WINDOWS_1252, UTF_16BE, b"\xFE\xFF", &[], &[1]);
5169 sniff_to_utf16(WINDOWS_1252, UTF_16LE, b"\xFF\xFE", &[], &[]);
5170 sniff_to_utf16(WINDOWS_1252, UTF_16LE, b"\xFF\xFE", &[], &[1]);
5171 }
5172
5173 #[test]
5174 fn test_output_encoding() {
5175 assert_eq!(REPLACEMENT.output_encoding(), UTF_8);
5176 assert_eq!(UTF_16BE.output_encoding(), UTF_8);
5177 assert_eq!(UTF_16LE.output_encoding(), UTF_8);
5178 assert_eq!(UTF_8.output_encoding(), UTF_8);
5179 assert_eq!(WINDOWS_1252.output_encoding(), WINDOWS_1252);
5180 assert_eq!(REPLACEMENT.new_encoder().encoding(), UTF_8);
5181 assert_eq!(UTF_16BE.new_encoder().encoding(), UTF_8);
5182 assert_eq!(UTF_16LE.new_encoder().encoding(), UTF_8);
5183 assert_eq!(UTF_8.new_encoder().encoding(), UTF_8);
5184 assert_eq!(WINDOWS_1252.new_encoder().encoding(), WINDOWS_1252);
5185 }
5186
5187 #[test]
5188 fn test_label_resolution() {
5189 assert_eq!(Encoding::for_label(b"utf-8"), Some(UTF_8));
5190 assert_eq!(Encoding::for_label(b"UTF-8"), Some(UTF_8));
5191 assert_eq!(
5192 Encoding::for_label(b" \t \n \x0C \n utf-8 \r \n \t \x0C "),
5193 Some(UTF_8)
5194 );
5195 assert_eq!(Encoding::for_label(b"utf-8 _"), None);
5196 assert_eq!(Encoding::for_label(b"bogus"), None);
5197 assert_eq!(Encoding::for_label(b"bogusbogusbogusbogus"), None);
5198 }
5199
5200 #[test]
5201 fn test_decode_valid_windows_1257_to_cow() {
5202 let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc\x80\xE4");
5203 match cow {
5204 Cow::Borrowed(_) => unreachable!(),
5205 Cow::Owned(s) => {
5206 assert_eq!(s, "abc\u{20AC}\u{00E4}");
5207 }
5208 }
5209 assert_eq!(encoding, WINDOWS_1257);
5210 assert!(!had_errors);
5211 }
5212
5213 #[test]
5214 fn test_decode_invalid_windows_1257_to_cow() {
5215 let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc\x80\xA1\xE4");
5216 match cow {
5217 Cow::Borrowed(_) => unreachable!(),
5218 Cow::Owned(s) => {
5219 assert_eq!(s, "abc\u{20AC}\u{FFFD}\u{00E4}");
5220 }
5221 }
5222 assert_eq!(encoding, WINDOWS_1257);
5223 assert!(had_errors);
5224 }
5225
5226 #[test]
5227 fn test_decode_ascii_only_windows_1257_to_cow() {
5228 let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"abc");
5229 match cow {
5230 Cow::Borrowed(s) => {
5231 assert_eq!(s, "abc");
5232 }
5233 Cow::Owned(_) => unreachable!(),
5234 }
5235 assert_eq!(encoding, WINDOWS_1257);
5236 assert!(!had_errors);
5237 }
5238
5239 #[test]
5240 fn test_decode_bomful_valid_utf8_as_windows_1257_to_cow() {
5241 let (cow, encoding, had_errors) = WINDOWS_1257.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5242 match cow {
5243 Cow::Borrowed(s) => {
5244 assert_eq!(s, "\u{20AC}\u{00E4}");
5245 }
5246 Cow::Owned(_) => unreachable!(),
5247 }
5248 assert_eq!(encoding, UTF_8);
5249 assert!(!had_errors);
5250 }
5251
5252 #[test]
5253 fn test_decode_bomful_invalid_utf8_as_windows_1257_to_cow() {
5254 let (cow, encoding, had_errors) =
5255 WINDOWS_1257.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4");
5256 match cow {
5257 Cow::Borrowed(_) => unreachable!(),
5258 Cow::Owned(s) => {
5259 assert_eq!(s, "\u{20AC}\u{FFFD}\u{00E4}");
5260 }
5261 }
5262 assert_eq!(encoding, UTF_8);
5263 assert!(had_errors);
5264 }
5265
5266 #[test]
5267 fn test_decode_bomful_valid_utf8_as_utf_8_to_cow() {
5268 let (cow, encoding, had_errors) = UTF_8.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5269 match cow {
5270 Cow::Borrowed(s) => {
5271 assert_eq!(s, "\u{20AC}\u{00E4}");
5272 }
5273 Cow::Owned(_) => unreachable!(),
5274 }
5275 assert_eq!(encoding, UTF_8);
5276 assert!(!had_errors);
5277 }
5278
5279 #[test]
5280 fn test_decode_bomful_invalid_utf8_as_utf_8_to_cow() {
5281 let (cow, encoding, had_errors) = UTF_8.decode(b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4");
5282 match cow {
5283 Cow::Borrowed(_) => unreachable!(),
5284 Cow::Owned(s) => {
5285 assert_eq!(s, "\u{20AC}\u{FFFD}\u{00E4}");
5286 }
5287 }
5288 assert_eq!(encoding, UTF_8);
5289 assert!(had_errors);
5290 }
5291
5292 #[test]
5293 fn test_decode_bomful_valid_utf8_as_utf_8_to_cow_with_bom_removal() {
5294 let (cow, had_errors) = UTF_8.decode_with_bom_removal(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5295 match cow {
5296 Cow::Borrowed(s) => {
5297 assert_eq!(s, "\u{20AC}\u{00E4}");
5298 }
5299 Cow::Owned(_) => unreachable!(),
5300 }
5301 assert!(!had_errors);
5302 }
5303
5304 #[test]
5305 fn test_decode_bomful_valid_utf8_as_windows_1257_to_cow_with_bom_removal() {
5306 let (cow, had_errors) =
5307 WINDOWS_1257.decode_with_bom_removal(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5308 match cow {
5309 Cow::Borrowed(_) => unreachable!(),
5310 Cow::Owned(s) => {
5311 assert_eq!(
5312 s,
5313 "\u{013C}\u{00BB}\u{00E6}\u{0101}\u{201A}\u{00AC}\u{0106}\u{00A4}"
5314 );
5315 }
5316 }
5317 assert!(!had_errors);
5318 }
5319
5320 #[test]
5321 fn test_decode_valid_windows_1257_to_cow_with_bom_removal() {
5322 let (cow, had_errors) = WINDOWS_1257.decode_with_bom_removal(b"abc\x80\xE4");
5323 match cow {
5324 Cow::Borrowed(_) => unreachable!(),
5325 Cow::Owned(s) => {
5326 assert_eq!(s, "abc\u{20AC}\u{00E4}");
5327 }
5328 }
5329 assert!(!had_errors);
5330 }
5331
5332 #[test]
5333 fn test_decode_invalid_windows_1257_to_cow_with_bom_removal() {
5334 let (cow, had_errors) = WINDOWS_1257.decode_with_bom_removal(b"abc\x80\xA1\xE4");
5335 match cow {
5336 Cow::Borrowed(_) => unreachable!(),
5337 Cow::Owned(s) => {
5338 assert_eq!(s, "abc\u{20AC}\u{FFFD}\u{00E4}");
5339 }
5340 }
5341 assert!(had_errors);
5342 }
5343
5344 #[test]
5345 fn test_decode_ascii_only_windows_1257_to_cow_with_bom_removal() {
5346 let (cow, had_errors) = WINDOWS_1257.decode_with_bom_removal(b"abc");
5347 match cow {
5348 Cow::Borrowed(s) => {
5349 assert_eq!(s, "abc");
5350 }
5351 Cow::Owned(_) => unreachable!(),
5352 }
5353 assert!(!had_errors);
5354 }
5355
5356 #[test]
5357 fn test_decode_bomful_valid_utf8_to_cow_without_bom_handling() {
5358 let (cow, had_errors) =
5359 UTF_8.decode_without_bom_handling(b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4");
5360 match cow {
5361 Cow::Borrowed(s) => {
5362 assert_eq!(s, "\u{FEFF}\u{20AC}\u{00E4}");
5363 }
5364 Cow::Owned(_) => unreachable!(),
5365 }
5366 assert!(!had_errors);
5367 }
5368
5369 #[test]
5370 fn test_decode_bomful_invalid_utf8_to_cow_without_bom_handling() {
5371 let (cow, had_errors) =
5372 UTF_8.decode_without_bom_handling(b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4");
5373 match cow {
5374 Cow::Borrowed(_) => unreachable!(),
5375 Cow::Owned(s) => {
5376 assert_eq!(s, "\u{FEFF}\u{20AC}\u{FFFD}\u{00E4}");
5377 }
5378 }
5379 assert!(had_errors);
5380 }
5381
5382 #[test]
5383 fn test_decode_valid_windows_1257_to_cow_without_bom_handling() {
5384 let (cow, had_errors) = WINDOWS_1257.decode_without_bom_handling(b"abc\x80\xE4");
5385 match cow {
5386 Cow::Borrowed(_) => unreachable!(),
5387 Cow::Owned(s) => {
5388 assert_eq!(s, "abc\u{20AC}\u{00E4}");
5389 }
5390 }
5391 assert!(!had_errors);
5392 }
5393
5394 #[test]
5395 fn test_decode_invalid_windows_1257_to_cow_without_bom_handling() {
5396 let (cow, had_errors) = WINDOWS_1257.decode_without_bom_handling(b"abc\x80\xA1\xE4");
5397 match cow {
5398 Cow::Borrowed(_) => unreachable!(),
5399 Cow::Owned(s) => {
5400 assert_eq!(s, "abc\u{20AC}\u{FFFD}\u{00E4}");
5401 }
5402 }
5403 assert!(had_errors);
5404 }
5405
5406 #[test]
5407 fn test_decode_ascii_only_windows_1257_to_cow_without_bom_handling() {
5408 let (cow, had_errors) = WINDOWS_1257.decode_without_bom_handling(b"abc");
5409 match cow {
5410 Cow::Borrowed(s) => {
5411 assert_eq!(s, "abc");
5412 }
5413 Cow::Owned(_) => unreachable!(),
5414 }
5415 assert!(!had_errors);
5416 }
5417
5418 #[test]
5419 fn test_decode_bomful_valid_utf8_to_cow_without_bom_handling_and_without_replacement() {
5420 match UTF_8.decode_without_bom_handling_and_without_replacement(
5421 b"\xEF\xBB\xBF\xE2\x82\xAC\xC3\xA4",
5422 ) {
5423 Some(cow) => match cow {
5424 Cow::Borrowed(s) => {
5425 assert_eq!(s, "\u{FEFF}\u{20AC}\u{00E4}");
5426 }
5427 Cow::Owned(_) => unreachable!(),
5428 },
5429 None => unreachable!(),
5430 }
5431 }
5432
5433 #[test]
5434 fn test_decode_bomful_invalid_utf8_to_cow_without_bom_handling_and_without_replacement() {
5435 assert!(UTF_8
5436 .decode_without_bom_handling_and_without_replacement(
5437 b"\xEF\xBB\xBF\xE2\x82\xAC\x80\xC3\xA4"
5438 )
5439 .is_none());
5440 }
5441
5442 #[test]
5443 fn test_decode_valid_windows_1257_to_cow_without_bom_handling_and_without_replacement() {
5444 match WINDOWS_1257.decode_without_bom_handling_and_without_replacement(b"abc\x80\xE4") {
5445 Some(cow) => match cow {
5446 Cow::Borrowed(_) => unreachable!(),
5447 Cow::Owned(s) => {
5448 assert_eq!(s, "abc\u{20AC}\u{00E4}");
5449 }
5450 },
5451 None => unreachable!(),
5452 }
5453 }
5454
5455 #[test]
5456 fn test_decode_invalid_windows_1257_to_cow_without_bom_handling_and_without_replacement() {
5457 assert!(WINDOWS_1257
5458 .decode_without_bom_handling_and_without_replacement(b"abc\x80\xA1\xE4")
5459 .is_none());
5460 }
5461
5462 #[test]
5463 fn test_decode_ascii_only_windows_1257_to_cow_without_bom_handling_and_without_replacement() {
5464 match WINDOWS_1257.decode_without_bom_handling_and_without_replacement(b"abc") {
5465 Some(cow) => match cow {
5466 Cow::Borrowed(s) => {
5467 assert_eq!(s, "abc");
5468 }
5469 Cow::Owned(_) => unreachable!(),
5470 },
5471 None => unreachable!(),
5472 }
5473 }
5474
5475 #[test]
5476 fn test_encode_ascii_only_windows_1257_to_cow() {
5477 let (cow, encoding, had_errors) = WINDOWS_1257.encode("abc");
5478 match cow {
5479 Cow::Borrowed(s) => {
5480 assert_eq!(s, b"abc");
5481 }
5482 Cow::Owned(_) => unreachable!(),
5483 }
5484 assert_eq!(encoding, WINDOWS_1257);
5485 assert!(!had_errors);
5486 }
5487
5488 #[test]
5489 fn test_encode_valid_windows_1257_to_cow() {
5490 let (cow, encoding, had_errors) = WINDOWS_1257.encode("abc\u{20AC}\u{00E4}");
5491 match cow {
5492 Cow::Borrowed(_) => unreachable!(),
5493 Cow::Owned(s) => {
5494 assert_eq!(s, b"abc\x80\xE4");
5495 }
5496 }
5497 assert_eq!(encoding, WINDOWS_1257);
5498 assert!(!had_errors);
5499 }
5500
5501 #[test]
5502 fn test_utf16_space_with_one_bom_byte() {
5503 let mut decoder = UTF_16LE.new_decoder();
5504 let mut dst = [0u16; 12];
5505 {
5506 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5507 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], false);
5508 assert_eq!(result, CoderResult::InputEmpty);
5509 }
5510 {
5511 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5512 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5513 assert_eq!(result, CoderResult::InputEmpty);
5514 }
5515 }
5516
5517 #[test]
5518 fn test_utf8_space_with_one_bom_byte() {
5519 let mut decoder = UTF_8.new_decoder();
5520 let mut dst = [0u16; 12];
5521 {
5522 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5523 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], false);
5524 assert_eq!(result, CoderResult::InputEmpty);
5525 }
5526 {
5527 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5528 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5529 assert_eq!(result, CoderResult::InputEmpty);
5530 }
5531 }
5532
5533 #[test]
5534 fn test_utf16_space_with_two_bom_bytes() {
5535 let mut decoder = UTF_16LE.new_decoder();
5536 let mut dst = [0u16; 12];
5537 {
5538 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5539 let (result, _, _, _) = decoder.decode_to_utf16(b"\xEF", &mut dst[..needed], false);
5540 assert_eq!(result, CoderResult::InputEmpty);
5541 }
5542 {
5543 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5544 let (result, _, _, _) = decoder.decode_to_utf16(b"\xBB", &mut dst[..needed], false);
5545 assert_eq!(result, CoderResult::InputEmpty);
5546 }
5547 {
5548 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5549 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5550 assert_eq!(result, CoderResult::InputEmpty);
5551 }
5552 }
5553
5554 #[test]
5555 fn test_utf8_space_with_two_bom_bytes() {
5556 let mut decoder = UTF_8.new_decoder();
5557 let mut dst = [0u16; 12];
5558 {
5559 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5560 let (result, _, _, _) = decoder.decode_to_utf16(b"\xEF", &mut dst[..needed], false);
5561 assert_eq!(result, CoderResult::InputEmpty);
5562 }
5563 {
5564 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5565 let (result, _, _, _) = decoder.decode_to_utf16(b"\xBB", &mut dst[..needed], false);
5566 assert_eq!(result, CoderResult::InputEmpty);
5567 }
5568 {
5569 let needed = decoder.max_utf16_buffer_length(1).unwrap();
5570 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF", &mut dst[..needed], true);
5571 assert_eq!(result, CoderResult::InputEmpty);
5572 }
5573 }
5574
5575 #[test]
5576 fn test_utf16_space_with_one_bom_byte_and_a_second_byte_in_same_call() {
5577 let mut decoder = UTF_16LE.new_decoder();
5578 let mut dst = [0u16; 12];
5579 {
5580 let needed = decoder.max_utf16_buffer_length(2).unwrap();
5581 let (result, _, _, _) = decoder.decode_to_utf16(b"\xFF\xFF", &mut dst[..needed], true);
5582 assert_eq!(result, CoderResult::InputEmpty);
5583 }
5584 }
5585
5586 #[test]
5587 fn test_too_short_buffer_with_iso_2022_jp_ascii_from_utf8() {
5588 let mut dst = [0u8; 8];
5589 let mut encoder = ISO_2022_JP.new_encoder();
5590 {
5591 let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..], false);
5592 assert_eq!(result, CoderResult::InputEmpty);
5593 }
5594 {
5595 let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..], true);
5596 assert_eq!(result, CoderResult::InputEmpty);
5597 }
5598 }
5599
5600 #[test]
5601 fn test_too_short_buffer_with_iso_2022_jp_roman_from_utf8() {
5602 let mut dst = [0u8; 16];
5603 let mut encoder = ISO_2022_JP.new_encoder();
5604 {
5605 let (result, _, _, _) = encoder.encode_from_utf8("\u{A5}", &mut dst[..], false);
5606 assert_eq!(result, CoderResult::InputEmpty);
5607 }
5608 {
5609 let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..8], false);
5610 assert_eq!(result, CoderResult::InputEmpty);
5611 }
5612 {
5613 let (result, _, _, _) = encoder.encode_from_utf8("", &mut dst[..8], true);
5614 assert_eq!(result, CoderResult::OutputFull);
5615 }
5616 }
5617
5618 #[test]
5619 fn test_buffer_end_iso_2022_jp_from_utf8() {
5620 let mut dst = [0u8; 18];
5621 {
5622 let mut encoder = ISO_2022_JP.new_encoder();
5623 let (result, _, _, _) =
5624 encoder.encode_from_utf8("\u{A5}\u{1F4A9}", &mut dst[..], false);
5625 assert_eq!(result, CoderResult::InputEmpty);
5626 }
5627 {
5628 let mut encoder = ISO_2022_JP.new_encoder();
5629 let (result, _, _, _) = encoder.encode_from_utf8("\u{A5}\u{1F4A9}", &mut dst[..], true);
5630 assert_eq!(result, CoderResult::OutputFull);
5631 }
5632 {
5633 let mut encoder = ISO_2022_JP.new_encoder();
5634 let (result, _, _, _) = encoder.encode_from_utf8("\u{1F4A9}", &mut dst[..13], false);
5635 assert_eq!(result, CoderResult::InputEmpty);
5636 }
5637 {
5638 let mut encoder = ISO_2022_JP.new_encoder();
5639 let (result, _, _, _) = encoder.encode_from_utf8("\u{1F4A9}", &mut dst[..13], true);
5640 assert_eq!(result, CoderResult::InputEmpty);
5641 }
5642 }
5643
5644 #[test]
5645 fn test_too_short_buffer_with_iso_2022_jp_ascii_from_utf16() {
5646 let mut dst = [0u8; 8];
5647 let mut encoder = ISO_2022_JP.new_encoder();
5648 {
5649 let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..], false);
5650 assert_eq!(result, CoderResult::InputEmpty);
5651 }
5652 {
5653 let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..], true);
5654 assert_eq!(result, CoderResult::InputEmpty);
5655 }
5656 }
5657
5658 #[test]
5659 fn test_too_short_buffer_with_iso_2022_jp_roman_from_utf16() {
5660 let mut dst = [0u8; 16];
5661 let mut encoder = ISO_2022_JP.new_encoder();
5662 {
5663 let (result, _, _, _) = encoder.encode_from_utf16(&[0xA5u16], &mut dst[..], false);
5664 assert_eq!(result, CoderResult::InputEmpty);
5665 }
5666 {
5667 let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..8], false);
5668 assert_eq!(result, CoderResult::InputEmpty);
5669 }
5670 {
5671 let (result, _, _, _) = encoder.encode_from_utf16(&[0u16; 0], &mut dst[..8], true);
5672 assert_eq!(result, CoderResult::OutputFull);
5673 }
5674 }
5675
5676 #[test]
5677 fn test_buffer_end_iso_2022_jp_from_utf16() {
5678 let mut dst = [0u8; 18];
5679 {
5680 let mut encoder = ISO_2022_JP.new_encoder();
5681 let (result, _, _, _) =
5682 encoder.encode_from_utf16(&[0xA5u16, 0xD83Du16, 0xDCA9u16], &mut dst[..], false);
5683 assert_eq!(result, CoderResult::InputEmpty);
5684 }
5685 {
5686 let mut encoder = ISO_2022_JP.new_encoder();
5687 let (result, _, _, _) =
5688 encoder.encode_from_utf16(&[0xA5u16, 0xD83Du16, 0xDCA9u16], &mut dst[..], true);
5689 assert_eq!(result, CoderResult::OutputFull);
5690 }
5691 {
5692 let mut encoder = ISO_2022_JP.new_encoder();
5693 let (result, _, _, _) =
5694 encoder.encode_from_utf16(&[0xD83Du16, 0xDCA9u16], &mut dst[..13], false);
5695 assert_eq!(result, CoderResult::InputEmpty);
5696 }
5697 {
5698 let mut encoder = ISO_2022_JP.new_encoder();
5699 let (result, _, _, _) =
5700 encoder.encode_from_utf16(&[0xD83Du16, 0xDCA9u16], &mut dst[..13], true);
5701 assert_eq!(result, CoderResult::InputEmpty);
5702 }
5703 }
5704
5705 #[test]
5706 fn test_buffer_end_utf16be() {
5707 let mut decoder = UTF_16BE.new_decoder_without_bom_handling();
5708 let mut dest = [0u8; 4];
5709
5710 assert_eq!(
5711 decoder.decode_to_utf8(&[0xD8, 0x00], &mut dest, false),
5712 (CoderResult::InputEmpty, 2, 0, false)
5713 );
5714
5715 let _ = decoder.decode_to_utf8(&[0xD8, 0x00], &mut dest, true);
5716 }
5717
5718 #[test]
5719 fn test_hash() {
5720 let mut encodings = ::alloc::collections::btree_set::BTreeSet::new();
5721 encodings.insert(UTF_8);
5722 encodings.insert(ISO_2022_JP);
5723 assert!(encodings.contains(UTF_8));
5724 assert!(encodings.contains(ISO_2022_JP));
5725 assert!(!encodings.contains(WINDOWS_1252));
5726 encodings.remove(ISO_2022_JP);
5727 assert!(!encodings.contains(ISO_2022_JP));
5728 }
5729
5730 #[test]
5731 fn test_iso_2022_jp_ncr_extra_from_utf16() {
5732 let mut dst = [0u8; 17];
5733 {
5734 let mut encoder = ISO_2022_JP.new_encoder();
5735 let (result, _, _, _) =
5736 encoder.encode_from_utf16(&[0x3041u16, 0xFFFFu16], &mut dst[..], true);
5737 assert_eq!(result, CoderResult::OutputFull);
5738 }
5739 }
5740
5741 #[test]
5742 fn test_iso_2022_jp_ncr_extra_from_utf8() {
5743 let mut dst = [0u8; 17];
5744 {
5745 let mut encoder = ISO_2022_JP.new_encoder();
5746 let (result, _, _, _) =
5747 encoder.encode_from_utf8("\u{3041}\u{FFFF}", &mut dst[..], true);
5748 assert_eq!(result, CoderResult::OutputFull);
5749 }
5750 }
5751
5752 #[test]
5753 fn test_max_length_with_bom_to_utf8() {
5754 let mut output = [0u8; 20];
5755 let mut decoder = REPLACEMENT.new_decoder();
5756 let input = b"\xEF\xBB\xBFA";
5757 {
5758 let needed = decoder
5759 .max_utf8_buffer_length_without_replacement(input.len())
5760 .unwrap();
5761 let (result, read, written) =
5762 decoder.decode_to_utf8_without_replacement(input, &mut output[..needed], true);
5763 assert_eq!(result, DecoderResult::InputEmpty);
5764 assert_eq!(read, input.len());
5765 assert_eq!(written, 1);
5766 assert_eq!(output[0], 0x41);
5767 }
5768 }
5769
5770 #[cfg(feature = "serde")]
5771 #[test]
5772 fn test_serde() {
5773 let demo = Demo {
5774 num: 42,
5775 name: "foo".into(),
5776 enc: UTF_8,
5777 };
5778
5779 let serialized = serde_json::to_string(&demo).unwrap();
5780
5781 let deserialized: Demo = serde_json::from_str(&serialized).unwrap();
5782 assert_eq!(deserialized, demo);
5783
5784 let bincoded = bincode::serialize(&demo).unwrap();
5785 let debincoded: Demo = bincode::deserialize(&bincoded[..]).unwrap();
5786 assert_eq!(debincoded, demo);
5787 }
5788
5789 #[test]
5790 fn test_is_single_byte() {
5791 assert!(!BIG5.is_single_byte());
5792 assert!(!EUC_JP.is_single_byte());
5793 assert!(!EUC_KR.is_single_byte());
5794 assert!(!GB18030.is_single_byte());
5795 assert!(!GBK.is_single_byte());
5796 assert!(!REPLACEMENT.is_single_byte());
5797 assert!(!SHIFT_JIS.is_single_byte());
5798 assert!(!UTF_8.is_single_byte());
5799 assert!(!UTF_16BE.is_single_byte());
5800 assert!(!UTF_16LE.is_single_byte());
5801 assert!(!ISO_2022_JP.is_single_byte());
5802
5803 assert!(IBM866.is_single_byte());
5804 assert!(ISO_8859_2.is_single_byte());
5805 assert!(ISO_8859_3.is_single_byte());
5806 assert!(ISO_8859_4.is_single_byte());
5807 assert!(ISO_8859_5.is_single_byte());
5808 assert!(ISO_8859_6.is_single_byte());
5809 assert!(ISO_8859_7.is_single_byte());
5810 assert!(ISO_8859_8.is_single_byte());
5811 assert!(ISO_8859_10.is_single_byte());
5812 assert!(ISO_8859_13.is_single_byte());
5813 assert!(ISO_8859_14.is_single_byte());
5814 assert!(ISO_8859_15.is_single_byte());
5815 assert!(ISO_8859_16.is_single_byte());
5816 assert!(ISO_8859_8_I.is_single_byte());
5817 assert!(KOI8_R.is_single_byte());
5818 assert!(KOI8_U.is_single_byte());
5819 assert!(MACINTOSH.is_single_byte());
5820 assert!(WINDOWS_874.is_single_byte());
5821 assert!(WINDOWS_1250.is_single_byte());
5822 assert!(WINDOWS_1251.is_single_byte());
5823 assert!(WINDOWS_1252.is_single_byte());
5824 assert!(WINDOWS_1253.is_single_byte());
5825 assert!(WINDOWS_1254.is_single_byte());
5826 assert!(WINDOWS_1255.is_single_byte());
5827 assert!(WINDOWS_1256.is_single_byte());
5828 assert!(WINDOWS_1257.is_single_byte());
5829 assert!(WINDOWS_1258.is_single_byte());
5830 assert!(X_MAC_CYRILLIC.is_single_byte());
5831 assert!(X_USER_DEFINED.is_single_byte());
5832 }
5833
5834 #[test]
5835 fn test_latin1_byte_compatible_up_to() {
5836 let buffer = b"a\x81\xB6\xF6\xF0\x82\xB4";
5837 assert_eq!(
5838 BIG5.new_decoder_without_bom_handling()
5839 .latin1_byte_compatible_up_to(buffer)
5840 .unwrap(),
5841 1
5842 );
5843 assert_eq!(
5844 EUC_JP
5845 .new_decoder_without_bom_handling()
5846 .latin1_byte_compatible_up_to(buffer)
5847 .unwrap(),
5848 1
5849 );
5850 assert_eq!(
5851 EUC_KR
5852 .new_decoder_without_bom_handling()
5853 .latin1_byte_compatible_up_to(buffer)
5854 .unwrap(),
5855 1
5856 );
5857 assert_eq!(
5858 GB18030
5859 .new_decoder_without_bom_handling()
5860 .latin1_byte_compatible_up_to(buffer)
5861 .unwrap(),
5862 1
5863 );
5864 assert_eq!(
5865 GBK.new_decoder_without_bom_handling()
5866 .latin1_byte_compatible_up_to(buffer)
5867 .unwrap(),
5868 1
5869 );
5870 assert!(REPLACEMENT
5871 .new_decoder_without_bom_handling()
5872 .latin1_byte_compatible_up_to(buffer)
5873 .is_none());
5874 assert_eq!(
5875 SHIFT_JIS
5876 .new_decoder_without_bom_handling()
5877 .latin1_byte_compatible_up_to(buffer)
5878 .unwrap(),
5879 1
5880 );
5881 assert_eq!(
5882 UTF_8
5883 .new_decoder_without_bom_handling()
5884 .latin1_byte_compatible_up_to(buffer)
5885 .unwrap(),
5886 1
5887 );
5888 assert!(UTF_16BE
5889 .new_decoder_without_bom_handling()
5890 .latin1_byte_compatible_up_to(buffer)
5891 .is_none());
5892 assert!(UTF_16LE
5893 .new_decoder_without_bom_handling()
5894 .latin1_byte_compatible_up_to(buffer)
5895 .is_none());
5896 assert_eq!(
5897 ISO_2022_JP
5898 .new_decoder_without_bom_handling()
5899 .latin1_byte_compatible_up_to(buffer)
5900 .unwrap(),
5901 1
5902 );
5903
5904 assert_eq!(
5905 IBM866
5906 .new_decoder_without_bom_handling()
5907 .latin1_byte_compatible_up_to(buffer)
5908 .unwrap(),
5909 1
5910 );
5911 assert_eq!(
5912 ISO_8859_2
5913 .new_decoder_without_bom_handling()
5914 .latin1_byte_compatible_up_to(buffer)
5915 .unwrap(),
5916 2
5917 );
5918 assert_eq!(
5919 ISO_8859_3
5920 .new_decoder_without_bom_handling()
5921 .latin1_byte_compatible_up_to(buffer)
5922 .unwrap(),
5923 2
5924 );
5925 assert_eq!(
5926 ISO_8859_4
5927 .new_decoder_without_bom_handling()
5928 .latin1_byte_compatible_up_to(buffer)
5929 .unwrap(),
5930 2
5931 );
5932 assert_eq!(
5933 ISO_8859_5
5934 .new_decoder_without_bom_handling()
5935 .latin1_byte_compatible_up_to(buffer)
5936 .unwrap(),
5937 2
5938 );
5939 assert_eq!(
5940 ISO_8859_6
5941 .new_decoder_without_bom_handling()
5942 .latin1_byte_compatible_up_to(buffer)
5943 .unwrap(),
5944 2
5945 );
5946 assert_eq!(
5947 ISO_8859_7
5948 .new_decoder_without_bom_handling()
5949 .latin1_byte_compatible_up_to(buffer)
5950 .unwrap(),
5951 2
5952 );
5953 assert_eq!(
5954 ISO_8859_8
5955 .new_decoder_without_bom_handling()
5956 .latin1_byte_compatible_up_to(buffer)
5957 .unwrap(),
5958 3
5959 );
5960 assert_eq!(
5961 ISO_8859_10
5962 .new_decoder_without_bom_handling()
5963 .latin1_byte_compatible_up_to(buffer)
5964 .unwrap(),
5965 2
5966 );
5967 assert_eq!(
5968 ISO_8859_13
5969 .new_decoder_without_bom_handling()
5970 .latin1_byte_compatible_up_to(buffer)
5971 .unwrap(),
5972 4
5973 );
5974 assert_eq!(
5975 ISO_8859_14
5976 .new_decoder_without_bom_handling()
5977 .latin1_byte_compatible_up_to(buffer)
5978 .unwrap(),
5979 4
5980 );
5981 assert_eq!(
5982 ISO_8859_15
5983 .new_decoder_without_bom_handling()
5984 .latin1_byte_compatible_up_to(buffer)
5985 .unwrap(),
5986 6
5987 );
5988 assert_eq!(
5989 ISO_8859_16
5990 .new_decoder_without_bom_handling()
5991 .latin1_byte_compatible_up_to(buffer)
5992 .unwrap(),
5993 4
5994 );
5995 assert_eq!(
5996 ISO_8859_8_I
5997 .new_decoder_without_bom_handling()
5998 .latin1_byte_compatible_up_to(buffer)
5999 .unwrap(),
6000 3
6001 );
6002 assert_eq!(
6003 KOI8_R
6004 .new_decoder_without_bom_handling()
6005 .latin1_byte_compatible_up_to(buffer)
6006 .unwrap(),
6007 1
6008 );
6009 assert_eq!(
6010 KOI8_U
6011 .new_decoder_without_bom_handling()
6012 .latin1_byte_compatible_up_to(buffer)
6013 .unwrap(),
6014 1
6015 );
6016 assert_eq!(
6017 MACINTOSH
6018 .new_decoder_without_bom_handling()
6019 .latin1_byte_compatible_up_to(buffer)
6020 .unwrap(),
6021 1
6022 );
6023 assert_eq!(
6024 WINDOWS_874
6025 .new_decoder_without_bom_handling()
6026 .latin1_byte_compatible_up_to(buffer)
6027 .unwrap(),
6028 2
6029 );
6030 assert_eq!(
6031 WINDOWS_1250
6032 .new_decoder_without_bom_handling()
6033 .latin1_byte_compatible_up_to(buffer)
6034 .unwrap(),
6035 4
6036 );
6037 assert_eq!(
6038 WINDOWS_1251
6039 .new_decoder_without_bom_handling()
6040 .latin1_byte_compatible_up_to(buffer)
6041 .unwrap(),
6042 1
6043 );
6044 assert_eq!(
6045 WINDOWS_1252
6046 .new_decoder_without_bom_handling()
6047 .latin1_byte_compatible_up_to(buffer)
6048 .unwrap(),
6049 5
6050 );
6051 assert_eq!(
6052 WINDOWS_1253
6053 .new_decoder_without_bom_handling()
6054 .latin1_byte_compatible_up_to(buffer)
6055 .unwrap(),
6056 3
6057 );
6058 assert_eq!(
6059 WINDOWS_1254
6060 .new_decoder_without_bom_handling()
6061 .latin1_byte_compatible_up_to(buffer)
6062 .unwrap(),
6063 4
6064 );
6065 assert_eq!(
6066 WINDOWS_1255
6067 .new_decoder_without_bom_handling()
6068 .latin1_byte_compatible_up_to(buffer)
6069 .unwrap(),
6070 3
6071 );
6072 assert_eq!(
6073 WINDOWS_1256
6074 .new_decoder_without_bom_handling()
6075 .latin1_byte_compatible_up_to(buffer)
6076 .unwrap(),
6077 1
6078 );
6079 assert_eq!(
6080 WINDOWS_1257
6081 .new_decoder_without_bom_handling()
6082 .latin1_byte_compatible_up_to(buffer)
6083 .unwrap(),
6084 4
6085 );
6086 assert_eq!(
6087 WINDOWS_1258
6088 .new_decoder_without_bom_handling()
6089 .latin1_byte_compatible_up_to(buffer)
6090 .unwrap(),
6091 4
6092 );
6093 assert_eq!(
6094 X_MAC_CYRILLIC
6095 .new_decoder_without_bom_handling()
6096 .latin1_byte_compatible_up_to(buffer)
6097 .unwrap(),
6098 1
6099 );
6100 assert_eq!(
6101 X_USER_DEFINED
6102 .new_decoder_without_bom_handling()
6103 .latin1_byte_compatible_up_to(buffer)
6104 .unwrap(),
6105 1
6106 );
6107
6108 assert!(UTF_8
6109 .new_decoder()
6110 .latin1_byte_compatible_up_to(buffer)
6111 .is_none());
6112
6113 let mut decoder = UTF_8.new_decoder();
6114 let mut output = [0u16; 4];
6115 let _ = decoder.decode_to_utf16(b"\xEF", &mut output, false);
6116 assert!(decoder.latin1_byte_compatible_up_to(buffer).is_none());
6117 let _ = decoder.decode_to_utf16(b"\xBB\xBF", &mut output, false);
6118 assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), Some(1));
6119 let _ = decoder.decode_to_utf16(b"\xEF", &mut output, false);
6120 assert_eq!(decoder.latin1_byte_compatible_up_to(buffer), None);
6121 }
6122 }