| 1 |
//===-- sanitizer_lzw.h -----------------------------------------*- C++ -*-===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// Lempel–Ziv–Welch encoding/decoding |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
|
| 13 |
#ifndef SANITIZER_LZW_H |
| 14 |
#define SANITIZER_LZW_H |
| 15 |
|
| 16 |
#include "sanitizer_dense_map.h" |
| 17 |
|
| 18 |
namespace __sanitizer { |
| 19 |
|
| 20 |
using LzwCodeType = u32; |
| 21 |
|
| 22 |
template <class T, class ItIn, class ItOut> |
| 23 |
ItOut LzwEncode(ItIn begin, ItIn end, ItOut out) { |
| 24 |
using Substring = |
| 25 |
detail::DenseMapPair<LzwCodeType /* Prefix */, T /* Next input */>; |
| 26 |
|
| 27 |
// Sentinel value for substrings of len 1. |
| 28 |
static constexpr LzwCodeType kNoPrefix = |
| 29 |
Min(DenseMapInfo<Substring>::getEmptyKey().first, |
| 30 |
DenseMapInfo<Substring>::getTombstoneKey().first) - |
| 31 |
1; |
| 32 |
DenseMap<Substring, LzwCodeType> prefix_to_code; |
| 33 |
{ |
| 34 |
// Add all substring of len 1 as initial dictionary. |
| 35 |
InternalMmapVector<T> dict_len1; |
| 36 |
for (auto it = begin; it != end; ++it) |
| 37 |
if (prefix_to_code.try_emplace({kNoPrefix, *it}, 0).second) |
| 38 |
dict_len1.push_back(*it); |
| 39 |
|
| 40 |
// Slightly helps with later delta encoding. |
| 41 |
Sort(dict_len1.data(), dict_len1.size()); |
| 42 |
|
| 43 |
// For large sizeof(T) we have to store dict_len1. Smaller types like u8 can |
| 44 |
// just generate them. |
| 45 |
*out = dict_len1.size(); |
| 46 |
++out; |
| 47 |
|
| 48 |
for (uptr i = 0; i != dict_len1.size(); ++i) { |
| 49 |
// Remap after the Sort. |
| 50 |
prefix_to_code[{kNoPrefix, dict_len1[i]}] = i; |
| 51 |
*out = dict_len1[i]; |
| 52 |
++out; |
| 53 |
} |
| 54 |
CHECK_EQ(prefix_to_code.size(), dict_len1.size()); |
| 55 |
} |
| 56 |
|
| 57 |
if (begin == end) |
| 58 |
return out; |
| 59 |
|
| 60 |
// Main LZW encoding loop. |
| 61 |
LzwCodeType match = prefix_to_code.find({kNoPrefix, *begin})->second; |
| 62 |
++begin; |
| 63 |
for (auto it = begin; it != end; ++it) { |
| 64 |
// Extend match with the new item. |
| 65 |
auto ins = prefix_to_code.try_emplace({match, *it}, prefix_to_code.size()); |
| 66 |
if (ins.second) { |
| 67 |
// This is a new substring, but emit the code for the current match |
| 68 |
// (before extend). This allows LZW decoder to recover the dictionary. |
| 69 |
*out = match; |
| 70 |
++out; |
| 71 |
// Reset the match to a single item, which must be already in the map. |
| 72 |
match = prefix_to_code.find({kNoPrefix, *it})->second; |
| 73 |
} else { |
| 74 |
// Already known, use as the current match. |
| 75 |
match = ins.first->second; |
| 76 |
} |
| 77 |
} |
| 78 |
|
| 79 |
*out = match; |
| 80 |
++out; |
| 81 |
|
| 82 |
return out; |
| 83 |
} |
| 84 |
|
| 85 |
template <class T, class ItIn, class ItOut> |
| 86 |
ItOut LzwDecode(ItIn begin, ItIn end, ItOut out) { |
| 87 |
if (begin == end) |
| 88 |
return out; |
| 89 |
|
| 90 |
// Load dictionary of len 1 substrings. Theses correspont to lowest codes. |
| 91 |
InternalMmapVector<T> dict_len1(*begin); |
| 92 |
++begin; |
| 93 |
|
| 94 |
if (begin == end) |
| 95 |
return out; |
| 96 |
|
| 97 |
for (auto& v : dict_len1) { |
| 98 |
v = *begin; |
| 99 |
++begin; |
| 100 |
} |
| 101 |
|
| 102 |
// Substrings of len 2 and up. Indexes are shifted because [0, |
| 103 |
// dict_len1.size()) stored in dict_len1. Substings get here after being |
| 104 |
// emitted to the output, so we can use output position. |
| 105 |
InternalMmapVector<detail::DenseMapPair<ItOut /* begin. */, ItOut /* end */>> |
| 106 |
code_to_substr; |
| 107 |
|
| 108 |
// Copies already emitted substrings into the output again. |
| 109 |
auto copy = [&code_to_substr, &dict_len1](LzwCodeType code, ItOut out) { |
| 110 |
if (code < dict_len1.size()) { |
| 111 |
*out = dict_len1[code]; |
| 112 |
++out; |
| 113 |
return out; |
| 114 |
} |
| 115 |
const auto& s = code_to_substr[code - dict_len1.size()]; |
| 116 |
|
| 117 |
for (ItOut it = s.first; it != s.second; ++it, ++out) *out = *it; |
| 118 |
return out; |
| 119 |
}; |
| 120 |
|
| 121 |
// Returns lens of the substring with the given code. |
| 122 |
auto code_to_len = [&code_to_substr, &dict_len1](LzwCodeType code) -> uptr { |
| 123 |
if (code < dict_len1.size()) |
| 124 |
return 1; |
| 125 |
const auto& s = code_to_substr[code - dict_len1.size()]; |
| 126 |
return s.second - s.first; |
| 127 |
}; |
| 128 |
|
| 129 |
// Main LZW decoding loop. |
| 130 |
LzwCodeType prev_code = *begin; |
| 131 |
++begin; |
| 132 |
out = copy(prev_code, out); |
| 133 |
for (auto it = begin; it != end; ++it) { |
| 134 |
LzwCodeType code = *it; |
| 135 |
auto start = out; |
| 136 |
if (code == dict_len1.size() + code_to_substr.size()) { |
| 137 |
// Special LZW case. The code is not in the dictionary yet. This is |
| 138 |
// possible only when the new substring is the same as previous one plus |
| 139 |
// the first item of the previous substring. We can emit that in two |
| 140 |
// steps. |
| 141 |
out = copy(prev_code, out); |
| 142 |
*out = *start; |
| 143 |
++out; |
| 144 |
} else { |
| 145 |
out = copy(code, out); |
| 146 |
} |
| 147 |
|
| 148 |
// Every time encoded emits the code, it also creates substing of len + 1 |
| 149 |
// including the first item of the just emmited substring. Do the same here. |
| 150 |
uptr len = code_to_len(prev_code); |
| 151 |
code_to_substr.push_back({start - len, start + 1}); |
| 152 |
|
| 153 |
prev_code = code; |
| 154 |
} |
| 155 |
return out; |
| 156 |
} |
| 157 |
|
| 158 |
} // namespace __sanitizer |
| 159 |
#endif |