| 1 |
/* |
| 2 |
* Huffman entropy encoding (32-bit Arm Neon) |
| 3 |
* |
| 4 |
* Copyright (C) 2020, Arm Limited. All Rights Reserved. |
| 5 |
* Copyright (C) 2024, D. R. Commander. All Rights Reserved. |
| 6 |
* |
| 7 |
* This software is provided 'as-is', without any express or implied |
| 8 |
* warranty. In no event will the authors be held liable for any damages |
| 9 |
* arising from the use of this software. |
| 10 |
* |
| 11 |
* Permission is granted to anyone to use this software for any purpose, |
| 12 |
* including commercial applications, and to alter it and redistribute it |
| 13 |
* freely, subject to the following restrictions: |
| 14 |
* |
| 15 |
* 1. The origin of this software must not be misrepresented; you must not |
| 16 |
* claim that you wrote the original software. If you use this software |
| 17 |
* in a product, an acknowledgment in the product documentation would be |
| 18 |
* appreciated but is not required. |
| 19 |
* 2. Altered source versions must be plainly marked as such, and must not be |
| 20 |
* misrepresented as being the original software. |
| 21 |
* 3. This notice may not be removed or altered from any source distribution. |
| 22 |
* |
| 23 |
* NOTE: All referenced figures are from |
| 24 |
* Recommendation ITU-T T.81 (1992) | ISO/IEC 10918-1:1994. |
| 25 |
*/ |
| 26 |
|
| 27 |
#define JPEG_INTERNALS |
| 28 |
#include "../../../src/jinclude.h" |
| 29 |
#include "../../../src/jpeglib.h" |
| 30 |
#include "../../../src/jsimd.h" |
| 31 |
#include "../../../src/jdct.h" |
| 32 |
#include "../../../src/jsimddct.h" |
| 33 |
#include "../../jsimd.h" |
| 34 |
#include "../jchuff.h" |
| 35 |
#include "neon-compat.h" |
| 36 |
|
| 37 |
#include <limits.h> |
| 38 |
|
| 39 |
#include <arm_neon.h> |
| 40 |
|
| 41 |
|
| 42 |
JOCTET *jsimd_huff_encode_one_block_neon(void *state, JOCTET *buffer, |
| 43 |
JCOEFPTR block, int last_dc_val, |
| 44 |
c_derived_tbl *dctbl, |
| 45 |
c_derived_tbl *actbl) |
| 46 |
{ |
| 47 |
uint8_t block_nbits[DCTSIZE2]; |
| 48 |
uint16_t block_diff[DCTSIZE2]; |
| 49 |
|
| 50 |
/* Load rows of coefficients from DCT block in zig-zag order. */ |
| 51 |
|
| 52 |
/* Compute DC coefficient difference value. (F.1.1.5.1) */ |
| 53 |
int16x8_t row0 = vdupq_n_s16(block[0] - last_dc_val); |
| 54 |
row0 = vld1q_lane_s16(block + 1, row0, 1); |
| 55 |
row0 = vld1q_lane_s16(block + 8, row0, 2); |
| 56 |
row0 = vld1q_lane_s16(block + 16, row0, 3); |
| 57 |
row0 = vld1q_lane_s16(block + 9, row0, 4); |
| 58 |
row0 = vld1q_lane_s16(block + 2, row0, 5); |
| 59 |
row0 = vld1q_lane_s16(block + 3, row0, 6); |
| 60 |
row0 = vld1q_lane_s16(block + 10, row0, 7); |
| 61 |
|
| 62 |
int16x8_t row1 = vld1q_dup_s16(block + 17); |
| 63 |
row1 = vld1q_lane_s16(block + 24, row1, 1); |
| 64 |
row1 = vld1q_lane_s16(block + 32, row1, 2); |
| 65 |
row1 = vld1q_lane_s16(block + 25, row1, 3); |
| 66 |
row1 = vld1q_lane_s16(block + 18, row1, 4); |
| 67 |
row1 = vld1q_lane_s16(block + 11, row1, 5); |
| 68 |
row1 = vld1q_lane_s16(block + 4, row1, 6); |
| 69 |
row1 = vld1q_lane_s16(block + 5, row1, 7); |
| 70 |
|
| 71 |
int16x8_t row2 = vld1q_dup_s16(block + 12); |
| 72 |
row2 = vld1q_lane_s16(block + 19, row2, 1); |
| 73 |
row2 = vld1q_lane_s16(block + 26, row2, 2); |
| 74 |
row2 = vld1q_lane_s16(block + 33, row2, 3); |
| 75 |
row2 = vld1q_lane_s16(block + 40, row2, 4); |
| 76 |
row2 = vld1q_lane_s16(block + 48, row2, 5); |
| 77 |
row2 = vld1q_lane_s16(block + 41, row2, 6); |
| 78 |
row2 = vld1q_lane_s16(block + 34, row2, 7); |
| 79 |
|
| 80 |
int16x8_t row3 = vld1q_dup_s16(block + 27); |
| 81 |
row3 = vld1q_lane_s16(block + 20, row3, 1); |
| 82 |
row3 = vld1q_lane_s16(block + 13, row3, 2); |
| 83 |
row3 = vld1q_lane_s16(block + 6, row3, 3); |
| 84 |
row3 = vld1q_lane_s16(block + 7, row3, 4); |
| 85 |
row3 = vld1q_lane_s16(block + 14, row3, 5); |
| 86 |
row3 = vld1q_lane_s16(block + 21, row3, 6); |
| 87 |
row3 = vld1q_lane_s16(block + 28, row3, 7); |
| 88 |
|
| 89 |
int16x8_t abs_row0 = vabsq_s16(row0); |
| 90 |
int16x8_t abs_row1 = vabsq_s16(row1); |
| 91 |
int16x8_t abs_row2 = vabsq_s16(row2); |
| 92 |
int16x8_t abs_row3 = vabsq_s16(row3); |
| 93 |
|
| 94 |
int16x8_t row0_lz = vclzq_s16(abs_row0); |
| 95 |
int16x8_t row1_lz = vclzq_s16(abs_row1); |
| 96 |
int16x8_t row2_lz = vclzq_s16(abs_row2); |
| 97 |
int16x8_t row3_lz = vclzq_s16(abs_row3); |
| 98 |
|
| 99 |
/* Compute number of bits required to represent each coefficient. */ |
| 100 |
uint8x8_t row0_nbits = vsub_u8(vdup_n_u8(16), |
| 101 |
vmovn_u16(vreinterpretq_u16_s16(row0_lz))); |
| 102 |
uint8x8_t row1_nbits = vsub_u8(vdup_n_u8(16), |
| 103 |
vmovn_u16(vreinterpretq_u16_s16(row1_lz))); |
| 104 |
uint8x8_t row2_nbits = vsub_u8(vdup_n_u8(16), |
| 105 |
vmovn_u16(vreinterpretq_u16_s16(row2_lz))); |
| 106 |
uint8x8_t row3_nbits = vsub_u8(vdup_n_u8(16), |
| 107 |
vmovn_u16(vreinterpretq_u16_s16(row3_lz))); |
| 108 |
|
| 109 |
vst1_u8(block_nbits + 0 * DCTSIZE, row0_nbits); |
| 110 |
vst1_u8(block_nbits + 1 * DCTSIZE, row1_nbits); |
| 111 |
vst1_u8(block_nbits + 2 * DCTSIZE, row2_nbits); |
| 112 |
vst1_u8(block_nbits + 3 * DCTSIZE, row3_nbits); |
| 113 |
|
| 114 |
uint16x8_t row0_mask = |
| 115 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row0, 15)), |
| 116 |
vnegq_s16(row0_lz)); |
| 117 |
uint16x8_t row1_mask = |
| 118 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row1, 15)), |
| 119 |
vnegq_s16(row1_lz)); |
| 120 |
uint16x8_t row2_mask = |
| 121 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row2, 15)), |
| 122 |
vnegq_s16(row2_lz)); |
| 123 |
uint16x8_t row3_mask = |
| 124 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row3, 15)), |
| 125 |
vnegq_s16(row3_lz)); |
| 126 |
|
| 127 |
uint16x8_t row0_diff = veorq_u16(vreinterpretq_u16_s16(abs_row0), row0_mask); |
| 128 |
uint16x8_t row1_diff = veorq_u16(vreinterpretq_u16_s16(abs_row1), row1_mask); |
| 129 |
uint16x8_t row2_diff = veorq_u16(vreinterpretq_u16_s16(abs_row2), row2_mask); |
| 130 |
uint16x8_t row3_diff = veorq_u16(vreinterpretq_u16_s16(abs_row3), row3_mask); |
| 131 |
|
| 132 |
/* Store diff values for rows 0, 1, 2, and 3. */ |
| 133 |
vst1q_u16(block_diff + 0 * DCTSIZE, row0_diff); |
| 134 |
vst1q_u16(block_diff + 1 * DCTSIZE, row1_diff); |
| 135 |
vst1q_u16(block_diff + 2 * DCTSIZE, row2_diff); |
| 136 |
vst1q_u16(block_diff + 3 * DCTSIZE, row3_diff); |
| 137 |
|
| 138 |
/* Load last four rows of coefficients from DCT block in zig-zag order. */ |
| 139 |
int16x8_t row4 = vld1q_dup_s16(block + 35); |
| 140 |
row4 = vld1q_lane_s16(block + 42, row4, 1); |
| 141 |
row4 = vld1q_lane_s16(block + 49, row4, 2); |
| 142 |
row4 = vld1q_lane_s16(block + 56, row4, 3); |
| 143 |
row4 = vld1q_lane_s16(block + 57, row4, 4); |
| 144 |
row4 = vld1q_lane_s16(block + 50, row4, 5); |
| 145 |
row4 = vld1q_lane_s16(block + 43, row4, 6); |
| 146 |
row4 = vld1q_lane_s16(block + 36, row4, 7); |
| 147 |
|
| 148 |
int16x8_t row5 = vld1q_dup_s16(block + 29); |
| 149 |
row5 = vld1q_lane_s16(block + 22, row5, 1); |
| 150 |
row5 = vld1q_lane_s16(block + 15, row5, 2); |
| 151 |
row5 = vld1q_lane_s16(block + 23, row5, 3); |
| 152 |
row5 = vld1q_lane_s16(block + 30, row5, 4); |
| 153 |
row5 = vld1q_lane_s16(block + 37, row5, 5); |
| 154 |
row5 = vld1q_lane_s16(block + 44, row5, 6); |
| 155 |
row5 = vld1q_lane_s16(block + 51, row5, 7); |
| 156 |
|
| 157 |
int16x8_t row6 = vld1q_dup_s16(block + 58); |
| 158 |
row6 = vld1q_lane_s16(block + 59, row6, 1); |
| 159 |
row6 = vld1q_lane_s16(block + 52, row6, 2); |
| 160 |
row6 = vld1q_lane_s16(block + 45, row6, 3); |
| 161 |
row6 = vld1q_lane_s16(block + 38, row6, 4); |
| 162 |
row6 = vld1q_lane_s16(block + 31, row6, 5); |
| 163 |
row6 = vld1q_lane_s16(block + 39, row6, 6); |
| 164 |
row6 = vld1q_lane_s16(block + 46, row6, 7); |
| 165 |
|
| 166 |
int16x8_t row7 = vld1q_dup_s16(block + 53); |
| 167 |
row7 = vld1q_lane_s16(block + 60, row7, 1); |
| 168 |
row7 = vld1q_lane_s16(block + 61, row7, 2); |
| 169 |
row7 = vld1q_lane_s16(block + 54, row7, 3); |
| 170 |
row7 = vld1q_lane_s16(block + 47, row7, 4); |
| 171 |
row7 = vld1q_lane_s16(block + 55, row7, 5); |
| 172 |
row7 = vld1q_lane_s16(block + 62, row7, 6); |
| 173 |
row7 = vld1q_lane_s16(block + 63, row7, 7); |
| 174 |
|
| 175 |
int16x8_t abs_row4 = vabsq_s16(row4); |
| 176 |
int16x8_t abs_row5 = vabsq_s16(row5); |
| 177 |
int16x8_t abs_row6 = vabsq_s16(row6); |
| 178 |
int16x8_t abs_row7 = vabsq_s16(row7); |
| 179 |
|
| 180 |
int16x8_t row4_lz = vclzq_s16(abs_row4); |
| 181 |
int16x8_t row5_lz = vclzq_s16(abs_row5); |
| 182 |
int16x8_t row6_lz = vclzq_s16(abs_row6); |
| 183 |
int16x8_t row7_lz = vclzq_s16(abs_row7); |
| 184 |
|
| 185 |
/* Compute number of bits required to represent each coefficient. */ |
| 186 |
uint8x8_t row4_nbits = vsub_u8(vdup_n_u8(16), |
| 187 |
vmovn_u16(vreinterpretq_u16_s16(row4_lz))); |
| 188 |
uint8x8_t row5_nbits = vsub_u8(vdup_n_u8(16), |
| 189 |
vmovn_u16(vreinterpretq_u16_s16(row5_lz))); |
| 190 |
uint8x8_t row6_nbits = vsub_u8(vdup_n_u8(16), |
| 191 |
vmovn_u16(vreinterpretq_u16_s16(row6_lz))); |
| 192 |
uint8x8_t row7_nbits = vsub_u8(vdup_n_u8(16), |
| 193 |
vmovn_u16(vreinterpretq_u16_s16(row7_lz))); |
| 194 |
|
| 195 |
vst1_u8(block_nbits + 4 * DCTSIZE, row4_nbits); |
| 196 |
vst1_u8(block_nbits + 5 * DCTSIZE, row5_nbits); |
| 197 |
vst1_u8(block_nbits + 6 * DCTSIZE, row6_nbits); |
| 198 |
vst1_u8(block_nbits + 7 * DCTSIZE, row7_nbits); |
| 199 |
|
| 200 |
uint16x8_t row4_mask = |
| 201 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row4, 15)), |
| 202 |
vnegq_s16(row4_lz)); |
| 203 |
uint16x8_t row5_mask = |
| 204 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row5, 15)), |
| 205 |
vnegq_s16(row5_lz)); |
| 206 |
uint16x8_t row6_mask = |
| 207 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row6, 15)), |
| 208 |
vnegq_s16(row6_lz)); |
| 209 |
uint16x8_t row7_mask = |
| 210 |
vshlq_u16(vreinterpretq_u16_s16(vshrq_n_s16(row7, 15)), |
| 211 |
vnegq_s16(row7_lz)); |
| 212 |
|
| 213 |
uint16x8_t row4_diff = veorq_u16(vreinterpretq_u16_s16(abs_row4), row4_mask); |
| 214 |
uint16x8_t row5_diff = veorq_u16(vreinterpretq_u16_s16(abs_row5), row5_mask); |
| 215 |
uint16x8_t row6_diff = veorq_u16(vreinterpretq_u16_s16(abs_row6), row6_mask); |
| 216 |
uint16x8_t row7_diff = veorq_u16(vreinterpretq_u16_s16(abs_row7), row7_mask); |
| 217 |
|
| 218 |
/* Store diff values for rows 4, 5, 6, and 7. */ |
| 219 |
vst1q_u16(block_diff + 4 * DCTSIZE, row4_diff); |
| 220 |
vst1q_u16(block_diff + 5 * DCTSIZE, row5_diff); |
| 221 |
vst1q_u16(block_diff + 6 * DCTSIZE, row6_diff); |
| 222 |
vst1q_u16(block_diff + 7 * DCTSIZE, row7_diff); |
| 223 |
|
| 224 |
/* Construct bitmap to accelerate encoding of AC coefficients. A set bit |
| 225 |
* means that the corresponding coefficient != 0. |
| 226 |
*/ |
| 227 |
uint8x8_t row0_nbits_gt0 = vcgt_u8(row0_nbits, vdup_n_u8(0)); |
| 228 |
uint8x8_t row1_nbits_gt0 = vcgt_u8(row1_nbits, vdup_n_u8(0)); |
| 229 |
uint8x8_t row2_nbits_gt0 = vcgt_u8(row2_nbits, vdup_n_u8(0)); |
| 230 |
uint8x8_t row3_nbits_gt0 = vcgt_u8(row3_nbits, vdup_n_u8(0)); |
| 231 |
uint8x8_t row4_nbits_gt0 = vcgt_u8(row4_nbits, vdup_n_u8(0)); |
| 232 |
uint8x8_t row5_nbits_gt0 = vcgt_u8(row5_nbits, vdup_n_u8(0)); |
| 233 |
uint8x8_t row6_nbits_gt0 = vcgt_u8(row6_nbits, vdup_n_u8(0)); |
| 234 |
uint8x8_t row7_nbits_gt0 = vcgt_u8(row7_nbits, vdup_n_u8(0)); |
| 235 |
|
| 236 |
/* { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 } */ |
| 237 |
const uint8x8_t bitmap_mask = |
| 238 |
vreinterpret_u8_u64(vmov_n_u64(0x0102040810204080)); |
| 239 |
|
| 240 |
row0_nbits_gt0 = vand_u8(row0_nbits_gt0, bitmap_mask); |
| 241 |
row1_nbits_gt0 = vand_u8(row1_nbits_gt0, bitmap_mask); |
| 242 |
row2_nbits_gt0 = vand_u8(row2_nbits_gt0, bitmap_mask); |
| 243 |
row3_nbits_gt0 = vand_u8(row3_nbits_gt0, bitmap_mask); |
| 244 |
row4_nbits_gt0 = vand_u8(row4_nbits_gt0, bitmap_mask); |
| 245 |
row5_nbits_gt0 = vand_u8(row5_nbits_gt0, bitmap_mask); |
| 246 |
row6_nbits_gt0 = vand_u8(row6_nbits_gt0, bitmap_mask); |
| 247 |
row7_nbits_gt0 = vand_u8(row7_nbits_gt0, bitmap_mask); |
| 248 |
|
| 249 |
uint8x8_t bitmap_rows_10 = vpadd_u8(row1_nbits_gt0, row0_nbits_gt0); |
| 250 |
uint8x8_t bitmap_rows_32 = vpadd_u8(row3_nbits_gt0, row2_nbits_gt0); |
| 251 |
uint8x8_t bitmap_rows_54 = vpadd_u8(row5_nbits_gt0, row4_nbits_gt0); |
| 252 |
uint8x8_t bitmap_rows_76 = vpadd_u8(row7_nbits_gt0, row6_nbits_gt0); |
| 253 |
uint8x8_t bitmap_rows_3210 = vpadd_u8(bitmap_rows_32, bitmap_rows_10); |
| 254 |
uint8x8_t bitmap_rows_7654 = vpadd_u8(bitmap_rows_76, bitmap_rows_54); |
| 255 |
uint8x8_t bitmap = vpadd_u8(bitmap_rows_7654, bitmap_rows_3210); |
| 256 |
|
| 257 |
/* Shift left to remove DC bit. */ |
| 258 |
bitmap = vreinterpret_u8_u64(vshl_n_u64(vreinterpret_u64_u8(bitmap), 1)); |
| 259 |
/* Move bitmap to 32-bit scalar registers. */ |
| 260 |
uint32_t bitmap_1_32 = vget_lane_u32(vreinterpret_u32_u8(bitmap), 1); |
| 261 |
uint32_t bitmap_33_63 = vget_lane_u32(vreinterpret_u32_u8(bitmap), 0); |
| 262 |
|
| 263 |
/* Set up state and bit buffer for output bitstream. */ |
| 264 |
working_state *state_ptr = (working_state *)state; |
| 265 |
int free_bits = state_ptr->cur.free_bits; |
| 266 |
size_t put_buffer = state_ptr->cur.put_buffer; |
| 267 |
|
| 268 |
/* Encode DC coefficient. */ |
| 269 |
|
| 270 |
unsigned int nbits = block_nbits[0]; |
| 271 |
/* Emit Huffman-coded symbol and additional diff bits. */ |
| 272 |
unsigned int diff = block_diff[0]; |
| 273 |
PUT_CODE(dctbl->ehufco[nbits], dctbl->ehufsi[nbits], diff) |
| 274 |
|
| 275 |
/* Encode AC coefficients. */ |
| 276 |
|
| 277 |
unsigned int r = 0; /* r = run length of zeros */ |
| 278 |
unsigned int i = 1; /* i = number of coefficients encoded */ |
| 279 |
/* Code and size information for a run length of 16 zero coefficients */ |
| 280 |
const unsigned int code_0xf0 = actbl->ehufco[0xf0]; |
| 281 |
const unsigned int size_0xf0 = actbl->ehufsi[0xf0]; |
| 282 |
|
| 283 |
while (bitmap_1_32 != 0) { |
| 284 |
r = BUILTIN_CLZ(bitmap_1_32); |
| 285 |
i += r; |
| 286 |
bitmap_1_32 <<= r; |
| 287 |
nbits = block_nbits[i]; |
| 288 |
diff = block_diff[i]; |
| 289 |
while (r > 15) { |
| 290 |
/* If run length > 15, emit special run-length-16 codes. */ |
| 291 |
PUT_BITS(code_0xf0, size_0xf0) |
| 292 |
r -= 16; |
| 293 |
} |
| 294 |
/* Emit Huffman symbol for run length / number of bits. (F.1.2.2.1) */ |
| 295 |
unsigned int rs = (r << 4) + nbits; |
| 296 |
PUT_CODE(actbl->ehufco[rs], actbl->ehufsi[rs], diff) |
| 297 |
i++; |
| 298 |
bitmap_1_32 <<= 1; |
| 299 |
} |
| 300 |
|
| 301 |
r = 33 - i; |
| 302 |
i = 33; |
| 303 |
|
| 304 |
while (bitmap_33_63 != 0) { |
| 305 |
unsigned int leading_zeros = BUILTIN_CLZ(bitmap_33_63); |
| 306 |
r += leading_zeros; |
| 307 |
i += leading_zeros; |
| 308 |
bitmap_33_63 <<= leading_zeros; |
| 309 |
nbits = block_nbits[i]; |
| 310 |
diff = block_diff[i]; |
| 311 |
while (r > 15) { |
| 312 |
/* If run length > 15, emit special run-length-16 codes. */ |
| 313 |
PUT_BITS(code_0xf0, size_0xf0) |
| 314 |
r -= 16; |
| 315 |
} |
| 316 |
/* Emit Huffman symbol for run length / number of bits. (F.1.2.2.1) */ |
| 317 |
unsigned int rs = (r << 4) + nbits; |
| 318 |
PUT_CODE(actbl->ehufco[rs], actbl->ehufsi[rs], diff) |
| 319 |
r = 0; |
| 320 |
i++; |
| 321 |
bitmap_33_63 <<= 1; |
| 322 |
} |
| 323 |
|
| 324 |
/* If the last coefficient(s) were zero, emit an end-of-block (EOB) code. |
| 325 |
* The value of RS for the EOB code is 0. |
| 326 |
*/ |
| 327 |
if (i != 64) { |
| 328 |
PUT_BITS(actbl->ehufco[0], actbl->ehufsi[0]) |
| 329 |
} |
| 330 |
|
| 331 |
state_ptr->cur.put_buffer = put_buffer; |
| 332 |
state_ptr->cur.free_bits = free_bits; |
| 333 |
|
| 334 |
return buffer; |
| 335 |
} |