Debian Codegraph @

src/fcml_modrm_decoder.c

raw
1 /*
2 * FCML - Free Code Manipulation Library.
3 * Copyright (C) 2010-2020 Slawomir Wojtasiak
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20 #include "fcml_modrm_decoder.h"
21
22 #include <fcml_errors.h>
23
24 #include "fcml_env_int.h"
25 #include "fcml_utils.h"
26 #include "fcml_disp8_n.h"
27
28 /* Register configurations used for 16 bit addressing form decoding */
29
30 struct fcml_st_register fcml_modrm_addressing_form_reg_16[8][2] = {
31 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_BX, FCML_FALSE },
32 { FCML_REG_GPR, FCML_DS_16, FCML_REG_SI, FCML_FALSE } },
33 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_BX, FCML_FALSE },
34 { FCML_REG_GPR, FCML_DS_16, FCML_REG_DI, FCML_FALSE } },
35 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_BP, FCML_FALSE },
36 { FCML_REG_GPR, FCML_DS_16, FCML_REG_SI, FCML_FALSE } },
37 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_BP, FCML_FALSE },
38 { FCML_REG_GPR, FCML_DS_16, FCML_REG_DI, FCML_FALSE } },
39 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_SI, FCML_FALSE },
40 { FCML_REG_UNDEFINED, 0, 0, FCML_FALSE } },
41 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_DI, FCML_FALSE },
42 { FCML_REG_UNDEFINED, 0, 0, FCML_FALSE } },
43 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_BP, FCML_FALSE },
44 { FCML_REG_UNDEFINED, 0, 0, FCML_FALSE } },
45 { { FCML_REG_GPR, FCML_DS_16, FCML_REG_BX, FCML_FALSE },
46 { FCML_REG_UNDEFINED, 0, 0, FCML_FALSE } }
47 };
48
49 static fcml_uint32_t decode_compressed_disp8(
50 fcml_st_modrm_decoder_context *context, fcml_size size,
51 fcml_st_integer *displacement) {
52 /* Handle compressed disp8. */
53 fcml_uint32_t n = 0;
54 if (context->is_evex && size == FCML_DS_8) {
55 /* Notice that vector length here is calculated using EVEX.L'L only.
56 * For now it's not a problem, because register to register
57 * instructions which take static rounding mode into account while
58 * calculating vector length don't use memory addressing.
59 */
60 n = fcml_fn_d8n_calculate_n(context->tuple_type,
61 context->b, context->input_size,
62 context->vector_length);
63 if (n) {
64 fcml_int32_t disp = displacement->int8 * n;
65 fcml_st_integer decompressedDisp8 = {0};
66 decompressedDisp8.is_signed = FCML_TRUE;
67 decompressedDisp8.size = FCML_DS_32;
68 decompressedDisp8.int32 = disp;
69 *displacement = decompressedDisp8;
70 }
71 }
72 return n;
73 }
74
75 static fcml_ceh_error decode_displacement(
76 fcml_st_modrm_decoder_context *context,
77 fcml_st_memory_stream *stream, fcml_st_integer *displacement,
78 fcml_st_modrm_displacement *raw_displacement, fcml_st_offset *offset,
79 fcml_usize size, fcml_usize offset_extension_size,
80 fcml_usize displacement_extension_size) {
81
82 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
83
84 fcml_st_integer integer = {0};
85
86 error = fcml_fn_utils_decode_integer(stream, &integer, size);
87 if (error) {
88 return error;
89 }
90
91 /* Offsets and displacement are signed integers.*/
92 integer.is_signed = FCML_TRUE;
93
94 raw_displacement->displacement = integer;
95
96 /* This method modifies displacement in place when needed. */
97 fcml_uint32_t n = decode_compressed_disp8(context, size,
98 &integer);
99 if (n) {
100 raw_displacement->N.is_not_null = FCML_TRUE;
101 raw_displacement->N.value = n;
102 }
103
104 if (displacement_extension_size) {
105 fcml_fn_utils_extend_int(&integer, displacement_extension_size);
106 }
107
108 if (displacement) {
109 *displacement = integer;
110 }
111
112 if (offset) {
113
114 if (offset_extension_size) {
115 fcml_fn_utils_extend_int(&integer, offset_extension_size);
116 }
117
118 error = fcml_fn_utils_integer_to_offset(&integer, offset);
119 if (error) {
120 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
121 }
122
123 }
124
125 return error;
126 }
127
128 static fcml_usize get_displacement_extension(
129 fcml_st_modrm_decoder_context *context,
130 fcml_uint8_t flags) {
131 return flags & FCML_MODRM_DEC_FLAG_EXTEND_DISPLACEMENT_TO_ASA ?
132 context->effective_address_size : 0;
133 }
134
135 static fcml_ceh_error decode_16bit(fcml_st_modrm_decoder_context *context,
136 fcml_st_modrm_source *modrm_source, fcml_st_modrm *decoded_modrm,
137 fcml_st_modrm_details *modrm_details, fcml_uint8_t flags ) {
138
139 fcml_bool result;
140 fcml_st_memory_stream *stream = modrm_source->stream;
141
142 /* Just in case.*/
143 if (context->op_mode == FCML_OM_64_BIT) {
144 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
145 }
146
147 fcml_st_address *address = &(decoded_modrm->address);
148 fcml_st_effective_address *effective_address =
149 &(address->effective_address);
150
151 /* Gets ModR/M byte from stream.*/
152 fcml_uint8_t mod_rm = fcml_fn_stream_read(stream, &result);
153 if (!result) {
154 return FCML_CEH_GEC_EOF;
155 }
156
157 modrm_details->modrm = mod_rm;
158
159 /* Decode ModRM.*/
160 fcml_uint8_t f_mod = FCML_MODRM_DEC_MOD(mod_rm);
161 fcml_uint8_t f_rm = FCML_MODRM_DEC_RM(mod_rm);
162 fcml_uint8_t f_reg_opcode = FCML_MODRM_DEC_REG_OPCODE(mod_rm);
163
164 if (modrm_source->is_vsib) {
165 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
166 }
167
168 address->address_form = FCML_AF_COMBINED;
169
170 if (f_mod == 0 && f_rm == 6) {
171
172 /* disp16.*/
173 fcml_ceh_error error = decode_displacement(context,
174 stream, &(effective_address->displacement),
175 &(modrm_details->displacement), &( address->offset), FCML_DS_16,
176 context->effective_address_size,
177 get_displacement_extension(context, flags));
178 if (error) {
179 return error;
180 }
181
182 address->address_form = FCML_AF_OFFSET;
183
184 } else if (f_mod < 3) {
185 effective_address->base = fcml_modrm_addressing_form_reg_16[f_rm][0];
186 effective_address->index = fcml_modrm_addressing_form_reg_16[f_rm][1];
187 if (f_mod > 0) {
188 fcml_ceh_error error = decode_displacement(context,
189 stream, &(effective_address->displacement),
190 &(modrm_details->displacement), NULL,
191 (f_mod == 1) ? FCML_DS_8 : FCML_DS_16, 0,
192 get_displacement_extension(context, flags));
193 if ( error ) {
194 return error;
195 }
196 }
197 } else {
198 /* Straight copy of registers.*/
199 decoded_modrm->reg.is_not_null = FCML_TRUE;
200 decoded_modrm->reg.value = f_rm;
201 }
202
203 decoded_modrm->reg_opcode = f_reg_opcode;
204
205 return FCML_CEH_GEC_NO_ERROR;
206 }
207
208 static fcml_ceh_error decode_sib(fcml_st_modrm_decoder_context *context,
209 fcml_uint8_t mod_rm, fcml_st_modrm_source *modrm_source,
210 fcml_st_modrm *decoded_modrm, fcml_st_modrm_details *modrm_details,
211 fcml_uint8_t flags) {
212
213 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
214 fcml_st_memory_stream *stream = modrm_source->stream;
215 fcml_st_address *address = &(decoded_modrm->address);
216 fcml_st_effective_address *effective_address =
217 &(address->effective_address);
218
219 /* Effective address size affects index register.*/
220 fcml_usize effective_address_size = context->effective_address_size;
221
222 /* ModR/M fields.*/
223 fcml_uint8_t f_base;
224 fcml_uint8_t f_index;
225 fcml_uint8_t f_scale;
226 fcml_uint8_t f_mod;
227
228 /* Get SIB.*/
229 fcml_bool sib_found = FCML_FALSE;
230 fcml_uint8_t sib = fcml_fn_stream_read(stream, &sib_found);
231 if (!sib_found) {
232 return FCML_CEH_GEC_EOF;
233 }
234
235 modrm_details->sib.is_not_null = FCML_TRUE;
236 modrm_details->sib.value = sib;
237
238 f_mod = FCML_MODRM_DEC_MOD(mod_rm);
239 f_base = FCML_MODRM_SIB_BASE(sib) | (modrm_source->ext_B << 3);
240 f_index = FCML_MODRM_SIB_INDEX(sib) | (modrm_source->ext_X << 3);
241 f_scale = FCML_MODRM_SIB_SS(sib);
242
243 /* Index register and scale.*/
244 if (FCML_MODRM_SIB_INDEX(sib) != 4) {
245 if (modrm_source->is_vsib) {
246 /* If EVEX is available, VSIB index register is extended
247 * by EVEX.V' bit.
248 */
249 if (context->is_evex) {
250 f_index |= (modrm_source->ext_V_prim << 4);
251 }
252 effective_address->index.type = FCML_REG_SIMD;
253 effective_address->index.size = modrm_source->vsib_index_size;
254 } else {
255 effective_address->index.type = FCML_REG_GPR;
256 effective_address->index.size = (effective_address_size ==
257 FCML_DS_64 ) ? FCML_DS_64 : FCML_DS_32;
258 }
259 effective_address->index.reg = f_index;
260 /* Scale. (f_scale * 2) */
261 effective_address->scale_factor = f_scale ? (1 << f_scale) : 0;
262 }
263
264 address->address_form = FCML_AF_COMBINED;
265
266 /* Base register and displacement.*/
267 if (f_mod == 0 && FCML_MODRM_SIB_BASE(sib) == 5) {
268 if (f_index == 4) {
269 address->address_form = FCML_AF_OFFSET;
270 }
271
272 /* In this case base register doesn't exist.*/
273 error = decode_displacement(context,
274 stream, &(effective_address->displacement),
275 &(modrm_details->displacement), &(address->offset), FCML_DS_32,
276 effective_address_size, 0);
277 } else {
278 /* Effective address size affects base register.*/
279 effective_address->base.type = FCML_REG_GPR;
280 effective_address->base.size = (effective_address_size == FCML_DS_64)
281 ? FCML_DS_64 : FCML_DS_32;
282 effective_address->base.reg = f_base;
283
284 /* There is no displacement for mod == 0.*/
285 if (f_mod > 0) {
286 error = decode_displacement(context, stream,
287 &(effective_address->displacement),
288 &(modrm_details->displacement), NULL,
289 (f_mod == 1) ? FCML_DS_8 : FCML_DS_32, 0,
290 get_displacement_extension(context, flags));
291 }
292 }
293
294 return error;
295 }
296
297 static fcml_ceh_error decode_3264bit(fcml_st_modrm_decoder_context *context,
298 fcml_st_modrm_source *modrm_source, fcml_st_modrm *decoded_modrm,
299 fcml_st_modrm_details *modrm_details, fcml_uint8_t flags ) {
300
301 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
302
303 fcml_st_memory_stream *stream = modrm_source->stream;
304
305 fcml_st_address *address = &( decoded_modrm->address );
306 fcml_st_effective_address *effective_address =
307 &(address->effective_address);
308
309 fcml_uint8_t f_mod;
310 fcml_uint8_t f_rm;
311
312 fcml_usize effective_address_size = context->effective_address_size;
313
314 /* Gets ModR/M byte from stream.*/
315 fcml_bool result;
316 fcml_uint8_t mod_rm = fcml_fn_stream_read(stream, &result);
317 if (!result) {
318 return FCML_CEH_GEC_EOF;
319 }
320
321 modrm_details->modrm = mod_rm;
322
323 /* Decode ModRM.*/
324 f_mod = FCML_MODRM_DEC_MOD(mod_rm);
325 f_rm = FCML_MODRM_DEC_RM(mod_rm) | (modrm_source->ext_B << 3) |
326 (modrm_source->ext_X << 4);
327
328 if (modrm_source->is_vsib && FCML_MODRM_DEC_RM(mod_rm) != 4) {
329 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
330 }
331
332 address->address_form = FCML_AF_COMBINED;
333
334 if (f_mod == 3) {
335 /* Registers.*/
336 decoded_modrm->reg.is_not_null = FCML_TRUE;
337 decoded_modrm->reg.value = f_rm;
338 } else if (FCML_MODRM_DEC_RM(mod_rm) == 4) {
339 /* Decode SIB addressing format.*/
340 error = decode_sib( context, mod_rm,
341 modrm_source, decoded_modrm, modrm_details, flags);
342 } else if (f_mod == 0 && FCML_MODRM_DEC_RM(mod_rm) == 5) {
343
344 address->address_form = FCML_AF_OFFSET;
345
346 /* disp32.*/
347 if (context->op_mode == FCML_OM_64_BIT) {
348 decoded_modrm->is_rip = FCML_TRUE;
349 /* In case of RIP we also set base register to IP.*/
350 decoded_modrm->address.effective_address.base.type = FCML_REG_IP;
351 decoded_modrm->address.effective_address.base.size = FCML_DS_64;
352 }
353
354 /* Only displacement value is decoded here. RIP offset is
355 * calculated in post processors.
356 */
357 error = decode_displacement(context, stream,
358 &(effective_address->displacement),
359 &(modrm_details->displacement), &(address->offset),
360 FCML_DS_32, decoded_modrm->is_rip
361 ? 0 : effective_address_size, 0);
362 if (error) {
363 return error;
364 }
365
366 } else {
367 /* Base register.*/
368 effective_address->base.type = FCML_REG_GPR;
369 effective_address->base.size = (effective_address_size == FCML_DS_64)
370 ? FCML_DS_64 : FCML_DS_32;
371 effective_address->base.reg = f_rm;
372 /* Displacement.*/
373 if (f_mod != 0) {
374 error = decode_displacement(context, stream,
375 &(effective_address->displacement),
376 &(modrm_details->displacement), NULL,
377 (f_mod == 1) ? FCML_DS_8 : FCML_DS_32, 0,
378 get_displacement_extension(context, flags));
379 if (error) {
380 return error;
381 }
382 }
383
384 /* [EBP] can be only encoded with displacement 0, so do not return
385 * it as it was used only in order to encode [EBP]. */
386 if (effective_address->base.reg == FCML_REG_EBP
387 && is_integer_zero(&(effective_address->displacement))) {
388 fcml_fn_env_memory_clear(&(effective_address->displacement),
389 sizeof(fcml_st_integer));
390 }
391 }
392
393 /* Decodes register if something needs it.*/
394 decoded_modrm->reg_opcode = FCML_MODRM_DEC_REG_OPCODE(mod_rm) |
395 (modrm_source->ext_R << 3) | (modrm_source->ext_R_prim << 4);
396
397 return error;
398 }
399
400 fcml_ceh_error fcml_fn_modrm_decode_rip(
401 fcml_uint64_t rip,
402 fcml_usize effective_address_size,
403 fcml_st_offset *offset,
404 fcml_st_offset *address) {
405
406 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
407
408 fcml_int64_t effective_address = (fcml_int64_t) rip + offset->off32;
409
410 if (effective_address_size == FCML_DS_32) {
411 effective_address &= 0x00000000FFFFFFFFUL;
412 } else if (effective_address_size == FCML_DS_16) {
413 error = FCML_CEH_GEC_INVALID_ADDRESS_SIZE;
414 }
415
416 if (!error) {
417 address->off64 = effective_address;
418 address->size = FCML_DS_64;
419 }
420
421 return error;
422 }
423
424 fcml_ceh_error fcml_fn_modrm_decode(
425 fcml_st_modrm_decoder_context *context,
426 fcml_st_modrm_source *modrm_source,
427 fcml_st_modrm *decoded_modrm,
428 fcml_st_modrm_details *modrm_details,
429 fcml_uint8_t flags) {
430 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
431 fcml_fn_env_memory_clear(decoded_modrm, sizeof(fcml_st_modrm));
432 if (context->effective_address_size == FCML_DS_16) {
433 error = decode_16bit(
434 context, modrm_source, decoded_modrm, modrm_details, flags);
435 } else if (context->effective_address_size == FCML_DS_32 ||
436 context->effective_address_size == FCML_DS_64) {
437 error = decode_3264bit( context, modrm_source,
438 decoded_modrm, modrm_details, flags);
439 } else {
440 /* Unknown addressing mode.*/
441 error = FCML_CEH_GEC_INVALID_OPERATING_MODE;
442 }
443 return error;
444 }
445
446