Debian Codegraph @

src/fcml_modrm_encoder.c

raw
1 /*
2 * FCML - Free Code Manipulation Library.
3 * Copyright (C) 2010-2020 Slawomir Wojtasiak
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2.1 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
14 *
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 */
19
20 #include "fcml_modrm_encoder.h"
21
22 #include <fcml_common.h>
23 #include <fcml_errors.h>
24 #include <fcml_types.h>
25
26 #include "fcml_stream.h"
27 #include "fcml_utils.h"
28 #include "fcml_disp8_n.h"
29
30 static fcml_st_memory_stream map_displacement_to_stream(
31 fcml_st_encoded_modrm *encoded_modrm) {
32 fcml_st_memory_stream stream;
33 stream.base_address = &(encoded_modrm->displacement);
34 stream.offset = 0;
35 stream.size = sizeof(encoded_modrm->displacement);
36 return stream;
37 }
38
39 static inline fcml_ceh_error is_displacement(const fcml_st_address *address) {
40 return address->offset.size || address->effective_address.displacement.size;
41 }
42
43 static fcml_ceh_error convert_absolute_address_to_integer(
44 const fcml_st_address *address, fcml_st_integer *integer) {
45 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
46 if (address->address_form == FCML_AF_OFFSET) {
47 if (address->offset.size) {
48 error = fcml_fn_utils_offset_to_integer(&(address->offset),
49 integer);
50 } else {
51 *integer = address->effective_address.displacement;
52 }
53 } else {
54 /* Combined effective address can not be treated as absolute offset. */
55 return FCML_CEH_GEC_INVALID_INPUT;
56 }
57 return error;
58 }
59
60 static fcml_ceh_error encode_compress_disp8(
61 const fcml_st_modrm_encoder_context *context,
62 const fcml_st_integer *displacement, fcml_st_integer *dest,
63 fcml_usize asa) {
64
65 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
66 fcml_uint32_t n = fcml_fn_d8n_calculate_n(context->tuple_type,
67 context->b, context->input_size, context->vector_length);
68
69 if (n) {
70 fcml_st_integer compressed_disp;
71 fcml_int64_t remainder = fcml_fn_utils_divide_integer(
72 displacement, &compressed_disp, n);
73
74 if (remainder == 0) {
75 /* Check if we can still store compressed displacement
76 * on 8-bits. */
77 error = fcml_fn_utils_conv_int_to_int(
78 &compressed_disp, dest, asa, FCML_DS_8);
79 } else {
80 error = FCML_CEH_GEC_VALUE_OUT_OF_RANGE;
81 }
82 }
83
84 if (error == FCML_CEH_GEC_VALUE_OUT_OF_RANGE || !n) {
85 /* Displacement cannot be compressed, so force disp32. */
86 error = fcml_fn_utils_conv_int_to_int(displacement, dest,
87 asa, asa == FCML_DS_16 ? FCML_DS_16 : FCML_DS_32);
88 }
89
90 return error;
91 }
92
93 /* Optionally extends and encodes displacement.*/
94 static fcml_ceh_error encode_displacement(
95 fcml_st_modrm_encoder_context *context,
96 const fcml_st_integer *displacement,
97 fcml_st_encoded_modrm *encoded_modrm, fcml_usize asa,
98 fcml_usize *disp_size, fcml_bool is_rip) {
99
100 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
101
102 fcml_st_integer dest;
103
104 /* Displacement value is always signed, so change it to signed value
105 * if it has expected size.
106 */
107 fcml_st_integer src = *displacement;
108 if (src.size == asa) {
109 src.is_signed = FCML_TRUE;
110 }
111
112 if (*disp_size == FCML_DS_UNDEF) {
113 /* Check if compressed disp8 has to be used. Notice that for EVEX
114 * encoded instructions disp8 is always compressed. */
115 if (!is_rip && context->is_evex && asa != FCML_DS_16) {
116 error = encode_compress_disp8(context, &src, &dest,
117 asa);
118 } else {
119
120 /* Convert displacement to 8 bits value.*/
121 error = fcml_fn_utils_conv_int_to_int(&src, &dest, asa,
122 FCML_DS_8);
123
124 if (error == FCML_CEH_GEC_VALUE_OUT_OF_RANGE) {
125 /* Convert displacement to 32 bits value.*/
126 error = fcml_fn_utils_conv_int_to_int(&src, &dest,
127 asa, asa == FCML_DS_16 ? FCML_DS_16 : FCML_DS_32);
128 }
129 }
130 } else {
131 error = fcml_fn_utils_conv_int_to_int(&src, &dest, asa,
132 *disp_size);
133 }
134
135 if (error) {
136 return error;
137 }
138
139 /* Gets displacement as stream. */
140 fcml_st_memory_stream stream = map_displacement_to_stream(
141 encoded_modrm);
142
143 /* Writes displacement to the stream. */
144 error = fcml_fn_utils_encode_integer(&stream, &dest);
145 if (error) {
146 return error;
147 }
148
149 *disp_size = dest.size;
150
151 encoded_modrm->displacement_size = stream.offset;
152
153 return error;
154 }
155
156 static fcml_ceh_error encode_16bit(fcml_st_modrm_encoder_context *context,
157 const fcml_st_modrm *decoded_modrm,
158 fcml_st_encoded_modrm *encoded_modrm) {
159
160 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
161
162 const fcml_st_address *address = &(decoded_modrm->address);
163 const fcml_st_effective_address *effective_address =
164 &(address->effective_address);
165
166 /* ModR/M fields.*/
167 fcml_uint8_t f_reg = 0;
168 fcml_uint8_t f_mod = 0;
169 fcml_uint8_t f_rm = 0;
170
171 if (context->op_mode == FCML_OM_64_BIT) {
172 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
173 }
174
175 /* Check if there is disp16 addressing mode encoded.*/
176 if (!effective_address->base.type
177 && (effective_address->displacement.size || address->offset.size)) {
178 /* Sign extends displacement to 16 bits if there is such need,
179 * and encode it. Remember that in 16 bit addressing mode address
180 * form doesn't matter.
181 */
182 fcml_st_integer absolute_address;
183
184 error = convert_absolute_address_to_integer(address,
185 &absolute_address);
186 if (!error) {
187 fcml_usize disp_size = FCML_DS_16;
188 error = encode_displacement(context,
189 &absolute_address, encoded_modrm, FCML_DS_16, &disp_size,
190 FCML_FALSE);
191 }
192
193 f_rm = 0x06;
194 } else if (effective_address->base.type) {
195
196 /* Registers are allowed only in COMBINED addressing form.*/
197 if (address->address_form != FCML_AF_COMBINED) {
198 return FCML_CEH_GEC_INVALID_INPUT;
199 }
200
201 if (effective_address->base.type
202 && effective_address->base.size != FCML_DS_16) {
203 error = FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
204 }
205
206 if (effective_address->index.type
207 && effective_address->index.size != FCML_DS_16) {
208 error = FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
209 }
210
211 if (!error) {
212 /* There is base register set.*/
213 switch (effective_address->base.reg) {
214 case FCML_REG_BX:
215 if (effective_address->index.type) {
216 if (effective_address->index.reg == FCML_REG_DI) {
217 f_rm = 0x01;
218 }
219 } else {
220 f_rm = 0x07;
221 }
222 break;
223 case FCML_REG_BP:
224 if (effective_address->index.type) {
225 switch (effective_address->index.reg) {
226 case FCML_REG_SI:
227 f_rm = 0x02;
228 break;
229 case FCML_REG_DI:
230 f_rm = 0x03;
231 break;
232 }
233 } else if (!effective_address->displacement.size) {
234 /* BP is not allowed without displacement.*/
235 error = FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
236 } else {
237 f_rm = 0x06;
238 }
239 break;
240 case FCML_REG_SI:
241 f_rm = 0x04;
242 break;
243 case FCML_REG_DI:
244 f_rm = 0x05;
245 break;
246 }
247
248 if (!error) {
249 /* Encode displacement if there is any.*/
250 fcml_usize disp_size = FCML_DS_UNDEF;
251 if (effective_address->displacement.size) {
252 error = encode_displacement(context,
253 &(effective_address->displacement), encoded_modrm,
254 FCML_DS_16, &disp_size, FCML_FALSE);
255 }
256
257 if (!error) {
258 /* Now encode displacement, is there is any.*/
259 switch (disp_size) {
260 case FCML_DS_UNDEF:
261 f_mod = 0x00;
262 break;
263 case FCML_DS_8:
264 f_mod = 0x01;
265 break;
266 case FCML_DS_16:
267 f_mod = 0x02;
268 break;
269 default:
270 /* Only disp8 and disp16 is supported in 16 bit
271 * addressing mode.
272 */
273 error = FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
274 break;
275 }
276 }
277 }
278 }
279 } else if (decoded_modrm->reg.is_not_null) {
280 f_mod = 0x03;
281 f_rm = decoded_modrm->reg.value;
282 } else {
283 /* There is no base register and displacement, so*/
284 error = FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
285 }
286
287 if (!error) {
288 /* Encode reg/opcode.*/
289 f_reg = decoded_modrm->reg_opcode;
290 /* Encode calculated ModR/M byte.*/
291 encoded_modrm->modrm = FCML_MODRM_ENC(f_mod, f_reg, f_rm);
292 }
293
294 return error;
295 }
296
297 /* 32 and 64 bit addressing mode */
298 static fcml_ceh_error encode_3264bit(fcml_st_modrm_encoder_context *context,
299 const fcml_st_modrm *decoded_modrm,
300 fcml_st_encoded_modrm *encoded_modrm) {
301
302 fcml_uint8_t f_mod = 0;
303 fcml_uint8_t f_rm = 0;
304 fcml_uint8_t f_reg = 0;
305
306 fcml_uint8_t f_ext_R = 0;
307 fcml_uint8_t f_ext_R_prim = 0;
308 fcml_uint8_t f_ext_X = 0;
309 fcml_uint8_t f_ext_B = 0;
310 fcml_uint8_t f_ext_V_prim = 0;
311
312 fcml_usize disp_size = 0;
313
314 fcml_bool choose_sib = FCML_FALSE;
315
316 const fcml_st_address *address = &(decoded_modrm->address);
317 const fcml_st_effective_address *effective_address =
318 &(address->effective_address);
319 fcml_st_integer displacement = address->effective_address.displacement;
320
321 /* Check if addressing mode and effective address size are compatible.*/
322 if ((context->chosen_effective_address_size == FCML_DS_64
323 && context->op_mode != FCML_OM_64_BIT)
324 || (context->op_mode == FCML_OM_64_BIT
325 && context->chosen_effective_address_size == FCML_DS_16)) {
326 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
327 }
328
329 fcml_bool is_disp = is_displacement(address);
330 fcml_bool is_base = effective_address->base.type != FCML_REG_UNDEFINED;
331 fcml_bool is_index = effective_address->index.type != FCML_REG_UNDEFINED;
332
333 /* Should be treated like standard displacement when RIP addressing is
334 * not available.
335 */
336 fcml_bool is_rip_disp = effective_address->base.type == FCML_REG_IP;
337
338 /* Sanity check. */
339 if ((is_base || is_index) && address->address_form != FCML_AF_COMBINED) {
340 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
341 }
342
343 fcml_bool is_rip = FCML_FALSE;
344
345 /* Check if there is SIB alternative.*/
346 if (!is_rip_disp) {
347
348 if (is_index) {
349 choose_sib = FCML_TRUE;
350 } else {
351 if (!is_base && is_disp) {
352
353 /* disp32 or RIP*/
354
355 fcml_bool choose_rip = context->choose_rip_encoding
356 && (context->op_mode == FCML_OM_64_BIT);
357
358 if (address->address_form != FCML_AF_OFFSET) {
359 choose_rip = FCML_FALSE;
360 }
361
362 if (choose_rip) {
363 /* Relative.*/
364 is_rip = FCML_TRUE;
365 context->is_sib_alternative = FCML_FALSE;
366 } else {
367 /* In case of 64 bit addressing SIB is needed to encode
368 * absolute address. */
369 choose_sib = (context->op_mode == FCML_OM_64_BIT) ?
370 FCML_TRUE : context->choose_sib_encoding;
371 context->is_sib_alternative = FCML_TRUE;
372 }
373
374 } else if (is_base) {
375 /* [esp],edx can be only encoded with using of SIB byte. */
376 if (effective_address->base.reg == FCML_REG_ESP) {
377 choose_sib = FCML_TRUE;
378 } else if (effective_address->base.reg == FCML_REG_EBP
379 && !is_disp) {
380 /* This addressing mode can be only encoded with
381 * displacement set to 0. */
382 disp_size = FCML_DS_8;
383 displacement.int8 = 0;
384 displacement.size = FCML_DS_8;
385 is_disp = FCML_TRUE;
386 } else {
387 /* RIP relative addressing can not be encoded using SIB.*/
388 if (effective_address->base.type != FCML_REG_IP) {
389 /* [base]+[disp8/disp32]*/
390 choose_sib = context->choose_sib_encoding;
391 context->is_sib_alternative = FCML_TRUE;
392 }
393 }
394 }
395 }
396 }
397
398 if (choose_sib) {
399
400 /* SIB needed.*/
401
402 fcml_uint8_t f_base = 0;
403 fcml_uint8_t f_ss = 0;
404 fcml_uint8_t f_index = 0;
405
406 /* Base register.*/
407 if (is_base) {
408 if (context->chosen_effective_address_size
409 != effective_address->base.size) {
410 /* Wrong size of base register, it has to be equal to EASA. */
411 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
412 }
413 f_base = effective_address->base.reg;
414 if (f_base > 7) {
415 f_ext_B = 0x01;
416 f_base &= 0x07;
417 }
418 } else {
419 /* SIB without base register, there is only one addressing
420 * mode with this combination and it needs disp32.
421 */
422 if (!is_disp) {
423 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
424 }
425 disp_size = FCML_DS_32;
426 f_base = 0x05;
427 }
428
429 /* Index register.*/
430 if (is_index) {
431 /* Check if index register size is correct. */
432 if (effective_address->index.type != FCML_REG_SIMD
433 && context->chosen_effective_address_size
434 != effective_address->index.size) {
435 /* Wrong size of index register, it has to be equal to EASA. */
436 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
437 }
438 /* ESP is not allowed as index register. */
439 if (effective_address->index.reg == FCML_REG_ESP) {
440 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
441 }
442 f_index = effective_address->index.reg;
443
444 f_ext_X = (f_index >> 3) & 0x01;
445 f_ext_V_prim = (f_index >> 4) & 0x01;
446 f_index &= 0x07;
447 } else {
448 f_index = 0x04;
449 }
450
451 /* Scale factor.*/
452 switch (effective_address->scale_factor) {
453 case 2:
454 f_ss = 1;
455 break;
456 case 4:
457 f_ss = 2;
458 break;
459 case 8:
460 f_ss = 3;
461 break;
462 }
463
464 /* Encode SIB.*/
465 encoded_modrm->sib.is_not_null = FCML_TRUE;
466 encoded_modrm->sib.value = FCML_MODRM_SIB_ENC(f_ss, f_index, f_base);
467
468 /* RM when SIB is used.*/
469 f_rm = 0x04;
470
471 } else {
472
473 /* SIB not needed.*/
474
475 if (is_base && !is_rip_disp) {
476 if (context->chosen_effective_address_size
477 != effective_address->base.size) {
478 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
479 }
480 f_rm = effective_address->base.reg;
481 } else if (decoded_modrm->reg.is_not_null) {
482 f_mod = 0x03;
483 f_rm = decoded_modrm->reg.value;
484 if (effective_address->displacement.size) {
485 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
486 }
487 } else if (is_disp) {
488 /* Absolute address or RIP. Disp8 is not supported in
489 * this case, so force disp32. */
490 disp_size = FCML_DS_32;
491 f_rm = 0x05;
492 } else {
493 /* Only reg_opcode is encoded.*/
494 f_mod = 0x03;
495 }
496
497 f_ext_B = (f_rm >> 3) & 0x01;
498 f_ext_X = (f_rm >> 4) & 0x01;
499 f_rm &= 0x07;
500 }
501
502 if (is_rip) {
503
504 /* RIP given ad absolute address that has to be converted to
505 * relative displacement.
506 */
507
508 fcml_st_integer rip_address;
509
510 /* Offset is not available, so displacement was used instead.*/
511 if (!address->offset.size) {
512 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
513 }
514
515 fcml_ceh_error error = fcml_fn_utils_offset_to_integer(
516 &(address->offset), &rip_address);
517 if (error) {
518 return error;
519 }
520
521 /* Extends RIP address to 64 bit value.*/
522 if (rip_address.size != FCML_DS_64) {
523 error = fcml_fn_utils_extend_int(&rip_address, FCML_DS_64);
524 if (error) {
525 return error;
526 }
527 }
528
529 /* Store RIP offset. It can be used then to calculate displacement*/
530 encoded_modrm->is_rip = FCML_TRUE;
531 encoded_modrm->rip_address = rip_address.int64;
532
533 /* Check if 32 bit ASA should be used, if so check if RIP address is
534 * not out of range.
535 */
536 fcml_usize asa =
537 (context->chosen_effective_address_size) ?
538 context->chosen_effective_address_size :
539 context->effective_address_size;
540 if (asa == FCML_DS_32) {
541 if (rip_address.is_signed) {
542 if (rip_address.int64 > FCML_INT32_MAX
543 || rip_address.int64 < FCML_INT32_MIN) {
544 return FCML_CEH_GEC_VALUE_OUT_OF_RANGE;
545 }
546 } else {
547 if ((fcml_uint64_t) rip_address.int64 > FCML_UINT32_MAX) {
548 return FCML_CEH_GEC_VALUE_OUT_OF_RANGE;
549 }
550 }
551 }
552
553 f_mod = 0x00;
554
555 } else if (is_disp || is_rip_disp) {
556
557 if (is_base || is_index) {
558
559 fcml_ceh_error error = encode_displacement(context,
560 &displacement, encoded_modrm,
561 context->chosen_effective_address_size, &disp_size,
562 is_rip_disp);
563 if (error) {
564 return error;
565 }
566
567 /* Encode displacement.*/
568 if (disp_size == FCML_DS_8) {
569 f_mod = 0x01;
570 } else if (disp_size == FCML_DS_32 && is_base && !is_rip_disp) {
571 f_mod = 0x02;
572 } else {
573 f_mod = 0x00;
574 }
575
576 /* Displacement has been already encoded, so post processing is
577 * not needed anymore.
578 */
579 if (is_rip_disp) {
580 encoded_modrm->is_rip = FCML_TRUE;
581 encoded_modrm->is_rip_encoded = FCML_TRUE;
582 encoded_modrm->rip_address = 0LL;
583 }
584
585 } else {
586
587 /* Absolute address or RIP relative displacement. */
588 fcml_st_integer absolute_address;
589
590 /* Convert absolute address to generic integer value in
591 * order to convert it to ASA size.
592 */
593 fcml_usize error =
594 convert_absolute_address_to_integer(address,
595 &absolute_address);
596 if (error) {
597 return error;
598 }
599
600 /* Absolute address without base or index register set
601 * cannot be encoded as disp8. */
602 disp_size = FCML_DS_32;
603
604 error = encode_displacement(context,
605 &absolute_address, encoded_modrm,
606 context->chosen_effective_address_size, &disp_size,
607 is_rip_disp);
608 if (error) {
609 return error;
610 }
611
612 f_mod = 0x00;
613 }
614
615 }
616
617 /* Encode reg/opcode.*/
618 f_reg = decoded_modrm->reg_opcode;
619 f_ext_R = (f_reg >> 3) & 0x01;
620 f_ext_R_prim = (f_reg >> 4) & 0x01;
621 f_reg &= 0x07;
622
623 if (context->op_mode != FCML_OM_64_BIT && (f_ext_R || f_ext_X || f_ext_B ||
624 f_ext_R_prim || f_ext_V_prim)) {
625 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
626 }
627
628 encoded_modrm->modrm = FCML_MODRM_ENC(f_mod, f_reg, f_rm);
629 encoded_modrm->ext_R = f_ext_R;
630 encoded_modrm->ext_R_prim = f_ext_R_prim;
631 encoded_modrm->ext_X = f_ext_X;
632 encoded_modrm->ext_B = f_ext_B;
633 encoded_modrm->ext_V_prim = f_ext_V_prim;
634
635 return FCML_CEH_GEC_NO_ERROR;
636 }
637
638 fcml_ceh_error fcml_fn_modrm_encode(fcml_st_modrm_encoder_context *context,
639 const fcml_st_modrm *decoded_modrm,
640 fcml_st_encoded_modrm *encoded_modrm) {
641
642 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
643
644 fcml_usize easa = context->effective_address_size;
645 fcml_en_operating_mode op_mode = context->op_mode;
646
647 const fcml_st_effective_address *effective_address =
648 &(decoded_modrm->address.effective_address);
649
650 /* Sanity check. */
651 if (context->is_evex && context->input_size == FCML_DS_UNDEF &&
652 context->vector_length == FCML_DS_UNDEF) {
653 return FCML_CEH_GEC_INVALID_OPPERAND_SIZE;
654 }
655
656 /* Check if there is ambiguity.*/
657 if (effective_address->base.size == FCML_DS_16
658 || effective_address->index.size == FCML_DS_16) {
659 if (op_mode == FCML_OM_64_BIT) {
660 /* 16 bit addressing is not supported in 64 bit mode.*/
661 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
662 }
663 easa = FCML_DS_16;
664 } else if (effective_address->base.size == FCML_DS_64
665 || effective_address->index.size == FCML_DS_64) {
666 if (op_mode == FCML_OM_16_BIT) {
667 /* 64 bit addressing is not supported in 16 bit mode.*/
668 return FCML_CEH_GEC_INVALID_ADDRESSING_FORM;
669 }
670 easa = FCML_DS_64;
671 }
672
673 context->chosen_effective_address_size = easa;
674
675 if (easa == FCML_DS_16) {
676 error = encode_16bit(context, decoded_modrm,
677 encoded_modrm);
678 } else if (easa == FCML_DS_32 || easa == FCML_DS_64) {
679 error = encode_3264bit(context, decoded_modrm,
680 encoded_modrm);
681 } else {
682 /* Unknown addressing mode.*/
683 error = FCML_CEH_GEC_INVALID_OPERATING_MODE;
684 }
685 return error;
686 }
687
688 fcml_ceh_error fcml_fn_modrm_encode_rip_offset(fcml_st_memory_stream *stream,
689 fcml_int64_t rip, fcml_uint8_t instruction_size,
690 const fcml_st_encoded_modrm *encoded_modrm) {
691
692 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
693
694 if (encoded_modrm->is_rip) {
695 fcml_int64_t rip_offset =
696 (encoded_modrm->rip_address - instruction_size)
697 - (fcml_int64_t) rip;
698
699 if (rip_offset < FCML_INT32_MIN || rip_offset > FCML_INT32_MAX) {
700 error = FCML_CEH_GEC_VALUE_OUT_OF_RANGE;
701 } else {
702
703 fcml_st_integer integer;
704 integer.int32 = (fcml_int32_t) rip_offset;
705 integer.size = FCML_DS_32;
706
707 /* Encodes and encodes displacement to given stream.*/
708 error = fcml_fn_utils_encode_integer(stream, &integer);
709
710 }
711 } else {
712 error = FCML_CEH_GEC_INVALID_INPUT;
713 }
714
715 return error;
716 }
717
718 fcml_ceh_error fcml_fn_modrm_calculate_effective_address_size(
719 const fcml_st_modrm *decoded_modrm,
720 fcml_flags *effective_address_size) {
721
722 fcml_ceh_error error = FCML_CEH_GEC_NO_ERROR;
723
724 /* Calculate EOSA.*/
725
726 const fcml_st_address *address = &(decoded_modrm->address);
727 const fcml_st_effective_address *effective_address =
728 &(address->effective_address);
729
730 fcml_flags easa = 0;
731
732 fcml_st_register reg;
733 if (effective_address->base.type) {
734 reg = effective_address->base;
735 } else {
736 reg = effective_address->index;
737 }
738 if (reg.type) {
739 switch (reg.size) {
740 case FCML_DS_16:
741 case FCML_DS_32:
742 case FCML_DS_64:
743 easa = reg.size >> 4;
744 break;
745 case FCML_DS_128:
746 case FCML_DS_256:
747 case FCML_DS_512:
748 easa = FCML_EN_ASF_32 | FCML_EN_ASF_64;
749 break;
750 default:
751 error = FCML_CEH_GEC_INVALID_OPERATING_MODE;
752 break;
753 }
754 } else {
755 /* Remember that -1 can be represented as int64 value, so displacement
756 * nor offset size cannot be taken into account here.
757 */
758 easa = FCML_EN_ASF_ALL;
759 }
760
761 if (error) {
762 return error;
763 }
764
765 /* Calculate EASA.*/
766
767 *effective_address_size = easa;
768
769 return error;
770 }
771