Debian Codegraph @

visa/iga/IGALibrary/Frontend/KernelParser.cpp

raw
1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2017-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "KernelParser.hpp"
10 #include "../IR/InstBuilder.hpp"
11 #include "../IR/Types.hpp"
12 #include "../strings.hpp"
13 #include "BufferedLexer.hpp"
14 #include "Floats.hpp"
15 #include "Lexemes.hpp"
16 #include "Parser.hpp"
17
18 #include <algorithm>
19 #include <limits>
20 #include <string>
21 #include <unordered_map>
22 #include <vector>
23
24 using namespace iga;
25
26 static const IdentMap<FlagModifier> FLAGMODS{
27 {"lt", FlagModifier::LT},
28 {"le", FlagModifier::LE},
29 {"gt", FlagModifier::GT},
30 {"ge", FlagModifier::GE},
31 {"eq", FlagModifier::EQ},
32 {"ne", FlagModifier::NE},
33 {"ov", FlagModifier::OV},
34 {"un", FlagModifier::UN},
35 {"eo", FlagModifier::EO},
36 // aliased by different operations
37 {"ze", FlagModifier::EQ},
38 {"nz", FlagModifier::NE},
39 };
40 static const IdentMap<FlagModifier> FLAGMODS_LEGACY{
41 {"l", FlagModifier::LT}, {"g", FlagModifier::GT}, {"e", FlagModifier::EQ},
42 {"o", FlagModifier::OV}, {"u", FlagModifier::UN}, {"z", FlagModifier::EQ},
43 };
44 static const IdentMap<Type> SRC_TYPES{
45 // dpas types
46 {"u1", Type::U1},
47 {"u2", Type::U2},
48 {"u4", Type::U4},
49 {"s2", Type::S2},
50 {"s4", Type::S4},
51 {"b", Type::B},
52 {"ub", Type::UB},
53 {"w", Type::W},
54 {"uw", Type::UW},
55 {"d", Type::D},
56 {"ud", Type::UD},
57 {"q", Type::Q},
58 {"uq", Type::UQ},
59 {"hf", Type::HF},
60 {"f", Type::F},
61 {"df", Type::DF},
62 {"v", Type::V},
63 {"uv", Type::UV},
64 {"vf", Type::VF},
65 {"nf", Type::NF},
66 {"bf", Type::BF},
67 {"bf16", Type::BF},
68 {"bf8", Type::BF8},
69 {"hf8", Type::HF8},
70 {"qf", Type::QF},
71 {"tf32", Type::TF32},
72 ///////////////////////////////////////////
73 // non-standard names
74 {"u8", Type::UB},
75 {"u16", Type::UW},
76 {"u32", Type::UD},
77 {"u64", Type::UQ},
78 {"s8", Type::B},
79 {"s16", Type::W},
80 {"s32", Type::D},
81 {"s64", Type::Q},
82 {"f16", Type::HF},
83 {"f32", Type::F},
84 {"f64", Type::DF},
85 {"e2m1", Type::E2M1},
86 };
87 static const IdentMap<Type> DST_TYPES{
88 {"b", Type::B},
89 {"ub", Type::UB},
90 {"w", Type::W},
91 {"uw", Type::UW},
92 {"d", Type::D},
93 {"ud", Type::UD},
94 {"q", Type::Q},
95 {"uq", Type::UQ},
96 //
97 {"hf", Type::HF},
98 {"bf", Type::BF},
99 {"bf16", Type::BF},
100 {"bf8", Type::BF8},
101 {"hf8", Type::HF8},
102 {"qf", Type::QF},
103 {"tf32", Type::TF32},
104 {"f", Type::F},
105 {"df", Type::DF},
106 {"nf", Type::NF},
107 ///////////////////////////////////////////
108 // non-standard names
109 {"u8", Type::UB},
110 {"u16", Type::UW},
111 {"u32", Type::UD},
112 {"u64", Type::UQ},
113 {"s8", Type::B},
114 {"s16", Type::W},
115 {"s32", Type::D},
116 {"s64", Type::Q},
117 {"f16", Type::HF},
118 {"f32", Type::F},
119 {"f64", Type::DF},
120 };
121 static const IdentMap<MathMacroExt> MATHMACROREGS = {
122 {"mme0", MathMacroExt::MME0}, {"mme1", MathMacroExt::MME1},
123 {"mme2", MathMacroExt::MME2}, {"mme3", MathMacroExt::MME3},
124 {"mme4", MathMacroExt::MME4}, {"mme5", MathMacroExt::MME5},
125 {"mme6", MathMacroExt::MME6}, {"mme7", MathMacroExt::MME7},
126 {"nomme", MathMacroExt::NOMME},
127 };
128 static const IdentMap<MathMacroExt> MATHMACROREGS_OLDSTYLE = {
129 {"acc2", MathMacroExt::MME0}, {"acc3", MathMacroExt::MME1},
130 {"acc4", MathMacroExt::MME2}, {"acc5", MathMacroExt::MME3},
131 {"acc6", MathMacroExt::MME4}, {"acc7", MathMacroExt::MME5},
132 {"acc8", MathMacroExt::MME6}, {"acc9", MathMacroExt::MME7},
133 {"noacc", MathMacroExt::NOMME},
134 };
135
136 GenParser::GenParser(const Model &model, InstBuilder &handler,
137 const std::string &inp, ErrorHandler &eh,
138 const ParseOpts &pots)
139 : Parser(inp, eh), m_model(model), m_builder(handler), m_opts(pots) {
140 initSymbolMaps();
141 }
142
143 bool GenParser::LookupReg(const std::string &str, const RegInfo *&ri,
144 int &reg) {
145 ri = nullptr;
146 reg = 0;
147
148 // given something like "r13", parse the "r"
149 // "cr0" -> "cr"
150 size_t len = 0;
151 while (len < str.length() && !isdigit(str[len]))
152 len++;
153 if (len == 0)
154 return false;
155 auto itr = m_regmap.find(str.substr(0, len));
156 if (itr == m_regmap.end()) {
157 return false;
158 }
159 ri = itr->second;
160 reg = 0;
161 if (ri->numRegs > 0) {
162 // if it's a numbered register like "r13" or "cr1", then
163 // parse the number part
164 size_t off = len;
165 while (off < str.size() && isdigit(str[off])) {
166 char c = str[off++];
167 reg = 10 * reg + c - '0';
168 }
169 if (off < str.size()) {
170 // we have something like "r13xyz"; we don't treat this
171 // as a register, but fallback so it can be treated as an
172 // identifier (an immediate reference)
173 reg = 0;
174 ri = nullptr;
175 return false;
176 }
177 } // else it was something like "null" or "ip"
178 // either way, we are done
179 return true;
180 }
181
182 bool GenParser::PeekReg(const RegInfo *&regInfo, int &regNum) {
183 const Token &tk = Next();
184 if (tk.lexeme != IDENT) {
185 return false;
186 } else if (LookupReg(GetTokenAsString(tk), regInfo, regNum)) {
187 // helpful translation that permits acc2-acc9 and translates them
188 // to mme0-7 with a warning (or whatever they map to on the given
189 // platform
190 if (regInfo->regName == RegName::ARF_ACC && regNum >= regInfo->numRegs) {
191 const RegInfo *mme = m_model.lookupRegInfoByRegName(RegName::ARF_MME);
192 IGA_ASSERT(mme != nullptr, "unable to find MMR on this platform");
193 if (mme) {
194 // switch acc to mme
195 regInfo = mme;
196 // adjust the reg num (e.g. acc2 -> mme0 on GEN8)
197 regNum -= mme->regNumBase;
198 WarningAtT(tk.loc,
199 "old-style access to mme via acc "
200 "(use mme",
201 regNum, " for acc", regNum + mme->regNumBase, ")");
202 }
203 }
204 if (!regInfo->isRegNumberValid(regNum)) {
205 WarningS(tk.loc, "register number out of bounds");
206 }
207 return true;
208 } else {
209 return false;
210 }
211 }
212
213 bool GenParser::ConsumeReg(const RegInfo *&regInfo, int &regNum) {
214 const Token &tk = Next();
215 if (tk.lexeme != IDENT) {
216 return false;
217 }
218 if (PeekReg(regInfo, regNum)) {
219 Skip();
220 return true;
221 } else {
222 return false;
223 }
224 }
225
226 static bool isFloating(const ImmVal &v) {
227 switch (v.kind) {
228 case ImmVal::Kind::F16:
229 case ImmVal::Kind::F32:
230 case ImmVal::Kind::F64:
231 return true;
232 default:
233 return false;
234 }
235 }
236 static bool isSignedInt(const ImmVal &v) {
237 switch (v.kind) {
238 case ImmVal::Kind::S8:
239 case ImmVal::Kind::S16:
240 case ImmVal::Kind::S32:
241 case ImmVal::Kind::S64:
242 return true;
243 default:
244 return false;
245 }
246 }
247 static bool isUnsignedInt(const ImmVal &v) {
248 switch (v.kind) {
249 case ImmVal::Kind::U8:
250 case ImmVal::Kind::U16:
251 case ImmVal::Kind::U32:
252 case ImmVal::Kind::U64:
253 return true;
254 default:
255 return false;
256 }
257 }
258 static bool isIntegral(const ImmVal &v) {
259 return isSignedInt(v) || isUnsignedInt(v);
260 }
261
262 // expression parsing
263 // &,|
264 // <<,>>
265 // +,-
266 // *,/,%
267 // -(unary neg)
268 bool GenParser::TryParseConstExpr(ImmVal &v) {
269 return TryParseConstExpr(ExprParseOpts(), v);
270 }
271 bool GenParser::TryParseConstExpr(const ExprParseOpts &pos, ImmVal &v) {
272 return parseBitwiseExpr(pos, false, v);
273 }
274 //
275 bool GenParser::TryParseIntConstExpr(ImmVal &v, const char *forWhat) {
276 return TryParseIntConstExpr(ExprParseOpts(), v, forWhat);
277 }
278 bool GenParser::TryParseIntConstExpr(const ExprParseOpts &pos, ImmVal &v,
279 const char *forWhat) {
280 // allow floats in the parse, but check the result (better error)
281 Loc loc = NextLoc();
282 ExprParseOpts epos = pos; // allow floats, throw tantrum at end
283 epos.allowFloat = true;
284 bool z = TryParseConstExpr(epos, v);
285 if (!z) {
286 return false;
287 } else if (!isIntegral(v)) {
288 std::stringstream ss;
289 if (forWhat) {
290 ss << forWhat << " must be a constant integer expression";
291 } else {
292 ss << "expected constant integer expression";
293 }
294 FailS(loc, ss.str());
295 }
296 return true;
297 }
298 bool GenParser::TryParseIntConstExprAdd(ImmVal &v, const char *forWhat) {
299 return TryParseIntConstExprPrimary(ExprParseOpts(), v, forWhat);
300 }
301 bool GenParser::TryParseIntConstExprAdd(const ExprParseOpts &pos, ImmVal &v,
302 const char *forWhat) {
303 Loc loc = NextLoc();
304 ExprParseOpts epos = pos; // allow floats, throw tantrum at end
305 epos.allowFloat = true;
306 bool z = parseAddExpr(epos, false, v);
307 if (!z) {
308 return false;
309 } else if (!isIntegral(v)) {
310 std::stringstream ss;
311 if (forWhat) {
312 ss << forWhat << " must be a constant integer expression";
313 } else {
314 ss << "expected constant integer expression";
315 }
316 FailS(loc, ss.str());
317 }
318 return true;
319 }
320 bool GenParser::TryParseIntConstExprPrimary(ImmVal &v, const char *forWhat) {
321 return TryParseIntConstExprPrimary(ExprParseOpts(), v, forWhat);
322 }
323 bool GenParser::TryParseIntConstExprPrimary(const ExprParseOpts &pos, ImmVal &v,
324 const char *forWhat) {
325 Loc loc = NextLoc();
326 ExprParseOpts epos = pos; // allow floats, throw tantrum at end
327 epos.allowFloat = true;
328 bool z = parsePrimaryExpr(epos, false, v);
329 if (!z) {
330 return false;
331 } else if (!isIntegral(v)) {
332 std::stringstream ss;
333 if (forWhat) {
334 ss << forWhat << " must be a constant integer expression";
335 } else {
336 ss << "expected constant integer expression";
337 }
338 FailS(loc, ss.str());
339 }
340 return true;
341 }
342 bool GenParser::TryParseIntConstExprAddChain(const ExprParseOpts &pos,
343 ImmVal &v, const char *forWhat) {
344 Loc loc = NextLoc();
345 ExprParseOpts epos = pos; // allow floats, throw tantrum at end
346 epos.allowFloat = true;
347
348 // treat this as
349 // 0 + .... (or 0 - ...)
350 v = (int64_t)0;
351 while (LookingAtAnyOf(ADD, SUB)) {
352 Token t = Next();
353 Skip();
354 ImmVal r;
355 parseMulExpr(pos, true, r);
356 v = evalBinExpr(v, t, r);
357 }
358 if (!isIntegral(v)) {
359 std::stringstream ss;
360 if (forWhat) {
361 ss << forWhat << " must be a constant integer expression";
362 } else {
363 ss << "expected constant integer expression";
364 }
365 FailS(loc, ss.str());
366 }
367 return true;
368 }
369 bool GenParser::TryParseIntConstExprAddChain(ImmVal &v, const char *forWhat) {
370 return TryParseIntConstExprAddChain(ExprParseOpts(), v, forWhat);
371 }
372
373 void GenParser::ensureIntegral(const Token &t, const ImmVal &v) {
374 if (!isIntegral(v)) {
375 FailS(t.loc, "argument to operator must be integral");
376 }
377 }
378 void GenParser::checkNumTypes(const ImmVal &v1, const Token &op,
379 const ImmVal &v2) {
380 if (isFloating(v1) && !isFloating(v2)) {
381 FailAtT(op.loc, "right operand to ", GetTokenAsString(op),
382 " must be floating point");
383 } else if (isFloating(v2) && !isFloating(v1)) {
384 FailAtT(op.loc, "left operand to ", GetTokenAsString(op),
385 " must be floating point");
386 }
387 }
388 // target must be float
389 void GenParser::checkIntTypes(const ImmVal &v1, const Token &op,
390 const ImmVal &v2) {
391 if (isFloating(v1)) {
392 FailAtT(op.loc, "left operand to ", GetTokenAsString(op),
393 " must be integral");
394 } else if (isFloating(v2)) {
395 FailAtT(op.loc, "right operand to ", GetTokenAsString(op),
396 " must be integral");
397 }
398 }
399
400 ImmVal GenParser::evalBinExpr(const ImmVal &v1, const Token &op,
401 const ImmVal &v2) {
402 bool isF = isFloating(v1) || isFloating(v2);
403 bool isU = isUnsignedInt(v1) || isUnsignedInt(v2);
404 ImmVal result = v1;
405 switch (op.lexeme) {
406 case AMP:
407 case CIRC:
408 case PIPE:
409 case LSH:
410 case RSH:
411 case MOD:
412 // integral only operations
413 checkIntTypes(v1, op, v2);
414 switch (op.lexeme) {
415 case AMP:
416 if (isU) {
417 result.u64 &= v2.u64;
418 } else {
419 result.s64 &= v2.s64;
420 }
421 break;
422 case CIRC:
423 if (isU) {
424 result.u64 ^= v2.u64;
425 } else {
426 result.s64 ^= v2.s64;
427 }
428 break;
429 case PIPE:
430 if (isU) {
431 result.u64 |= v2.u64;
432 } else {
433 result.s64 |= v2.s64;
434 }
435 break;
436 case LSH:
437 if (isU) {
438 result.u64 <<= v2.u64;
439 } else {
440 result.s64 <<= v2.s64;
441 }
442 break;
443 case RSH:
444 if (isU) {
445 result.u64 >>= v2.u64;
446 } else {
447 result.s64 >>= v2.s64;
448 }
449 break;
450 case MOD:
451 if (isU) {
452 result.u64 %= v2.u64;
453 } else {
454 result.s64 %= v2.s64;
455 }
456 break;
457 default:
458 break;
459 }
460 break;
461 case ADD:
462 case SUB:
463 case MUL:
464 case DIV:
465 checkNumTypes(v1, op, v2);
466 switch (op.lexeme) {
467 case ADD:
468 if (isF) {
469 result.f64 += v2.f64;
470 } else if (isU) {
471 result.u64 += v2.u64;
472 } else {
473 result.s64 += v2.s64;
474 }
475 break;
476 case SUB:
477 if (isF) {
478 result.f64 -= v2.f64;
479 } else if (isU) {
480 result.u64 -= v2.u64;
481 } else {
482 result.s64 -= v2.s64;
483 }
484 break;
485 case MUL:
486 if (isF) {
487 result.f64 *= v2.f64;
488 } else if (isU) {
489 result.u64 *= v2.u64;
490 } else {
491 result.s64 *= v2.s64;
492 }
493 break;
494 case DIV:
495 if (isF) {
496 result.f64 /= v2.f64;
497 } else {
498 if (v2.u64 == 0) {
499 FailS(op.loc, "(integral) division by zero");
500 }
501 if (isU) {
502 result.u64 /= v2.u64;
503 } else {
504 result.s64 /= v2.s64;
505 }
506 }
507 break;
508 default:
509 break;
510 }
511 break;
512 default:
513 break;
514 }
515 return result;
516 }
517
518 bool GenParser::parseBitwiseExpr(const ExprParseOpts &pos, bool consumed,
519 ImmVal &v) {
520 return parseBitwiseORExpr(pos, consumed, v);
521 }
522 // E -> E ('|' E)*
523 bool GenParser::parseBitwiseORExpr(const ExprParseOpts &pos, bool consumed,
524 ImmVal &v) {
525 if (!parseBitwiseXORExpr(pos, consumed, v)) {
526 return false;
527 }
528 while (LookingAt(PIPE)) {
529 Token t = Next();
530 Skip();
531 ImmVal r;
532 parseBitwiseXORExpr(pos, true, r);
533 v = evalBinExpr(v, t, r);
534 }
535 return true;
536 }
537 // E -> E ('^' E)*
538 bool GenParser::parseBitwiseXORExpr(const ExprParseOpts &pos, bool consumed,
539 ImmVal &v) {
540 if (!parseBitwiseANDExpr(pos, consumed, v)) {
541 return false;
542 }
543 while (LookingAt(CIRC)) {
544 Token t = Next();
545 Skip();
546 ImmVal r;
547 parseBitwiseANDExpr(pos, true, r);
548 v = evalBinExpr(v, t, r);
549 }
550 return true;
551 }
552 // E -> E ('&' E)*
553 bool GenParser::parseBitwiseANDExpr(const ExprParseOpts &pos, bool consumed,
554 ImmVal &v) {
555 if (!parseShiftExpr(pos, consumed, v)) {
556 return false;
557 }
558 while (LookingAt(AMP)) {
559 Token t = Next();
560 Skip();
561 ImmVal r;
562 parseShiftExpr(pos, true, r);
563 v = evalBinExpr(v, t, r);
564 }
565 return true;
566 }
567
568 // E -> E (('<<'|'>>') E)*
569 bool GenParser::parseShiftExpr(const ExprParseOpts &pos, bool consumed,
570 ImmVal &v) {
571 if (!parseAddExpr(pos, consumed, v)) {
572 return false;
573 }
574 while (LookingAtAnyOf(LSH, RSH)) {
575 Token t = Next();
576 Skip();
577 ImmVal r;
578 parseAddExpr(pos, true, r);
579 v = evalBinExpr(v, t, r);
580 }
581 return true;
582 }
583 // E -> E (('+'|'-') E)*
584 bool GenParser::parseAddExpr(const ExprParseOpts &pos, bool consumed,
585 ImmVal &v) {
586 if (!parseMulExpr(pos, consumed, v)) {
587 return false;
588 }
589 while (LookingAtAnyOf(ADD, SUB)) {
590 Token t = Next();
591 Skip();
592 ImmVal r;
593 parseMulExpr(pos, true, r);
594 v = evalBinExpr(v, t, r);
595 }
596 return true;
597 }
598
599 // E -> E (('*'|'/'|'%') E)*
600 bool GenParser::parseMulExpr(const ExprParseOpts &pos, bool consumed,
601 ImmVal &v) {
602 if (!parseUnExpr(pos, consumed, v)) {
603 return false;
604 }
605 while (LookingAtAnyOf({MUL, DIV, MOD})) {
606 Token t = Next();
607 Skip();
608 ImmVal r;
609 parseUnExpr(pos, true, r);
610 v = evalBinExpr(v, t, r);
611 }
612 return true;
613 }
614
615 // E -> ('-'|'~') E
616 bool GenParser::parseUnExpr(const ExprParseOpts &pos, bool consumed,
617 ImmVal &v) {
618 if (!LookingAtAnyOf(SUB, TILDE)) {
619 if (!parsePrimaryExpr(pos, consumed, v)) {
620 return false;
621 }
622 } else {
623 Token t = Next();
624 Skip();
625 parsePrimaryExpr(pos, true, v);
626 switch (t.lexeme) {
627 case SUB:
628 v.Negate();
629 break;
630 case TILDE:
631 ensureIntegral(t, v);
632 v.u64 = ~v.u64;
633 break;
634 default:
635 break;
636 }
637 }
638 return true;
639 }
640
641 // special symbol (e.g. nan, inf, ...)
642 // grouped expression (E)
643 // literal
644 bool GenParser::parsePrimaryExpr(const ExprParseOpts &pos, bool consumed,
645 ImmVal &v) {
646 Token t = Next();
647 bool isQuietNaN = false;
648 if (pos.allowFloat && LookingAtIdentEq(t, "nan")) {
649 WarningT("nan is deprecated, us snan(...) or qnan(...)");
650 v.kind = ImmVal::Kind::F64;
651 v.f64 = std::numeric_limits<double>::signaling_NaN();
652 Skip();
653 } else if (pos.allowFloat && ((isQuietNaN = LookingAtIdentEq(t, "qnan")) ||
654 LookingAtIdentEq(t, "snan"))) {
655 auto nanSymLoc = NextLoc();
656 Skip();
657 if (Consume(LPAREN)) {
658 auto payloadLoc = NextLoc();
659 ImmVal payload;
660 payload.u64 = 0;
661 parseBitwiseExpr(pos, true, payload);
662 if (payload.u64 >= F64_QNAN_BIT) {
663 FailS(payloadLoc, "NaN payload overflows");
664 } else if (payload.u64 == 0 && !isQuietNaN) {
665 // signaling NaN has a 0 in the high mantissa, so we must
666 // ensure that at least one bit in the mantissa is set
667 // otherwise the entire mantissa is 0's and the pattern
668 // would be an "infinity" pattern, not a NaN
669 FailS(payloadLoc, "NaN payload must be nonzero for snan");
670 }
671 ConsumeOrFail(RPAREN, "expected )");
672 v.u64 = payload.u64 | F64_EXP_MASK;
673 if (isQuietNaN) {
674 v.u64 |= F64_QNAN_BIT;
675 }
676 } else {
677 WarningS(nanSymLoc, "bare qnan and snan tokens deprecated"
678 " (pass in a valid payload)");
679 v.u64 = F64_EXP_MASK;
680 if (isQuietNaN) {
681 v.u64 |= F64_QNAN_BIT; // the quiet bit suffices
682 } else {
683 v.u64 |= 1; // set something other than the quiet bit
684 // non-zero in the payload
685 }
686 }
687 v.kind = ImmVal::Kind::F64;
688 } else if (pos.allowFloat && LookingAtIdentEq(t, "inf")) {
689 v.f64 = std::numeric_limits<double>::infinity();
690 v.kind = ImmVal::Kind::F64;
691 Skip();
692 } else if (pos.allowFloat && LookingAt(FLTLIT)) {
693 ParseFltFrom(t.loc, v.f64);
694 v.kind = ImmVal::Kind::F64;
695 Skip();
696 } else if (LookingAtAnyOf({INTLIT02, INTLIT10, INTLIT16})) {
697 // we parse as unsigned, but tag as signed for negation etc...
698 ParseIntFrom<uint64_t>(t.loc, v.u64);
699 v.kind = ImmVal::Kind::S64;
700 Skip();
701 } else if (Consume(LPAREN)) {
702 // (E)
703 parseBitwiseExpr(pos, true, v);
704 Consume(RPAREN);
705 } else if (LookingAt(IDENT)) {
706 // TEST CASES
707 // LABEL:
708 // // jmpi LABEL // passes
709 // // goto (16) (2 + LABEL) LABEL // fails
710 // // goto (16) LABEL LABEL // passes
711 // // join (16) LABEL // passes
712 // // mov (1) r65:ud LABEL:ud // fails
713 // // mov (1) r65:ud (2 + LABEL):ud // fails (poor diagnostic)
714 if (!pos.symbols.empty()) {
715 std::string str = GetTokenAsString();
716 auto itr = pos.symbols.find(str);
717 if (itr != pos.symbols.end()) {
718 v = itr->second;
719 return true;
720 }
721 }
722 if (consumed)
723 FailT("unbound identifier");
724 return false;
725 } else {
726 // something else: error unless we haven't consumed anything
727 if (consumed) {
728 FailT("syntax error in constant expression");
729 }
730 return false;
731 }
732 return true;
733 } // parsePrimary
734
735 void GenParser::initSymbolMaps() {
736 // map the register names
737 // this maps just the non-number part.
738 // e.g. with cr0, this maps "cr"; see LookupReg()
739 int tableLen;
740 const RegInfo *table = GetRegisterSpecificationTable(tableLen);
741 for (int i = 0; i < tableLen; i++) {
742 const RegInfo *ri = table + i;
743 if (ri->supportedOn(platform())) {
744 m_regmap[ri->syntax] = ri;
745 }
746 }
747 }
748
749 class KernelParser : GenParser {
750 // maps mnemonics and registers for faster lookup
751 std::unordered_map<std::string, const OpSpec *> opmap;
752
753 ExecSize m_defaultExecutionSize;
754 Type m_defaultRegisterType;
755
756 // instruction state
757 const OpSpec *m_opSpec = nullptr;
758 bool m_hasWrEn = false;
759 Type m_unifType = {};
760 const Token *m_unifTypeTk = nullptr;
761 RegRef m_flagReg;
762 ExecSize m_execSize = {};
763 ChannelOffset m_chOff = {};
764 Loc m_execSizeLoc;
765 Loc m_mnemonicLoc;
766 Operand::Kind m_srcKinds[3] = {};
767 Loc m_srcLocs[3];
768 int m_sendSrcLens[2]; // send message lengths
769 Loc m_sendSrcLenLocs[2]; // locations so we can referee
770 bool m_implicitExBSO = false; // send src1 suffixed with length implies exBSO, but
771 // ensure the inst opt is given
772
773 private:
774 void addNullGatherSendSrc1() {
775 bool checkForBadSrc1 = true;
776 if (checkForBadSrc1 && LookingAt(IDENT)) {
777 // friendly error if the user includes src1 on a gather send
778 // send... ... r[s0..] null:0 ExDesc Desc
779 // ^^^^^^ oops
780 // send... ... r[s0..] a0.2 Desc
781 // ^^^^ okay (a0.2 is the ExDesc not src1)
782 // sendg... ... r[s0..] s0.0 s0.16 Desc
783 // ^^^^ okay (s0.0 is ID0 not src1)
784 auto at = NextLoc();
785 const RegInfo *regInfo = nullptr;
786 int regNum = 0;
787 m_lexer.Mark();
788 if (ConsumeReg(regInfo, regNum) &&
789 (regInfo->regName == RegName::GRF_R ||
790 regInfo->regName == RegName::ARF_NULL))
791 {
792 FailAtT(at, "src1 operand forbidden for Gathering Sends");
793 }
794 m_lexer.Reset();
795 }
796
797 // For instructions those have implicit types, match the type to
798 // the expected one. Otherwise, ARF_NULL operand should have Type::INVALID
799 Type type = Type::INVALID;
800 m_opSpec->implicitSrcTypeVal(1, false, type);
801 m_builder.InstSrcOpRegDirect(1,
802 m_srcLocs[0], // use src0 loc
803 SrcModifier::NONE, RegName::ARF_NULL,
804 REGREF_ZERO_ZERO, Region::INVALID, type);
805 }
806
807 // A helper function to check if the instruction under parsing is a gather
808 // send. This function is only meaningful after opSpec and src0 is parsed
809 bool isParsingGatherSend() {
810 IGA_ASSERT(m_opSpec && m_srcKinds[0] != Operand::Kind::INVALID,
811 "isParsingGatherSend cannot be called before src0 is parsed");
812 return m_opSpec && m_opSpec->isAnySendFormat() &&
813 m_srcKinds[0] == Operand::Kind::INDIRECT;
814 }
815
816 public:
817 KernelParser(const Model &model, InstBuilder &handler, const std::string &inp,
818 ErrorHandler &eh, const ParseOpts &pots)
819 : GenParser(model, handler, inp, eh, pots),
820 m_defaultExecutionSize(ExecSize::SIMD1),
821 m_defaultRegisterType(Type::INVALID) {
822 initSymbolMaps();
823 }
824
825 void ParseListing() { ParseProgram(); }
826
827 void initSymbolMaps() {
828 // map mnemonics names to their ops
829 // subops only get mapped by their fully qualified names in this pass
830 for (const OpSpec *os : m_model.ops()) {
831 if (os->isValid()) {
832 opmap[os->mnemonic] = os;
833 }
834 }
835 }
836
837 // Program = (Label? Insts* (Label Insts))?
838 void ParseProgram() {
839 m_builder.ProgramStart();
840
841 // parse default type directives
842 if (m_opts.supportLegacyDirectives) {
843 // e.g. .default_execution_size...
844 ParseLegacyDirectives();
845 }
846
847 // first block doesn't need a label
848 if (!LookingAtLabelDef() && !EndOfFile()) {
849 ParseBlock(NextLoc(), ""); // unnamed
850 }
851 // successive blocks need a label
852 std::string label;
853 Loc lblLoc = NextLoc();
854 while (ConsumeLabelDef(label)) {
855 ParseBlock(lblLoc, label);
856 lblLoc = NextLoc();
857 }
858 if (!EndOfFile()) {
859 FailT("expected instruction, block, or EOF");
860 }
861
862 m_builder.ProgramEnd();
863 }
864
865 // fix to support .default_execution_size
866 bool ParseLegacyDirectives() {
867 int parsed = 0;
868 try {
869 while (LookingAtSeq(Lexeme::DOT, Lexeme::IDENT)) {
870 Skip();
871 parsed++;
872 if (ConsumeIdentEq("default_execution_size")) {
873 Consume(LPAREN);
874 auto loc = NextLoc();
875 int dftExecSize;
876 if (!ConsumeIntLit<int>(dftExecSize)) {
877 FailT("expected SIMD width (integral value)");
878 }
879 if (dftExecSize != 1 && dftExecSize != 2 && dftExecSize != 4 &&
880 dftExecSize != 8 && dftExecSize != 16 && dftExecSize != 32) {
881 FailS(loc, "invalid default execution size; "
882 "must be 1, 2, 4, 8, 16, 32");
883 }
884 m_defaultExecutionSize = (ExecSize)dftExecSize;
885 Consume(RPAREN);
886 Consume(NEWLINE);
887 } else if (ConsumeIdentEq("default_register_type")) {
888 m_defaultRegisterType = TryParseOpType(DST_TYPES);
889 if (m_defaultRegisterType == Type::INVALID) {
890 FailT("expected default register type");
891 }
892 Consume(NEWLINE);
893 } else {
894 FailT("unexpected directive name");
895 }
896 } // while
897 } catch (const SyntaxError &s) {
898 RecoverFromSyntaxError(s);
899 // returning true will make ParseBlock restart the loop
900 // (so more directives get another shot)
901 return true;
902 }
903 return parsed > 0;
904 }
905
906 void RecoverFromSyntaxError(const SyntaxError &s) {
907 // record the error in this instruction
908 m_errorHandler.reportError(s.loc, s.message);
909 // bail if we've reached the max number of errors
910 if (m_errorHandler.getErrors().size() >= m_opts.maxSyntaxErrors) {
911 throw s;
912 }
913 // attempt resync by skipping until we pass a newline
914 // DumpLookaheads();
915 while (!Consume(Lexeme::NEWLINE) &&
916 // !Consume(Lexeme::SEMI) &&
917 // don't sync on ; since this is typically part of a
918 // region rather than an operand separator
919 // ... r12.0<8;8,1>:d
920 // ^ error ^ syncs here otherwise
921 !LookingAt(Lexeme::END_OF_FILE)) {
922 // DumpLookaheads();
923 Skip();
924 }
925 }
926
927 void ParseBlock(const Loc &lblLoc, const std::string &label) {
928 m_builder.BlockStart(lblLoc, label);
929 auto lastInst = NextLoc();
930 while (true) {
931 if (Consume(Lexeme::NEWLINE) || Consume(Lexeme::SEMI)) {
932 continue;
933 } else if (m_opts.supportLegacyDirectives && ParseLegacyDirectives()) {
934 // .default_execution_size...
935 continue;
936 }
937 // if we make it here, then we are not looking at a legacy
938 // directive and we are not looking at a newline
939 if (LookingAtLabelDef() || EndOfFile()) {
940 break;
941 }
942 // not at EOF and not looking at the beginning of new block
943 // so we better be looking at an instruction
944 ParseInst();
945 lastInst = NextLoc(-1); // peek at the end of the last instruction
946 }
947
948 m_builder.BlockEnd(ExtentTo(lblLoc, lastInst));
949 }
950
951 void ParseInst() {
952 try {
953 // parse one instruction
954 // DumpLookaheads();
955 ParseInstCanThrowException();
956 } catch (SyntaxError &s) {
957 // we resync here and return from ParseInst
958 RecoverFromSyntaxError(s);
959 }
960 }
961
962 // reset KernelParser information
963 void reset() {
964 m_flagReg = REGREF_INVALID;
965 m_execSizeLoc = Loc::INVALID;
966 m_opSpec = nullptr;
967 m_unifType = Type::INVALID;
968 m_unifTypeTk = nullptr;
969 m_mnemonicLoc = Loc::INVALID;
970 for (size_t i = 0; i < sizeof(m_srcKinds) / sizeof(m_srcKinds[0]); i++) {
971 m_srcLocs[i] = Loc::INVALID;
972 m_srcKinds[i] = Operand::Kind::INVALID;
973 }
974 for (size_t i = 0; i < sizeof(m_sendSrcLens) / sizeof(m_sendSrcLens[0]);
975 i++) {
976 m_sendSrcLens[i] = -1;
977 m_sendSrcLenLocs[i] = Loc::INVALID;
978 }
979 m_implicitExBSO = false;
980 }
981
982 // try if the instruction is an inline-binary-instruction
983 // return true if it is
984 bool tryParseInlineBinaryInst() {
985 if (!Consume(Lexeme::DOT))
986 return false;
987 if (!ConsumeIdentEq("inline_inst"))
988 return false;
989
990 // .inline_inst 0x123 0x123 0x123 0x123
991 Instruction::InlineBinaryType val = {0};
992 for (size_t i = 0; i < 4; ++i) {
993 if (LookingAt(INTLIT16)) {
994 ParseIntFrom<uint32_t>(NextLoc(), val[i]);
995 Skip();
996 } else {
997 FailT("Inline binary instruction must contain four 32b hexadecimal "
998 "values");
999 }
1000 }
1001 m_builder.InstInlineBinary(val);
1002 // set ILLEGAL as inline-binary-instruction's opSpec so that other passes
1003 // such as SWSBSetter can bypass it
1004 m_opSpec = &m_model.lookupOpSpec(Op::ILLEGAL);
1005 m_builder.InstOp(m_opSpec);
1006 return true;
1007 }
1008
1009 void ParseRegularInst() {
1010 // (W&~f0) mov (8|M0) r1 r2
1011 // ^
1012 ParseWrEnPred();
1013
1014 // (W&~f0) mov (8|M0) r1 r2
1015 // ^
1016 //
1017 // math.sqrt (8|M0) ...
1018 // ^
1019 m_mnemonicLoc = NextLoc();
1020 m_opSpec = ParseMnemonic();
1021 if (m_opSpec) {
1022 // looking at a regular instruction (non special-ld-st inst)
1023 m_builder.InstOp(m_opSpec);
1024 FinishNonLdStInstBody();
1025 } else if (!ParseLdStInst()) {
1026 FailS(m_mnemonicLoc, "invalid mnemonic");
1027 }
1028
1029 // .... {...}
1030 ParseInstOpts();
1031 }
1032
1033 // Instruction = RegularInst | LdStInst
1034 // RegularInst = Predication? Mnemonic UniformType? EMask
1035 // ConditionModifier? Operands InstOptions?
1036 // LdStInst = ... comes from Sends/Interface.hpp ...
1037 void ParseInstCanThrowException() {
1038 // ShowCurrentLexicalContext();
1039 const Loc startLoc = NextLoc();
1040 m_builder.InstStart(startLoc);
1041 reset();
1042
1043 if (!tryParseInlineBinaryInst())
1044 ParseRegularInst();
1045
1046 if (!LookingAt(Lexeme::NEWLINE) && !LookingAt(Lexeme::SEMI) &&
1047 !LookingAt(Lexeme::END_OF_FILE)) {
1048 FailAfterPrev("expected newline, ';', or EOF");
1049 }
1050
1051 m_builder.InstEnd(ExtentToPrevEnd(startLoc));
1052 }
1053
1054 // ExecInfo = '(' ExecSize EmOffNm? ')'
1055 // where EmOffNm = '|' EmOff (',' 'NM')?
1056 // | '|' 'NM'
1057 // EmOff = 'M0' | 'M4' | ...
1058 // ExecSize = '1' | '2' | ... | '32'
1059 void ParseExecInfo(ExecSize dftExecSize, ExecSize &execSize,
1060 ChannelOffset &chOff) {
1061 Loc execSizeLoc = m_execSizeLoc = NextLoc(0);
1062 Loc execOffsetLoc = NextLoc(0);
1063 // we are careful here since we might have things like:
1064 // jmpi (1*16)
1065 // jmpi (1|M0) ...
1066 // jmpi (1) ...
1067 // We resolve that by looking ahead two symbols
1068 int execSizeVal = 1;
1069 if (LookingAt(LPAREN) &&
1070 (LookingAtFrom(2, RPAREN) || LookingAtFrom(2, PIPE))) {
1071 Skip();
1072 execSizeLoc = NextLoc();
1073 ConsumeIntLitOrFail(execSizeVal, "expected SIMD width");
1074
1075 if (Consume(PIPE)) {
1076 static const IdentMap<ChannelOffset> EM_OFFS{
1077 {"M0", ChannelOffset::M0}, {"M4", ChannelOffset::M4},
1078 {"M8", ChannelOffset::M8}, {"M12", ChannelOffset::M12},
1079 {"M16", ChannelOffset::M16}, {"M20", ChannelOffset::M20},
1080 {"M24", ChannelOffset::M24}, {"M28", ChannelOffset::M28}};
1081 execOffsetLoc = NextLoc();
1082 ConsumeIdentOneOfOrFail(EM_OFFS, chOff, "expected ChOff",
1083 "invalid ChOff");
1084 // if (m_chOff % m_execSize != 0) {
1085 // Fail(execOffsetLoc,
1086 // "invalid execution mask offset for execution size");
1087 // } else if (m_chOff + m_execSize > 32) {
1088 // Fail(execOffsetLoc,
1089 // "invalid execution mask offset for execution size");
1090 // }
1091 } else {
1092 chOff = ChannelOffset::M0;
1093 }
1094 ConsumeOrFail(RPAREN, "expected )");
1095 } else {
1096 if (m_opSpec && m_opSpec->hasImpicitEm()) {
1097 chOff = ChannelOffset::M0;
1098 execSizeVal = 1;
1099 } else if (m_opts.supportLegacyDirectives) {
1100 chOff = ChannelOffset::M0;
1101 execSizeVal = (int)dftExecSize;
1102 } else {
1103 FailT("expected '(' (start of execution size info)");
1104 }
1105 }
1106
1107 switch (execSizeVal) {
1108 case 1:
1109 execSize = ExecSize::SIMD1;
1110 break;
1111 case 2:
1112 execSize = ExecSize::SIMD2;
1113 break;
1114 case 4:
1115 execSize = ExecSize::SIMD4;
1116 break;
1117 case 8:
1118 execSize = ExecSize::SIMD8;
1119 break;
1120 case 16:
1121 execSize = ExecSize::SIMD16;
1122 break;
1123 case 32:
1124 execSize = ExecSize::SIMD32;
1125 break;
1126 default:
1127 FailT("invalid SIMD width");
1128 }
1129
1130 m_builder.InstExecInfo(execSizeLoc, execSize, execOffsetLoc, chOff);
1131 }
1132
1133 Type SendOperandDefaultType(int srcIx) const {
1134 auto t = srcIx == 1 ? Type::INVALID : Type::UD;
1135 if (srcIx < 0) {
1136 if (m_opSpec->hasImplicitDstType())
1137 t = m_opSpec->implicitDstType();
1138 } else {
1139 if (m_opSpec->hasImplicitSrcType(srcIx, false))
1140 t = m_opSpec->implicitSrcType(srcIx, false);
1141 }
1142 return t;
1143 }
1144
1145 Type ParseSendOperandTypeWithDefault(int srcIx) {
1146 // sends's second parameter doesn't have a valid type
1147 Type t = Type::INVALID;
1148 if (Consume(COLON)) {
1149 if (!LookingAt(IDENT)) {
1150 FailT("expected a send operand type");
1151 }
1152 if (!IdentLookupFrom(0, DST_TYPES, t)) {
1153 FailT("unexpected operand type for send");
1154 }
1155 Skip();
1156 } else {
1157 t = SendOperandDefaultType(srcIx);
1158 }
1159 return t;
1160 }
1161
1162 bool ParseLdStInst();
1163 void ParseLdStOpControls(Loc mneLoc, const SendOpDefinition &opInfo,
1164 VectorMessageArgs &vma);
1165
1166 // e.g. add (8|M8) ...
1167 // ^
1168 // e.g. nop ...
1169 // ^
1170 void FinishNonLdStInstBody() {
1171 // if (Consume(COLON)) {
1172 // m_unifTypeTk = &Next(0);
1173 // ConsumeIdentOneOfOrFail(
1174 // dstTypes,
1175 // m_unifType,
1176 // "expected uniform type",
1177 // "invalid uniform type");
1178 // }
1179
1180 // (W&~f0) mov (8|M0) (le)f0.0 r1:f r2:f
1181 // ^
1182 ChannelOffset chOff;
1183 ParseExecInfo(m_defaultExecutionSize, m_execSize, chOff); // sets m_execSize
1184
1185 ParseFlagModOpt();
1186 switch (m_opSpec->format) {
1187 case OpSpec::NULLARY:
1188 // nop ...
1189 // illegal ...
1190 // thryld ...
1191 break; // fallthrough to instruction options
1192 case OpSpec::BASIC_UNARY_REG:
1193 case OpSpec::BASIC_UNARY_REGIMM:
1194 ParseDstOp();
1195 ParseSrcOp(0);
1196 if (m_opSpec->format == OpSpec::BASIC_UNARY_REG &&
1197 m_srcKinds[0] != Operand::Kind::DIRECT &&
1198 m_srcKinds[0] != Operand::Kind::INDIRECT) {
1199 FailS(m_srcLocs[0], "src0 must be a register");
1200 }
1201 break;
1202 case OpSpec::SYNC_UNARY:
1203 // implicit destination
1204 ParseSyncSrc0Op();
1205 if (m_model.supportsWait() && m_srcKinds[0] != Operand::Kind::DIRECT) {
1206 FailS(m_srcLocs[0], "src0 must be a notification register");
1207 }
1208 break;
1209 case OpSpec::SEND_UNARY:
1210 // <=Gen11
1211 ParseSendDstOp();
1212 ParseSendSrcOp(0, false);
1213 ParseSendDescsLegacy();
1214 break;
1215 case OpSpec::SEND_BINARY:
1216 if (platform() <= Platform::XE) {
1217 ParseSendInstructionLegacy();
1218 } else if (m_opSpec->isSendgFormat()) {
1219 ParseSendgInstructionXe3p();
1220 } else if (platform() >= Platform::XE3) {
1221 ParseSendInstructionXe3();
1222 } else if (platform() == Platform::XE2) {
1223 ParseSendInstructionXe2();
1224 } else if (platform() >= Platform::XE_HP) {
1225 ParseSendInstructionXeHP();
1226 } else {
1227 IGA_ASSERT_FALSE("invalid format for platform");
1228 }
1229 break;
1230 case OpSpec::BASIC_BINARY_REG_IMM:
1231 case OpSpec::BASIC_BINARY_REG_REG:
1232 case OpSpec::BASIC_BINARY_REG_REGIMM:
1233 ParseDstOp();
1234 ParseSrcOp(0);
1235 ParseSrcOp(1);
1236 break;
1237 case OpSpec::MATH_BINARY_REG_REGIMM:
1238 ParseDstOp();
1239 ParseSrcOp(0);
1240 if (m_opSpec->getSourceCount(m_builder.getSubfunction()) > 1) {
1241 // math sometimes has only one operand
1242 ParseSrcOp(1);
1243 }
1244 break;
1245 case OpSpec::TERNARY_REGIMM_REG_REGIMM:
1246 if (m_opSpec->isDpasFormat()) {
1247 ParseDpasOp(-1, true); // dst
1248 ParseDpasOp(0, false); // src0
1249 ParseDpasOp(1, false); // src1
1250 ParseDpasOp(2, false); // src2
1251 break;
1252 }
1253 ParseDstOp();
1254 ParseSrcOp(0);
1255 ParseSrcOp(1);
1256 ParseSrcOp(2);
1257 break;
1258 case OpSpec::QUINARY_SRC:
1259 assert(m_opSpec->is(Op::BDPAS) && "only bdpas has 5 sources");
1260 ParseDpasOp(-1, true); // dst
1261 ParseDpasOp(0, false); // src0
1262 ParseDpasOp(1, false); // src1
1263 ParseDpasOp(2, false); // src2
1264 ParseDpasOp(3, false); // src3
1265 ParseDpasOp(4, false); // src4
1266 break;
1267 case OpSpec::JUMP_UNARY_IMM:
1268 ParseSrcOpLabel(0);
1269 break;
1270 case OpSpec::JUMP_UNARY_REGIMM:
1271 // e.g. jmpi or brd (registers or labels)
1272 ParseSrcOp(0);
1273 break;
1274 case OpSpec::JUMP_UNARY_REG:
1275 // e.g. ret (..) r12
1276 ParseSrcOp(0);
1277 if (m_srcKinds[0] != Operand::Kind::DIRECT &&
1278 m_srcKinds[0] != Operand::Kind::INDIRECT) {
1279 FailS(m_srcLocs[0], "src0 must be a register");
1280 }
1281 break;
1282 case OpSpec::JUMP_BINARY_BRC: {
1283 // brc (...) lbl lbl
1284 // brc (...) reg null
1285 //
1286 // for legacy reasons we support a unary form
1287 // brc (...) reg
1288 // we do add an explicit null parameter for src1
1289 // it's very possible there are some simulator tests floating
1290 // aroudn that require this old form.
1291 ParseSrcOp(0);
1292 if (m_srcKinds[0] == Operand::Kind::IMMEDIATE ||
1293 m_srcKinds[0] == Operand::Kind::LABEL || LookingAtIdentEq("null")) {
1294 ParseSrcOp(1);
1295 } else {
1296 // legacy format
1297 // add an explicit null operand
1298 //
1299 // trb: this is no longer a warning, but we still set the
1300 // parameter in the IR for the time being
1301 //
1302 // if (m_parseOpts.deprecatedSyntaxWarnings) {
1303 // Warning(brcStart,
1304 // "deprecated syntax: register version "
1305 // "of brc takes an explicit null parameter");
1306 // }
1307 // m_handler.InstSrcOpRegDirect(
1308 // 1,
1309 // brcStart,
1310 // SrcModifier::NONE,
1311 // RegName::ARF_NULL,
1312 // REGREF_ZERO_ZERO,
1313 // Region::SRC010,
1314 // Type::UD);
1315 }
1316 break;
1317 }
1318 case OpSpec::JUMP_UNARY_CALL_REGIMM:
1319 // call (..) dst src
1320 // calla (..) dst src
1321 ParseDstOp();
1322 ParseSrcOp(0);
1323 break;
1324 case OpSpec::JUMP_BINARY_IMM_IMM:
1325 // e.g. else, cont, break, goto, halt
1326 ParseSrcOpLabel(0);
1327 ParseSrcOpLabel(1);
1328 break;
1329 default:
1330 IGA_ASSERT_FALSE("unhandled syntax class in parser");
1331 }
1332 }
1333
1334 // Original binary send <=XE (sends, sendsc)
1335 void ParseSendInstructionLegacy() {
1336 ParseSendDstOp();
1337 ParseSendSrcOp(0, false);
1338 ParseSendSrcOp(1, m_model.supportsUnarySend() &&
1339 m_opts.supportLegacyDirectives);
1340 ParseSendDescsLegacy();
1341 }
1342
1343 // XE_HP offers new : syntax for separate src1Len for certain messages
1344 // XE_HPG, XE_HPC require : syntax
1345 // We are flexible here and permit src1Len to come out of the immediate
1346 // descriptor for certain cases (with a warning).
1347 void ParseSendInstructionXeHP() {
1348 ParseSendDstOp();
1349 ParseSendSrcOp(0, false);
1350 int src1Len = -1;
1351 ParseSendSrc1OpWithOptLen(src1Len);
1352 ParseSendDescsWithOptSrc1Len(src1Len);
1353 }
1354
1355 // same as prior sends, but has ExDescImm (c.f. ParseSendDescsXe2)
1356 void ParseSendInstructionXe2() {
1357 ParseSendDstOp();
1358 ParseSendSrcOp(0, false);
1359 int src1Len = -1;
1360 ParseSendSrc1OpWithOptLen(src1Len);
1361 ParseSendDescsXe2(src1Len);
1362 }
1363
1364 // backwards compatible send with Gather Send support
1365 // same as Xe2, but allows gather send (src0 indirect operand)
1366 void ParseSendInstructionXe3() {
1367 ParseSendDstOpXe3(); // use dst type INVALID on >=XE3
1368 bool isGathering = ParseSendSrc0OpXe3();
1369 int src1Len = -1;
1370 if (LookingAtSeq(COLON, IDENT)) {
1371 // send ... r0:ud ...
1372 // ^^^ forbidden on XE3+
1373 // send ... r[s0.0]:ud ...
1374 // ^^^ forbidden on XE3
1375 FailS(NextLoc(), "types are forbidden on send operands on this platform");
1376 }
1377 if (isGathering) {
1378 if (LookingAtSeq(COLON, INTLIT10)) {
1379 // send ... r[s0.0]:4 ...
1380 // ^^ oops needs to be descriptor for this platform
1381 FailS(NextLoc(), "Src0.Len should come from the descriptor");
1382 }
1383 m_srcLocs[1] = NextLoc();
1384 // create null src1 for Gather Send (src1 won't be printed in asm)
1385 addNullGatherSendSrc1();
1386 // set src1Len to 0 to indicate ExBSO mode on
1387 src1Len = 0;
1388 } else {
1389 ParseSendSrc1OpWithOptLen(src1Len);
1390 }
1391 ParseSendDescsXe2(src1Len);
1392 }
1393
1394 void ParseSendgInstructionXe3p() {
1395 ParseSendDstOpXe3();
1396
1397 bool gatheringSrc0 = ParseSendgSrc0OpXe3p();
1398
1399 if (gatheringSrc0) {
1400 // create null src1 for Gather Send (src1 won't be printed in asm)
1401 addNullGatherSendSrc1();
1402 m_builder.InstSendSrc1Length(0);
1403 } else {
1404 ParseSendgSrc1OpXe3();
1405 }
1406 ParseSendgDescsXe3();
1407 }
1408
1409
1410 // Predication = ('(' WrEnPred ')')?
1411 // where WrEnPred = WrEn
1412 // | Pred
1413 // | WrEn '&' Pred
1414 // WrEn = 'W'
1415 // Pred = '~'? FlagReg ('.' PreCtrl?)
1416 void ParseWrEnPred() {
1417 m_hasWrEn = false;
1418 if (Consume(LPAREN)) {
1419 const Loc nmLoc = NextLoc(0);
1420 if (ConsumeIdentEq("W")) {
1421 m_hasWrEn = true;
1422 m_builder.InstNoMask(nmLoc);
1423 if (Consume(AMP)) {
1424 ParsePred();
1425 }
1426 } else {
1427 ParsePred();
1428 }
1429 ConsumeOrFail(RPAREN, "expected )");
1430 }
1431 }
1432
1433 // Pred = '~'? FlagReg ('.' PreCtrl?)
1434 void ParsePred() {
1435 static const IdentMap<PredCtrl> PREDCTRLS{
1436 {"xyzw", PredCtrl::SEQ}, // lack of a specific pred control defaults to
1437 // SEQ
1438 {"anyv", PredCtrl::ANYV}, {"allv", PredCtrl::ALLV},
1439 {"any2h", PredCtrl::ANY2H}, {"all2h", PredCtrl::ALL2H},
1440 {"any4h", PredCtrl::ANY4H}, {"all4h", PredCtrl::ALL4H},
1441 {"any8h", PredCtrl::ANY8H}, {"all8h", PredCtrl::ALL8H},
1442 {"any16h", PredCtrl::ANY16H}, {"all16h", PredCtrl::ALL16H},
1443 {"any32h", PredCtrl::ANY32H}, {"all32h", PredCtrl::ALL32H},
1444 {"any", PredCtrl::ANY}, {"all", PredCtrl::ALL},
1445 };
1446
1447 const Loc prLoc = NextLoc(0);
1448 bool predInv = Consume(TILDE);
1449 ParseFlagRegRef(m_flagReg);
1450 PredCtrl predCtrl = PredCtrl::NONE;
1451 if (Consume(DOT)) {
1452 ConsumeIdentOneOfOrFail(PREDCTRLS, predCtrl,
1453 "expected predication control",
1454 "invalid predication control");
1455 } else {
1456 predCtrl = PredCtrl::SEQ;
1457 }
1458 m_builder.InstPredication(prLoc, predInv, m_flagReg, predCtrl);
1459 }
1460
1461 const OpSpec *TryConsumeMmenonic() {
1462 const Token &tk = Next();
1463 if (tk.lexeme != IDENT) {
1464 return nullptr;
1465 }
1466 const char *p = &m_lexer.GetSource()[tk.loc.offset];
1467 std::string s;
1468 s.reserve((size_t)tk.loc.extent + 1);
1469 for (size_t i = 0; i < tk.loc.extent; i++) {
1470 s += *p++;
1471 }
1472 auto itr = opmap.find(s);
1473 if (itr == opmap.end()) {
1474 return nullptr;
1475 } else {
1476 Skip();
1477 return itr->second;
1478 }
1479 }
1480
1481 #if 0
1482 // returns the number of characters off
1483 size_t similarityR(
1484 size_t mismatches, size_t shortestStrLen,
1485 const std::string &left, size_t li,
1486 const std::string &right, size_t ri)
1487 {
1488 if (mismatches >= shortestStrLen) { // everything mismatches
1489 return shortestStrLen;
1490 } else if (li == left.size() - 1) { // the rest of right mismatches
1491 return mismatches + right.size() - ri;
1492 } else if (ri == right.size() - 1) { // the rest of left mismatches
1493 return mismatches + left.size() - li;
1494 } else {
1495 if (left[li] != right[ri]) {
1496 mismatches++;
1497 }
1498 // 1. delete both
1499 auto db = similarityR(
1500 mismatches, shortestStrLen,
1501 left, li + 1,
1502 right, ri + 1);
1503 // 2. delete left
1504 auto dr = similarityR(
1505 mismatches, shortestStrLen,
1506 left, li,
1507 right, ri + 1);
1508 // 3. delete right
1509 auto dl = similarityR(
1510 mismatches, shortestStrLen,
1511 left, li + 1,
1512 right, ri);
1513 return std::min<size_t>(db, std::min<size_t>(dr, dl));
1514 }
1515 }
1516 // roughly the number of characters
1517 float similarity(const std::string &left, const std::string &right) {
1518 if (left.size() > 8 || right.size() > 8)
1519 return 0;
1520 auto shortestLen = std::min<size_t>(left.size(), right.size());
1521 size_t minEdits = similarityR(0, shortestLen, left, 0, right, 0);
1522 return 1.0f - (float)minEdits/(float)shortestLen;
1523 }
1524 #endif
1525 void failWithUnexpectedSubfunction(const Loc &loc, const std::string &) {
1526 FailAtT(loc, "unexpected subfunction for op");
1527 #if 0
1528 std::vector<std::pair<float,const char *>> matches;
1529 for (int i = (int)m_opSpec->op + 1;
1530 i < (int)m_opSpec->op + m_opSpec->subopsLength;
1531 i++)
1532 {
1533 const OpSpec &sf = m_model.lookupOpSpec((iga::Op)i);
1534 if (sf.isValid()) {
1535 auto sim = similarity(sfIdent, sf.mnemonic.str());
1536 std::cout << sf.mnemonic << ": " << sim << "\n";
1537 if (sim >= 0.66f) {
1538 matches.emplace_back(sim, sf.mnemonic.str());
1539 }
1540 }
1541 }
1542 std::sort(matches.begin(), matches.end(),
1543 std::greater<std::pair<float,const char *>>());
1544 if (!matches.empty()) {
1545 ss << "; did you mean ";
1546 if (matches.size() > 2) {
1547 ss << "one of: ";
1548 }
1549 // only show the top five
1550 size_t len = std::min<size_t>(5, matches.size());
1551 for (size_t i = 0; i < len; i++) {
1552 if (i > 0 && i == len - 1) {
1553 if (len > 2) {
1554 ss << ", or ";
1555 } else {
1556 ss << " or ";
1557 }
1558 } else {
1559 ss << ", ";
1560 }
1561 ss << matches[i].second;
1562 }
1563 }
1564 #endif
1565 }
1566
1567 // Parse BFN expressions.
1568 //
1569 // BFNExpr = BFNBinExpr
1570 // where BFNBinExpr = (BFNUnExpr ('&'|'^'|'|') BFNUnExpr)*
1571 // regular boolean precedence: & > ^ > |
1572 // BFNUnExpr = '~'? BFNPrimExpr
1573 // BFNPrimExpr = '(' BFNExpr ')' | 's0' | 's1' | 's2'
1574 //
1575 // E.g.
1576 // bfn.(s0^(s1|~s2)) ...
1577 // ^
1578 uint8_t ParseBfnMnemonicSubfuncExpr() {
1579 Skip(1); // skip LPAREN
1580 uint32_t val = ParseBfnMnemonicSubfuncExprOR();
1581 ConsumeOrFail(RPAREN, "expected )");
1582 return (uint8_t)val;
1583 }
1584 uint32_t ParseBfnMnemonicSubfuncExprOR() {
1585 uint32_t val = ParseBfnMnemonicSubfuncExprXOR();
1586 while (Consume(PIPE)) {
1587 val |= ParseBfnMnemonicSubfuncExprXOR();
1588 }
1589 return val;
1590 }
1591 uint32_t ParseBfnMnemonicSubfuncExprXOR() {
1592 uint32_t val = ParseBfnMnemonicSubfuncExprAND();
1593 while (Consume(CIRC)) {
1594 val ^= ParseBfnMnemonicSubfuncExprAND();
1595 }
1596 return val;
1597 }
1598 uint32_t ParseBfnMnemonicSubfuncExprAND() {
1599 uint32_t val = ParseBfnMnemonicSubfuncExprNEG();
1600 while (Consume(AMP)) {
1601 val &= ParseBfnMnemonicSubfuncExprNEG();
1602 }
1603 return val;
1604 }
1605 uint32_t ParseBfnMnemonicSubfuncExprNEG() {
1606 bool c = Consume(TILDE);
1607 uint32_t val = ParseBfnMnemonicSubfuncExprATOM();
1608 if (c) {
1609 val = (~val & 0xFF);
1610 }
1611 return val;
1612 }
1613 uint32_t ParseBfnMnemonicSubfuncExprATOM() {
1614 if (Consume(LPAREN)) {
1615 uint32_t val = ParseBfnMnemonicSubfuncExprOR();
1616 ConsumeOrFail(RPAREN, "expected )");
1617 return val;
1618 } else if (LookingAt(IDENT) || LookingAt(INTLIT10)) {
1619 auto symTkn = Next();
1620 auto sfIdent = GetTokenAsString(symTkn);
1621 Skip();
1622 if (sfIdent == "s0") {
1623 return 0xAA;
1624 } else if (sfIdent == "s1") {
1625 return 0xCC;
1626 } else if (sfIdent == "s2") {
1627 return 0xF0;
1628 } else if (sfIdent == "zeros" || sfIdent == "0") {
1629 return 0x00;
1630 } else if (sfIdent == "ones" || sfIdent == "1") {
1631 return 0xFF;
1632 } else {
1633 FailAtT(symTkn.loc, "syntax error in symbolic expression");
1634 }
1635 } else {
1636 FailT("syntax error in symbolic expression");
1637 }
1638 return 0;
1639 }
1640
1641 //
1642 // Mnemoninc
1643 // = Ident SubMnemonic?
1644 // | Ident BrCtl
1645 // | Ident '(' BFNExpr ')'
1646 // SubMnemoninc
1647 // = '.' SfIdent
1648 // | '.' HEX_INT | '.' DEC_INT
1649 //
1650 // SfIdent =
1651 // | BFNExpr = see above
1652 // | BrnchCtl
1653 // | SFID
1654 // | SyncFc
1655 // | ... other subfunction identifier
1656 const OpSpec *ParseMnemonic() {
1657 const Loc mnemonicLoc = NextLoc();
1658 const OpSpec *pOs = TryConsumeMmenonic();
1659 if (!pOs) {
1660 return nullptr;
1661 }
1662 m_builder.InstOp(pOs);
1663 // GED will reject this otherwise
1664 if (!m_hasWrEn && pOs->op == Op::JMPI) {
1665 WarningAtT(mnemonicLoc,
1666 "jmpi must have (W) specified (automatically adding)");
1667 m_builder.InstNoMask(mnemonicLoc);
1668 }
1669
1670 // TODO: abstract this all into a something that's contained in
1671 // the IGA bxml enums files.
1672 // e.g. use OpSpec::supportsSubfunction()
1673 // bool Subfunction::FromString(Op, Subfunction&)
1674 if (pOs->supportsBranchCtrl()) {
1675 BranchCntrl brctl = BranchCntrl::OFF;
1676 if (Consume(DOT)) {
1677 if (!ConsumeIdentEq("b")) {
1678 FailT("expected 'b' (branch control)");
1679 }
1680 brctl = BranchCntrl::ON;
1681 }
1682 m_builder.InstSubfunction(brctl);
1683 } else if (pOs->is(Op::MATH)) {
1684 // e.g. math.*, send.*, etc...
1685 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnum<MathFC>());
1686 } else if (pOs->is(Op::SYNC)) {
1687 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnum<SyncFC>());
1688 } else if (platform() >= Platform::XE && pOs->isAnySendFormat()) {
1689 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnum<SFID>());
1690 // else: it's part of ExDesc
1691 } else if (pOs->is(Op::BFN)) {
1692 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnumBfnFC());
1693 } else if (pOs->isDpasFormat()) {
1694 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnum<DpasFC>());
1695 } else if (pOs->is(Op::SHFL)) {
1696 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnum<ShuffleFC>());
1697 } else if (pOs->is(Op::LFSR)) {
1698 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnum<LfsrFC>());
1699 } else if (pOs->is(Op::DNSCL)){
1700 m_builder.InstSubfunction(ParseSubfunctionFromBxmlEnumDnsclFC());
1701 } else {
1702 // isOldSend: pre XE will have a subfunction,
1703 // but we pull it from ExDesc
1704 bool isOldSend = platform() < Platform::XE && pOs->isAnySendFormat();
1705 IGA_ASSERT(!pOs->supportsSubfunction() || isOldSend,
1706 "INTERNAL ERROR: subfunction expected");
1707 }
1708
1709 if (LookingAt(DOT)) {
1710 // maybe an old condition modifier or saturation
1711 FlagModifier fm;
1712 if (LookingAtIdentEq(1, "sat")) {
1713 FailT("saturation flag prefixes destination operand: "
1714 "e.g. op (..) (sat)dst ...");
1715 } else if (IdentLookupFrom(1, FLAGMODS, fm) ||
1716 IdentLookupFrom(1, FLAGMODS_LEGACY, fm)) {
1717 FailT("conditional modifier follows execution mask "
1718 "info: e.g. op (16|M0) (le)f0.0 ...");
1719 } else {
1720 // didn't match flag modifier
1721 FailT("unexpected . (expected execution size)");
1722 }
1723 }
1724
1725 return pOs;
1726 }
1727
1728 template <typename T> T ParseSubfunctionFromBxmlEnum() {
1729 if (!Consume(DOT)) {
1730 FailAfterPrev("expected operation subfunction");
1731 }
1732
1733 auto sfLoc = NextLoc();
1734 if (LookingAt(IDENT)) {
1735 auto loc = NextLoc();
1736 const std::string sfIdent = GetTokenAsString();
1737 Skip();
1738 auto x = FromSyntax<T>(sfIdent);
1739 if (x == T::INVALID) {
1740 FailAtT(loc, "invalid subfunction");
1741 }
1742 return x;
1743 } else if (LookingAtAnyOf(INTLIT10, INTLIT16)) {
1744 uint32_t val;
1745 (void)ConsumeIntLit(val);
1746 return (T)val;
1747 }
1748 FailAtT(sfLoc, "invalid subfunction");
1749 return (T)-1;
1750 }
1751
1752 // we treat BFN a little different since we permit a BFN expression
1753 // e.g. bfn.(s0&~s1^s2)
1754 //
1755 // template <> BfnFC ParseSubfunctionFromBxmlEnum()
1756 // gcc: doesn't permit this specialization
1757 BfnFC ParseSubfunctionFromBxmlEnumBfnFC() {
1758 if (!Consume(DOT)) {
1759 FailAfterPrev("expected subfunction");
1760 }
1761 uint8_t fc = 0;
1762 if (LookingAt(LPAREN)) {
1763 fc = ParseBfnMnemonicSubfuncExpr();
1764 } else if (LookingAtAnyOf(INTLIT10, INTLIT16)) {
1765 auto loc = NextLoc();
1766 uint32_t val = 0;
1767 (void)ConsumeIntLit(val);
1768 if (val > 0xFF) {
1769 FailAtT(loc, "subfunction value is out of bounds");
1770 }
1771 fc = (uint8_t)val;
1772 } else {
1773 FailT("invalid subexpression");
1774 }
1775 return BfnFC(fc);
1776 }
1777
1778 DnsclFC ParseSubfunctionFromBxmlEnumDnsclFC() {
1779 if (!Consume(DOT)) {
1780 FailAfterPrev("expected operation subfunction");
1781 }
1782 // DnsclFC is composed by 4 parts. dnscl op is in the syntax:
1783 // dnscl.<ConvSrcDataType>to<ConvDstDataType>.<DnsclMode>.<RoundingMode>
1784 // for example: dnscl.bftoE2M1.mode0.srnd
1785 ConvSrcDataType srcTy = ConvSrcDataType::INVALID;
1786 ConvDstDataType dstTy = ConvDstDataType::INVALID;
1787 DnsclMode dMode = DnsclMode::INVALID;
1788 RoundingMode rMode = RoundingMode::INVALID;
1789
1790 // parse <ConvSrcDataType>to<ConvDstDataType>
1791 auto loc = NextLoc();
1792 if (LookingAt(IDENT)) {
1793 auto srcDstTyLoc = NextLoc();
1794 std::string str = GetTokenAsString();
1795 Skip();
1796 auto toPos = str.find("to");
1797 if (toPos == std::string::npos)
1798 FailAtT(srcDstTyLoc, "invalid subfunction");
1799 std::string srcStr = str.substr(0, toPos);
1800 srcTy = FromSyntax<ConvSrcDataType>(srcStr);
1801 std::string dstStr = str.substr(toPos + 2);
1802 dstTy = FromSyntax<ConvDstDataType>(dstStr);
1803 if (srcTy == ConvSrcDataType::INVALID ||
1804 dstTy == ConvDstDataType::INVALID) {
1805 FailAtT(srcDstTyLoc, "invalid conv type");
1806 }
1807 } else {
1808 FailAtT(loc, "expected operation subfunction");
1809 }
1810
1811 // parse <DnsclMode>
1812 if (!Consume(DOT)) {
1813 // FIXME: vague error message to avoid IP issue
1814 FailAfterPrev("expected mode");
1815 }
1816 loc = NextLoc();
1817 if (LookingAt(IDENT)) {
1818 auto dmLoc = NextLoc();
1819 std::string str = GetTokenAsString();
1820 Skip();
1821 dMode = FromSyntax<DnsclMode>(str);
1822 if (dMode == DnsclMode::INVALID)
1823 FailAtT(dmLoc, "invalid mode");
1824 } else {
1825 // FIXME: vague error message to avoid IP issue
1826 FailAtT(loc, "expected mode");
1827 }
1828
1829 // parse <RoundingMode>
1830 if (!Consume(DOT)) {
1831 FailAfterPrev("expected rounding mode");
1832 }
1833 loc = NextLoc();
1834 if (LookingAt(IDENT)) {
1835 auto rmLoc = NextLoc();
1836 std::string str = GetTokenAsString();
1837 Skip();
1838 rMode = FromSyntax<RoundingMode>(str);
1839 if (rMode == RoundingMode::INVALID)
1840 FailAtT(rmLoc, "invalid mode");
1841 } else {
1842 FailAtT(loc, "expected rounding mode");
1843 }
1844
1845 return DnsclFC(srcTy, dstTy, dMode, rMode);
1846 }
1847
1848 // FlagModifierOpt = '(' FlagModif ')' FlagReg
1849 void ParseFlagModOpt() {
1850 Loc loc = NextLoc();
1851 if (Consume(LBRACK)) {
1852 // try the full form [(le)f0.0]
1853 FlagModifier flagMod;
1854 if (!TryParseFlagModFlag(flagMod)) {
1855 FailT("expected flag modifier function");
1856 }
1857 ParseFlagModFlagReg();
1858 ConsumeOrFail(RBRACK, "expected ]");
1859 if (m_opts.deprecatedSyntaxWarnings) {
1860 WarningAtT(loc, "deprecated flag modifier syntax (omit the brackets)");
1861 }
1862 m_builder.InstFlagModifier(m_flagReg, flagMod);
1863 } else {
1864 // try the short form
1865 // (le)f0.0
1866 FlagModifier flagMod;
1867 if (!TryParseFlagModFlag(flagMod)) {
1868 // soft fail (we might be looking at "(sat)r0.0")
1869 return;
1870 }
1871 ParseFlagModFlagReg();
1872 m_builder.InstFlagModifier(m_flagReg, flagMod);
1873 }
1874 }
1875 // e.g. "(le)"
1876 bool TryParseFlagModFlag(FlagModifier &flagMod) {
1877 if (!LookingAt(LPAREN)) {
1878 return false;
1879 }
1880 if (!IdentLookupFrom(1, FLAGMODS, flagMod)) {
1881 Loc loc = NextLoc(1);
1882 if (!IdentLookupFrom(1, FLAGMODS_LEGACY, flagMod)) {
1883 return false;
1884 } else if (m_opts.deprecatedSyntaxWarnings) {
1885 // deprecated syntax
1886 WarningAtT(loc,
1887 "deprecated flag modifier syntax: "
1888 "use ",
1889 ToSyntax(flagMod), " for this function");
1890 }
1891 }
1892 Skip(2);
1893 ConsumeOrFail(RPAREN, "expected )");
1894 return true;
1895 }
1896 // e.g. "f0.1"
1897 void ParseFlagModFlagReg() {
1898 const Loc flregLoc = NextLoc();
1899 RegRef fr;
1900 ParseFlagRegRef(fr);
1901 if (m_flagReg.regNum != REGREF_INVALID.regNum &&
1902 (m_flagReg.regNum != fr.regNum ||
1903 m_flagReg.subRegNum != fr.subRegNum)) {
1904 FailAtT(flregLoc, "flag register must be same for predication "
1905 "and flag modifier");
1906 }
1907 m_flagReg = fr;
1908 }
1909
1910 // Examples:
1911 // r13.4<2>:hf
1912 // (sat)r13.0<1>:f
1913 // r[a0.3,16]<1>:ud
1914 void ParseDstOp() {
1915 // We want to track the beginning of the "operand", not just the register.
1916 // That includes the saturate flag prefix(sat) on the dst.
1917 // That is we need:
1918 // (sat)r13.3:ud
1919 // ^ ^
1920 // | | not here
1921 // |
1922 // | here
1923 const Loc opStart = NextLoc(0);
1924 // ParseSatOpt increments m_offset.
1925 if (ParseSatOpt()) {
1926 m_builder.InstDstOpSaturate();
1927 }
1928 // Note that, if there is SAT token, opStart != regStart.
1929 // This is because m_offset changed.
1930 const Loc regStart = NextLoc(0);
1931 if (ConsumeIdentEq("r")) {
1932 ParseDstOpRegInd(opStart, 0);
1933 } else {
1934 const RegInfo *regInfo;
1935 int regNum;
1936 if (!ConsumeReg(regInfo, regNum)) {
1937 FailT("invalid destination register");
1938 }
1939 if (regInfo && !regInfo->isRegNumberValid(regNum)) {
1940 FailT("invalid destination register number "
1941 "(",
1942 regInfo->syntax, " only has ", regInfo->numRegs,
1943 " registers on this platform)");
1944 }
1945 if (LookingAt(LBRACK)) {
1946 ParseDstOpRegInd(opStart, regNum * 32);
1947 } else {
1948 assert(regInfo != nullptr);
1949 FinishDstOpRegDirSubRegRgnTy(opStart, regStart, *regInfo, regNum);
1950 }
1951 }
1952 }
1953
1954 // r13
1955 // null
1956 // r13:w (non-standard)
1957 void ParseSendDstOp() {
1958 const Loc regStart = NextLoc(0);
1959 if (ConsumeIdentEq("r")) {
1960 ParseDstOpRegInd(regStart, 0);
1961 } else {
1962 const RegInfo *ri = NULL;
1963 int regNum = 0;
1964 if (!ConsumeReg(ri, regNum)) {
1965 FailT("invalid send destination register");
1966 }
1967 if (!ri->isRegNumberValid(regNum)) {
1968 FailT("invalid destination register number "
1969 "(",
1970 ri->syntax, " only has ", ri->numRegs,
1971 " registers on this platform)");
1972 }
1973 if (LookingAt(LBRACK)) {
1974 FailT("this form of indirect (r3[a0.0,16]) is invalid for "
1975 "send dst operand; use regular form: r[a0.0,16]");
1976 } else {
1977 FinishDstOpRegDirSubRegRgnTy(regStart, regStart, *ri, regNum);
1978 }
1979 }
1980 }
1981
1982 void ParseSendDstOpXe3() {
1983 OperandInfo opInfo;
1984 m_srcLocs[1] = NextLoc();
1985 m_srcKinds[1] = opInfo.kind = Operand::Kind::DIRECT;
1986 auto reg = ParseReg();
1987 // for sendgx, check if the destination is ARF_NULL
1988 // if ARF_NULL, then change to r511
1989 if ((m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) &&
1990 reg.first->regName == RegName::ARF_NULL) {
1991 opInfo.regOpName = RegName::ARF_NULL;
1992 opInfo.regOpReg.regNum = (uint16_t)0;
1993 opInfo.regOpReg.subRegNum = 0;
1994 }
1995 else {
1996 opInfo.regOpName = reg.first->regName;
1997 opInfo.regOpReg.regNum = reg.second;
1998 opInfo.regOpReg.subRegNum = 0;
1999 }
2000 m_builder.InstDstOp(opInfo);
2001 if (LookingAtSeq(COLON, IDENT)) {
2002 // send r10:ud ...
2003 // ^^^ forbidden on XE3+
2004 FailS(NextLoc(), "types are forbidden on send operands on this platform");
2005 }
2006 }
2007
2008 // (sat)
2009 bool ParseSatOpt() {
2010 // TODO: expand to tokens so (sat) can be imm expr
2011 return Consume(SAT);
2012 }
2013
2014 // .mme2 or .nomme
2015 MathMacroExt ParseMathMacroReg() {
2016 auto loc = NextLoc();
2017 MathMacroExt mme = MathMacroExt::INVALID;
2018 const char *EXPECTED = "expected math macro register "
2019 "(e.g. .mme0, ..., .mme7, or .nomme)";
2020 ConsumeOrFail(DOT, EXPECTED);
2021 if (ConsumeIdentOneOf<MathMacroExt>(MATHMACROREGS, mme)) {
2022 return mme;
2023 }
2024 // determine if we support the old-style
2025 bool supportOldStyleAcc2ToAcc7 = true;
2026 supportOldStyleAcc2ToAcc7 = platform() < Platform::XE;
2027
2028 if (supportOldStyleAcc2ToAcc7) {
2029 if (ConsumeIdentOneOf<MathMacroExt>(MATHMACROREGS_OLDSTYLE, mme)) {
2030 // warn?
2031 WarningS(loc, "old-style math macro register (use mme)");
2032 return mme;
2033 }
2034 }
2035
2036 // favor the new error message
2037 FailS(loc, EXPECTED);
2038 return mme;
2039 }
2040
2041 // REG ('.' INT)? DstRgn (':' DstTy)?
2042 // where DstRgn = '<' INT '>'
2043 //
2044 // E.g. r13.4<2>:t
2045 void FinishDstOpRegDirSubRegRgnTy(const Loc &opStart, const Loc &regnameLoc,
2046 const RegInfo &ri, int regNum) {
2047 // subregister or implicit accumulator operand
2048 // e.g. .4 or .acc3
2049 Loc subregLoc = NextLoc(0);
2050 int subregNum = 0;
2051
2052 // <1>
2053 Region::Horz rgnHz = Region::Horz::HZ_1;
2054
2055 MathMacroExt mmeReg = MathMacroExt::INVALID;
2056 if (m_opSpec->isAnySendFormat() && Consume(DOT)) {
2057 ConsumeIntLitOrFail(subregNum, "expected subregister");
2058 // whine about them using a subregister on a send operand
2059 if (m_opts.deprecatedSyntaxWarnings) {
2060 WarningAtT(subregLoc, "send operand subregisters have no effect"
2061 " and are deprecated syntax");
2062 }
2063 if (m_opSpec->isAnySendFormat() && LookingAt(LANGLE)) {
2064 if (m_opts.deprecatedSyntaxWarnings) {
2065 // whine about them using a region
2066 WarningT("send operand region has no effect and is"
2067 " deprecated syntax");
2068 }
2069 }
2070 rgnHz = ParseDstOpRegion();
2071 } else if (m_builder.isMacroOp()) {
2072 // implicit accumulator operand
2073 mmeReg = ParseMathMacroReg();
2074 } else {
2075 // non-send with subregister.
2076 // regular subregister
2077 if (Consume(DOT)) {
2078 ConsumeIntLitOrFail(subregNum, "expected subregister");
2079 } else {
2080 // implicit subregister
2081 // E.g. r12:d ...
2082 subregNum = 0;
2083 }
2084 }
2085
2086 // <1>
2087 rgnHz = ParseDstOpRegion();
2088
2089 // :t
2090 Type dty = Type::INVALID;
2091 if (m_opSpec->isAnySendFormat()) {
2092 // special handling for send types
2093 dty = ParseSendOperandTypeWithDefault(-1);
2094 } else {
2095 dty = ParseDstOpTypeWithDefault();
2096 }
2097
2098 // ensure the subregister is not out of bounds
2099 if (dty != Type::INVALID) {
2100 int typeSize = TypeSizeInBits(dty) / 8;
2101 if (!ri.isSubRegByteOffsetValid(regNum, subregNum * typeSize,
2102 m_model.getGRFByteSize())) {
2103 if (ri.regName == RegName::GRF_R) {
2104 ErrorAtT(subregLoc, "subregister out of bounds for data type ",
2105 ToSyntax(dty));
2106 } else {
2107 // Print warning for non-GRF register, in case for those
2108 // valid but strange case e.g. null0.20:f
2109 WarningAtT(subregLoc, "subregister out of bounds for data type");
2110 }
2111 } else if (typeSize < ri.accGran) {
2112 WarningAtT(regnameLoc, "access granularity too small for data type");
2113 }
2114 }
2115
2116 if (m_builder.isMacroOp()) {
2117 m_builder.InstDstOpRegMathMacroExtReg(opStart, ri.regName, regNum, mmeReg,
2118 rgnHz, dty);
2119 } else {
2120 RegRef reg(regNum, subregNum);
2121 m_builder.InstDstOpRegDirect(opStart, ri.regName, reg, rgnHz, dty);
2122 }
2123 }
2124
2125 // e.g. [a0.4, 16]
2126 // or [a0.4 + 16]
2127 // or [a0.4 - 16]
2128 void ParseIndOpArgs(RegRef &addrRegRef, int &addrOff, RegName &regName) {
2129 ConsumeOrFail(LBRACK, "expected [");
2130 if (!ParseAddrRegRefOpt(addrRegRef, regName)) {
2131 FailT("expected address subregister");
2132 }
2133 Loc addrOffLoc = NextLoc();
2134 if (Consume(COMMA)) {
2135 bool neg = Consume(SUB);
2136 ConsumeIntLitOrFail(addrOff, "expected indirect address offset");
2137 if (neg)
2138 addrOff *= -1;
2139 } else if (Consume(ADD)) {
2140 ConsumeIntLitOrFail(addrOff, "expected indirect address offset");
2141 } else if (Consume(SUB)) {
2142 ConsumeIntLitOrFail(addrOff, "expected indirect address offset");
2143 addrOff *= -1;
2144 } else {
2145 addrOff = 0;
2146 }
2147
2148 // check if the imm offset out of bound
2149 int addroffMax = 511;
2150 int addroffMin = -512;
2151 if (platform() >= Platform::XE_HPC) {
2152 addroffMax = 1023;
2153 addroffMin = -1024;
2154 }
2155 if (addrOff < addroffMin || addrOff > addroffMax) {
2156 FailAtT(addrOffLoc,
2157 "immediate offset is out of range; must be in "
2158 "[",
2159 addroffMin, ",", std::to_string(addroffMax), "]");
2160 }
2161
2162 ConsumeOrFail(RBRACK, "expected ]");
2163 }
2164
2165 // '[' 'a0' '.' INT (',' INT)? ']' ('<' INT '>')?)? TY?
2166 // e.g [a0.3,16]<1>:t
2167 void ParseDstOpRegInd(const Loc &opStart, int baseAddr) {
2168 // [a0.4,16]
2169 int addrOff;
2170 RegRef addrRegRef;
2171 RegName regName = RegName::INVALID;
2172 ParseIndOpArgs(addrRegRef, addrOff, regName);
2173 addrOff += baseAddr;
2174 //
2175 // <1>
2176 Region::Horz rgnHz = ParseDstOpRegion();
2177 //
2178 // :t
2179 Type dty = ParseDstOpTypeWithDefault();
2180 m_builder.InstDstOpRegIndirect(opStart, addrRegRef, addrOff, rgnHz, dty);
2181 }
2182
2183 // '<' INT '>'
2184 Region::Horz ParseDstOpRegion() {
2185 if (!LookingAt(LANGLE)) {
2186 if (m_opSpec->hasImplicitDstRegion(m_builder.isMacroOp())) {
2187 return m_opSpec->implicitDstRegion(m_builder.isMacroOp()).getHz();
2188 } else {
2189 return Region::Horz::HZ_1;
2190 }
2191 }
2192
2193 Region::Horz rgnHz = Region::Horz::HZ_1;
2194 if (Consume(LANGLE)) {
2195 const Loc loc = NextLoc();
2196 int rgnHzInt;
2197 ConsumeIntLitOrFail(rgnHzInt, "destination region argument");
2198 switch (rgnHzInt) {
2199 case 1:
2200 rgnHz = Region::Horz::HZ_1;
2201 break;
2202 case 2:
2203 rgnHz = Region::Horz::HZ_2;
2204 break;
2205 case 4:
2206 rgnHz = Region::Horz::HZ_4;
2207 break;
2208 default:
2209 FailS(loc, "invalid destination region");
2210 }
2211 ConsumeOrFail(RANGLE, "expected >");
2212 }
2213
2214 return rgnHz;
2215 }
2216
2217 // E.g. 3 in "a0.3"
2218 bool ParseAddrRegRefOpt(RegRef &addrReg, RegName &regName) {
2219 const RegInfo *ri;
2220 int regNum;
2221 regName = RegName::INVALID;
2222 if (!ConsumeReg(ri, regNum)) {
2223 return false;
2224 }
2225 regName = ri->regName;
2226 if (regNum != 0 || (ri->regName != RegName::ARF_A
2227 && ri->regName != RegName::ARF_S // r[s0.0]
2228 )) {
2229 FailT("expected address register for indirect access (a0)");
2230 }
2231 if (!Consume(DOT)) {
2232 FailT("expected .");
2233 }
2234 addrReg.regNum = addrReg.subRegNum = 0;
2235 ConsumeIntLitOrFail(addrReg.subRegNum,
2236 "expected address register subregister");
2237 // TODO: range-check against RegInfo for "a"
2238 return true;
2239 }
2240
2241 // E.g. 3 in "s0.2"
2242 bool ParseScalarRegRefOpt(RegRef &addrReg) {
2243 const RegInfo *ri;
2244 int regNum;
2245 if (!ConsumeReg(ri, regNum)) {
2246 return false;
2247 }
2248 if (regNum != 0 || ri->regName != RegName::ARF_S) {
2249 FailT("expected scalar register for indirect access (s0.#)");
2250 }
2251 if (!Consume(DOT)) {
2252 FailT("expected .");
2253 }
2254 addrReg.regNum = addrReg.subRegNum = 0;
2255 ConsumeIntLitOrFail(addrReg.subRegNum,
2256 "expected scalar register subregister");
2257 return true;
2258 }
2259
2260 // same as ParseSrcOp, but semantically chekcs that it's a label
2261 void ParseSrcOpLabel(int srcOpIx) {
2262 ParseSrcOp(srcOpIx);
2263 if (m_srcKinds[srcOpIx] != Operand::Kind::LABEL &&
2264 m_srcKinds[srcOpIx] != Operand::Kind::IMMEDIATE) {
2265 FailAtT(m_srcLocs[srcOpIx], "src", srcOpIx,
2266 " must be an immediate label");
2267 }
2268 }
2269
2270 void ParseSrcOp(int srcOpIx) {
2271 m_srcLocs[srcOpIx] = NextLoc();
2272
2273 // TODO: sink this down to the immediate literal case specially
2274 // For now, we leave it here since TryParseConstExpr() will otherwise
2275 // fail on "-r12:f" would fail: - as unary negation then r12 fails to
2276 // resolve
2277 const Token regnameTk = Next();
2278 const RegInfo *regInfo;
2279 int regNum;
2280
2281 // first try and parse as register
2282 m_lexer.Mark();
2283 bool pipeAbs = false;
2284 SrcModifier srcMods = ParseSrcModifierOpt(pipeAbs);
2285 if (ConsumeIdentEq("r")) {
2286 // canonical register indirect
2287 // r[a0.4,-32]
2288 m_srcKinds[srcOpIx] = Operand::Kind::INDIRECT;
2289 ParseSrcOpInd(srcOpIx, m_srcLocs[srcOpIx], srcMods, 0);
2290 // register, symbolic immediate, label, ...
2291 if (pipeAbs) {
2292 ConsumeOrFailAfterPrev(PIPE, "expected |");
2293 }
2294 if (!m_opSpec->supportsSourceModifiers() &&
2295 srcMods != SrcModifier::NONE) {
2296 FailS(m_srcLocs[srcOpIx], "source modifier not supported");
2297 }
2298 } else if (ConsumeReg(regInfo, regNum)) {
2299 // normal register access
2300 // r13.3<0;1,0>
2301 // acc3
2302 m_srcKinds[srcOpIx] = Operand::Kind::DIRECT;
2303 if (!m_opSpec->supportsSourceModifiers() &&
2304 srcMods != SrcModifier::NONE) {
2305 FailS(m_srcLocs[srcOpIx], "source modifier not supported");
2306 }
2307 FinishSrcOpRegDirSubRegRgnTy(srcOpIx, m_srcLocs[srcOpIx], regnameTk.loc,
2308 srcMods, *regInfo, regNum);
2309 if (pipeAbs) {
2310 ConsumeOrFailAfterPrev(PIPE, "expected |");
2311 }
2312 } else {
2313 // backtrack to before any "source modifier"
2314 m_lexer.Reset();
2315 if (pipeAbs) {
2316 Skip(1);
2317 }
2318
2319 // try as raw label first
2320 if ((m_opSpec->isBranching() || m_opSpec->op == Op::MOV) &&
2321 LookingAt(IDENT) && !LookingAtIdentEq("snan") &&
2322 !LookingAtIdentEq("qnan") && !LookingAtIdentEq("nan") &&
2323 !LookingAtIdentEq("inf")) {
2324 if (pipeAbs) {
2325 FailS(regnameTk.loc, "unexpected |");
2326 }
2327 // e.g. LABEL64 (mustn't be an expression)
2328 m_srcKinds[srcOpIx] = Operand::Kind::LABEL;
2329 std::string str = GetTokenAsString();
2330 Skip(1);
2331 FinishSrcOpImmLabel(srcOpIx, m_srcLocs[srcOpIx], regnameTk.loc, str);
2332 } else {
2333 ImmVal immVal;
2334 if (TryParseConstExpr(immVal)) {
2335 // (does not match labels)
2336 m_srcKinds[srcOpIx] = Operand::Kind::IMMEDIATE;
2337 if (pipeAbs) {
2338 immVal.Abs();
2339 }
2340 FinishSrcOpImmValue(srcOpIx, m_srcLocs[srcOpIx], regnameTk, immVal);
2341 if (pipeAbs) {
2342 ConsumeOrFailAfterPrev(PIPE, "expected |");
2343 }
2344 }
2345 }
2346 }
2347 } // ParseSrcOp
2348
2349 // .allrd/.allwr SBID set first
2350 // "($11,$2,$7)" means ((1 << 11) | (1 << 2) | (1 << 7))
2351 // we do also permit
2352 // "()" as the empty set
2353 void ParseSyncSrc0Op() {
2354 auto sf = m_builder.getSubfunction().sync;
2355 bool isSyncSetOp = sf == SyncFC::ALLRD || sf == SyncFC::ALLWR;
2356 bool lookingAtPfx =
2357 LookingAtSeq(LPAREN, DOLLAR) || LookingAtSeq(LPAREN, RPAREN);
2358 if (!isSyncSetOp || !lookingAtPfx) {
2359 // e.g. other sync op, or immediate literal, or null
2360 ParseSrcOp(0);
2361 return;
2362 }
2363 m_srcLocs[0] = NextLoc();
2364 Skip(); // LPAREN
2365 uint32_t sbidSet = 0x0;
2366 auto parseSbid = [&]() {
2367 uint32_t sbid = 0;
2368 ConsumeOrFail(DOLLAR, "expected SBID");
2369 auto nxt = NextLoc();
2370 ConsumeIntLitOrFail(sbid, "expected SBID");
2371 if (sbid >= 32)
2372 FailAtT(nxt, "SBID out of bounds");
2373 sbidSet |= 1 << sbid;
2374 return true;
2375 };
2376 while (LookingAt(DOLLAR)) {
2377 parseSbid();
2378 if (!Consume(COMMA)) {
2379 break;
2380 }
2381 }
2382 ConsumeOrFail(RPAREN, "expected )");
2383 ImmVal val;
2384 val = sbidSet;
2385 auto srcTy = ParseSrcOpTypeWithoutDefault(0, true);
2386 m_builder.InstSrcOpImmValue(0, m_srcLocs[0], val, srcTy);
2387 }
2388
2389 void ParseSrcOpInd(int srcOpIx, const Loc &opStart,
2390 const SrcModifier &srcMods, int baseOff) {
2391 m_srcKinds[srcOpIx] = Operand::Kind::INDIRECT;
2392
2393 // [a0.4 + 16]
2394 int addrOff;
2395 RegRef addrRegRef;
2396 RegName regName = RegName::INVALID;
2397 ParseIndOpArgs(addrRegRef, addrOff, regName);
2398 addrOff += baseOff;
2399
2400 // regioning... <V;W,H> or <V,H>
2401 Region rgn = ParseSrcOpRegionInd(srcOpIx, opStart);
2402
2403 // :t
2404 Type sty = ParseSrcOpTypeWithDefault(srcOpIx, true);
2405
2406 IGA_ASSERT(regName != RegName::INVALID,
2407 "SrcOpInd must not have INVALID register name");
2408 m_builder.InstSrcOpRegIndirect(srcOpIx, opStart, srcMods, regName,
2409 addrRegRef, addrOff, rgn, sty);
2410 }
2411
2412 // REG ('.' INT)? SrcRgn? (':' SrcTy)?
2413 // ^ HERE
2414 //
2415 // E.g. r13.4<2>:f
2416 // E.g. r13.4<0;1,0>:f
2417 // E.g. r13.4<8;8,1>:f
2418 // E.g. r13.acc2:f
2419 // E.g. ce:ud
2420 // E.g. ip:ud
2421 // E.g. a0.1
2422 void FinishSrcOpRegDirSubRegRgnTy(int srcOpIx, const Loc &opStart,
2423 const Loc &regnameLoc,
2424 const SrcModifier &srcMod,
2425 const RegInfo &ri, int regNum) {
2426 // subregister or implicit accumulator operand
2427 // e.g. .4 or .acc3
2428 int subregNum;
2429 Region rgn;
2430 Loc subregLoc = NextLoc(1);
2431 MathMacroExt mme = MathMacroExt::INVALID;
2432 bool hasExplicitSubreg = false;
2433 if (m_builder.isMacroOp()) {
2434 // implicit accumulator operand
2435 // r13.acc2:f
2436 subregNum = 0;
2437 mme = ParseMathMacroReg();
2438 } else {
2439 // regular src with subregister
2440 // r13.4....
2441 // ^
2442 if (Consume(DOT)) {
2443 ConsumeIntLitOrFail(subregNum, "expected subregister");
2444 hasExplicitSubreg = true;
2445 } else { // default to 0
2446 subregLoc = NextLoc(0);
2447 subregNum = 0;
2448 }
2449 }
2450
2451 // for ternary ops <V;H> or <H>
2452 // for other regions <V;W,H>
2453 if (m_opSpec->isTernary()) {
2454 if (srcOpIx < 2) {
2455 rgn = ParseSrcOpRegionVH(srcOpIx, hasExplicitSubreg);
2456 } else {
2457 rgn = ParseSrcOpRegionH(srcOpIx, hasExplicitSubreg);
2458 }
2459 } else {
2460 rgn = ParseSrcOpRegionVWH(ri, srcOpIx, hasExplicitSubreg);
2461 }
2462
2463 // :t
2464 Type sty = Type::INVALID;
2465 if (m_opSpec->isAnySendFormat()) {
2466 sty = ParseSendOperandTypeWithDefault(srcOpIx);
2467 } else {
2468 sty = ParseSrcOpTypeWithDefault(srcOpIx, false);
2469 }
2470
2471 if (sty != Type::INVALID) {
2472 // ensure the subregister is not out of bounds
2473 int typeSize = TypeSizeInBits(sty) / 8;
2474 if (ri.isRegNumberValid(regNum) &&
2475 !ri.isSubRegByteOffsetValid(regNum, subregNum * typeSize,
2476 m_model.getGRFByteSize())) {
2477 // don't add an extra error if the parent register is
2478 // already out of bounds
2479 WarningS(subregLoc, "subregister out of bounds");
2480 } else if (typeSize < ri.accGran) {
2481 WarningS(regnameLoc, "register access granularity too small type");
2482 }
2483 }
2484
2485 if (m_builder.isMacroOp()) {
2486 m_builder.InstSrcOpRegMathMacroExtReg(srcOpIx, opStart, srcMod,
2487 ri.regName, regNum, mme, rgn, sty);
2488 } else {
2489 RegRef reg((uint16_t)regNum, (uint16_t)subregNum);
2490 m_builder.InstSrcOpRegDirect(srcOpIx, opStart, srcMod, ri.regName, reg,
2491 rgn, sty);
2492 }
2493 }
2494
2495 // '<' INT ';' INT ',' INT '>' (VxWxH)
2496 Region ParseSrcOpRegionVWH(const RegInfo &ri, int srcOpIx,
2497 bool hasExplicitSubreg) {
2498 if (m_opSpec->hasImplicitSrcRegion(srcOpIx, m_execSize,
2499 m_builder.isMacroOp())) {
2500 if (!LookingAt(LANGLE)) {
2501 return m_opSpec->implicitSrcRegion(srcOpIx, m_execSize,
2502 m_builder.isMacroOp());
2503 } else {
2504 WarningT(m_opSpec->mnemonic.str(), ".Src", srcOpIx,
2505 " region should be implicit");
2506 }
2507 }
2508
2509 Region rgn = Region::SRC010;
2510 if (Consume(LANGLE)) {
2511 rgn.set(ParseRegionVert());
2512 ConsumeOrFailAfterPrev(SEMI, "expected ;");
2513 rgn.set(ParseRegionWidth());
2514 ConsumeOrFailAfterPrev(COMMA, "expected ,");
2515 rgn.set(ParseRegionHorz());
2516 ConsumeOrFailAfterPrev(RANGLE, "expected >");
2517 } else if (m_opSpec->isAnySendFormat()) {
2518 rgn = defaultSendOperandRegion(ri.regName, srcOpIx);
2519 } else if (ri.supportsRegioning()) {
2520 // N.B. <1;1,0> won't coissue on PreGEN11
2521 rgn = hasExplicitSubreg || m_execSize == ExecSize::SIMD1 ? Region::SRC010
2522 : Region::SRC110;
2523 } else {
2524 rgn = Region::SRC010;
2525 }
2526 return rgn;
2527 }
2528
2529 // '<' INT ';' INT '>' (CNL Align1 ternary)
2530 Region ParseSrcOpRegionVH(int srcOpIx, bool hasExplicitSubreg) {
2531 if (m_opSpec->hasImplicitSrcRegion(srcOpIx, m_execSize,
2532 m_builder.isMacroOp())) {
2533 if (!LookingAt(LANGLE)) {
2534 return m_opSpec->implicitSrcRegion(srcOpIx, m_execSize,
2535 m_builder.isMacroOp());
2536 } else if (m_opts.deprecatedSyntaxWarnings) {
2537 WarningT(m_opSpec->mnemonic.str(), ".Src", srcOpIx,
2538 " region should be implicit");
2539 }
2540 }
2541 Region rgn = Region::SRC010;
2542 if (Consume(LANGLE)) {
2543 rgn.set(ParseRegionVert());
2544 ConsumeOrFailAfterPrev(SEMI, "expected ;");
2545 rgn.set(Region::Width::WI_INVALID);
2546 rgn.set(ParseRegionHorz());
2547 ConsumeOrFailAfterPrev(RANGLE, "expected >");
2548 } else {
2549 bool scalarAccess = hasExplicitSubreg || m_execSize == ExecSize::SIMD1;
2550 if (scalarAccess) {
2551 rgn = Region::SRC0X0;
2552 } else {
2553 // packed access
2554 rgn = platform() >= Platform::XE
2555 ? Region::SRC1X0
2556 : Region::SRC2X1; // most conservative mux
2557 }
2558 }
2559 return rgn;
2560 }
2561
2562 // '<' INT '>' (CNL Align1 ternary src2)
2563 Region ParseSrcOpRegionH(int srcOpIx, bool hasExplicitSubreg) {
2564 if (m_opSpec->hasImplicitSrcRegion(srcOpIx, m_execSize,
2565 m_builder.isMacroOp())) {
2566 if (!LookingAt(LANGLE)) {
2567 return m_opSpec->implicitSrcRegion(srcOpIx, m_execSize,
2568 m_builder.isMacroOp());
2569 } else if (m_opts.deprecatedSyntaxWarnings) {
2570 WarningT(m_opSpec->mnemonic.str(), ".Src", srcOpIx,
2571 " region should be implicit");
2572 }
2573 }
2574
2575 Region rgn;
2576 rgn.bits = 0; // needed to clear _padding
2577 if (Consume(LANGLE)) {
2578 rgn.set(Region::Vert::VT_INVALID, Region::Width::WI_INVALID,
2579 ParseRegionHorz());
2580 ConsumeOrFailAfterPrev(RANGLE, "expected >");
2581 } else {
2582 rgn = hasExplicitSubreg || m_execSize == ExecSize::SIMD1 ? Region::SRCXX0
2583 : Region::SRCXX1;
2584 }
2585 return rgn;
2586 }
2587
2588 Region defaultSendOperandRegion(RegName rn, int opIx) const {
2589 Region r = Region::INVALID;
2590 if (opIx < 0) {
2591 if (m_opSpec->hasImplicitDstRegion(m_builder.isMacroOp())) {
2592 r = m_opSpec->implicitDstRegion(m_builder.isMacroOp());
2593 }
2594 } else {
2595 if (m_opSpec->hasImplicitSrcRegion(opIx, m_execSize,
2596 m_builder.isMacroOp())) {
2597 r = m_opSpec->implicitSrcRegion(0, m_execSize, m_builder.isMacroOp());
2598 } else if (rn == RegName::ARF_NULL) {
2599 r = Region::SRC010;
2600 } else {
2601 r = Region::SRC110;
2602 }
2603 }
2604 return r;
2605 }
2606
2607 // '<' INT ',' INT '>' (VxH mode)
2608 // '<' INT ';' INT ',' INT '>'
2609 Region ParseSrcOpRegionInd(int srcOpIx, const Loc &opStart) {
2610 if (m_opSpec->hasImplicitSrcRegion(srcOpIx, m_execSize,
2611 m_builder.isMacroOp())) {
2612 if (!LookingAt(LANGLE)) {
2613 return m_opSpec->implicitSrcRegion(srcOpIx, m_execSize,
2614 m_builder.isMacroOp());
2615 } else if (m_opts.deprecatedSyntaxWarnings) {
2616 WarningT(m_opSpec->mnemonic.str(), ".Src", srcOpIx,
2617 " region should be implicit");
2618 }
2619 }
2620
2621 Region rgn;
2622 rgn.bits = 0;
2623 if (Consume(LANGLE)) {
2624 Loc arg1Loc = NextLoc();
2625 int arg1;
2626 ConsumeIntLitOrFail(arg1, "syntax error in source region");
2627 if (Consume(COMMA)) {
2628 // <W,H>
2629 rgn.set(Region::Vert::VT_VxH);
2630 switch (arg1) {
2631 case 1:
2632 case 2:
2633 case 4:
2634 case 8:
2635 case 16:
2636 rgn.w = arg1;
2637 break;
2638 default:
2639 FailAtT(arg1Loc, "invalid region width");
2640 rgn.set(Region::Width::WI_INVALID);
2641 }
2642 rgn.set(ParseRegionHorz());
2643 } else {
2644 // <V;W,H>
2645 ConsumeOrFailAfterPrev(SEMI, "expected ;");
2646 switch (arg1) {
2647 case 0:
2648 case 1:
2649 case 2:
2650 case 4:
2651 case 8:
2652 case 16:
2653 case 32:
2654 rgn.v = arg1;
2655 break;
2656 default:
2657 FailAtT(arg1Loc, "invalid region vertical stride");
2658 rgn.set(Region::Vert::VT_INVALID);
2659 }
2660 rgn.set(ParseRegionWidth());
2661 ConsumeOrFailAfterPrev(COMMA, "expected ,");
2662 rgn.set(ParseRegionHorz());
2663 }
2664 ConsumeOrFailAfterPrev(RANGLE, "expected >");
2665 } else {
2666 rgn = Region::SRC110;
2667 }
2668 return rgn;
2669 }
2670
2671 Region::Vert ParseRegionVert() {
2672 Loc loc = NextLoc();
2673 int x;
2674 ConsumeIntLitOrFail(x, "syntax error in region (vertical stride)");
2675 Region::Vert vs;
2676 switch (x) {
2677 case 0:
2678 vs = Region::Vert::VT_0;
2679 break;
2680 case 1:
2681 vs = Region::Vert::VT_1;
2682 break;
2683 case 2:
2684 vs = Region::Vert::VT_2;
2685 break;
2686 case 4:
2687 vs = Region::Vert::VT_4;
2688 break;
2689 case 8:
2690 vs = Region::Vert::VT_8;
2691 break;
2692 case 16:
2693 vs = Region::Vert::VT_16;
2694 break;
2695 case 32:
2696 vs = Region::Vert::VT_32;
2697 break;
2698 default:
2699 FailAtT(loc, "invalid region vertical stride");
2700 vs = Region::Vert::VT_INVALID;
2701 }
2702 return vs;
2703 }
2704
2705 Region::Width ParseRegionWidth() {
2706 Loc loc = NextLoc();
2707 int x;
2708 ConsumeIntLitOrFail(x, "syntax error in region (width)");
2709 Region::Width wi;
2710 switch (x) {
2711 case 1:
2712 wi = Region::Width::WI_1;
2713 break;
2714 case 2:
2715 wi = Region::Width::WI_2;
2716 break;
2717 case 4:
2718 wi = Region::Width::WI_4;
2719 break;
2720 case 8:
2721 wi = Region::Width::WI_8;
2722 break;
2723 case 16:
2724 wi = Region::Width::WI_16;
2725 break;
2726 default:
2727 FailAtT(loc, "invalid region width");
2728 wi = Region::Width::WI_INVALID;
2729 }
2730 return wi;
2731 }
2732
2733 Region::Horz ParseRegionHorz() {
2734 Loc loc = NextLoc();
2735 int x;
2736 ConsumeIntLitOrFail(x, "syntax error in region (horizontal stride)");
2737 Region::Horz hz;
2738 switch (x) {
2739 case 0:
2740 hz = Region::Horz::HZ_0;
2741 break;
2742 case 1:
2743 hz = Region::Horz::HZ_1;
2744 break;
2745 case 2:
2746 hz = Region::Horz::HZ_2;
2747 break;
2748 case 4:
2749 hz = Region::Horz::HZ_4;
2750 break;
2751 default:
2752 FailAtT(loc, "invalid region horizontal stride");
2753 hz = Region::Horz::HZ_INVALID;
2754 }
2755 return hz;
2756 }
2757
2758 void CheckLiteralBounds(const Loc &opStart, Type type, const ImmVal &val,
2759 int64_t mn, int64_t mx) {
2760 if (val.s64 < mn || val.s64 > mx) {
2761 WarningAtT(opStart, "literal is out of bounds for type ", ToSyntax(type));
2762 }
2763 }
2764
2765 void FinishSrcOpImmValue(int srcOpIx, const Loc &opStart,
2766 const Token &valToken, ImmVal &val) {
2767 // convert to the underlying data type
2768 Type sty = ParseSrcOpTypeWithoutDefault(srcOpIx, true);
2769
2770 if (sty == Type::BF) {
2771 // ensure there's no BF type imm value
2772 ErrorAtT(opStart, "Imm operand with BF type is not allowed");
2773 }
2774
2775 switch (sty) {
2776 case Type::B:
2777 case Type::UB:
2778 case Type::W:
2779 case Type::UW:
2780 case Type::D:
2781 case Type::UD:
2782 case Type::Q:
2783 case Type::UQ:
2784 case Type::V:
2785 case Type::UV:
2786 case Type::VF:
2787 if (!val.isS64()) {
2788 ErrorAtT(opStart, "literal must be integral for type ", ToSyntax(sty));
2789 }
2790 break;
2791 case Type::HF:
2792 if (val.isS64()) { // The value was parsed as integral
2793 if (valToken.lexeme == INTLIT10 && val.u64 != 0) {
2794 // base10
2795 // examples: 2:f or (1+2):f
2796 FailAtT(opStart, "immediate integer floating point literals must be "
2797 "in hex or binary (e.g. 0x7F800000:f)");
2798 }
2799 // base2 or base16
2800 // e.g. 0x1234:hf (preserve it)
2801 if (~0xFFFFull & val.u64) {
2802 ErrorAtT(opStart, "hex literal too big for type");
2803 }
2804 // no copy needed, the half float is already encoded as such
2805 } else { // ==ImmVal::F64
2806 // it's an fp64 literal, we need to narrow to fp16
2807 // e.g. "(1.0/10.0):hf"
2808 uint64_t DROPPED_PAYLOAD =
2809 ~((uint64_t)F16_MANT_MASK) & (F64_MANT_MASK >> 1);
2810 if (IS_NAN(val.f64) && (val.u64 & DROPPED_PAYLOAD)) {
2811 FailAtT(opStart, "NaN payload value overflows");
2812 }
2813 // uint64_t orginalValue = val.u64;
2814 val.u64 = ConvertDoubleToHalf(val.f64); // copy over u64 to clobber all
2815 val.kind = ImmVal::Kind::F16;
2816 // uint64_t newValue = FloatToBits(
2817 // ConvertFloatToDouble(ConvertHalfToFloat(val.u16)));
2818 // if (orginalValue != newValue) {
2819 // Warning(opStart,
2820 // "precision lost in literal conversion to fp16");
2821 // }
2822 }
2823 break;
2824 case Type::F:
2825 case Type::DF:
2826 if (val.isS64()) {
2827 if (valToken.lexeme == INTLIT10 && val.u64 != 0) {
2828 // base10
2829 // examples: 2:f or (1+2):f
2830 FailAtT(opStart, "immediate integer floating point literals must be "
2831 "in hex or binary (e.g. 0x7F800000:f)");
2832 }
2833 // base2 or base16 float bits
2834 // examples:
2835 // 0x3F000000:f (0.5)
2836 // 0x7F801010:f (qnan(0x01010))
2837 // 0xFFC00000:f (-qnan(...))
2838 // 0x7FC00000:f (qnan())
2839 // 0x7F800000:f (infinity)
2840 if (sty == Type::F) {
2841 // preserve the bits
2842 val.u32 = (uint32_t)val.s64;
2843 val.kind = ImmVal::Kind::F32;
2844 } else {
2845 // leave :df alone, bits are already set
2846 val.kind = ImmVal::Kind::F64;
2847 }
2848 } else { // ==ImmVal::F64
2849 if (sty == Type::F) {
2850 // parsed as double e.g. "3.1414:f"
2851 // need to narrow to fp32
2852 // any NaN payload must fit in the smaller mantissa
2853 // The bits we will remove
2854 // ~((uint64_t)F32_MANT_MASK) &
2855 // (F64_MANT_MASK >> 1)
2856 // the mantissa bits that will get truncated
2857 uint64_t DROPPED_PAYLOAD =
2858 ~((uint64_t)F32_MANT_MASK) & (F64_MANT_MASK >> 1);
2859 if (IS_NAN(val.f64) && (val.u64 & DROPPED_PAYLOAD)) {
2860 FailS(opStart, "NaN payload value overflows");
2861 }
2862 //
2863 // Use a raw bitwise assignment; some compilers will clear
2864 // the NaN bit by making an assignment
2865 val.u64 = ConvertDoubleToFloatBits(val.f64);
2866 val.kind = ImmVal::Kind::F32;
2867 // the below would be wrong
2868 // val = ConvertDoubleToFloat(val.f64);
2869 } // else: sty == Type::DF (nothing needed)
2870 }
2871 break;
2872 default:
2873 break;
2874 }
2875
2876 // check literal bounds of integral values
2877 // literals may be signed floating in general
2878 switch (sty) {
2879 case Type::B:
2880 // if (val.kind == ImmVal::S64 &&
2881 // ((val.u64 & 0xFFFFFFFFFFFFFF00ull) == 0xFFFFFFFFFFFFFF00ull))
2882 // {
2883 // // negative value, but not too negative for 16 bits
2884 // val.u64 &= 0xFFull;
2885 // }
2886 CheckLiteralBounds(opStart, sty, val, -128, 127);
2887 val.kind = ImmVal::Kind::S8;
2888 val.s64 = val.s8; // sign extend to a 64-bit value
2889 break;
2890 case Type::UB:
2891 // CheckLiteralBounds(opStart, sty, val, 0, 0xFF);
2892 val.kind = ImmVal::Kind::U8;
2893 val.u64 = val.u8; // could &= by 0xFF
2894 break;
2895 case Type::W:
2896 // if (val.kind == ImmVal::S64 &&
2897 // ((val.u64 & 0xFFFFFFFFFFFF0000ull) == 0xFFFFFFFFFFFF0000ull))
2898 // {
2899 // // negative value, but not too negative for 16 bits
2900 // val.u64 &= 0xFFFFull;
2901 // }
2902 val.s64 = val.s16; // sign extend to a 64-bit value
2903 CheckLiteralBounds(opStart, sty, val, -32768, 32767);
2904 val.kind = ImmVal::Kind::S16;
2905 break;
2906 case Type::UW:
2907 // fails ~1:ub
2908 // CheckLiteralBounds(opStart, sty, val, 0, 0xFFFF);
2909 val.kind = ImmVal::Kind::U16;
2910 val.u64 = val.u16; // truncate to 16-bit: // could &= by 0xFFFF
2911 break;
2912 case Type::D:
2913 val.s64 = val.s32; // sign extend to a 64-bit value
2914 CheckLiteralBounds(opStart, sty, val, -2147483648ll, 2147483647ll);
2915 val.kind = ImmVal::Kind::S32;
2916 break;
2917 case Type::UD:
2918 // CheckLiteralBounds(opStart, sty, val, 0, 0xFFFFFFFF);
2919 // only check if input is signed??? Or reject signed input
2920 // if (val.kind == ImmVal::S64 &&
2921 // ((val.u64 & 0xFFFFFFFF00000000ull) == 0xFFFFFFFF00000000ull))
2922 // {
2923 // // negative value, but we can reprsent it as u32
2924 // }
2925 // val.u64 &= 0xFFFFFFFF;
2926 val.u64 = val.u32; // truncate top bits
2927 val.kind = ImmVal::Kind::U32;
2928 break;
2929 case Type::Q:
2930 // no conversion needed
2931 val.kind = ImmVal::Kind::S64;
2932 CheckLiteralBounds(opStart, sty, val, 0x8000000000000000ll,
2933 0x7FFFFFFFFFFFFFFFll);
2934 break;
2935 case Type::UQ:
2936 // no conversion needed
2937 val.kind = ImmVal::Kind::U64;
2938 break;
2939 case Type::HF:
2940 val.kind = ImmVal::Kind::F16;
2941 break;
2942 case Type::F:
2943 val.kind = ImmVal::Kind::F32;
2944 break;
2945 case Type::DF:
2946 val.kind = ImmVal::Kind::F64;
2947 break;
2948 case Type::UV:
2949 case Type::V:
2950 case Type::VF:
2951 val.kind = ImmVal::Kind::U32;
2952 break;
2953 default:
2954 break;
2955 }
2956
2957 if (m_opSpec->isBranching()) {
2958 if (m_opSpec->isJipAbsolute()) {
2959 m_builder.InstSrcOpImmLabelAbsolute(srcOpIx, opStart, val.s64, sty);
2960 } else {
2961 m_builder.InstSrcOpImmLabelRelative(srcOpIx, opStart, val.s64, sty);
2962 }
2963 } else {
2964 m_builder.InstSrcOpImmValue(srcOpIx, opStart, val, sty);
2965 }
2966 }
2967
2968 void FinishSrcOpImmLabel(int srcOpIx, const Loc &opStart,
2969 const Loc /* lblLoc */, const std::string &lbl) {
2970 Type type = ParseSrcOpTypeWithDefault(srcOpIx, true, true);
2971 m_builder.InstSrcOpImmLabel(srcOpIx, opStart, lbl, type);
2972 }
2973
2974 // = '-' | '~' | '(abs)' | '-(abs)' | '~(abs)'
2975 // = or start of "|r3|" like
2976 SrcModifier ParseSrcModifierOpt(bool &pipeAbs) {
2977 SrcModifier srcMod = SrcModifier::NONE;
2978 if (Consume(SUB) || Consume(TILDE)) { // same lexeme as -
2979 srcMod = SrcModifier::NEG;
2980 }
2981 pipeAbs = LookingAt(PIPE);
2982 if (Consume(ABS) || Consume(PIPE)) {
2983 srcMod =
2984 srcMod == SrcModifier::NEG ? SrcModifier::NEG_ABS : SrcModifier::ABS;
2985 }
2986 return srcMod;
2987 }
2988
2989 // Special handling for >=XE_HP src1 op, we rely on the src1 op format
2990 // to determine if this is the new XE_HP encoding. If the format is
2991 // r2:4 then should set ExBSO to true later on.
2992 //
2993 // In XeHP: only ExBSO descriptors take Src1.Length out of the descriptor
2994 // and only if the descriptor is a register (e.g. a0.2)
2995 //
2996 // The syntax:
2997 // REG
2998 // bare register: "r13" or "null"
2999 // given a reg ExDesc means to not use ExBSO
3000 // NOTE: users are encouraged to use the explicit InstOpt {ExBSO}
3001 //
3002 // REG (COLON|HASH) INT
3003 // src one length
3004 // given a reg ExDesc, this means to use ExBSO
3005 // NOTE: users are encouraged to use the explicit InstOpt {ExBSO}
3006 //
3007 // REG COLON TYPE
3008 // r13:ud or null:ud
3009 //
3010 // REG COLON TYPE (COLON|HASH) INT
3011 // REG (COLON|HASH) INT COLON TYPE
3012 // compatibility with old asm
3013 // r13:ud:1 or null:ud:0
3014 void ParseSendSrc1OpWithOptLen(int &src1Len) {
3015 m_srcLocs[1] = NextLoc();
3016 src1Len = -1;
3017 const RegInfo *regInfo;
3018 int regNum;
3019 bool parsedType = false;
3020
3021 if (ConsumeReg(regInfo, regNum)) {
3022 // send src1 must not have region and subreg
3023 m_srcKinds[1] = Operand::Kind::DIRECT;
3024 if (regInfo->regName != RegName::ARF_NULL &&
3025 regInfo->regName != RegName::GRF_R) {
3026 FailAtT(m_srcLocs[1], "invalid src1 register");
3027 }
3028 //
3029 // parse the type
3030 Type sty = Type::INVALID;
3031 if (LookingAtSeq(COLON, IDENT)) {
3032 if (platform() >= Platform::XE3) {
3033 FailS(NextLoc(),
3034 "types are forbidden on send operands on this platform");
3035 }
3036 // this disables expressions for the length sizes
3037 // send ... null:(2*0)
3038 sty = ParseSendOperandTypeWithDefault(1);
3039 parsedType = true;
3040 } else {
3041 sty = SendOperandDefaultType(1);
3042 }
3043
3044 const Token &tk = Next();
3045 if (Consume(COLON) || Consume(HASH)) {
3046 // e.g. r12:4 or r12#4 (the latter is legacy only)
3047 // (we added COLON so we could use this with a preproecssor)
3048 if (platform() < Platform::XE_HP) {
3049 FailAtT(tk.loc, "Send with Imm ExDesc should not have Src1.Length "
3050 "given on src1 (e.g. r0#4); Src1.Length should be "
3051 "in ExDesc[10:6]");
3052 }
3053 m_sendSrcLenLocs[1] = NextLoc();
3054 //
3055 ImmVal v;
3056 if (!TryParseIntConstExpr(v, "extended descriptor")) {
3057 FailT("expected extended send descriptor");
3058 }
3059 src1Len = (int)v.s64;
3060 m_sendSrcLens[1] = src1Len;
3061 if (src1Len < 0 || src1Len > 0x1F) {
3062 FailAtT(tk.loc, "Src1.Length is out of range");
3063 }
3064 if (regNum + src1Len - 1 > 255) {
3065 FailAtT(tk.loc, "Src1.Length extends past GRF end");
3066 }
3067 } else if (regInfo->regName == RegName::GRF_R &&
3068 LookingAtSeq(SUB, INTLIT10)) {
3069 // enabling a range based syntax as well for src1Len
3070 // r12-14 => r12#3
3071 // users that require the C-preprocessor get tripped up using
3072 // # as a decorator (e.g. since it's a cpp operator)
3073 Skip();
3074 int lastReg = 0;
3075 ConsumeIntLit(lastReg);
3076 if (lastReg < regNum)
3077 FailAtT(tk.loc, "Src1Len: ending register must be >= start reg");
3078 else if (lastReg > 255)
3079 FailAtT(tk.loc, "Src1Len: ending register must be <= 255");
3080 src1Len = lastReg - regNum + 1;
3081 }
3082
3083 // null#0:ud
3084 if (!parsedType)
3085 sty = ParseSendOperandTypeWithDefault(1);
3086
3087 Region dfltRgn = Region::SRC010;
3088 if (m_opSpec->hasImplicitSrcRegion(1, m_execSize, false)) {
3089 dfltRgn = m_opSpec->implicitSrcRegion(1, m_execSize, false);
3090 }
3091
3092 // construct the op directly
3093 m_builder.InstSrcOpRegDirect(1, m_srcLocs[1], SrcModifier::NONE,
3094 regInfo->regName, RegRef(regNum, 0), dfltRgn,
3095 sty);
3096 } else {
3097 FailT("syntax error in send src1");
3098 }
3099 }
3100
3101 // e.g. "r13" or "r13:f"
3102 void ParseSendSrcOp(int srcOpIx, bool enableImplicitOperand) {
3103 m_srcLocs[srcOpIx] = NextLoc();
3104
3105 const RegInfo *regInfo;
3106 int regNum;
3107
3108 // For XE send now supports src1; sends goes away
3109 // To support some older syntax floating around
3110 // underneath IGA will insert a null src1 if needed
3111 if (enableImplicitOperand) {
3112 bool isSuccess = PeekReg(regInfo, regNum);
3113 if (!isSuccess || regInfo->regName == RegName::ARF_A) {
3114 m_builder.InstSrcOpRegDirect(
3115 srcOpIx, m_srcLocs[srcOpIx], SrcModifier::NONE, RegName::ARF_NULL,
3116 REGREF_ZERO_ZERO, Region::SRC010, Type::INVALID);
3117 return;
3118 }
3119 }
3120
3121 ParseSrcOp(srcOpIx);
3122 }
3123 // e.g. "r13" or "r[s0.2]" (no suffix Src0Len)
3124 // returns if the operand is indirect (gathering)
3125 bool ParseSendSrc0OpXe3() {
3126 m_srcLocs[0] = NextLoc();
3127
3128 OperandInfo opInfo;
3129 if (ConsumeIdentEq("r")) {
3130 m_srcKinds[0] = opInfo.kind = Operand::Kind::INDIRECT;
3131 ConsumeOrFail(LBRACK, "expected [");
3132 auto sreg = ParseReg();
3133 if (sreg.first->regName != RegName::ARF_S) {
3134 FailAtT(m_srcLocs[0], "gathering send expects s0.# index");
3135 }
3136 ConsumeOrFail(DOT, "expected subregister");
3137 int subregNum = 0;
3138 ConsumeIntLitOrFail(subregNum, "expected subregister");
3139 opInfo.regOpIndOff = 0;
3140 opInfo.regOpReg = RegRef(sreg.second, (uint16_t)subregNum);
3141 opInfo.regOpName = RegName::ARF_S;
3142 ConsumeOrFail(RBRACK, "expected ]");
3143 } else {
3144 m_srcKinds[0] = opInfo.kind = Operand::Kind::DIRECT;
3145 auto reg = ParseReg();
3146 opInfo.regOpName = reg.first->regName;
3147 opInfo.regOpReg.regNum = reg.second;
3148 opInfo.regOpReg.subRegNum = 0;
3149 }
3150
3151 m_builder.InstSrcOp(0, opInfo);
3152 return opInfo.kind == Operand::Kind::INDIRECT;
3153 }
3154
3155 // e.g. "r13:2" or "r[s0.2]:8" (suffix Src0Len)
3156 // returns if the operand is indirect (gathering)
3157 bool ParseSendgSrc0OpXe3p() {
3158 m_srcLocs[0] = NextLoc();
3159
3160 OperandInfo opInfo;
3161 if (ConsumeIdentEq("r")) {
3162 m_srcKinds[0] = opInfo.kind = Operand::Kind::INDIRECT;
3163 ConsumeOrFail(LBRACK, "expected [");
3164 auto sreg = ParseReg();
3165 if (sreg.first->regName != RegName::ARF_S) {
3166 FailAtT(m_srcLocs[0], "gathering send expects s0.# index");
3167 }
3168 ConsumeOrFail(DOT, "expected subregister");
3169 int subregNum = 0;
3170 ConsumeIntLitOrFail(subregNum, "expected subregister");
3171 opInfo.regOpIndOff = 0;
3172 opInfo.regOpReg = RegRef(sreg.second, (uint16_t)subregNum);
3173 opInfo.regOpName = RegName::ARF_S;
3174 ConsumeOrFail(RBRACK, "expected ]");
3175 if (m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) {
3176 FailAtT(m_srcLocs[0], "Sendgx/sendgxc forbids gather src0 operand");
3177 }
3178 } else {
3179 m_srcKinds[0] = opInfo.kind = Operand::Kind::DIRECT;
3180 auto reg = ParseReg();
3181 // for sendgx, check is src0 is ARF_NULL
3182 // if ARF_NULL, change to r511
3183 if ((m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) &&
3184 reg.first->regName == RegName::ARF_NULL) {
3185 opInfo.regOpName = RegName::ARF_NULL;
3186 opInfo.regOpReg.regNum = (uint16_t)0;
3187 opInfo.regOpReg.subRegNum = 0;
3188 } else {
3189 opInfo.regOpName = reg.first->regName;
3190 opInfo.regOpReg.regNum = reg.second;
3191 opInfo.regOpReg.subRegNum = 0;
3192 }
3193 }
3194 int src0Len = 0;
3195 if (LookingAt(COLON) || opInfo.regOpName == RegName::GRF_R)
3196 src0Len = ParseSrcOpLenSuffix((int)opInfo.regOpReg.regNum);
3197 m_builder.InstSrcOp(0, opInfo);
3198 m_builder.InstSendSrc0Length(src0Len);
3199
3200 return opInfo.kind == Operand::Kind::INDIRECT;
3201 }
3202 void ParseSendgSrc1OpXe3() {
3203 OperandInfo opInfo;
3204 m_srcLocs[1] = NextLoc();
3205 m_srcKinds[1] = opInfo.kind = Operand::Kind::DIRECT;
3206 auto reg = ParseReg();
3207 // for sendgx, check if src1 is ARF_NULL
3208 // if ARF_NULL, then change to r511
3209 if ((m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) &&
3210 reg.first->regName == RegName::ARF_NULL) {
3211 opInfo.regOpName = RegName::ARF_NULL;
3212 opInfo.regOpReg.regNum = (uint16_t)0;
3213 opInfo.regOpReg.subRegNum = 0;
3214 }
3215 else
3216 {
3217 opInfo.regOpName = reg.first->regName;
3218 opInfo.regOpReg.regNum = reg.second;
3219 opInfo.regOpReg.subRegNum = 0;
3220 }
3221 int src1Len = 0;
3222 if (LookingAt(COLON) || opInfo.regOpName != RegName::ARF_NULL)
3223 src1Len = ParseSrcOpLenSuffix((int)opInfo.regOpReg.regNum);
3224 m_builder.InstSrcOp(1, opInfo);
3225 m_builder.InstSendSrc1Length(src1Len);
3226 }
3227
3228
3229 std::pair<const RegInfo *, uint16_t> ParseReg() {
3230 const RegInfo *regInfo = nullptr;
3231 int regNum = -1;
3232 if (!ConsumeReg(regInfo, regNum)) {
3233 FailT("expected register");
3234 } else if (!regInfo->isRegNumberValid(regNum)) {
3235 FailT("invalid register number");
3236 }
3237 if (m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) {
3238 if (regNum == 511) {
3239 IGA_ASSERT(false, "r511 is null and should not be referenced directly");
3240 }
3241 }
3242 return std::pair<const RegInfo *, uint16_t>(regInfo, (uint16_t)regNum);
3243 }
3244
3245 int ParseSrcOpLenSuffix(int regNum) {
3246 int srcLen = -1;
3247 ConsumeOrFail(COLON, "expected payload length suffix (e.g. :2)");
3248
3249 int regLimit = 255;
3250 if (m_opSpec->is(Op::SENDGX) || m_opSpec->is(Op::SENDGXC)) {
3251 regLimit = 511;
3252 }
3253
3254 // explicit length syntax
3255 // e.g. "r13:4"
3256 // e.g. "r13:(2*2)"
3257 // NOTE: permit : so we can use this in the preprocessor
3258 // e.g. r13:4
3259 auto at = NextLoc();
3260 ImmVal v;
3261 if (!TryParseIntConstExpr(v, "extended descriptor")) {
3262 FailT("expected extended send descriptor");
3263 }
3264 if (v.s64 < 0 || v.s64 > 0x1F) {
3265 FailAtT(at, "SrcLen out of range");
3266 } else if (regNum + (int)v.s64 - 1 > regLimit) {
3267 WarningAtT(at, "SrcLen register range extends past GRF end");
3268 }
3269 if (v.s64 > 32) { // could constrain this more
3270 FailAtT(at, "invalid payload length");
3271 }
3272 srcLen = (int)v.s64;
3273
3274 return srcLen;
3275 }
3276
3277 Type ParseDstOpTypeWithDefault() {
3278 if (m_opSpec->hasImplicitDstType()) {
3279 if (LookingAt(COLON)) {
3280 if (m_opts.deprecatedSyntaxWarnings)
3281 WarningT("implicit type on dst should be omitted");
3282 // parse the type but ignore it
3283 ParseOpTypeWithDefault(DST_TYPES, "expected destination type");
3284 }
3285 // use the implicit type anyway
3286 return m_opSpec->implicitDstType();
3287 }
3288 return ParseOpTypeWithDefault(DST_TYPES, "expected destination type");
3289 }
3290
3291 Type ParseSrcOpTypeWithDefault(int srcOpIx, bool immOrLbl,
3292 bool isLabel = false) {
3293 if (m_opSpec->hasImplicitSrcType(srcOpIx, immOrLbl)) {
3294 if (LookingAt(COLON)) {
3295 if (m_opts.deprecatedSyntaxWarnings)
3296 WarningT("implicit type on src", srcOpIx, " should be omitted");
3297 // parse the type but ignore it
3298 ParseOpTypeWithDefault(SRC_TYPES, "expected source type");
3299 }
3300 // use the implicit type anyway
3301 return m_opSpec->implicitSrcType(srcOpIx, immOrLbl);
3302 } else if (m_opSpec->op == Op::MOV && immOrLbl && isLabel) {
3303 // support mov label without giving label's type
3304 return Type::UD;
3305 }
3306
3307 return ParseOpTypeWithDefault(SRC_TYPES, "expected source type");
3308 }
3309 Type ParseSrcOpTypeWithoutDefault(int srcOpIx, bool immOrLbl) {
3310 if (m_opSpec->hasImplicitSrcType(srcOpIx, immOrLbl)) {
3311 if (LookingAt(COLON)) {
3312 if (m_opts.deprecatedSyntaxWarnings)
3313 WarningT("implicit type on src", srcOpIx, " should be omitted");
3314 // parse the type but ignore it
3315 TryParseOpType(SRC_TYPES);
3316 }
3317 // use the implicit type anyway
3318 return m_opSpec->implicitSrcType(srcOpIx, immOrLbl);
3319 }
3320 Type t = TryParseOpType(SRC_TYPES);
3321 if (t == Type::INVALID &&
3322 !(m_opSpec->isBranching() && !m_model.supportsSimplifiedBranches())) {
3323 FailT("expected source type");
3324 }
3325 return t;
3326 }
3327 Type ParseOpTypeWithDefault(const IdentMap<Type> types,
3328 const char *expectedErr) {
3329 auto t = TryParseOpType(types);
3330 if (t == Type::INVALID) {
3331 if (m_defaultRegisterType != Type::INVALID) {
3332 t = m_defaultRegisterType;
3333 } else if (m_opSpec->isAnySendFormat()) {
3334 t = Type::UD;
3335 } else if (m_opSpec->isBranching() &&
3336 m_model.supportsSimplifiedBranches()) {
3337 // no more types for branching
3338 t = Type::UD;
3339 } else if (m_opSpec->is(Op::SYNC)) {
3340 // we allow implicit type for sync reg32 (grf or null)
3341 t = Type::UB;
3342 } else {
3343 FailT(expectedErr);
3344 }
3345 }
3346 return t;
3347 }
3348 Type TryParseOpType(const IdentMap<Type> types) {
3349 if (!LookingAt(COLON)) {
3350 return Type::INVALID;
3351 }
3352 Type type = Type::INVALID;
3353 if (IdentLookupFrom(1, types, type)) {
3354 Skip(2);
3355 }
3356 return type;
3357 }
3358
3359 // (INTEXPR|AddrRegRef) (INTEXPR|AddrRegRef)
3360 //
3361 // This function is the same as ParseSendDescs, except for it's handling
3362 // of ExBSO and Src1Len.
3363 void ParseSendDescsWithOptSrc1Len(int src1Length) {
3364 const Loc exDescLoc = NextLoc();
3365 SendDesc exDesc;
3366 RegName regName = RegName::INVALID;
3367 if (ParseAddrRegRefOpt(exDesc.reg, regName)) { // ExDesc is register
3368 IGA_ASSERT(regName == RegName::ARF_A,
3369 "Indirect send exDesc must be ARF_A");
3370 exDesc.type = SendDesc::Kind::REG32A;
3371 if (src1Length >= 0) {
3372 m_implicitExBSO = true;
3373 m_builder.InstOptAdd(InstOpt::EXBSO);
3374 }
3375 } else { // ExDesc is imm
3376 exDesc.type = SendDesc::Kind::IMM;
3377
3378 // constant integral expression
3379 ImmVal v;
3380 if (!TryParseIntConstExpr(v, "extended descriptor")) {
3381 FailT("expected extended send descriptor");
3382 }
3383 exDesc.imm = (uint32_t)v.s64;
3384
3385 if (src1Length >= 0) {
3386 // In XeHP, immediate descriptors still treat Src1.Length as
3387 // ExDesc[10:6]
3388 if (platform() == Platform::XE_HP) {
3389 WarningT("Send with immediate ExDesc should not have "
3390 "Src1.Length given on src1 (e.g. r10:4) "
3391 "on this platform");
3392 exDesc.imm |= (((uint32_t)src1Length & 0x1F) << 6);
3393 }
3394 }
3395 if (platform() >= Platform::XE_HPG) {
3396 // XeHPG+: Src1.Length is not part of ExDesc for ExDesc.IsImm,
3397 // but are explicit bits in the EU ISA.
3398 int exDescSrc1Len = (int)(exDesc.imm >> 6) & 0x1F;
3399 if (src1Length < 0) {
3400 WarningAtT(exDescLoc,
3401 "Src1.Length should suffix src1 register (e.g. r10:4)");
3402 // take it from the descriptor
3403 src1Length = exDescSrc1Len;
3404 } else if (exDescSrc1Len != 0) { // src1Length >= 0
3405 // they set Src1.Length as part of the descriptor
3406 WarningAtT(exDescLoc,
3407 "Send ExDesc[10:6] is not Src1.Length; suffix length "
3408 "on src1 reg (e.g. r10:4)");
3409 // if Src1.Length was also set on the register, then ensure
3410 // that it at least matches the descriptor value
3411 IGA_ASSERT(src1Length >= 0, "Unexpected value");
3412 // throw a fit if they mismatch
3413 if (exDescSrc1Len != src1Length)
3414 FailAtT(exDescLoc, "mismatch of Src1.Length suffix and "
3415 "ExDesc[10:6]");
3416 } // else: Src1Length >= 0 && exDescSrc1Len == 0 (prefer reg op)
3417
3418 if (exDesc.imm & 0x7FF) {
3419 WarningAtT(exDescLoc, "ExDesc[10:0] must be zero");
3420 exDesc.imm &= ~0x7FF; // [10:0] MBZ
3421 }
3422 } // else <=XeHPG (everything is in the imm descriptor)
3423
3424 if (platform() >= Platform::XE && (exDesc.imm & 0xF)) {
3425 FailAtT(exDescLoc, "ExDesc[3:0] must be 0's; SFID is expressed "
3426 "as a function control value (e.g. send.dc0 ...)");
3427 }
3428 }
3429 if (LookingAt(COLON)) {
3430 FailAtT(NextLoc(), "extended message descriptor is typeless");
3431 }
3432 //
3433 const Loc descLoc = NextLoc();
3434 SendDesc desc;
3435 if (ParseAddrRegRefOpt(desc.reg, regName)) {
3436 IGA_ASSERT(regName == RegName::ARF_A, "Indirect send desc must be ARF_A");
3437 desc.type = SendDesc::Kind::REG32A;
3438 } else {
3439 // constant integral expression
3440 ImmVal v;
3441 if (!TryParseConstExpr(v)) {
3442 FailT("expected extended send descriptor");
3443 }
3444 if (!v.isI64()) {
3445 FailAtT(descLoc, "immediate descriptor expression must be integral");
3446 }
3447 desc.imm = (uint32_t)v.s64;
3448 desc.type = SendDesc::Kind::IMM;
3449 }
3450 if (LookingAt(COLON)) {
3451 FailT("Message Descriptor should not have a type");
3452 }
3453 //
3454 m_builder.InstSendDescs(exDescLoc, exDesc, descLoc, desc);
3455 //
3456 m_builder.InstSendSrc1Length(src1Length);
3457 }
3458
3459 SendDesc ParseDesc(const char *which) {
3460 SendDesc sd;
3461 RegName regName = RegName::INVALID;
3462 if (ParseAddrRegRefOpt(sd.reg, regName)) { // ExDesc is register
3463 IGA_ASSERT(regName == RegName::ARF_A, "Indirect send Desc must be ARF_A");
3464 sd.type = SendDesc::Kind::REG32A;
3465
3466 } else { // ExDesc is imm
3467 sd.type = SendDesc::Kind::IMM;
3468
3469 // constant integral expression
3470 ImmVal v;
3471 if (!TryParseIntConstExprPrimary(ExprParseOpts(), v, which)) {
3472 FailT("expected ", which);
3473 }
3474 sd.imm = (uint32_t)v.s64;
3475 }
3476 return sd;
3477 }
3478
3479 // Xe2 version of descriptors:
3480 // (IntExpr COLON)?(INTEXPR|AddrRegRef) (INTEXPR|AddrRegRef)
3481 // Examples:
3482 // 0x1234 0x56780002
3483 // a0.2 a0.0
3484 // 0x100:a0.2 0x56780001
3485 void ParseSendDescsXe2(int src1Len) {
3486 const Loc exDescLoc = NextLoc();
3487 ImmVal v;
3488 uint32_t exImmOffDesc = 0;
3489 SendDesc exDesc, desc;
3490
3491 bool z = TryParseIntConstExprPrimary(
3492 v, "extended descriptor or immediate offset");
3493 if (z) {
3494 if (Consume(COLON)) {
3495 exDesc = ParseDesc("extended descriptor");
3496 exImmOffDesc = (uint32_t)v.s64;
3497 } else {
3498 exDesc.type = SendDesc::Kind::IMM;
3499 exDesc.imm = (uint32_t)v.s64;
3500 }
3501 } else {
3502 exDesc = ParseDesc("extended descriptor");
3503 }
3504
3505 // src1Length set implies ExBSO is set when ExBSO is applicable
3506 // ExBSO is applicable when exDesc is reg and non-UGM
3507 if (src1Len >= 0 && exDesc.type != SendDesc::Kind::IMM &&
3508 m_builder.getSubfunction().send != SFID::UGM) {
3509 m_implicitExBSO = true;
3510 m_builder.InstOptAdd(InstOpt::EXBSO);
3511 }
3512
3513 const Loc descLoc = NextLoc();
3514 desc = ParseDesc("Descriptor");
3515
3516 m_builder.InstSendDescs(exDescLoc, exImmOffDesc, exDesc, descLoc, desc);
3517 m_builder.InstSendSrc1Length(src1Len);
3518 }
3519
3520 void ParseSendgDescsXe3() {
3521 RegRef id0{REGREF_INVALID};
3522 RegRef id1{REGREF_INVALID};
3523 auto at = NextLoc();
3524 if (ParseScalarRegRefOpt(id0)) {
3525 ParseScalarRegRefOpt(id1);
3526 }
3527 const Loc exDescLoc = NextLoc();
3528 ImmVal v;
3529 bool z = TryParseIntConstExprPrimary(v, "descriptor");
3530 if (!z)
3531 FailAtT(exDescLoc, "expected send descriptor");
3532 m_builder.InstSendDescs(v.u64, id0, id1);
3533 }
3534
3535 //
3536 // (INTEXPR|AddrRegRef) (INTEXPR|AddrRegRef)
3537 void ParseSendDescsLegacy() {
3538 IGA_ASSERT(platform() <= Platform::XE, "wrong platform for function");
3539
3540 const Loc exDescLoc = NextLoc();
3541 SendDesc exDesc;
3542 RegName regName = RegName::INVALID;
3543 if (ParseAddrRegRefOpt(exDesc.reg, regName)) {
3544 IGA_ASSERT(regName == RegName::ARF_A,
3545 "Indirect send exDesc must be ARF_A");
3546 exDesc.type = SendDesc::Kind::REG32A;
3547 if (platform() < Platform::XE)
3548 m_builder.InstSubfunction(SFID::A0REG);
3549 // subfunc is already set as part of opcode (e.g. send.gtwy)
3550 } else {
3551 // constant integral expression
3552 ImmVal v;
3553 if (!TryParseConstExpr(v)) {
3554 FailT("expected extended send descriptor");
3555 }
3556 if (!v.isI64()) {
3557 FailAtT(exDescLoc, "immediate descriptor expression must be integral");
3558 }
3559 exDesc.imm = (uint32_t)v.s64;
3560 exDesc.type = SendDesc::Kind::IMM;
3561 if (platform() >= Platform::XE && (exDesc.imm & 0xF)) {
3562 FailAtT(exDescLoc, "ExDesc[3:0] must be 0's; SFID is expressed "
3563 "as a function control value (e.g. send.dc0 ...)");
3564 }
3565 if (platform() < Platform::XE) {
3566 SFID sfid = sfidFromEncoding(platform(), exDesc.imm & 0xF);
3567 m_builder.InstSubfunction(sfid);
3568 }
3569 }
3570
3571 if (LookingAt(COLON)) {
3572 FailT("extended message descriptor is typeless");
3573 }
3574
3575 const Loc descLoc = NextLoc();
3576 SendDesc desc;
3577 if (ParseAddrRegRefOpt(desc.reg, regName)) {
3578 IGA_ASSERT(regName == RegName::ARF_A, "Indirect send Desc must be ARF_A");
3579 desc.type = SendDesc::Kind::REG32A;
3580 } else {
3581 // constant integral expression
3582 ImmVal v;
3583 if (!TryParseConstExpr(v)) {
3584 FailT("expected extended send descriptor");
3585 }
3586 if (!v.isI64()) {
3587 FailAtT(descLoc, "immediate descriptor expression must be integral");
3588 }
3589 desc.imm = (uint32_t)v.s64;
3590 desc.type = SendDesc::Kind::IMM;
3591 }
3592
3593 m_builder.InstSendDescs(exDescLoc, exDesc, descLoc, desc);
3594
3595 if (LookingAt(COLON)) {
3596 FailT("Message Descriptor is typeless");
3597 }
3598 }
3599
3600 // ('{' InstOptOrDepInfo (',' IDENT)* '}')?
3601 // InstOptOrDepInfo = DepInfo | InstOpt
3602 // InstOpt = 'AccWrEn' | 'Atomic' | ...
3603 // DepInfo = 'NoDDChk' | 'NoDDClr' | ...
3604 void ParseInstOpts() {
3605 InstOptSet instOpts;
3606 instOpts.clear();
3607
3608 if (Consume(LBRACE)) {
3609 if (!LookingAt(RBRACE))
3610 ParseInstOptOrFail(instOpts); // else: could be empty list "{}"
3611 while (Consume(COMMA)) {
3612 ParseInstOptOrFail(instOpts);
3613 }
3614 ConsumeOrFail(RBRACE, "expected }");
3615 }
3616
3617 bool src1LengthSuffixSet = m_sendSrcLens[1] != -1;
3618 if (instOpts.contains(InstOpt::EXBSO) && !src1LengthSuffixSet
3619 && !isParsingGatherSend() // Gather Send must have ExBSO set but it has
3620 // no explicit src1 length
3621 ) {
3622 // GOOD: send ... r10:2 a0.# ... {ExBSO}
3623 // ERROR: send ... r10 a0.# ... {ExBSO}
3624 // Src1.Length comes from EU bits a0.# holds 26b offset
3625 // We throw a tantrum if length is absent
3626 auto loc = m_srcLocs[1].isValid() ? m_srcLocs[1] : m_mnemonicLoc;
3627 FailAtT(loc, "send with ExBSO option should have "
3628 "Src1.Length suffixing parameter (e.g. r10:4)");
3629 ////////////////////////////
3630 // else if:
3631 // platform() >= Platform::XE_HPG
3632 // m_builder.getExDesc().isImm() &&
3633 // src1LengthSuffixSet
3634 // GOOD: send ... r10:2 IMM ... {}
3635 // GOOD: send ... r10 a0.# ... {}
3636 // this is already checked in ExDesc parser
3637 //
3638 } else if (instOpts.contains(InstOpt::EXBSO) &&
3639 m_builder.getExDesc().isImm()) {
3640 // ERROR: send ... r10 IMM ... {ExBSO}
3641 // ExBSO only makes sense when using a0.#
3642 auto loc = m_srcLocs[1].isValid() ? m_srcLocs[1] : m_mnemonicLoc;
3643 FailAtT(loc, "send with immediate exdesc forbids ExBSO");
3644 }
3645
3646 m_builder.InstOptsAdd(instOpts);
3647
3648 if (m_implicitExBSO && !instOpts.contains(InstOpt::EXBSO)
3649 && !isParsingGatherSend() // gather-send must have ExBSO set but it has
3650 // no explicit src1 length
3651 ) {
3652 WarningAtT(m_mnemonicLoc, "send src1 length implicitly added "
3653 "(include {ExBSO})");
3654 }
3655 }
3656
3657 void ParseInstOptOrFail(InstOptSet &instOpts) {
3658 auto loc = NextLoc();
3659 if (!tryParseInstOptToken(instOpts) && !tryParseInstOptDepInfo(instOpts)) {
3660 FailAtT(loc, "invalid instruction option");
3661 }
3662 }
3663
3664 bool tryParseInstOptToken(InstOptSet &instOpts) {
3665 auto loc = NextLoc();
3666 InstOpt newOpt = InstOpt::ACCWREN;
3667 if (ConsumeIdentEq("AccWrEn")) {
3668 newOpt = InstOpt::ACCWREN;
3669 if (platform() >= Platform::XE_HPC) {
3670 FailAtT(loc, "AccWrEn not supported on this platform");
3671 }
3672 } else if (ConsumeIdentEq("Atomic")) {
3673 if (platform() < Platform::GEN7) {
3674 FailAtT(loc, "Atomic mot supported on given platform");
3675 }
3676 newOpt = InstOpt::ATOMIC;
3677 if (instOpts.contains(InstOpt::SWITCH)) {
3678 FailAtT(loc, "Atomic mutually exclusive with Switch");
3679 } else if (instOpts.contains(InstOpt::NOPREEMPT)) {
3680 FailAtT(loc, "Atomic mutually exclusive with NoPreempt");
3681 }
3682 } else if (ConsumeIdentEq("Breakpoint")) {
3683 newOpt = InstOpt::BREAKPOINT;
3684 } else if (ConsumeIdentEq("Compacted")) {
3685 newOpt = InstOpt::COMPACTED;
3686 if (instOpts.contains(InstOpt::NOCOMPACT)) {
3687 FailAtT(loc, "Compacted mutually exclusive with "
3688 "Uncompacted/NoCompact");
3689 }
3690 } else if (ConsumeIdentEq("EOT")) {
3691 newOpt = InstOpt::EOT;
3692 if (!m_opSpec->isAnySendFormat()) {
3693 FailAtT(loc, "EOT is only allowed on send instructions");
3694 }
3695 } else if (ConsumeIdentEq("NoCompact") || ConsumeIdentEq("Uncompacted")) {
3696 newOpt = InstOpt::NOCOMPACT;
3697 if (instOpts.contains(InstOpt::COMPACTED)) {
3698 FailAtT(loc, "Uncomapcted/NoCompact mutually exclusive "
3699 "with Compacted");
3700 }
3701 } else if (ConsumeIdentEq("Serialize")) {
3702 newOpt = InstOpt::SERIALIZE;
3703 } else if (ConsumeIdentEq("NoMask")) {
3704 FailAtT(loc, "NoMask goes precedes predication as (W) for WrEn: "
3705 "e.g. (W) op (..) ... or (W&f0.0) op (..) ..");
3706 } else if (ConsumeIdentEq("H1")) {
3707 FailAtT(loc, "H1 is obsolete; use M0 in execution offset: "
3708 "e.g. op (16|M0) ...");
3709 } else if (ConsumeIdentEq("H2")) {
3710 FailAtT(loc, "H2 is obsolete; use M16 in execution offset: "
3711 "e.g. op (16|M16) ...");
3712 } else if (ConsumeIdentEq("Q1")) {
3713 FailAtT(loc, "Q1 is obsolete; use M0 in execution offset: "
3714 "e.g. op (8|M0) ...");
3715 } else if (ConsumeIdentEq("Q2")) {
3716 FailAtT(loc, "Q2 is obsolete; use M8 in execution offset: "
3717 "e.g. op (8|M8) ...");
3718 } else if (ConsumeIdentEq("Q3")) {
3719 FailAtT(loc, "Q3 is obsolete; use M16 in execution offset: "
3720 "e.g. op (8|M16) ...");
3721 } else if (ConsumeIdentEq("Q4")) {
3722 FailAtT(loc, "Q4 is obsolete; use M24 in execution offset: "
3723 "e.g. op (8|M24) ...");
3724 } else if (ConsumeIdentEq("N1")) {
3725 FailAtT(loc, "N1 is obsolete; use M0 in execution offset: "
3726 "e.g. op (4|M0) ...");
3727 } else if (ConsumeIdentEq("N2")) {
3728 FailAtT(loc, "N2 is obsolete; use M4 in execution offset: "
3729 "e.g. op (4|M4) ...");
3730 } else if (ConsumeIdentEq("N3")) {
3731 FailAtT(loc, "N3 is obsolete; use M8 in execution offset: "
3732 "e.g. op (4|M8) ...");
3733 } else if (ConsumeIdentEq("N4")) {
3734 FailAtT(loc, "N4 is obsolete; use M12 in execution offset: "
3735 "e.g. op (4|M12) ...");
3736 } else if (ConsumeIdentEq("N5")) {
3737 FailAtT(loc, "N5 is obsolete; use M16 in execution offset: "
3738 "e.g. op (4|M16) ...");
3739 } else if (ConsumeIdentEq("N6")) {
3740 FailAtT(loc, "N6 is obsolete; use M20 in execution offset: "
3741 "e.g. op (4|M20) ...");
3742 } else if (ConsumeIdentEq("N7")) {
3743 FailAtT(loc, "N7 is obsolete; use M24 in execution offset: "
3744 "e.g. op (4|M24) ...");
3745 } else if (ConsumeIdentEq("N8")) {
3746 FailAtT(loc, "N8 is obsolete; use M28 in execution offset: "
3747 "e.g. op (4|M28) ...");
3748 } else {
3749 return false;
3750 }
3751
3752 if (!instOpts.add(newOpt)) {
3753 // adding the option doesn't change the set... (it's duplicate)
3754 FailAtT(loc, "duplicate instruction options");
3755 }
3756 return true;
3757 }
3758
3759 bool tryParseInstOptDepInfo(InstOptSet &instOpts) {
3760 auto loc = NextLoc();
3761 if (LookingAt(IDENT)) {
3762 InstOpt newOpt = InstOpt::ACCWREN;
3763 bool isSwsbOpt = false;
3764 // classic instruction option that affects instruction dependency
3765 // scheduling etc...
3766 if (ConsumeIdentEq("NoDDChk")) {
3767 newOpt = InstOpt::NODDCHK;
3768 if (!m_model.supportsHwDeps()) {
3769 FailAtT(loc, "NoDDChk not supported on given platform");
3770 }
3771 } else if (ConsumeIdentEq("NoDDClr")) {
3772 newOpt = InstOpt::NODDCLR;
3773 if (!m_model.supportsHwDeps()) {
3774 FailAtT(loc, "NoDDClr not supported on given platform");
3775 }
3776 } else if (ConsumeIdentEq("NoPreempt")) {
3777 if (m_model.supportsNoPreempt()) {
3778 newOpt = InstOpt::NOPREEMPT;
3779 } else {
3780 FailAtT(loc, "NoPreempt not supported on given platform");
3781 }
3782 } else if (ConsumeIdentEq("NoSrcDepSet")) {
3783 if (m_model.supportNoSrcDepSet()) {
3784 newOpt = InstOpt::NOSRCDEPSET;
3785 } else {
3786 FailAtT(loc, "NoSrcDep not supported on given platform");
3787 }
3788 } else if (ConsumeIdentEq("Switch")) {
3789 newOpt = InstOpt::SWITCH;
3790 if (!m_model.supportsHwDeps()) {
3791 WarningAtT(loc, "ignoring unsupported instruction option {Switch}");
3792 }
3793 } else if (ConsumeIdentEq("ExBSO")) {
3794 if (platform() < Platform::XE_HP) {
3795 FailAtT(loc, "ExBSO not supported on this platform");
3796 } else if (!m_opSpec->isAnySendFormat()) {
3797 FailAtT(loc, "ExBSO is not allowed for non-send instructions");
3798 }
3799 newOpt = InstOpt::EXBSO;
3800 } else if (ConsumeIdentEq("CPS")) {
3801 if (platform() < Platform::XE_HP || platform() >= Platform::XE2) {
3802 FailAtT(loc, "CPS not supported on this platform");
3803 } else if (!m_opSpec->isAnySendFormat()) {
3804 FailAtT(loc, "CPS is not allowed for non-send instructions");
3805 }
3806 newOpt = InstOpt::CPS;
3807 } else if (ConsumeIdentEq("Fwd")) {
3808 if (!m_opSpec->isDpasFormat())
3809 FailAtT(loc, "FWD is not allowed for non-dpas instructions");
3810 if (platform() <= Platform::XE3)
3811 FailAtT(loc, "FWD not supported on this platform");
3812 newOpt = InstOpt::FWD;
3813 } else if (ConsumeIdentEq("NoAccSBSet")) {
3814 // try swsb special token: NoAccSBSet
3815 m_builder.InstDepInfoSpecialToken(loc, SWSB::SpecialToken::NOACCSBSET);
3816 isSwsbOpt = true;
3817 } else if (m_opts.swsbEncodeMode >= SWSB_ENCODE_MODE::ThreeDistPipe) {
3818 // swsb XE encoding: the distance could be A@, F@, L@, I@, M@
3819 // same pipe dist (e.g. @4) will parse further down
3820 auto tryParsePipe = [&](const char *distPipeSymbol,
3821 SWSB::DistType distPipe) {
3822 if (LookingAtSeq(IDENT, AT) && ConsumeIdentEq(distPipeSymbol) &&
3823 Consume(AT)) {
3824 int32_t regDist = 0;
3825 Loc distLoc = NextLoc();
3826 ConsumeIntLitOrFail(regDist, "expected register distance value");
3827 m_builder.InstDepInfoDist(distLoc, distPipe, regDist);
3828 return true;
3829 } else {
3830 return false;
3831 }
3832 };
3833 //
3834 bool parsedNamedPipe =
3835 tryParsePipe("S", SWSB::DistType::REG_DIST_SCALAR) ||
3836 tryParsePipe("F", SWSB::DistType::REG_DIST_FLOAT) ||
3837 tryParsePipe("I", SWSB::DistType::REG_DIST_INT) ||
3838 tryParsePipe("L", SWSB::DistType::REG_DIST_LONG) ||
3839 tryParsePipe("A", SWSB::DistType::REG_DIST_ALL) ||
3840 tryParsePipe("M", SWSB::DistType::REG_DIST_MATH);
3841 if (parsedNamedPipe && m_model.supportsHwDeps()) {
3842 FailAtT(loc, "software dependencies not supported on this platform");
3843 }
3844
3845 isSwsbOpt = true;
3846 if (!parsedNamedPipe)
3847 return false; // unrecognized swsb setting
3848 } else {
3849 return false; // unrecognized option
3850 }
3851 if (!isSwsbOpt && !instOpts.add(newOpt)) {
3852 // adding the option doesn't change the set... (it's a duplicate)
3853 FailAtT(loc, "duplicate instruction options");
3854 }
3855 } else if (Consume(DOLLAR)) {
3856 // sbid directive
3857 if (m_model.supportsHwDeps()) {
3858 FailAtT(loc, "software dependencies not supported on this platform");
3859 }
3860 int32_t sbid;
3861 ConsumeIntLitOrFail(sbid, "expected SBID token");
3862 // $4
3863 if (Consume(DOT)) {
3864 if (ConsumeIdentEq("dst")) {
3865 // $4.dst
3866 m_builder.InstDepInfoSBidDst(loc, sbid);
3867 } else if (ConsumeIdentEq("src")) {
3868 // $4.src
3869 m_builder.InstDepInfoSBidSrc(loc, sbid);
3870 } else if (ConsumeIdentEq("inc")) {
3871 // $4.inc
3872 m_builder.InstDepInfoSBidCntr(loc, sbid);
3873 } else {
3874 FailT("invalid SBID directive expecting 'dst' or 'src'");
3875 }
3876 } else if (!LookingAtAnyOf(COMMA, RBRACE)) {
3877 FailT("syntax error in SBID directive"
3878 " (expected '.' ',' or '}')");
3879 } else {
3880 // $4 (allocate/.set)
3881 m_builder.InstDepInfoSBidAlloc(loc, sbid);
3882 }
3883 } else if (Consume(AT)) {
3884 // min dependency distance @3
3885 if (m_model.supportsHwDeps()) {
3886 FailAtT(loc, "software dependencies not supported on this platform");
3887 }
3888 int32_t dist;
3889 auto loc = NextLoc();
3890 ConsumeIntLitOrFail(dist, "expected register min distance value");
3891 m_builder.InstDepInfoDist(loc, SWSB::DistType::REG_DIST, dist);
3892 } else {
3893 return false;
3894 }
3895 return true;
3896 }
3897
3898 // FlagRegRef = ('f0'|'f1'|'f2'|'f3') ('.' ('0'|'1'))?
3899 void ParseFlagRegRef(RegRef &freg) {
3900 if (!LookingAt(IDENT)) {
3901 FailT("expected flag register");
3902 }
3903
3904 if (ConsumeIdentEq("f0")) {
3905 freg.regNum = 0;
3906 } else if (ConsumeIdentEq("f1")) {
3907 freg.regNum = 1;
3908 } else if (ConsumeIdentEq("f2")) {
3909 freg.regNum = 2;
3910 } else if (ConsumeIdentEq("f3")) {
3911 freg.regNum = 3;
3912 } else {
3913 FailT("unexpected flag register number");
3914 }
3915
3916 if (LookingAtSeq(DOT, INTLIT10)) {
3917 // e.g. f0.1
3918 // ^
3919 // protects predication's use in short predication code
3920 // (f0.any2h)
3921 Skip();
3922 auto srLoc = NextLoc();
3923 ConsumeIntLitOrFail(freg.subRegNum, "expected flag subregister");
3924 if (freg.subRegNum > 1) {
3925 ErrorAtT(srLoc, "flag sub-register out of bounds");
3926 }
3927 } else {
3928 // e.g. f0.any2h (treat as f0.0.any2h)
3929 freg.subRegNum = 0;
3930 }
3931 }
3932
3933 bool ConsumeDstType(Type &dty) { return ConsumeIdentOneOf(DST_TYPES, dty); }
3934
3935 bool ConsumeSrcType(Type &sty) { return ConsumeIdentOneOf(SRC_TYPES, sty); }
3936
3937 // See LookingAtLabelDef for definition of a label
3938 bool ConsumeLabelDef(std::string &label) {
3939 if (LookingAtLabelDef()) {
3940 label = GetTokenAsString();
3941 (void)Skip(2);
3942 return true;
3943 }
3944 return false;
3945 }
3946
3947 // Label = IDENT ':' (not followed by identifier)
3948 // mov:ud (16) -- not a label
3949 //
3950 // mov: -- label
3951 // ud (16) ...
3952 // mov: -- label
3953 // (f0) ud (16) ...
3954 //
3955 // lbl1: -- label
3956 // lbl2:
3957 //
3958 // For now if we get IDENT COLON IDENT, we resolve by:
3959 // * if on same line, then it's an instruction
3960 // * otherwise it's a label
3961 bool LookingAtLabelDef() {
3962 if (LookingAtSeq(IDENT, COLON)) {
3963 // ensure not a uniform type
3964 // mov:ud (..)
3965 const Token &t2 = Next(2);
3966 if (t2.lexeme != IDENT || Next(0).loc.line != t2.loc.line) {
3967 return true;
3968 }
3969 }
3970 return false;
3971 }
3972
3973 //
3974 // DPAS-specific parsing functions (XE+)
3975 //
3976 // common function for non-standard operands such as
3977 // r13:d (type) or r13:u1 (OperandPrecision)
3978 // Both dst & src uses this function. 'isDst' tells if
3979 // it is a dst or src.
3980 void ParseDpasOp(int opIx, bool isDst) {
3981 const Loc regStart = NextLoc(0);
3982 if (!isDst) {
3983 m_srcLocs[opIx] = regStart;
3984 }
3985
3986 if (isDst) {
3987 // ParseSatOpt increments m_offset.
3988 if (ParseSatOpt()) {
3989 m_builder.InstDstOpSaturate();
3990 }
3991 }
3992
3993 // make sure indirect reg addressing is not allowed.
3994 if (LookingAtIdentEq("r")) {
3995 FailT("Indirect register addressing not allowed");
3996 }
3997
3998 // Match grf such as r10
3999 const RegInfo *ri = nullptr;
4000 int regNum = 0;
4001 if (!ConsumeReg(ri, regNum)) {
4002 if (isDst)
4003 FailT("invalid dst");
4004 else
4005 FailT("invalid src", opIx);
4006 }
4007 if (ri->regName != RegName::GRF_R) {
4008 // under bdpas, src0, src3 and src4 can be null (meaning +0)
4009 if (m_opSpec->is(Op::BDPAS)) {
4010 if ((opIx != 0 && opIx != 3 && opIx != 4) || ri->regName != RegName::ARF_NULL)
4011 FailT("src", opIx, ": invalid register",
4012 (opIx == 0 || opIx == 3 || opIx == 4) ?
4013 " (must be GRF or null)" : " (must be GRF)");
4014 } else
4015 // dpas must be GRF except src0, which can be null (meaning +0)
4016 if (opIx != 0 || ri->regName != RegName::ARF_NULL)
4017 FailT("src", opIx, ": invalid register",
4018 opIx == 0 ? " (must be GRF or null)" : " (must be GRF)");
4019 } else if (!ri->isRegNumberValid(regNum)) {
4020 FailT("src", opIx, "register number too large", " (", ri->syntax,
4021 " only has ", ri->numRegs, " on this platform)");
4022 }
4023
4024 Loc subRegLoc = NextLoc();
4025 int subregNum = 0;
4026 // parse subreg for src2
4027 //
4028 // all operands except src2 have an implicit subreg 0. No explicit
4029 // subreg is allowed. For src2, its subreg can be either implicit
4030 // or explicit. If it is explicit, the subreg must be half-grf
4031 // aligned.
4032 if (LookingAt(DOT)) {
4033 Skip();
4034 ConsumeIntLitOrFail(subregNum, "expected subregister");
4035 }
4036
4037 // check for reg region, which is not forbidden.
4038 if (LookingAt(LANGLE)) {
4039 FailT("instruction does not support regioning");
4040 }
4041
4042 // match :<Type> or :<SubBytePrecision>
4043 Type ty = TryParseOpType(SRC_TYPES);
4044 if (ty == Type::INVALID) {
4045 FailT("invalid type");
4046 }
4047 if (opIx == 1 && subregNum != 0) {
4048 // dpas src1 subreg must be 0
4049 WarningAtT(subRegLoc, "src1 subregister must be GRF aligned for this op");
4050 }
4051
4052
4053 // TODO: we could ensure src2 is half-grf aligned
4054 // int byteOff = SubRegToBytesOffset(subregNum, RegName::GRF_R, ty);
4055
4056 RegRef reg(regNum, subregNum);
4057 if (isDst) {
4058 m_builder.InstDpasDstOp(regStart, ri->regName, reg, ty);
4059 } else {
4060 m_builder.InstDpasSrcOp(opIx, regStart, ri->regName, reg, ty);
4061 }
4062 }
4063
4064 void ParseSdpasMeta() {
4065 const Loc regStart = NextLoc(0);
4066
4067 // Match grf such as r10
4068 const RegInfo *ri = nullptr;
4069 int regNum = 0;
4070 if (!ConsumeReg(ri, regNum)) {
4071 FailT("invalid meta operand");
4072 }
4073 if (ri && !ri->isRegNumberValid(regNum)) {
4074 FailT("invalid register number: ", regNum, " (", ri->syntax, " only has ",
4075 ri->numRegs, " registers on this platform)");
4076 }
4077
4078 int subregNum = 0;
4079 // match subreg such as .4 (for src2 only).
4080 //
4081 // all operands except src2 has an implicit subreg 0. No explicit
4082 // subreg is allowed. For src2, its subreg can be either implicit
4083 // or explicit. If it is explicit, the subreg must be either 0 or 4
4084 // as src2 is half-grf aligned.
4085 if (LookingAt(DOT)) {
4086 Skip();
4087 ConsumeIntLitOrFail(subregNum, "expected subregister");
4088 }
4089 RegRef regRef(regNum, subregNum);
4090 IGA_ASSERT(ri != nullptr, "nullptr RegInfo");
4091 m_builder.InstSdpasSrcMeta(regStart, ri->regName, regRef);
4092 }
4093 }; // class KernelParser
4094
4095 ///////////////////////////////////////////////////////////////////////////////
4096 struct ParsedPayloadInfo {
4097 Loc regLoc = Loc::INVALID;
4098 int regNum = -1;
4099 RegName regName = RegName::INVALID;
4100 Loc regLenLoc = Loc::INVALID;
4101 int regLen = 0;
4102 bool regLenWasImplicit = false;
4103 };
4104 struct ParsedAddrOperand {
4105 Loc surfArgLoc = Loc::INVALID;
4106 SendDesc surfArg; // won't be set for flat
4107 //
4108 Loc scaleLoc = Loc::INVALID;
4109 int scale = 1;
4110 //
4111 ParsedPayloadInfo payload;
4112 //
4113 Loc immOffsetLoc = Loc::INVALID;
4114 int immOffset = 0;
4115 };
4116
4117 //
4118 // parses a subset of LSC ops
4119 // EXAMPLES:
4120 // load.slm (32|M0) r10:2:d32 [r20:2]:a32
4121 // load_strided.ugm.ca.ca (32|M0) r10:4:d32x2 [r20:4]:a64
4122 //
4123 // store.slm (32|M0) [r20:2]:a32 r10:2:d32
4124 // store_quad.slm (32|M0) [r20:4]:a64 r10:4:d32.xw
4125 //
4126 // atomic_fadd.ugm.uc.uc (32|M0) r10:2:d32 [r20:4]:a64 null
4127 //
4128 void KernelParser::ParseLdStOpControls(Loc mneLoc,
4129 const SendOpDefinition &opInfo,
4130 VectorMessageArgs &vma) {
4131 auto tryParseAddrSize = [&](std::string ctrl) {
4132 if (ctrl == "a16") {
4133 vma.addrSize = 16;
4134 } else if (ctrl == "a32") {
4135 vma.addrSize = 32;
4136 } else if (ctrl == "a64") {
4137 vma.addrSize = 64;
4138 } else {
4139 return false;
4140 }
4141 return true;
4142 };
4143 //
4144 auto tryParseDataType = [&](Loc dtLoc, const std::string& dt) {
4145 if (dt.size() < 2 || dt[0] != 'd' || !isdigit(dt[1])) {
4146 return false;
4147 }
4148 //
4149 auto errorAtOffset = [&](size_t off, std::string msg) {
4150 dtLoc.col += (uint32_t)off;
4151 dtLoc.offset += (PC)off;
4152 FailS(dtLoc, msg);
4153 };
4154 size_t ix = 1;
4155 auto skip = [&](size_t i) { ix = std::max<size_t>(i + ix, dt.size()); };
4156 auto lookingAt = [&](const char *str) { return dt.find(str, ix) == ix; };
4157 auto consumeIf = [&](const char *str) {
4158 if (lookingAt(str)) {
4159 skip(iga::stringLength(str));
4160 return true;
4161 }
4162 return false;
4163 };
4164 //////////////////////////////////////////////////////////
4165 // data size
4166 // e.g. d32... or d8u32...
4167 int dataSize = 0;
4168 for (; ix < dt.size() && isdigit(dt[ix]); ix++) {
4169 dataSize = 10 * dataSize + dt[ix] - '0';
4170 }
4171 switch (dataSize) {
4172 case 8:
4173 case 16:
4174 case 32:
4175 case 64:
4176 case 128:
4177 case 256:
4178 break;
4179 default:
4180 errorAtOffset(0, "invalid data type size");
4181 }
4182 // could be d8u32
4183 vma.dataSizeExpandHigh = false;
4184 vma.dataSizeReg = vma.dataSizeMem = dataSize;
4185 if (consumeIf("u32h")) {
4186 vma.dataSizeReg = 32;
4187 vma.dataSizeExpandHigh = true;
4188 } else if (consumeIf("u32")) {
4189 vma.dataSizeReg = 32;
4190 }
4191
4192 //////////////////////////////////////////////////////////
4193 // vector size
4194 // ...x4 or ...x64t or ....xyw (for quad)
4195 if (opInfo.hasChMask()) {
4196 if (ix != dt.size())
4197 errorAtOffset(ix, "unexpected data type");
4198 // e.g. d32.xz
4199 // ^
4200 if (LookingAtSeq(DOT, IDENT)) {
4201 Skip();
4202 auto cmask = ConsumeIdentOrFail("expected op control symbols");
4203 for (size_t i = 0; i < cmask.size(); i++) {
4204 auto setElem = [&](int off) {
4205 if (vma.dataComponentMask & (1 << off)) {
4206 errorAtOffset(ix + i, "duplicate component mask element");
4207 }
4208 vma.dataComponentMask |= (1 << off);
4209 };
4210 if (cmask[i] == 'x') {
4211 setElem(0);
4212 } else if (cmask[i] == 'y') {
4213 setElem(1);
4214 } else if (cmask[i] == 'z') {
4215 setElem(2);
4216 } else if (cmask[i] == 'w') {
4217 setElem(3);
4218 } else {
4219 errorAtOffset(ix + i, "invalid component mask symbol");
4220 }
4221 }
4222 } else {
4223 FailS(NextLoc(), "expected component mask");
4224 }
4225 } else {
4226 // regular vector type
4227 // e.g. d8, d8u32 or d32x16t
4228 auto vix = ix;
4229 int vecSize = 1;
4230 if (ix < dt.size() - 1 && dt[ix] == 'x' && isdigit(dt[ix + 1])) {
4231 ix++;
4232 vecSize = 0;
4233 for (; ix < dt.size() && isdigit(dt[ix]); ix++)
4234 vecSize = 10 * vecSize + dt[ix] - '0';
4235 switch (vecSize) {
4236 case 1:
4237 case 2:
4238 case 3:
4239 case 4:
4240 case 8:
4241 case 16:
4242 case 32:
4243 case 64:
4244 break;
4245 default:
4246 errorAtOffset(vix, "invalid vector size");
4247 break;
4248 }
4249 }
4250 vma.dataVectorSize = (short)vecSize;
4251 vma.dataTranspose = false;
4252 //
4253 // maybe a 't' following
4254 for (; ix < dt.size(); ix++) {
4255 if (dt[ix] == 't') {
4256 if (vma.dataTranspose)
4257 errorAtOffset(ix, "transpose already set");
4258 vma.dataTranspose = true;
4259 } else if (dt[ix] == 'v') {
4260 if (vma.dataVnni) {
4261 errorAtOffset(ix, "vnni already set");
4262 } else if (vma.op != SendOp::LOAD_BLOCK2D) {
4263 errorAtOffset(ix, "vnni only permitted on load_block2d");
4264 } else {
4265 vma.dataVnni = true;
4266 }
4267 } else {
4268 errorAtOffset(ix, "malformed data type suffix");
4269 }
4270 } // data type suffix
4271 }
4272 //
4273 return true;
4274 };
4275 //
4276 auto tryCachingSymbol = [](const std::string& cc, CacheOpt &co) {
4277 if (cc == "df") {
4278 co = CacheOpt::DEFAULT;
4279 } else if (cc == "ri") {
4280 co = CacheOpt::READINVALIDATE;
4281 } else if (cc == "ca") {
4282 co = CacheOpt::CACHED;
4283 } else if (cc == "uc") {
4284 co = CacheOpt::UNCACHED;
4285 } else if (cc == "st") {
4286 co = CacheOpt::STREAMING;
4287 } else if (cc == "wb") {
4288 co = CacheOpt::WRITEBACK;
4289 } else if (cc == "wt") {
4290 co = CacheOpt::WRITETHROUGH;
4291 } else if (cc == "cc") {
4292 co = CacheOpt::CONSTCACHED;
4293 } else {
4294 return false;
4295 }
4296 return true;
4297 };
4298 //
4299 Loc cacheControlLoc = mneLoc;
4300 vma.cachingL1 = CacheOpt::DEFAULT;
4301 vma.cachingL3 = CacheOpt::DEFAULT;
4302 //
4303 bool cachingSet = false;
4304 bool isAddrSizeSet = false;
4305 bool isDataSizeSet = false;
4306 //
4307 while (Consume(DOT)) {
4308 Loc ctrlLoc = NextLoc();
4309 auto ctrl = ConsumeIdentOrFail("expected op control symbol");
4310 if (tryParseAddrSize(ctrl)) {
4311 if (isAddrSizeSet) {
4312 FailAtT(ctrlLoc, "duplciate address size");
4313 }
4314 isAddrSizeSet = true;
4315 } else if (tryCachingSymbol(ctrl, vma.cachingL1)) {
4316 if (cachingSet) {
4317 FailAtT(ctrlLoc, "duplicate caching options");
4318 }
4319 cachingSet = true;
4320 cacheControlLoc = ctrlLoc;
4321 if (!LookingAtSeq(DOT, IDENT)) {
4322 FailT("expected .");
4323 }
4324 Skip();
4325 auto l3sym = GetTokenAsString();
4326 if (!tryCachingSymbol(l3sym, vma.cachingL3)) {
4327 FailT("expected caching option");
4328 }
4329 Skip();
4330 } else if (tryParseDataType(ctrlLoc, ctrl)) {
4331 if (isDataSizeSet)
4332 FailS(ctrlLoc, "data size already set");
4333 isDataSizeSet = true;
4334 } else {
4335 FailT("invalid ld/st option");
4336 }
4337 }
4338 if (!isAddrSizeSet) {
4339 FailAtT(mneLoc, "address size not set");
4340 }
4341 if (!isDataSizeSet) {
4342 FailAtT(mneLoc, "data size not set");
4343 }
4344 bool isDft =
4345 vma.cachingL1 != CacheOpt::DEFAULT && vma.cachingL3 != CacheOpt::DEFAULT;
4346 if (isDft && vma.sfid == SFID::SLM) {
4347 FailAtT(cacheControlLoc, "SLM forbids cache control options");
4348 }
4349 }
4350
4351 bool KernelParser::ParseLdStInst() {
4352 if (!LookingAtSeq(IDENT, DOT, IDENT)) {
4353 return false;
4354 }
4355
4356 VectorMessageArgs vma; // vma {}; crashes VS2019 without dft constructor!
4357 //
4358 const auto mneLoc = NextLoc();
4359 auto mne = GetTokenAsString();
4360 const SendOpDefinition &opInfo = lookupSendOp(mne.c_str());
4361 if (!opInfo.isValid())
4362 return false; // doesn't match a known op
4363 vma.op = opInfo.op;
4364 //
4365 const auto sfidSym = GetTokenAsString(Next(2));
4366 vma.sfid = FromSyntax<SFID>(sfidSym);
4367 if (vma.sfid == SFID::INVALID) {
4368 // better error if we they forget the sfid: e.g. load.d32... instead of load.ugm
4369 // otherwise sendOpSupportsSyntax() will say the op "isn't supported"
4370 // the user thinks "BS that "load" isn't supported
4371 FailS(NextLoc(2), "invalid SFID");
4372 }
4373 if (!sendOpSupportsSyntax(platform(), vma.op, vma.sfid)) {
4374 FailS(NextLoc(), "op not yet supported for ld/st parse");
4375 }
4376 if (platform() >= Platform::XE3)
4377 FailAtT(mneLoc, "load/store syntax not enabled yet for this platform");
4378
4379 // IGA op
4380 m_opSpec =
4381 &m_model.lookupOpSpec(platform() < Platform::XE ? Op::SENDS : Op::SEND);
4382 if (!m_opSpec->isValid()) {
4383 FailS(NextLoc(), "INTERNAL ERROR: unable to lookup iga::Op");
4384 }
4385 m_builder.InstOp(m_opSpec);
4386 m_builder.InstSubfunction(vma.sfid);
4387 //
4388 Skip(3);
4389 //
4390 ParseLdStOpControls(mneLoc, opInfo, vma);
4391 //
4392 ChannelOffset chOff = ChannelOffset::M0; // sets m_execSize
4393 const auto execSizeLoc = NextLoc();
4394 ParseExecInfo(m_defaultExecutionSize, m_execSize, chOff);
4395 vma.execSize = static_cast<int>(m_execSize);
4396 //
4397 ///////////////////////////////////////////////////////////////////////////
4398 vma.addrType = AddrType::FLAT;
4399 //
4400 // r13 or null
4401 auto parseReg = [&](ParsedPayloadInfo &ppi) {
4402 ppi.regLoc = NextLoc();
4403 const RegInfo *regInfo;
4404 int regNum;
4405 if (!ConsumeReg(regInfo, regNum)) {
4406 FailT("expected register");
4407 return;
4408 } else if (regInfo->regName != RegName::GRF_R &&
4409 regInfo->regName != RegName::ARF_NULL) {
4410 FailAtT(ppi.regLoc, "register must be GRF or null");
4411 return;
4412 }
4413 ppi.regName = regInfo->regName;
4414 ppi.regNum = (uint16_t)regNum;
4415 };
4416
4417 // r13:0, r13 (meaning r13:1)
4418 // or null:0, or null meaning (null:0)
4419 auto parsePayload = [&](ParsedPayloadInfo &ppi) {
4420 auto regLoc = NextLoc();
4421 parseReg(ppi);
4422 if (Consume(COLON)) {
4423 auto tk = Next();
4424 ppi.regLenLoc = tk.loc;
4425 ImmVal val;
4426 if (!TryParseIntConstExprPrimary(val, "payload suffix expression")) {
4427 FailAtT(tk.loc, "expected integral payload suffix expression");
4428 }
4429 ensureIntegral(tk, val);
4430 if (val.s64 < 0 || val.s64 > 0x1F) {
4431 FailAtT(tk.loc, "payload size overflow");
4432 }
4433 ppi.regLen = (int)val.s64;
4434 } else {
4435 // implicitly 0 or 1 depending on register
4436 ppi.regLen = ppi.regName == RegName::ARF_NULL ? 0 : 1;
4437 ppi.regLenLoc = regLoc.endOfToken();
4438 ppi.regLenWasImplicit = true;
4439 }
4440 };
4441
4442 // the formatter deduces the length just to be nice to the reader
4443 auto parsePayloadDst = [&](ParsedPayloadInfo &ppi) {
4444 parsePayload(ppi);
4445 auto rgn = m_opSpec->hasImplicitDstRegion(m_builder.isMacroOp())
4446 ? m_opSpec->implicitDstRegion(m_builder.isMacroOp()).getHz()
4447 : Region::Horz::HZ_1;
4448 auto ty = SendOperandDefaultType(-1);
4449 m_builder.InstDstOpRegDirect(ppi.regLoc, ppi.regName, ppi.regNum, rgn, ty);
4450 };
4451
4452 auto parsePayloadSrc1 = [&](ParsedPayloadInfo &ppi) {
4453 parsePayload(ppi);
4454 m_sendSrcLens[1] = ppi.regLen;
4455 m_sendSrcLenLocs[1] = ppi.regLenLoc;
4456 //
4457 const auto rgn = defaultSendOperandRegion(ppi.regName, 1);
4458 const auto ty = SendOperandDefaultType(1);
4459 m_builder.InstSrcOpRegDirect(1, ppi.regLoc, ppi.regName, (int)ppi.regNum,
4460 rgn, ty);
4461 };
4462
4463 auto parseA0RegOrImm = [&]() {
4464 SendDesc surf;
4465 ConsumeOrFail(LBRACK, "expected [");
4466 RegName regName = RegName::INVALID;
4467 if (ParseAddrRegRefOpt(surf.reg, regName)) {
4468 IGA_ASSERT(regName == RegName::ARF_A, "Indirect send Desc must be ARF_A");
4469 // surface is a register
4470 surf.type = SendDesc::Kind::REG32A;
4471 if (surf.reg.subRegNum & 1) {
4472 FailT("a0 subregister must be even (values are word aligned)");
4473 }
4474 } else {
4475 // surface is a constant integral expression
4476 auto loc = NextLoc();
4477 surf.type = SendDesc::Kind::IMM;
4478 ImmVal v;
4479 if (!TryParseIntConstExpr(v, "extended descriptor")) {
4480 FailT("expected surface state offset");
4481 }
4482 if (v.s64 >= 0x03FFFFFF) { // 26b
4483 FailAtT(loc, "immediate surface state offset is out of bounds");
4484 }
4485 surf.imm = (uint32_t)v.s64;
4486 }
4487 ConsumeOrFail(RBRACK, "expected ]");
4488 return surf;
4489 };
4490
4491 Loc exDescLoc = mneLoc;
4492 auto parseAddrOperand = [&](ParsedAddrOperand &addr) {
4493 addr.surfArgLoc = exDescLoc = NextLoc();
4494 if (ConsumeIdentEq("bti")) {
4495 vma.addrType = AddrType::BTI;
4496 vma.addrSurface = parseA0RegOrImm();
4497 } else if (ConsumeIdentEq("bss")) {
4498 vma.addrType = AddrType::BSS;
4499 vma.addrSurface = parseA0RegOrImm();
4500 } else if (ConsumeIdentEq("ss")) {
4501 vma.addrType = AddrType::SS;
4502 vma.addrSurface = parseA0RegOrImm();
4503 } else {
4504 // the "flat" token is optional
4505 // (if addrtype is absent, we assume flat)
4506 if (!ConsumeIdentEq("flat") && !LookingAt(LBRACK)) {
4507 FailT("expected address type or [");
4508 }
4509 vma.addrType = AddrType::FLAT;
4510 vma.addrSurface = 0x0;
4511 }
4512 //
4513 Consume(LBRACK);
4514 //
4515 ImmVal v;
4516 auto scaleTk = Next();
4517 if (!LookingAt(IDENT) && TryParseIntConstExprPrimary(v)) {
4518 // LookingAt(..) to avoid failure as r13 parses as IDENT
4519 // FIXME: parser needs better context to work around this
4520 // deficiency
4521 ensureIntegral(scaleTk, v);
4522 int scale = (int)v.s64;
4523 ConsumeOrFail(MUL, "expected *");
4524 //
4525 // sanity check the size here so we can give good error
4526 // location info
4527 int vlen = vma.elementsPerAddress();
4528 int bytesPerElem = vma.dataSizeMem * vlen / 8;
4529 if (bytesPerElem > 32) {
4530 FailS(scaleTk.loc, "scaling factor is too large");
4531 } else if (scale == bytesPerElem) {
4532 vma.addrScale = bytesPerElem;
4533 } else if (scale == 2 * bytesPerElem) {
4534 vma.addrScale = 2 * bytesPerElem;
4535 } else if (scale == 4 * bytesPerElem) {
4536 vma.addrScale = 4 * bytesPerElem;
4537 } else {
4538 FailAtT(scaleTk.loc, "invalid scaling factor (must be ",
4539 1 * bytesPerElem, ", ", 2 * bytesPerElem, ", or ",
4540 4 * bytesPerElem, ")");
4541 }
4542 }
4543 //
4544 parsePayload(addr.payload);
4545 m_sendSrcLens[0] = addr.payload.regLen;
4546 m_sendSrcLenLocs[0] = addr.payload.regLenLoc;
4547 //
4548 vma.addrOffset = 0x0;
4549 auto addrOffLoc = NextLoc();
4550 if (LookingAt(ADD) || LookingAt(SUB)) {
4551 auto immOffLoc = NextLoc();
4552 auto rangeCheck = [&](int bits, const char *which) {
4553 if (v.s64 > (1 << (bits - 1)) - 1 || v.s64 < -(1 << (bits - 1))) {
4554 FailAtT(immOffLoc, "immediate offset exceeds ", bits, "b for ",
4555 which);
4556 }
4557 };
4558
4559 exDescLoc = addrOffLoc;
4560 if (vma.op == SendOp::LOAD_BLOCK2D || vma.op == SendOp::STORE_BLOCK2D) {
4561 // special handling for block 2d
4562 bool isNeg = LookingAt(SUB);
4563 Skip(1);
4564 ConsumeOrFail(LPAREN, "expected (");
4565 if (!TryParseIntConstExpr(v, "block2d x-offset")) {
4566 FailT("syntax error in block2d x-offset");
4567 }
4568 if (isNeg) {
4569 v.s64 = -v.s64;
4570 }
4571 rangeCheck(10, "block2d x-offset");
4572 vma.addrOffsetX = (int)v.s64;
4573 //
4574 ConsumeOrFail(COMMA, "expected ,");
4575 if (!TryParseIntConstExpr(v, "block2d y-offset")) {
4576 FailT("syntax error in block2d y-offset");
4577 }
4578 if (isNeg) {
4579 v.s64 = -v.s64;
4580 }
4581 rangeCheck(10, "block2d y-offset");
4582 vma.addrOffsetY = (int)v.s64;
4583 ConsumeOrFail(RPAREN, "expected )");
4584 } else if (TryParseIntConstExprAddChain(v, "address immediate offset")) {
4585 if (vma.addrType == AddrType::FLAT) {
4586 rangeCheck(20, "flat addresses");
4587 } else if (vma.addrType == AddrType::BSS ||
4588 vma.addrType == AddrType::SS) {
4589 if (vma.sfid == SFID::UGM) {
4590 rangeCheck(17, "ugm bss/ss addresses");
4591 } else {
4592 rangeCheck(16, "non-ugm bss/ss addresses");
4593 }
4594 }
4595 if (v.s64 % 4 != 0)
4596 FailAtT(immOffLoc, "immediate offset must be 32b aligned");
4597 vma.addrOffset = (int)v.s64;
4598 } else {
4599 FailAtT(immOffLoc, "expected immediate offset expression");
4600 }
4601 }
4602 //
4603 Consume(RBRACK);
4604 //
4605 const auto rgn = defaultSendOperandRegion(addr.payload.regName, 0);
4606 const auto ty = SendOperandDefaultType(0);
4607 m_builder.InstSrcOpRegDirect(0, addr.surfArgLoc, addr.payload.regName,
4608 (int)addr.payload.regNum, rgn, ty);
4609 };
4610
4611 auto setOpToNull = [&](int opIx, ParsedPayloadInfo &ppi, Loc loc) {
4612 ppi.regLen = 0;
4613 ppi.regLoc = ppi.regLenLoc = loc;
4614 ppi.regName = RegName::ARF_NULL;
4615 ppi.regNum = 0;
4616 //
4617 const auto rgn = defaultSendOperandRegion(ppi.regName, opIx);
4618 const auto ty = SendOperandDefaultType(opIx);
4619 if (opIx < 0) {
4620 m_builder.InstDstOpRegDirect(ppi.regLoc, ppi.regName, ppi.regNum,
4621 rgn.getHz(), ty);
4622 } else {
4623 m_builder.InstSrcOpRegDirect(opIx, ppi.regLoc, ppi.regName, ppi.regNum,
4624 rgn, ty);
4625 m_sendSrcLens[opIx] = 0;
4626 m_sendSrcLenLocs[opIx] = loc;
4627 }
4628 };
4629
4630 ///////////////////////////////////////////////////////////////////////////
4631 // actual parsing
4632 ParsedPayloadInfo dstData;
4633 ParsedAddrOperand src0Addr;
4634 ParsedPayloadInfo src1Data;
4635 if (vma.isLoad()) {
4636 parsePayloadDst(dstData);
4637 parseAddrOperand(src0Addr);
4638 setOpToNull(1, src1Data, mneLoc);
4639 } else if (vma.isStore()) {
4640 setOpToNull(-1, dstData, mneLoc); // build a null operand
4641 parseAddrOperand(src0Addr);
4642 parsePayloadSrc1(src1Data);
4643 } else if (vma.isAtomic()) {
4644 parsePayloadDst(dstData);
4645 parseAddrOperand(src0Addr);
4646 parsePayloadSrc1(src1Data);
4647 } else {
4648 FailAtT(mneLoc, "unsupported operation for load/store syntax");
4649 }
4650
4651 const int GRF_BYTES = platform() >= Platform::XE_HPC ? 64 : 32;
4652 const int DEFAULT_SIMD = GRF_BYTES / 2;
4653
4654 int expectedDataRegs = -1;
4655 int expectedAddrRegs = -1;
4656 if (vma.op == SendOp::LOAD_BLOCK2D || vma.op == SendOp::STORE_BLOCK2D) {
4657 expectedAddrRegs = 1;
4658 // expectedDataRegs = unknown (function of the payload)
4659 } else if (vma.dataTranspose) {
4660 // a block operation with a single register payload
4661 // technically execSize on transpose will be 1, but the
4662 // logical extension for higher SIMD's exists too;
4663 // for now we reject it
4664 if (vma.dataTranspose && vma.execSize != 1) {
4665 FailAtT(execSizeLoc, "transpose messages must be SIMD1");
4666 }
4667 int totalBitsPerSimd = vma.dataSizeReg * vma.elementsPerAddress();
4668 expectedDataRegs =
4669 vma.execSize * std::max<int>(totalBitsPerSimd / 8 / GRF_BYTES, 1);
4670 expectedAddrRegs = 1;
4671 } else {
4672 // e.g. SIMD4 rounds up to SIMD8 if messages are SIMD16 by default
4673 int execElems = std::max<int>(DEFAULT_SIMD / 2, vma.execSize);
4674 int grfsPerComponent =
4675 std::max<int>(vma.dataSizeReg * execElems / 8 / GRF_BYTES, 1);
4676 expectedDataRegs = grfsPerComponent * vma.elementsPerAddress();
4677 if (opInfo.isAddressScalar()) {
4678 expectedAddrRegs = 1;
4679 } else {
4680 expectedAddrRegs =
4681 std::max<int>(execElems * vma.addrSize / 8 / GRF_BYTES, 1);
4682 }
4683 }
4684 ///////////////////////////////////////////////////////////////////////////
4685 // for almost all messages we can deduce the number of address registers
4686 // needed by the message; one exception is in typed, which uses the address
4687 // register count to determine which of U, V, R, and LOD are included
4688 bool checkAddrPayloadSize = true;
4689 bool hasUvrl =
4690 vma.sfid == SFID::TGM && (vma.op == SendOp::LOAD_QUAD ||
4691 vma.op == SendOp::STORE_QUAD ||
4692 vma.op == SendOp::LOAD_QUAD_MSRT ||
4693 vma.op == SendOp::STORE_QUAD_MSRT ||
4694 vma.isAtomic());
4695 hasUvrl |= vma.sfid == SFID::TGM && vma.op == SendOp::STORE_UNCOMPRESSED_QUAD;
4696 bool addrLenIsCorrect = src0Addr.payload.regLen == expectedAddrRegs;
4697 addrLenIsCorrect |=
4698 hasUvrl && (src0Addr.payload.regLen != 2 * expectedAddrRegs ||
4699 src0Addr.payload.regLen != 3 * expectedAddrRegs ||
4700 src0Addr.payload.regLen != 4 * expectedAddrRegs);
4701 if (checkAddrPayloadSize && !addrLenIsCorrect) {
4702 WarningAtT(src0Addr.payload.regLoc, "Src0.Length: should be ",
4703 expectedAddrRegs);
4704 if (src0Addr.payload.regLenWasImplicit) {
4705 src0Addr.payload.regLen = expectedAddrRegs;
4706 }
4707 }
4708 //
4709 if (vma.isLoad() || vma.isAtomic()) {
4710 // if prefetch or atomic with no return, then
4711 int expectDstRegs =
4712 dstData.regName == RegName::ARF_NULL ? 0 : expectedDataRegs;
4713 if (!dstData.regLenWasImplicit && expectedDataRegs >= 0 &&
4714 dstData.regLen != expectDstRegs) {
4715 // if given an explicit number of dst regs, ensure it is correct
4716 WarningAtT(dstData.regLenLoc, "Dst.Length: should be ", expectDstRegs);
4717 } else if (dstData.regLenWasImplicit) {
4718 if (dstData.regName != RegName::ARF_NULL) {
4719 // assign it, but issue a warning
4720 dstData.regLen = expectDstRegs;
4721 WarningAtT(dstData.regLenLoc, "Dst.Length: :", expectDstRegs,
4722 " should suffix operand");
4723 } else {
4724 dstData.regLen = 0;
4725 }
4726 }
4727 }
4728 if (vma.isStore() || vma.isAtomic()) {
4729 int expectSrc1Regs =
4730 src1Data.regName == RegName::ARF_NULL ? 0 : expectedDataRegs;
4731 if (vma.isAtomic())
4732 expectSrc1Regs *= opInfo.numAtomicArgs();
4733 if (!src1Data.regLenWasImplicit && expectSrc1Regs >= 0 &&
4734 src1Data.regLen != expectSrc1Regs) {
4735 WarningAtT(src1Data.regLenLoc, "Src1.Length: should be ", expectSrc1Regs);
4736 } else if (src1Data.regLenWasImplicit &&
4737 src1Data.regName != RegName::ARF_NULL) {
4738 bool hasSrc1LenSfx = platform() <= Platform::XE2;
4739 if (hasSrc1LenSfx && vma.addrSurface.isReg()) {
4740 // with a0 exdesc must omit Src1.Length
4741 // if ExBSO is set
4742 expectSrc1Regs = -1;
4743 } else {
4744 WarningAtT(src1Data.regLenLoc, "Src1.Length: :", expectSrc1Regs,
4745 " should suffix operand");
4746 }
4747 src1Data.regLen = expectSrc1Regs;
4748 }
4749 }
4750
4751 SendDesc exDesc(0x0), desc(0x0);
4752 uint32_t exImmOffDesc = 0;
4753
4754 std::string err;
4755 if (!encodeDescriptors(platform(), vma, exImmOffDesc, exDesc, desc, err)) {
4756 if (err.empty())
4757 err = "unknown error translating load/store op";
4758 FailS(mneLoc, err);
4759 }
4760
4761 desc.imm |= ((uint32_t)src0Addr.payload.regLen & 0xF) << 25;
4762 desc.imm |= ((uint32_t)dstData.regLen & 0x1F) << 20;
4763 if (platform() < Platform::XE_HP && exDesc.isImm()) {
4764 // ExDesc[10:6] on <= XE_HP
4765 exDesc.imm |= (((uint32_t)src1Data.regLen & 0x1F) << 6);
4766 }
4767
4768 m_builder.InstSendSrc0Length(src0Addr.payload.regLen);
4769 m_builder.InstSendSrc1Length(src1Data.regLen);
4770
4771 m_builder.InstSendDescs(mneLoc, exImmOffDesc, exDesc, mneLoc, desc);
4772
4773 return true;
4774 } // KernelParser::ParseLscOp
4775
4776 Kernel *iga::ParseGenKernel(const Model &m, const char *inp,
4777 iga::ErrorHandler &e, const ParseOpts &popts) {
4778 Kernel *k = new Kernel(m);
4779
4780 InstBuilder h(k, e);
4781 if (popts.swsbEncodeMode != SWSB_ENCODE_MODE::SWSBInvalidMode)
4782 h.setSWSBEncodingMode(popts.swsbEncodeMode);
4783
4784 KernelParser p(m, h, inp, e, popts);
4785 try {
4786 p.ParseListing();
4787 } catch (SyntaxError &) {
4788 // no need to handle it (error handler has already recorded the errors)
4789 delete k;
4790 return nullptr;
4791 }
4792
4793 auto &insts = h.getInsts();
4794 auto blockStarts = Block::inferBlocks(e, k->getMemManager(), insts);
4795 int id = 0;
4796 for (const auto &bitr : blockStarts) {
4797 bitr.second->setID(id++);
4798 k->appendBlock(bitr.second);
4799 }
4800
4801 #if 0
4802 std::stringstream ss;
4803 ss << "PARSED BLOCKS\n\n";
4804 for (auto b : k->getBlockList()) {
4805 ss << "BLOCK[" << b->getID() << "] at pc " <<
4806 b->getPC() << ":\n";
4807 ss << " targeted by {";
4808 int totalTargets = 0;
4809 for (auto b2 : k->getBlockList()) {
4810 for (auto i2 : b2->getInstList()) {
4811 for (unsigned srcIx = 0; srcIx < i2->getSourceCount(); srcIx++) {
4812 const iga::Operand &src = i2->getSource(srcIx);
4813 if (src.getTargetBlock() == b) {
4814 if (totalTargets++ > 0)
4815 ss << ", ";
4816 ss << "." << i2->getPC();
4817 break;
4818 }
4819 }
4820 }
4821 }
4822 ss << "}\n";
4823 for (auto i : b->getInstList()) {
4824 ss << " ." << i->getPC() << " is " <<
4825 i->getOpSpec().fullMnemonic << "\n";
4826 }
4827 ss << "\n";
4828 }
4829 std::cout << ss.str();
4830 #endif
4831
4832 return k;
4833 }