| 1 |
//===- SveEmitter.cpp - Generate arm_sve.h for use with clang -*- C++ -*-===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// This tablegen backend is responsible for emitting arm_sve.h, which includes |
| 10 |
// a declaration and definition of each function specified by the ARM C/C++ |
| 11 |
// Language Extensions (ACLE). |
| 12 |
// |
| 13 |
// For details, visit: |
| 14 |
// https://developer.arm.com/architectures/system-architectures/software-standards/acle |
| 15 |
// |
| 16 |
// Each SVE instruction is implemented in terms of 1 or more functions which |
| 17 |
// are suffixed with the element type of the input vectors. Functions may be |
| 18 |
// implemented in terms of generic vector operations such as +, *, -, etc. or |
| 19 |
// by calling a __builtin_-prefixed function which will be handled by clang's |
| 20 |
// CodeGen library. |
| 21 |
// |
| 22 |
// See also the documentation in include/clang/Basic/arm_sve.td. |
| 23 |
// |
| 24 |
//===----------------------------------------------------------------------===// |
| 25 |
|
| 26 |
#include "llvm/ADT/STLExtras.h" |
| 27 |
#include "llvm/ADT/StringMap.h" |
| 28 |
#include "llvm/ADT/ArrayRef.h" |
| 29 |
#include "llvm/ADT/StringExtras.h" |
| 30 |
#include "llvm/TableGen/Record.h" |
| 31 |
#include "llvm/TableGen/Error.h" |
| 32 |
#include <string> |
| 33 |
#include <sstream> |
| 34 |
#include <set> |
| 35 |
#include <cctype> |
| 36 |
#include <tuple> |
| 37 |
|
| 38 |
using namespace llvm; |
| 39 |
|
| 40 |
enum ClassKind { |
| 41 |
ClassNone, |
| 42 |
ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix |
| 43 |
ClassG, // Overloaded name without type suffix |
| 44 |
}; |
| 45 |
|
| 46 |
using TypeSpec = std::string; |
| 47 |
|
| 48 |
namespace { |
| 49 |
|
| 50 |
class ImmCheck { |
| 51 |
unsigned Arg; |
| 52 |
unsigned Kind; |
| 53 |
unsigned ElementSizeInBits; |
| 54 |
|
| 55 |
public: |
| 56 |
ImmCheck(unsigned Arg, unsigned Kind, unsigned ElementSizeInBits = 0) |
| 57 |
: Arg(Arg), Kind(Kind), ElementSizeInBits(ElementSizeInBits) {} |
| 58 |
ImmCheck(const ImmCheck &Other) = default; |
| 59 |
~ImmCheck() = default; |
| 60 |
|
| 61 |
unsigned getArg() const { return Arg; } |
| 62 |
unsigned getKind() const { return Kind; } |
| 63 |
unsigned getElementSizeInBits() const { return ElementSizeInBits; } |
| 64 |
}; |
| 65 |
|
| 66 |
class SVEType { |
| 67 |
TypeSpec TS; |
| 68 |
bool Float, Signed, Immediate, Void, Constant, Pointer, BFloat; |
| 69 |
bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp; |
| 70 |
unsigned Bitwidth, ElementBitwidth, NumVectors; |
| 71 |
|
| 72 |
public: |
| 73 |
SVEType() : SVEType(TypeSpec(), 'v') {} |
| 74 |
|
| 75 |
SVEType(TypeSpec TS, char CharMod) |
| 76 |
: TS(TS), Float(false), Signed(true), Immediate(false), Void(false), |
| 77 |
Constant(false), Pointer(false), BFloat(false), DefaultType(false), |
| 78 |
IsScalable(true), Predicate(false), PredicatePattern(false), |
| 79 |
PrefetchOp(false), Bitwidth(128), ElementBitwidth(~0U), NumVectors(1) { |
| 80 |
if (!TS.empty()) |
| 81 |
applyTypespec(); |
| 82 |
applyModifier(CharMod); |
| 83 |
} |
| 84 |
|
| 85 |
bool isPointer() const { return Pointer; } |
| 86 |
bool isVoidPointer() const { return Pointer && Void; } |
| 87 |
bool isSigned() const { return Signed; } |
| 88 |
bool isImmediate() const { return Immediate; } |
| 89 |
bool isScalar() const { return NumVectors == 0; } |
| 90 |
bool isVector() const { return NumVectors > 0; } |
| 91 |
bool isScalableVector() const { return isVector() && IsScalable; } |
| 92 |
bool isChar() const { return ElementBitwidth == 8; } |
| 93 |
bool isVoid() const { return Void & !Pointer; } |
| 94 |
bool isDefault() const { return DefaultType; } |
| 95 |
bool isFloat() const { return Float && !BFloat; } |
| 96 |
bool isBFloat() const { return BFloat && !Float; } |
| 97 |
bool isFloatingPoint() const { return Float || BFloat; } |
| 98 |
bool isInteger() const { return !isFloatingPoint() && !Predicate; } |
| 99 |
bool isScalarPredicate() const { |
| 100 |
return !isFloatingPoint() && Predicate && NumVectors == 0; |
| 101 |
} |
| 102 |
bool isPredicateVector() const { return Predicate; } |
| 103 |
bool isPredicatePattern() const { return PredicatePattern; } |
| 104 |
bool isPrefetchOp() const { return PrefetchOp; } |
| 105 |
bool isConstant() const { return Constant; } |
| 106 |
unsigned getElementSizeInBits() const { return ElementBitwidth; } |
| 107 |
unsigned getNumVectors() const { return NumVectors; } |
| 108 |
|
| 109 |
unsigned getNumElements() const { |
| 110 |
assert(ElementBitwidth != ~0U); |
| 111 |
return Bitwidth / ElementBitwidth; |
| 112 |
} |
| 113 |
unsigned getSizeInBits() const { |
| 114 |
return Bitwidth; |
| 115 |
} |
| 116 |
|
| 117 |
/// Return the string representation of a type, which is an encoded |
| 118 |
/// string for passing to the BUILTIN() macro in Builtins.def. |
| 119 |
std::string builtin_str() const; |
| 120 |
|
| 121 |
/// Return the C/C++ string representation of a type for use in the |
| 122 |
/// arm_sve.h header file. |
| 123 |
std::string str() const; |
| 124 |
|
| 125 |
private: |
| 126 |
/// Creates the type based on the typespec string in TS. |
| 127 |
void applyTypespec(); |
| 128 |
|
| 129 |
/// Applies a prototype modifier to the type. |
| 130 |
void applyModifier(char Mod); |
| 131 |
}; |
| 132 |
|
| 133 |
|
| 134 |
class SVEEmitter; |
| 135 |
|
| 136 |
/// The main grunt class. This represents an instantiation of an intrinsic with |
| 137 |
/// a particular typespec and prototype. |
| 138 |
class Intrinsic { |
| 139 |
/// The unmangled name. |
| 140 |
std::string Name; |
| 141 |
|
| 142 |
/// The name of the corresponding LLVM IR intrinsic. |
| 143 |
std::string LLVMName; |
| 144 |
|
| 145 |
/// Intrinsic prototype. |
| 146 |
std::string Proto; |
| 147 |
|
| 148 |
/// The base type spec for this intrinsic. |
| 149 |
TypeSpec BaseTypeSpec; |
| 150 |
|
| 151 |
/// The base class kind. Most intrinsics use ClassS, which has full type |
| 152 |
/// info for integers (_s32/_u32), or ClassG which is used for overloaded |
| 153 |
/// intrinsics. |
| 154 |
ClassKind Class; |
| 155 |
|
| 156 |
/// The architectural #ifdef guard. |
| 157 |
std::string Guard; |
| 158 |
|
| 159 |
// The merge suffix such as _m, _x or _z. |
| 160 |
std::string MergeSuffix; |
| 161 |
|
| 162 |
/// The types of return value [0] and parameters [1..]. |
| 163 |
std::vector<SVEType> Types; |
| 164 |
|
| 165 |
/// The "base type", which is VarType('d', BaseTypeSpec). |
| 166 |
SVEType BaseType; |
| 167 |
|
| 168 |
uint64_t Flags; |
| 169 |
|
| 170 |
SmallVector<ImmCheck, 2> ImmChecks; |
| 171 |
|
| 172 |
public: |
| 173 |
Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy, |
| 174 |
StringRef MergeSuffix, uint64_t MemoryElementTy, StringRef LLVMName, |
| 175 |
uint64_t Flags, ArrayRef<ImmCheck> ImmChecks, TypeSpec BT, |
| 176 |
ClassKind Class, SVEEmitter &Emitter, StringRef Guard); |
| 177 |
|
| 178 |
~Intrinsic()=default; |
| 179 |
|
| 180 |
std::string getName() const { return Name; } |
| 181 |
std::string getLLVMName() const { return LLVMName; } |
| 182 |
std::string getProto() const { return Proto; } |
| 183 |
TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; } |
| 184 |
SVEType getBaseType() const { return BaseType; } |
| 185 |
|
| 186 |
StringRef getGuard() const { return Guard; } |
| 187 |
ClassKind getClassKind() const { return Class; } |
| 188 |
|
| 189 |
SVEType getReturnType() const { return Types[0]; } |
| 190 |
ArrayRef<SVEType> getTypes() const { return Types; } |
| 191 |
SVEType getParamType(unsigned I) const { return Types[I + 1]; } |
| 192 |
unsigned getNumParams() const { return Proto.size() - 1; } |
| 193 |
|
| 194 |
uint64_t getFlags() const { return Flags; } |
| 195 |
bool isFlagSet(uint64_t Flag) const { return Flags & Flag;} |
| 196 |
|
| 197 |
ArrayRef<ImmCheck> getImmChecks() const { return ImmChecks; } |
| 198 |
|
| 199 |
/// Return the type string for a BUILTIN() macro in Builtins.def. |
| 200 |
std::string getBuiltinTypeStr(); |
| 201 |
|
| 202 |
/// Return the name, mangled with type information. The name is mangled for |
| 203 |
/// ClassS, so will add type suffixes such as _u32/_s32. |
| 204 |
std::string getMangledName() const { return mangleName(ClassS); } |
| 205 |
|
| 206 |
/// Returns true if the intrinsic is overloaded, in that it should also generate |
| 207 |
/// a short form without the type-specifiers, e.g. 'svld1(..)' instead of |
| 208 |
/// 'svld1_u32(..)'. |
| 209 |
static bool isOverloadedIntrinsic(StringRef Name) { |
| 210 |
auto BrOpen = Name.find('['); |
| 211 |
auto BrClose = Name.find(']'); |
| 212 |
return BrOpen != std::string::npos && BrClose != std::string::npos; |
| 213 |
} |
| 214 |
|
| 215 |
/// Return true if the intrinsic takes a splat operand. |
| 216 |
bool hasSplat() const { |
| 217 |
// These prototype modifiers are described in arm_sve.td. |
| 218 |
return Proto.find_first_of("ajfrKLR@") != std::string::npos; |
| 219 |
} |
| 220 |
|
| 221 |
/// Return the parameter index of the splat operand. |
| 222 |
unsigned getSplatIdx() const { |
| 223 |
// These prototype modifiers are described in arm_sve.td. |
| 224 |
auto Idx = Proto.find_first_of("ajfrKLR@"); |
| 225 |
assert(Idx != std::string::npos && Idx > 0 && |
| 226 |
"Prototype has no splat operand"); |
| 227 |
return Idx - 1; |
| 228 |
} |
| 229 |
|
| 230 |
/// Emits the intrinsic declaration to the ostream. |
| 231 |
void emitIntrinsic(raw_ostream &OS) const; |
| 232 |
|
| 233 |
private: |
| 234 |
std::string getMergeSuffix() const { return MergeSuffix; } |
| 235 |
std::string mangleName(ClassKind LocalCK) const; |
| 236 |
std::string replaceTemplatedArgs(std::string Name, TypeSpec TS, |
| 237 |
std::string Proto) const; |
| 238 |
}; |
| 239 |
|
| 240 |
class SVEEmitter { |
| 241 |
private: |
| 242 |
// The reinterpret builtins are generated separately because they |
| 243 |
// need the cross product of all types (121 functions in total), |
| 244 |
// which is inconvenient to specify in the arm_sve.td file or |
| 245 |
// generate in CGBuiltin.cpp. |
| 246 |
struct ReinterpretTypeInfo { |
| 247 |
const char *Suffix; |
| 248 |
const char *Type; |
| 249 |
const char *BuiltinType; |
| 250 |
}; |
| 251 |
SmallVector<ReinterpretTypeInfo, 12> Reinterprets = { |
| 252 |
{"s8", "svint8_t", "q16Sc"}, {"s16", "svint16_t", "q8Ss"}, |
| 253 |
{"s32", "svint32_t", "q4Si"}, {"s64", "svint64_t", "q2SWi"}, |
| 254 |
{"u8", "svuint8_t", "q16Uc"}, {"u16", "svuint16_t", "q8Us"}, |
| 255 |
{"u32", "svuint32_t", "q4Ui"}, {"u64", "svuint64_t", "q2UWi"}, |
| 256 |
{"f16", "svfloat16_t", "q8h"}, {"bf16", "svbfloat16_t", "q8y"}, |
| 257 |
{"f32", "svfloat32_t", "q4f"}, {"f64", "svfloat64_t", "q2d"}}; |
| 258 |
|
| 259 |
RecordKeeper &Records; |
| 260 |
llvm::StringMap<uint64_t> EltTypes; |
| 261 |
llvm::StringMap<uint64_t> MemEltTypes; |
| 262 |
llvm::StringMap<uint64_t> FlagTypes; |
| 263 |
llvm::StringMap<uint64_t> MergeTypes; |
| 264 |
llvm::StringMap<uint64_t> ImmCheckTypes; |
| 265 |
|
| 266 |
public: |
| 267 |
SVEEmitter(RecordKeeper &R) : Records(R) { |
| 268 |
for (auto *RV : Records.getAllDerivedDefinitions("EltType")) |
| 269 |
EltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); |
| 270 |
for (auto *RV : Records.getAllDerivedDefinitions("MemEltType")) |
| 271 |
MemEltTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); |
| 272 |
for (auto *RV : Records.getAllDerivedDefinitions("FlagType")) |
| 273 |
FlagTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); |
| 274 |
for (auto *RV : Records.getAllDerivedDefinitions("MergeType")) |
| 275 |
MergeTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); |
| 276 |
for (auto *RV : Records.getAllDerivedDefinitions("ImmCheckType")) |
| 277 |
ImmCheckTypes[RV->getNameInitAsString()] = RV->getValueAsInt("Value"); |
| 278 |
} |
| 279 |
|
| 280 |
/// Returns the enum value for the immcheck type |
| 281 |
unsigned getEnumValueForImmCheck(StringRef C) const { |
| 282 |
auto It = ImmCheckTypes.find(C); |
| 283 |
if (It != ImmCheckTypes.end()) |
| 284 |
return It->getValue(); |
| 285 |
llvm_unreachable("Unsupported imm check"); |
| 286 |
} |
| 287 |
|
| 288 |
/// Returns the enum value for the flag type |
| 289 |
uint64_t getEnumValueForFlag(StringRef C) const { |
| 290 |
auto Res = FlagTypes.find(C); |
| 291 |
if (Res != FlagTypes.end()) |
| 292 |
return Res->getValue(); |
| 293 |
llvm_unreachable("Unsupported flag"); |
| 294 |
} |
| 295 |
|
| 296 |
// Returns the SVETypeFlags for a given value and mask. |
| 297 |
uint64_t encodeFlag(uint64_t V, StringRef MaskName) const { |
| 298 |
auto It = FlagTypes.find(MaskName); |
| 299 |
if (It != FlagTypes.end()) { |
| 300 |
uint64_t Mask = It->getValue(); |
| 301 |
unsigned Shift = llvm::countTrailingZeros(Mask); |
| 302 |
return (V << Shift) & Mask; |
| 303 |
} |
| 304 |
llvm_unreachable("Unsupported flag"); |
| 305 |
} |
| 306 |
|
| 307 |
// Returns the SVETypeFlags for the given element type. |
| 308 |
uint64_t encodeEltType(StringRef EltName) { |
| 309 |
auto It = EltTypes.find(EltName); |
| 310 |
if (It != EltTypes.end()) |
| 311 |
return encodeFlag(It->getValue(), "EltTypeMask"); |
| 312 |
llvm_unreachable("Unsupported EltType"); |
| 313 |
} |
| 314 |
|
| 315 |
// Returns the SVETypeFlags for the given memory element type. |
| 316 |
uint64_t encodeMemoryElementType(uint64_t MT) { |
| 317 |
return encodeFlag(MT, "MemEltTypeMask"); |
| 318 |
} |
| 319 |
|
| 320 |
// Returns the SVETypeFlags for the given merge type. |
| 321 |
uint64_t encodeMergeType(uint64_t MT) { |
| 322 |
return encodeFlag(MT, "MergeTypeMask"); |
| 323 |
} |
| 324 |
|
| 325 |
// Returns the SVETypeFlags for the given splat operand. |
| 326 |
unsigned encodeSplatOperand(unsigned SplatIdx) { |
| 327 |
assert(SplatIdx < 7 && "SplatIdx out of encodable range"); |
| 328 |
return encodeFlag(SplatIdx + 1, "SplatOperandMask"); |
| 329 |
} |
| 330 |
|
| 331 |
// Returns the SVETypeFlags value for the given SVEType. |
| 332 |
uint64_t encodeTypeFlags(const SVEType &T); |
| 333 |
|
| 334 |
/// Emit arm_sve.h. |
| 335 |
void createHeader(raw_ostream &o); |
| 336 |
|
| 337 |
/// Emit all the __builtin prototypes and code needed by Sema. |
| 338 |
void createBuiltins(raw_ostream &o); |
| 339 |
|
| 340 |
/// Emit all the information needed to map builtin -> LLVM IR intrinsic. |
| 341 |
void createCodeGenMap(raw_ostream &o); |
| 342 |
|
| 343 |
/// Emit all the range checks for the immediates. |
| 344 |
void createRangeChecks(raw_ostream &o); |
| 345 |
|
| 346 |
/// Create the SVETypeFlags used in CGBuiltins |
| 347 |
void createTypeFlags(raw_ostream &o); |
| 348 |
|
| 349 |
/// Create intrinsic and add it to \p Out |
| 350 |
void createIntrinsic(Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out); |
| 351 |
}; |
| 352 |
|
| 353 |
} // end anonymous namespace |
| 354 |
|
| 355 |
|
| 356 |
//===----------------------------------------------------------------------===// |
| 357 |
// Type implementation |
| 358 |
//===----------------------------------------------------------------------===// |
| 359 |
|
| 360 |
std::string SVEType::builtin_str() const { |
| 361 |
std::string S; |
| 362 |
if (isVoid()) |
| 363 |
return "v"; |
| 364 |
|
| 365 |
if (isScalarPredicate()) |
| 366 |
return "b"; |
| 367 |
|
| 368 |
if (isVoidPointer()) |
| 369 |
S += "v"; |
| 370 |
else if (!isFloatingPoint()) |
| 371 |
switch (ElementBitwidth) { |
| 372 |
case 1: S += "b"; break; |
| 373 |
case 8: S += "c"; break; |
| 374 |
case 16: S += "s"; break; |
| 375 |
case 32: S += "i"; break; |
| 376 |
case 64: S += "Wi"; break; |
| 377 |
case 128: S += "LLLi"; break; |
| 378 |
default: llvm_unreachable("Unhandled case!"); |
| 379 |
} |
| 380 |
else if (isFloat()) |
| 381 |
switch (ElementBitwidth) { |
| 382 |
case 16: S += "h"; break; |
| 383 |
case 32: S += "f"; break; |
| 384 |
case 64: S += "d"; break; |
| 385 |
default: llvm_unreachable("Unhandled case!"); |
| 386 |
} |
| 387 |
else if (isBFloat()) { |
| 388 |
assert(ElementBitwidth == 16 && "Not a valid BFloat."); |
| 389 |
S += "y"; |
| 390 |
} |
| 391 |
|
| 392 |
if (!isFloatingPoint()) { |
| 393 |
if ((isChar() || isPointer()) && !isVoidPointer()) { |
| 394 |
// Make chars and typed pointers explicitly signed. |
| 395 |
if (Signed) |
| 396 |
S = "S" + S; |
| 397 |
else if (!Signed) |
| 398 |
S = "U" + S; |
| 399 |
} else if (!isVoidPointer() && !Signed) { |
| 400 |
S = "U" + S; |
| 401 |
} |
| 402 |
} |
| 403 |
|
| 404 |
// Constant indices are "int", but have the "constant expression" modifier. |
| 405 |
if (isImmediate()) { |
| 406 |
assert(!isFloat() && "fp immediates are not supported"); |
| 407 |
S = "I" + S; |
| 408 |
} |
| 409 |
|
| 410 |
if (isScalar()) { |
| 411 |
if (Constant) S += "C"; |
| 412 |
if (Pointer) S += "*"; |
| 413 |
return S; |
| 414 |
} |
| 415 |
|
| 416 |
assert(isScalableVector() && "Unsupported type"); |
| 417 |
return "q" + utostr(getNumElements() * NumVectors) + S; |
| 418 |
} |
| 419 |
|
| 420 |
std::string SVEType::str() const { |
| 421 |
if (isPredicatePattern()) |
| 422 |
return "enum svpattern"; |
| 423 |
|
| 424 |
if (isPrefetchOp()) |
| 425 |
return "enum svprfop"; |
| 426 |
|
| 427 |
std::string S; |
| 428 |
if (Void) |
| 429 |
S += "void"; |
| 430 |
else { |
| 431 |
if (isScalableVector()) |
| 432 |
S += "sv"; |
| 433 |
if (!Signed && !isFloatingPoint()) |
| 434 |
S += "u"; |
| 435 |
|
| 436 |
if (Float) |
| 437 |
S += "float"; |
| 438 |
else if (isScalarPredicate() || isPredicateVector()) |
| 439 |
S += "bool"; |
| 440 |
else if (isBFloat()) |
| 441 |
S += "bfloat"; |
| 442 |
else |
| 443 |
S += "int"; |
| 444 |
|
| 445 |
if (!isScalarPredicate() && !isPredicateVector()) |
| 446 |
S += utostr(ElementBitwidth); |
| 447 |
if (!isScalableVector() && isVector()) |
| 448 |
S += "x" + utostr(getNumElements()); |
| 449 |
if (NumVectors > 1) |
| 450 |
S += "x" + utostr(NumVectors); |
| 451 |
if (!isScalarPredicate()) |
| 452 |
S += "_t"; |
| 453 |
} |
| 454 |
|
| 455 |
if (Constant) |
| 456 |
S += " const"; |
| 457 |
if (Pointer) |
| 458 |
S += " *"; |
| 459 |
|
| 460 |
return S; |
| 461 |
} |
| 462 |
void SVEType::applyTypespec() { |
| 463 |
for (char I : TS) { |
| 464 |
switch (I) { |
| 465 |
case 'P': |
| 466 |
Predicate = true; |
| 467 |
break; |
| 468 |
case 'U': |
| 469 |
Signed = false; |
| 470 |
break; |
| 471 |
case 'c': |
| 472 |
ElementBitwidth = 8; |
| 473 |
break; |
| 474 |
case 's': |
| 475 |
ElementBitwidth = 16; |
| 476 |
break; |
| 477 |
case 'i': |
| 478 |
ElementBitwidth = 32; |
| 479 |
break; |
| 480 |
case 'l': |
| 481 |
ElementBitwidth = 64; |
| 482 |
break; |
| 483 |
case 'h': |
| 484 |
Float = true; |
| 485 |
ElementBitwidth = 16; |
| 486 |
break; |
| 487 |
case 'f': |
| 488 |
Float = true; |
| 489 |
ElementBitwidth = 32; |
| 490 |
break; |
| 491 |
case 'd': |
| 492 |
Float = true; |
| 493 |
ElementBitwidth = 64; |
| 494 |
break; |
| 495 |
case 'b': |
| 496 |
BFloat = true; |
| 497 |
Float = false; |
| 498 |
ElementBitwidth = 16; |
| 499 |
break; |
| 500 |
default: |
| 501 |
llvm_unreachable("Unhandled type code!"); |
| 502 |
} |
| 503 |
} |
| 504 |
assert(ElementBitwidth != ~0U && "Bad element bitwidth!"); |
| 505 |
} |
| 506 |
|
| 507 |
void SVEType::applyModifier(char Mod) { |
| 508 |
switch (Mod) { |
| 509 |
case '2': |
| 510 |
NumVectors = 2; |
| 511 |
break; |
| 512 |
case '3': |
| 513 |
NumVectors = 3; |
| 514 |
break; |
| 515 |
case '4': |
| 516 |
NumVectors = 4; |
| 517 |
break; |
| 518 |
case 'v': |
| 519 |
Void = true; |
| 520 |
break; |
| 521 |
case 'd': |
| 522 |
DefaultType = true; |
| 523 |
break; |
| 524 |
case 'c': |
| 525 |
Constant = true; |
| 526 |
[[fallthrough]]; |
| 527 |
case 'p': |
| 528 |
Pointer = true; |
| 529 |
Bitwidth = ElementBitwidth; |
| 530 |
NumVectors = 0; |
| 531 |
break; |
| 532 |
case 'e': |
| 533 |
Signed = false; |
| 534 |
ElementBitwidth /= 2; |
| 535 |
break; |
| 536 |
case 'h': |
| 537 |
ElementBitwidth /= 2; |
| 538 |
break; |
| 539 |
case 'q': |
| 540 |
ElementBitwidth /= 4; |
| 541 |
break; |
| 542 |
case 'b': |
| 543 |
Signed = false; |
| 544 |
Float = false; |
| 545 |
BFloat = false; |
| 546 |
ElementBitwidth /= 4; |
| 547 |
break; |
| 548 |
case 'o': |
| 549 |
ElementBitwidth *= 4; |
| 550 |
break; |
| 551 |
case 'P': |
| 552 |
Signed = true; |
| 553 |
Float = false; |
| 554 |
BFloat = false; |
| 555 |
Predicate = true; |
| 556 |
Bitwidth = 16; |
| 557 |
ElementBitwidth = 1; |
| 558 |
break; |
| 559 |
case 's': |
| 560 |
case 'a': |
| 561 |
Bitwidth = ElementBitwidth; |
| 562 |
NumVectors = 0; |
| 563 |
break; |
| 564 |
case 'R': |
| 565 |
ElementBitwidth /= 2; |
| 566 |
NumVectors = 0; |
| 567 |
break; |
| 568 |
case 'r': |
| 569 |
ElementBitwidth /= 4; |
| 570 |
NumVectors = 0; |
| 571 |
break; |
| 572 |
case '@': |
| 573 |
Signed = false; |
| 574 |
Float = false; |
| 575 |
BFloat = false; |
| 576 |
ElementBitwidth /= 4; |
| 577 |
NumVectors = 0; |
| 578 |
break; |
| 579 |
case 'K': |
| 580 |
Signed = true; |
| 581 |
Float = false; |
| 582 |
BFloat = false; |
| 583 |
Bitwidth = ElementBitwidth; |
| 584 |
NumVectors = 0; |
| 585 |
break; |
| 586 |
case 'L': |
| 587 |
Signed = false; |
| 588 |
Float = false; |
| 589 |
BFloat = false; |
| 590 |
Bitwidth = ElementBitwidth; |
| 591 |
NumVectors = 0; |
| 592 |
break; |
| 593 |
case 'u': |
| 594 |
Predicate = false; |
| 595 |
Signed = false; |
| 596 |
Float = false; |
| 597 |
BFloat = false; |
| 598 |
break; |
| 599 |
case 'x': |
| 600 |
Predicate = false; |
| 601 |
Signed = true; |
| 602 |
Float = false; |
| 603 |
BFloat = false; |
| 604 |
break; |
| 605 |
case 'i': |
| 606 |
Predicate = false; |
| 607 |
Float = false; |
| 608 |
BFloat = false; |
| 609 |
ElementBitwidth = Bitwidth = 64; |
| 610 |
NumVectors = 0; |
| 611 |
Signed = false; |
| 612 |
Immediate = true; |
| 613 |
break; |
| 614 |
case 'I': |
| 615 |
Predicate = false; |
| 616 |
Float = false; |
| 617 |
BFloat = false; |
| 618 |
ElementBitwidth = Bitwidth = 32; |
| 619 |
NumVectors = 0; |
| 620 |
Signed = true; |
| 621 |
Immediate = true; |
| 622 |
PredicatePattern = true; |
| 623 |
break; |
| 624 |
case 'J': |
| 625 |
Predicate = false; |
| 626 |
Float = false; |
| 627 |
BFloat = false; |
| 628 |
ElementBitwidth = Bitwidth = 32; |
| 629 |
NumVectors = 0; |
| 630 |
Signed = true; |
| 631 |
Immediate = true; |
| 632 |
PrefetchOp = true; |
| 633 |
break; |
| 634 |
case 'k': |
| 635 |
Predicate = false; |
| 636 |
Signed = true; |
| 637 |
Float = false; |
| 638 |
BFloat = false; |
| 639 |
ElementBitwidth = Bitwidth = 32; |
| 640 |
NumVectors = 0; |
| 641 |
break; |
| 642 |
case 'l': |
| 643 |
Predicate = false; |
| 644 |
Signed = true; |
| 645 |
Float = false; |
| 646 |
BFloat = false; |
| 647 |
ElementBitwidth = Bitwidth = 64; |
| 648 |
NumVectors = 0; |
| 649 |
break; |
| 650 |
case 'm': |
| 651 |
Predicate = false; |
| 652 |
Signed = false; |
| 653 |
Float = false; |
| 654 |
BFloat = false; |
| 655 |
ElementBitwidth = Bitwidth = 32; |
| 656 |
NumVectors = 0; |
| 657 |
break; |
| 658 |
case 'n': |
| 659 |
Predicate = false; |
| 660 |
Signed = false; |
| 661 |
Float = false; |
| 662 |
BFloat = false; |
| 663 |
ElementBitwidth = Bitwidth = 64; |
| 664 |
NumVectors = 0; |
| 665 |
break; |
| 666 |
case 'w': |
| 667 |
ElementBitwidth = 64; |
| 668 |
break; |
| 669 |
case 'j': |
| 670 |
ElementBitwidth = Bitwidth = 64; |
| 671 |
NumVectors = 0; |
| 672 |
break; |
| 673 |
case 'f': |
| 674 |
Signed = false; |
| 675 |
ElementBitwidth = Bitwidth = 64; |
| 676 |
NumVectors = 0; |
| 677 |
break; |
| 678 |
case 'g': |
| 679 |
Signed = false; |
| 680 |
Float = false; |
| 681 |
BFloat = false; |
| 682 |
ElementBitwidth = 64; |
| 683 |
break; |
| 684 |
case 't': |
| 685 |
Signed = true; |
| 686 |
Float = false; |
| 687 |
BFloat = false; |
| 688 |
ElementBitwidth = 32; |
| 689 |
break; |
| 690 |
case 'z': |
| 691 |
Signed = false; |
| 692 |
Float = false; |
| 693 |
BFloat = false; |
| 694 |
ElementBitwidth = 32; |
| 695 |
break; |
| 696 |
case 'O': |
| 697 |
Predicate = false; |
| 698 |
Float = true; |
| 699 |
ElementBitwidth = 16; |
| 700 |
break; |
| 701 |
case 'M': |
| 702 |
Predicate = false; |
| 703 |
Float = true; |
| 704 |
BFloat = false; |
| 705 |
ElementBitwidth = 32; |
| 706 |
break; |
| 707 |
case 'N': |
| 708 |
Predicate = false; |
| 709 |
Float = true; |
| 710 |
ElementBitwidth = 64; |
| 711 |
break; |
| 712 |
case 'Q': |
| 713 |
Constant = true; |
| 714 |
Pointer = true; |
| 715 |
Void = true; |
| 716 |
NumVectors = 0; |
| 717 |
break; |
| 718 |
case 'S': |
| 719 |
Constant = true; |
| 720 |
Pointer = true; |
| 721 |
ElementBitwidth = Bitwidth = 8; |
| 722 |
NumVectors = 0; |
| 723 |
Signed = true; |
| 724 |
break; |
| 725 |
case 'W': |
| 726 |
Constant = true; |
| 727 |
Pointer = true; |
| 728 |
ElementBitwidth = Bitwidth = 8; |
| 729 |
NumVectors = 0; |
| 730 |
Signed = false; |
| 731 |
break; |
| 732 |
case 'T': |
| 733 |
Constant = true; |
| 734 |
Pointer = true; |
| 735 |
ElementBitwidth = Bitwidth = 16; |
| 736 |
NumVectors = 0; |
| 737 |
Signed = true; |
| 738 |
break; |
| 739 |
case 'X': |
| 740 |
Constant = true; |
| 741 |
Pointer = true; |
| 742 |
ElementBitwidth = Bitwidth = 16; |
| 743 |
NumVectors = 0; |
| 744 |
Signed = false; |
| 745 |
break; |
| 746 |
case 'Y': |
| 747 |
Constant = true; |
| 748 |
Pointer = true; |
| 749 |
ElementBitwidth = Bitwidth = 32; |
| 750 |
NumVectors = 0; |
| 751 |
Signed = false; |
| 752 |
break; |
| 753 |
case 'U': |
| 754 |
Constant = true; |
| 755 |
Pointer = true; |
| 756 |
ElementBitwidth = Bitwidth = 32; |
| 757 |
NumVectors = 0; |
| 758 |
Signed = true; |
| 759 |
break; |
| 760 |
case 'A': |
| 761 |
Pointer = true; |
| 762 |
ElementBitwidth = Bitwidth = 8; |
| 763 |
NumVectors = 0; |
| 764 |
Signed = true; |
| 765 |
break; |
| 766 |
case 'B': |
| 767 |
Pointer = true; |
| 768 |
ElementBitwidth = Bitwidth = 16; |
| 769 |
NumVectors = 0; |
| 770 |
Signed = true; |
| 771 |
break; |
| 772 |
case 'C': |
| 773 |
Pointer = true; |
| 774 |
ElementBitwidth = Bitwidth = 32; |
| 775 |
NumVectors = 0; |
| 776 |
Signed = true; |
| 777 |
break; |
| 778 |
case 'D': |
| 779 |
Pointer = true; |
| 780 |
ElementBitwidth = Bitwidth = 64; |
| 781 |
NumVectors = 0; |
| 782 |
Signed = true; |
| 783 |
break; |
| 784 |
case 'E': |
| 785 |
Pointer = true; |
| 786 |
ElementBitwidth = Bitwidth = 8; |
| 787 |
NumVectors = 0; |
| 788 |
Signed = false; |
| 789 |
break; |
| 790 |
case 'F': |
| 791 |
Pointer = true; |
| 792 |
ElementBitwidth = Bitwidth = 16; |
| 793 |
NumVectors = 0; |
| 794 |
Signed = false; |
| 795 |
break; |
| 796 |
case 'G': |
| 797 |
Pointer = true; |
| 798 |
ElementBitwidth = Bitwidth = 32; |
| 799 |
NumVectors = 0; |
| 800 |
Signed = false; |
| 801 |
break; |
| 802 |
default: |
| 803 |
llvm_unreachable("Unhandled character!"); |
| 804 |
} |
| 805 |
} |
| 806 |
|
| 807 |
|
| 808 |
//===----------------------------------------------------------------------===// |
| 809 |
// Intrinsic implementation |
| 810 |
//===----------------------------------------------------------------------===// |
| 811 |
|
| 812 |
Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy, |
| 813 |
StringRef MergeSuffix, uint64_t MemoryElementTy, |
| 814 |
StringRef LLVMName, uint64_t Flags, |
| 815 |
ArrayRef<ImmCheck> Checks, TypeSpec BT, ClassKind Class, |
| 816 |
SVEEmitter &Emitter, StringRef Guard) |
| 817 |
: Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()), |
| 818 |
BaseTypeSpec(BT), Class(Class), Guard(Guard.str()), |
| 819 |
MergeSuffix(MergeSuffix.str()), BaseType(BT, 'd'), Flags(Flags), |
| 820 |
ImmChecks(Checks.begin(), Checks.end()) { |
| 821 |
// Types[0] is the return value. |
| 822 |
for (unsigned I = 0; I < Proto.size(); ++I) { |
| 823 |
SVEType T(BaseTypeSpec, Proto[I]); |
| 824 |
Types.push_back(T); |
| 825 |
|
| 826 |
// Add range checks for immediates |
| 827 |
if (I > 0) { |
| 828 |
if (T.isPredicatePattern()) |
| 829 |
ImmChecks.emplace_back( |
| 830 |
I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31")); |
| 831 |
else if (T.isPrefetchOp()) |
| 832 |
ImmChecks.emplace_back( |
| 833 |
I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13")); |
| 834 |
} |
| 835 |
} |
| 836 |
|
| 837 |
// Set flags based on properties |
| 838 |
this->Flags |= Emitter.encodeTypeFlags(BaseType); |
| 839 |
this->Flags |= Emitter.encodeMemoryElementType(MemoryElementTy); |
| 840 |
this->Flags |= Emitter.encodeMergeType(MergeTy); |
| 841 |
if (hasSplat()) |
| 842 |
this->Flags |= Emitter.encodeSplatOperand(getSplatIdx()); |
| 843 |
} |
| 844 |
|
| 845 |
std::string Intrinsic::getBuiltinTypeStr() { |
| 846 |
std::string S = getReturnType().builtin_str(); |
| 847 |
for (unsigned I = 0; I < getNumParams(); ++I) |
| 848 |
S += getParamType(I).builtin_str(); |
| 849 |
|
| 850 |
return S; |
| 851 |
} |
| 852 |
|
| 853 |
std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS, |
| 854 |
std::string Proto) const { |
| 855 |
std::string Ret = Name; |
| 856 |
while (Ret.find('{') != std::string::npos) { |
| 857 |
size_t Pos = Ret.find('{'); |
| 858 |
size_t End = Ret.find('}'); |
| 859 |
unsigned NumChars = End - Pos + 1; |
| 860 |
assert(NumChars == 3 && "Unexpected template argument"); |
| 861 |
|
| 862 |
SVEType T; |
| 863 |
char C = Ret[Pos+1]; |
| 864 |
switch(C) { |
| 865 |
default: |
| 866 |
llvm_unreachable("Unknown predication specifier"); |
| 867 |
case 'd': |
| 868 |
T = SVEType(TS, 'd'); |
| 869 |
break; |
| 870 |
case '0': |
| 871 |
case '1': |
| 872 |
case '2': |
| 873 |
case '3': |
| 874 |
T = SVEType(TS, Proto[C - '0']); |
| 875 |
break; |
| 876 |
} |
| 877 |
|
| 878 |
// Replace templated arg with the right suffix (e.g. u32) |
| 879 |
std::string TypeCode; |
| 880 |
if (T.isInteger()) |
| 881 |
TypeCode = T.isSigned() ? 's' : 'u'; |
| 882 |
else if (T.isPredicateVector()) |
| 883 |
TypeCode = 'b'; |
| 884 |
else if (T.isBFloat()) |
| 885 |
TypeCode = "bf"; |
| 886 |
else |
| 887 |
TypeCode = 'f'; |
| 888 |
Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits())); |
| 889 |
} |
| 890 |
|
| 891 |
return Ret; |
| 892 |
} |
| 893 |
|
| 894 |
std::string Intrinsic::mangleName(ClassKind LocalCK) const { |
| 895 |
std::string S = getName(); |
| 896 |
|
| 897 |
if (LocalCK == ClassG) { |
| 898 |
// Remove the square brackets and everything in between. |
| 899 |
while (S.find('[') != std::string::npos) { |
| 900 |
auto Start = S.find('['); |
| 901 |
auto End = S.find(']'); |
| 902 |
S.erase(Start, (End-Start)+1); |
| 903 |
} |
| 904 |
} else { |
| 905 |
// Remove the square brackets. |
| 906 |
while (S.find('[') != std::string::npos) { |
| 907 |
auto BrPos = S.find('['); |
| 908 |
if (BrPos != std::string::npos) |
| 909 |
S.erase(BrPos, 1); |
| 910 |
BrPos = S.find(']'); |
| 911 |
if (BrPos != std::string::npos) |
| 912 |
S.erase(BrPos, 1); |
| 913 |
} |
| 914 |
} |
| 915 |
|
| 916 |
// Replace all {d} like expressions with e.g. 'u32' |
| 917 |
return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) + |
| 918 |
getMergeSuffix(); |
| 919 |
} |
| 920 |
|
| 921 |
void Intrinsic::emitIntrinsic(raw_ostream &OS) const { |
| 922 |
bool IsOverloaded = getClassKind() == ClassG && getProto().size() > 1; |
| 923 |
|
| 924 |
std::string FullName = mangleName(ClassS); |
| 925 |
std::string ProtoName = mangleName(getClassKind()); |
| 926 |
|
| 927 |
OS << (IsOverloaded ? "__aio " : "__ai ") |
| 928 |
<< "__attribute__((__clang_arm_builtin_alias(" |
| 929 |
<< "__builtin_sve_" << FullName << ")))\n"; |
| 930 |
|
| 931 |
OS << getTypes()[0].str() << " " << ProtoName << "("; |
| 932 |
for (unsigned I = 0; I < getTypes().size() - 1; ++I) { |
| 933 |
if (I != 0) |
| 934 |
OS << ", "; |
| 935 |
OS << getTypes()[I + 1].str(); |
| 936 |
} |
| 937 |
OS << ");\n"; |
| 938 |
} |
| 939 |
|
| 940 |
//===----------------------------------------------------------------------===// |
| 941 |
// SVEEmitter implementation |
| 942 |
//===----------------------------------------------------------------------===// |
| 943 |
uint64_t SVEEmitter::encodeTypeFlags(const SVEType &T) { |
| 944 |
if (T.isFloat()) { |
| 945 |
switch (T.getElementSizeInBits()) { |
| 946 |
case 16: |
| 947 |
return encodeEltType("EltTyFloat16"); |
| 948 |
case 32: |
| 949 |
return encodeEltType("EltTyFloat32"); |
| 950 |
case 64: |
| 951 |
return encodeEltType("EltTyFloat64"); |
| 952 |
default: |
| 953 |
llvm_unreachable("Unhandled float element bitwidth!"); |
| 954 |
} |
| 955 |
} |
| 956 |
|
| 957 |
if (T.isBFloat()) { |
| 958 |
assert(T.getElementSizeInBits() == 16 && "Not a valid BFloat."); |
| 959 |
return encodeEltType("EltTyBFloat16"); |
| 960 |
} |
| 961 |
|
| 962 |
if (T.isPredicateVector()) { |
| 963 |
switch (T.getElementSizeInBits()) { |
| 964 |
case 8: |
| 965 |
return encodeEltType("EltTyBool8"); |
| 966 |
case 16: |
| 967 |
return encodeEltType("EltTyBool16"); |
| 968 |
case 32: |
| 969 |
return encodeEltType("EltTyBool32"); |
| 970 |
case 64: |
| 971 |
return encodeEltType("EltTyBool64"); |
| 972 |
default: |
| 973 |
llvm_unreachable("Unhandled predicate element bitwidth!"); |
| 974 |
} |
| 975 |
} |
| 976 |
|
| 977 |
switch (T.getElementSizeInBits()) { |
| 978 |
case 8: |
| 979 |
return encodeEltType("EltTyInt8"); |
| 980 |
case 16: |
| 981 |
return encodeEltType("EltTyInt16"); |
| 982 |
case 32: |
| 983 |
return encodeEltType("EltTyInt32"); |
| 984 |
case 64: |
| 985 |
return encodeEltType("EltTyInt64"); |
| 986 |
default: |
| 987 |
llvm_unreachable("Unhandled integer element bitwidth!"); |
| 988 |
} |
| 989 |
} |
| 990 |
|
| 991 |
void SVEEmitter::createIntrinsic( |
| 992 |
Record *R, SmallVectorImpl<std::unique_ptr<Intrinsic>> &Out) { |
| 993 |
StringRef Name = R->getValueAsString("Name"); |
| 994 |
StringRef Proto = R->getValueAsString("Prototype"); |
| 995 |
StringRef Types = R->getValueAsString("Types"); |
| 996 |
StringRef Guard = R->getValueAsString("TargetGuard"); |
| 997 |
StringRef LLVMName = R->getValueAsString("LLVMIntrinsic"); |
| 998 |
uint64_t Merge = R->getValueAsInt("Merge"); |
| 999 |
StringRef MergeSuffix = R->getValueAsString("MergeSuffix"); |
| 1000 |
uint64_t MemEltType = R->getValueAsInt("MemEltType"); |
| 1001 |
std::vector<Record*> FlagsList = R->getValueAsListOfDefs("Flags"); |
| 1002 |
std::vector<Record*> ImmCheckList = R->getValueAsListOfDefs("ImmChecks"); |
| 1003 |
|
| 1004 |
int64_t Flags = 0; |
| 1005 |
for (auto FlagRec : FlagsList) |
| 1006 |
Flags |= FlagRec->getValueAsInt("Value"); |
| 1007 |
|
| 1008 |
// Create a dummy TypeSpec for non-overloaded builtins. |
| 1009 |
if (Types.empty()) { |
| 1010 |
assert((Flags & getEnumValueForFlag("IsOverloadNone")) && |
| 1011 |
"Expect TypeSpec for overloaded builtin!"); |
| 1012 |
Types = "i"; |
| 1013 |
} |
| 1014 |
|
| 1015 |
// Extract type specs from string |
| 1016 |
SmallVector<TypeSpec, 8> TypeSpecs; |
| 1017 |
TypeSpec Acc; |
| 1018 |
for (char I : Types) { |
| 1019 |
Acc.push_back(I); |
| 1020 |
if (islower(I)) { |
| 1021 |
TypeSpecs.push_back(TypeSpec(Acc)); |
| 1022 |
Acc.clear(); |
| 1023 |
} |
| 1024 |
} |
| 1025 |
|
| 1026 |
// Remove duplicate type specs. |
| 1027 |
llvm::sort(TypeSpecs); |
| 1028 |
TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()), |
| 1029 |
TypeSpecs.end()); |
| 1030 |
|
| 1031 |
// Create an Intrinsic for each type spec. |
| 1032 |
for (auto TS : TypeSpecs) { |
| 1033 |
// Collate a list of range/option checks for the immediates. |
| 1034 |
SmallVector<ImmCheck, 2> ImmChecks; |
| 1035 |
for (auto *R : ImmCheckList) { |
| 1036 |
int64_t Arg = R->getValueAsInt("Arg"); |
| 1037 |
int64_t EltSizeArg = R->getValueAsInt("EltSizeArg"); |
| 1038 |
int64_t Kind = R->getValueAsDef("Kind")->getValueAsInt("Value"); |
| 1039 |
assert(Arg >= 0 && Kind >= 0 && "Arg and Kind must be nonnegative"); |
| 1040 |
|
| 1041 |
unsigned ElementSizeInBits = 0; |
| 1042 |
if (EltSizeArg >= 0) |
| 1043 |
ElementSizeInBits = |
| 1044 |
SVEType(TS, Proto[EltSizeArg + /* offset by return arg */ 1]) |
| 1045 |
.getElementSizeInBits(); |
| 1046 |
ImmChecks.push_back(ImmCheck(Arg, Kind, ElementSizeInBits)); |
| 1047 |
} |
| 1048 |
|
| 1049 |
Out.push_back(std::make_unique<Intrinsic>( |
| 1050 |
Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, ImmChecks, |
| 1051 |
TS, ClassS, *this, Guard)); |
| 1052 |
|
| 1053 |
// Also generate the short-form (e.g. svadd_m) for the given type-spec. |
| 1054 |
if (Intrinsic::isOverloadedIntrinsic(Name)) |
| 1055 |
Out.push_back(std::make_unique<Intrinsic>( |
| 1056 |
Name, Proto, Merge, MergeSuffix, MemEltType, LLVMName, Flags, |
| 1057 |
ImmChecks, TS, ClassG, *this, Guard)); |
| 1058 |
} |
| 1059 |
} |
| 1060 |
|
| 1061 |
void SVEEmitter::createHeader(raw_ostream &OS) { |
| 1062 |
OS << "/*===---- arm_sve.h - ARM SVE intrinsics " |
| 1063 |
"-----------------------------------===\n" |
| 1064 |
" *\n" |
| 1065 |
" *\n" |
| 1066 |
" * Part of the LLVM Project, under the Apache License v2.0 with LLVM " |
| 1067 |
"Exceptions.\n" |
| 1068 |
" * See https://llvm.org/LICENSE.txt for license information.\n" |
| 1069 |
" * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n" |
| 1070 |
" *\n" |
| 1071 |
" *===-----------------------------------------------------------------" |
| 1072 |
"------===\n" |
| 1073 |
" */\n\n"; |
| 1074 |
|
| 1075 |
OS << "#ifndef __ARM_SVE_H\n"; |
| 1076 |
OS << "#define __ARM_SVE_H\n\n"; |
| 1077 |
|
| 1078 |
OS << "#if !defined(__LITTLE_ENDIAN__)\n"; |
| 1079 |
OS << "#error \"Big endian is currently not supported for arm_sve.h\"\n"; |
| 1080 |
OS << "#endif\n"; |
| 1081 |
|
| 1082 |
OS << "#include <stdint.h>\n\n"; |
| 1083 |
OS << "#ifdef __cplusplus\n"; |
| 1084 |
OS << "extern \"C\" {\n"; |
| 1085 |
OS << "#else\n"; |
| 1086 |
OS << "#include <stdbool.h>\n"; |
| 1087 |
OS << "#endif\n\n"; |
| 1088 |
|
| 1089 |
OS << "typedef __fp16 float16_t;\n"; |
| 1090 |
OS << "typedef float float32_t;\n"; |
| 1091 |
OS << "typedef double float64_t;\n"; |
| 1092 |
|
| 1093 |
OS << "typedef __SVInt8_t svint8_t;\n"; |
| 1094 |
OS << "typedef __SVInt16_t svint16_t;\n"; |
| 1095 |
OS << "typedef __SVInt32_t svint32_t;\n"; |
| 1096 |
OS << "typedef __SVInt64_t svint64_t;\n"; |
| 1097 |
OS << "typedef __SVUint8_t svuint8_t;\n"; |
| 1098 |
OS << "typedef __SVUint16_t svuint16_t;\n"; |
| 1099 |
OS << "typedef __SVUint32_t svuint32_t;\n"; |
| 1100 |
OS << "typedef __SVUint64_t svuint64_t;\n"; |
| 1101 |
OS << "typedef __SVFloat16_t svfloat16_t;\n\n"; |
| 1102 |
|
| 1103 |
OS << "typedef __SVBFloat16_t svbfloat16_t;\n"; |
| 1104 |
|
| 1105 |
OS << "#include <arm_bf16.h>\n"; |
| 1106 |
OS << "typedef __bf16 bfloat16_t;\n"; |
| 1107 |
|
| 1108 |
OS << "typedef __SVFloat32_t svfloat32_t;\n"; |
| 1109 |
OS << "typedef __SVFloat64_t svfloat64_t;\n"; |
| 1110 |
OS << "typedef __clang_svint8x2_t svint8x2_t;\n"; |
| 1111 |
OS << "typedef __clang_svint16x2_t svint16x2_t;\n"; |
| 1112 |
OS << "typedef __clang_svint32x2_t svint32x2_t;\n"; |
| 1113 |
OS << "typedef __clang_svint64x2_t svint64x2_t;\n"; |
| 1114 |
OS << "typedef __clang_svuint8x2_t svuint8x2_t;\n"; |
| 1115 |
OS << "typedef __clang_svuint16x2_t svuint16x2_t;\n"; |
| 1116 |
OS << "typedef __clang_svuint32x2_t svuint32x2_t;\n"; |
| 1117 |
OS << "typedef __clang_svuint64x2_t svuint64x2_t;\n"; |
| 1118 |
OS << "typedef __clang_svfloat16x2_t svfloat16x2_t;\n"; |
| 1119 |
OS << "typedef __clang_svfloat32x2_t svfloat32x2_t;\n"; |
| 1120 |
OS << "typedef __clang_svfloat64x2_t svfloat64x2_t;\n"; |
| 1121 |
OS << "typedef __clang_svint8x3_t svint8x3_t;\n"; |
| 1122 |
OS << "typedef __clang_svint16x3_t svint16x3_t;\n"; |
| 1123 |
OS << "typedef __clang_svint32x3_t svint32x3_t;\n"; |
| 1124 |
OS << "typedef __clang_svint64x3_t svint64x3_t;\n"; |
| 1125 |
OS << "typedef __clang_svuint8x3_t svuint8x3_t;\n"; |
| 1126 |
OS << "typedef __clang_svuint16x3_t svuint16x3_t;\n"; |
| 1127 |
OS << "typedef __clang_svuint32x3_t svuint32x3_t;\n"; |
| 1128 |
OS << "typedef __clang_svuint64x3_t svuint64x3_t;\n"; |
| 1129 |
OS << "typedef __clang_svfloat16x3_t svfloat16x3_t;\n"; |
| 1130 |
OS << "typedef __clang_svfloat32x3_t svfloat32x3_t;\n"; |
| 1131 |
OS << "typedef __clang_svfloat64x3_t svfloat64x3_t;\n"; |
| 1132 |
OS << "typedef __clang_svint8x4_t svint8x4_t;\n"; |
| 1133 |
OS << "typedef __clang_svint16x4_t svint16x4_t;\n"; |
| 1134 |
OS << "typedef __clang_svint32x4_t svint32x4_t;\n"; |
| 1135 |
OS << "typedef __clang_svint64x4_t svint64x4_t;\n"; |
| 1136 |
OS << "typedef __clang_svuint8x4_t svuint8x4_t;\n"; |
| 1137 |
OS << "typedef __clang_svuint16x4_t svuint16x4_t;\n"; |
| 1138 |
OS << "typedef __clang_svuint32x4_t svuint32x4_t;\n"; |
| 1139 |
OS << "typedef __clang_svuint64x4_t svuint64x4_t;\n"; |
| 1140 |
OS << "typedef __clang_svfloat16x4_t svfloat16x4_t;\n"; |
| 1141 |
OS << "typedef __clang_svfloat32x4_t svfloat32x4_t;\n"; |
| 1142 |
OS << "typedef __clang_svfloat64x4_t svfloat64x4_t;\n"; |
| 1143 |
OS << "typedef __SVBool_t svbool_t;\n\n"; |
| 1144 |
|
| 1145 |
OS << "typedef __clang_svbfloat16x2_t svbfloat16x2_t;\n"; |
| 1146 |
OS << "typedef __clang_svbfloat16x3_t svbfloat16x3_t;\n"; |
| 1147 |
OS << "typedef __clang_svbfloat16x4_t svbfloat16x4_t;\n"; |
| 1148 |
|
| 1149 |
OS << "enum svpattern\n"; |
| 1150 |
OS << "{\n"; |
| 1151 |
OS << " SV_POW2 = 0,\n"; |
| 1152 |
OS << " SV_VL1 = 1,\n"; |
| 1153 |
OS << " SV_VL2 = 2,\n"; |
| 1154 |
OS << " SV_VL3 = 3,\n"; |
| 1155 |
OS << " SV_VL4 = 4,\n"; |
| 1156 |
OS << " SV_VL5 = 5,\n"; |
| 1157 |
OS << " SV_VL6 = 6,\n"; |
| 1158 |
OS << " SV_VL7 = 7,\n"; |
| 1159 |
OS << " SV_VL8 = 8,\n"; |
| 1160 |
OS << " SV_VL16 = 9,\n"; |
| 1161 |
OS << " SV_VL32 = 10,\n"; |
| 1162 |
OS << " SV_VL64 = 11,\n"; |
| 1163 |
OS << " SV_VL128 = 12,\n"; |
| 1164 |
OS << " SV_VL256 = 13,\n"; |
| 1165 |
OS << " SV_MUL4 = 29,\n"; |
| 1166 |
OS << " SV_MUL3 = 30,\n"; |
| 1167 |
OS << " SV_ALL = 31\n"; |
| 1168 |
OS << "};\n\n"; |
| 1169 |
|
| 1170 |
OS << "enum svprfop\n"; |
| 1171 |
OS << "{\n"; |
| 1172 |
OS << " SV_PLDL1KEEP = 0,\n"; |
| 1173 |
OS << " SV_PLDL1STRM = 1,\n"; |
| 1174 |
OS << " SV_PLDL2KEEP = 2,\n"; |
| 1175 |
OS << " SV_PLDL2STRM = 3,\n"; |
| 1176 |
OS << " SV_PLDL3KEEP = 4,\n"; |
| 1177 |
OS << " SV_PLDL3STRM = 5,\n"; |
| 1178 |
OS << " SV_PSTL1KEEP = 8,\n"; |
| 1179 |
OS << " SV_PSTL1STRM = 9,\n"; |
| 1180 |
OS << " SV_PSTL2KEEP = 10,\n"; |
| 1181 |
OS << " SV_PSTL2STRM = 11,\n"; |
| 1182 |
OS << " SV_PSTL3KEEP = 12,\n"; |
| 1183 |
OS << " SV_PSTL3STRM = 13\n"; |
| 1184 |
OS << "};\n\n"; |
| 1185 |
|
| 1186 |
OS << "/* Function attributes */\n"; |
| 1187 |
OS << "#define __ai static __inline__ __attribute__((__always_inline__, " |
| 1188 |
"__nodebug__))\n\n"; |
| 1189 |
OS << "#define __aio static __inline__ __attribute__((__always_inline__, " |
| 1190 |
"__nodebug__, __overloadable__))\n\n"; |
| 1191 |
|
| 1192 |
// Add reinterpret functions. |
| 1193 |
for (auto ShortForm : { false, true } ) |
| 1194 |
for (const ReinterpretTypeInfo &From : Reinterprets) |
| 1195 |
for (const ReinterpretTypeInfo &To : Reinterprets) { |
| 1196 |
if (ShortForm) { |
| 1197 |
OS << "__aio __attribute__((target(\"sve\"))) " << From.Type |
| 1198 |
<< " svreinterpret_" << From.Suffix; |
| 1199 |
OS << "(" << To.Type << " op) {\n"; |
| 1200 |
OS << " return __builtin_sve_reinterpret_" << From.Suffix << "_" |
| 1201 |
<< To.Suffix << "(op);\n"; |
| 1202 |
OS << "}\n\n"; |
| 1203 |
} else |
| 1204 |
OS << "#define svreinterpret_" << From.Suffix << "_" << To.Suffix |
| 1205 |
<< "(...) __builtin_sve_reinterpret_" << From.Suffix << "_" |
| 1206 |
<< To.Suffix << "(__VA_ARGS__)\n"; |
| 1207 |
} |
| 1208 |
|
| 1209 |
SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; |
| 1210 |
std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| 1211 |
for (auto *R : RV) |
| 1212 |
createIntrinsic(R, Defs); |
| 1213 |
|
| 1214 |
// Sort intrinsics in header file by following order/priority: |
| 1215 |
// - Architectural guard (i.e. does it require SVE2 or SVE2_AES) |
| 1216 |
// - Class (is intrinsic overloaded or not) |
| 1217 |
// - Intrinsic name |
| 1218 |
std::stable_sort( |
| 1219 |
Defs.begin(), Defs.end(), [](const std::unique_ptr<Intrinsic> &A, |
| 1220 |
const std::unique_ptr<Intrinsic> &B) { |
| 1221 |
auto ToTuple = [](const std::unique_ptr<Intrinsic> &I) { |
| 1222 |
return std::make_tuple(I->getGuard(), (unsigned)I->getClassKind(), I->getName()); |
| 1223 |
}; |
| 1224 |
return ToTuple(A) < ToTuple(B); |
| 1225 |
}); |
| 1226 |
|
| 1227 |
// Actually emit the intrinsic declarations. |
| 1228 |
for (auto &I : Defs) |
| 1229 |
I->emitIntrinsic(OS); |
| 1230 |
|
| 1231 |
OS << "#define svcvtnt_bf16_x svcvtnt_bf16_m\n"; |
| 1232 |
OS << "#define svcvtnt_bf16_f32_x svcvtnt_bf16_f32_m\n"; |
| 1233 |
|
| 1234 |
OS << "#define svcvtnt_f16_x svcvtnt_f16_m\n"; |
| 1235 |
OS << "#define svcvtnt_f16_f32_x svcvtnt_f16_f32_m\n"; |
| 1236 |
OS << "#define svcvtnt_f32_x svcvtnt_f32_m\n"; |
| 1237 |
OS << "#define svcvtnt_f32_f64_x svcvtnt_f32_f64_m\n\n"; |
| 1238 |
|
| 1239 |
OS << "#define svcvtxnt_f32_x svcvtxnt_f32_m\n"; |
| 1240 |
OS << "#define svcvtxnt_f32_f64_x svcvtxnt_f32_f64_m\n\n"; |
| 1241 |
|
| 1242 |
OS << "#ifdef __cplusplus\n"; |
| 1243 |
OS << "} // extern \"C\"\n"; |
| 1244 |
OS << "#endif\n\n"; |
| 1245 |
OS << "#undef __ai\n\n"; |
| 1246 |
OS << "#undef __aio\n\n"; |
| 1247 |
OS << "#endif /* __ARM_SVE_H */\n"; |
| 1248 |
} |
| 1249 |
|
| 1250 |
void SVEEmitter::createBuiltins(raw_ostream &OS) { |
| 1251 |
std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| 1252 |
SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; |
| 1253 |
for (auto *R : RV) |
| 1254 |
createIntrinsic(R, Defs); |
| 1255 |
|
| 1256 |
// The mappings must be sorted based on BuiltinID. |
| 1257 |
llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, |
| 1258 |
const std::unique_ptr<Intrinsic> &B) { |
| 1259 |
return A->getMangledName() < B->getMangledName(); |
| 1260 |
}); |
| 1261 |
|
| 1262 |
OS << "#ifdef GET_SVE_BUILTINS\n"; |
| 1263 |
for (auto &Def : Defs) { |
| 1264 |
// Only create BUILTINs for non-overloaded intrinsics, as overloaded |
| 1265 |
// declarations only live in the header file. |
| 1266 |
if (Def->getClassKind() != ClassG) |
| 1267 |
OS << "TARGET_BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \"" |
| 1268 |
<< Def->getBuiltinTypeStr() << "\", \"n\", \"" << Def->getGuard() |
| 1269 |
<< "\")\n"; |
| 1270 |
} |
| 1271 |
|
| 1272 |
// Add reinterpret builtins |
| 1273 |
for (const ReinterpretTypeInfo &From : Reinterprets) |
| 1274 |
for (const ReinterpretTypeInfo &To : Reinterprets) |
| 1275 |
OS << "TARGET_BUILTIN(__builtin_sve_reinterpret_" << From.Suffix << "_" |
| 1276 |
<< To.Suffix << +", \"" << From.BuiltinType << To.BuiltinType |
| 1277 |
<< "\", \"n\", \"sve\")\n"; |
| 1278 |
|
| 1279 |
OS << "#endif\n\n"; |
| 1280 |
} |
| 1281 |
|
| 1282 |
void SVEEmitter::createCodeGenMap(raw_ostream &OS) { |
| 1283 |
std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| 1284 |
SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; |
| 1285 |
for (auto *R : RV) |
| 1286 |
createIntrinsic(R, Defs); |
| 1287 |
|
| 1288 |
// The mappings must be sorted based on BuiltinID. |
| 1289 |
llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, |
| 1290 |
const std::unique_ptr<Intrinsic> &B) { |
| 1291 |
return A->getMangledName() < B->getMangledName(); |
| 1292 |
}); |
| 1293 |
|
| 1294 |
OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n"; |
| 1295 |
for (auto &Def : Defs) { |
| 1296 |
// Builtins only exist for non-overloaded intrinsics, overloaded |
| 1297 |
// declarations only live in the header file. |
| 1298 |
if (Def->getClassKind() == ClassG) |
| 1299 |
continue; |
| 1300 |
|
| 1301 |
uint64_t Flags = Def->getFlags(); |
| 1302 |
auto FlagString = std::to_string(Flags); |
| 1303 |
|
| 1304 |
std::string LLVMName = Def->getLLVMName(); |
| 1305 |
std::string Builtin = Def->getMangledName(); |
| 1306 |
if (!LLVMName.empty()) |
| 1307 |
OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString |
| 1308 |
<< "),\n"; |
| 1309 |
else |
| 1310 |
OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n"; |
| 1311 |
} |
| 1312 |
OS << "#endif\n\n"; |
| 1313 |
} |
| 1314 |
|
| 1315 |
void SVEEmitter::createRangeChecks(raw_ostream &OS) { |
| 1316 |
std::vector<Record *> RV = Records.getAllDerivedDefinitions("Inst"); |
| 1317 |
SmallVector<std::unique_ptr<Intrinsic>, 128> Defs; |
| 1318 |
for (auto *R : RV) |
| 1319 |
createIntrinsic(R, Defs); |
| 1320 |
|
| 1321 |
// The mappings must be sorted based on BuiltinID. |
| 1322 |
llvm::sort(Defs, [](const std::unique_ptr<Intrinsic> &A, |
| 1323 |
const std::unique_ptr<Intrinsic> &B) { |
| 1324 |
return A->getMangledName() < B->getMangledName(); |
| 1325 |
}); |
| 1326 |
|
| 1327 |
|
| 1328 |
OS << "#ifdef GET_SVE_IMMEDIATE_CHECK\n"; |
| 1329 |
|
| 1330 |
// Ensure these are only emitted once. |
| 1331 |
std::set<std::string> Emitted; |
| 1332 |
|
| 1333 |
for (auto &Def : Defs) { |
| 1334 |
if (Emitted.find(Def->getMangledName()) != Emitted.end() || |
| 1335 |
Def->getImmChecks().empty()) |
| 1336 |
continue; |
| 1337 |
|
| 1338 |
OS << "case SVE::BI__builtin_sve_" << Def->getMangledName() << ":\n"; |
| 1339 |
for (auto &Check : Def->getImmChecks()) |
| 1340 |
OS << "ImmChecks.push_back(std::make_tuple(" << Check.getArg() << ", " |
| 1341 |
<< Check.getKind() << ", " << Check.getElementSizeInBits() << "));\n"; |
| 1342 |
OS << " break;\n"; |
| 1343 |
|
| 1344 |
Emitted.insert(Def->getMangledName()); |
| 1345 |
} |
| 1346 |
|
| 1347 |
OS << "#endif\n\n"; |
| 1348 |
} |
| 1349 |
|
| 1350 |
/// Create the SVETypeFlags used in CGBuiltins |
| 1351 |
void SVEEmitter::createTypeFlags(raw_ostream &OS) { |
| 1352 |
OS << "#ifdef LLVM_GET_SVE_TYPEFLAGS\n"; |
| 1353 |
for (auto &KV : FlagTypes) |
| 1354 |
OS << "const uint64_t " << KV.getKey() << " = " << KV.getValue() << ";\n"; |
| 1355 |
OS << "#endif\n\n"; |
| 1356 |
|
| 1357 |
OS << "#ifdef LLVM_GET_SVE_ELTTYPES\n"; |
| 1358 |
for (auto &KV : EltTypes) |
| 1359 |
OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; |
| 1360 |
OS << "#endif\n\n"; |
| 1361 |
|
| 1362 |
OS << "#ifdef LLVM_GET_SVE_MEMELTTYPES\n"; |
| 1363 |
for (auto &KV : MemEltTypes) |
| 1364 |
OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; |
| 1365 |
OS << "#endif\n\n"; |
| 1366 |
|
| 1367 |
OS << "#ifdef LLVM_GET_SVE_MERGETYPES\n"; |
| 1368 |
for (auto &KV : MergeTypes) |
| 1369 |
OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; |
| 1370 |
OS << "#endif\n\n"; |
| 1371 |
|
| 1372 |
OS << "#ifdef LLVM_GET_SVE_IMMCHECKTYPES\n"; |
| 1373 |
for (auto &KV : ImmCheckTypes) |
| 1374 |
OS << " " << KV.getKey() << " = " << KV.getValue() << ",\n"; |
| 1375 |
OS << "#endif\n\n"; |
| 1376 |
} |
| 1377 |
|
| 1378 |
namespace clang { |
| 1379 |
void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) { |
| 1380 |
SVEEmitter(Records).createHeader(OS); |
| 1381 |
} |
| 1382 |
|
| 1383 |
void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) { |
| 1384 |
SVEEmitter(Records).createBuiltins(OS); |
| 1385 |
} |
| 1386 |
|
| 1387 |
void EmitSveBuiltinCG(RecordKeeper &Records, raw_ostream &OS) { |
| 1388 |
SVEEmitter(Records).createCodeGenMap(OS); |
| 1389 |
} |
| 1390 |
|
| 1391 |
void EmitSveRangeChecks(RecordKeeper &Records, raw_ostream &OS) { |
| 1392 |
SVEEmitter(Records).createRangeChecks(OS); |
| 1393 |
} |
| 1394 |
|
| 1395 |
void EmitSveTypeFlags(RecordKeeper &Records, raw_ostream &OS) { |
| 1396 |
SVEEmitter(Records).createTypeFlags(OS); |
| 1397 |
} |
| 1398 |
|
| 1399 |
} // End namespace clang |