| 1 |
// RUN: mlir-opt -split-input-file \ |
| 2 |
// RUN: -transform-preload-library='transform-library-paths=%p/../td/vectorize-with-patterns.mlir' \ |
| 3 |
// RUN: -transform-interpreter=entry-point=vectorize_with_patterns %s | FileCheck %s |
| 4 |
|
| 5 |
//===----------------------------------------------------------------------===// |
| 6 |
// Contiguous load |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
|
| 9 |
#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> |
| 10 |
func.func @vectorize_nd_tensor_extract_transfer_read_basic( |
| 11 |
%arg0: tensor<3x3x3xf32>, |
| 12 |
%arg1: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> { |
| 13 |
|
| 14 |
%res = linalg.generic { |
| 15 |
indexing_maps = [#map], |
| 16 |
iterator_types = ["parallel", "parallel", "parallel"] |
| 17 |
} outs(%arg1 : tensor<1x1x3xf32>) { |
| 18 |
^bb0(%out: f32): |
| 19 |
%1 = linalg.index 0 : index |
| 20 |
%2 = linalg.index 1 : index |
| 21 |
%3 = linalg.index 2 : index |
| 22 |
%4 = tensor.extract %arg0[%1, %2, %3] : tensor<3x3x3xf32> |
| 23 |
linalg.yield %4 : f32 |
| 24 |
} -> tensor<1x1x3xf32> |
| 25 |
|
| 26 |
return %res : tensor<1x1x3xf32> |
| 27 |
} |
| 28 |
|
| 29 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_basic |
| 30 |
// CHECK-SAME: %[[ARG0:.*]]: tensor<3x3x3xf32> |
| 31 |
// CHECK-SAME: %[[ARG1:.*]]: tensor<1x1x3xf32> |
| 32 |
|
| 33 |
// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index |
| 34 |
// CHECK-DAG: %[[PV:.+]] = ub.poison : f32 |
| 35 |
|
| 36 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true]} : tensor<3x3x3xf32>, vector<1x1x3xf32> |
| 37 |
// CHECK: vector.transfer_write %[[READ]], %[[ARG1]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32> |
| 38 |
|
| 39 |
// ----- |
| 40 |
|
| 41 |
func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16xf32>, %arg0: index, %arg2: index, %arg1: index, %arg4: index, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 42 |
%c79 = arith.constant 79 : index |
| 43 |
%25 = linalg.generic { |
| 44 |
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>], |
| 45 |
iterator_types = ["parallel", "parallel"] |
| 46 |
} outs(%extracted_slice : tensor<1x4xf32>) { |
| 47 |
^bb0(%out: f32): |
| 48 |
%26 = linalg.index 0 : index |
| 49 |
%27 = arith.addi %arg0, %26 : index |
| 50 |
%28 = arith.addi %27, %arg2 : index |
| 51 |
%29 = linalg.index 1 : index |
| 52 |
%30 = arith.addi %arg1, %29 : index |
| 53 |
%31 = arith.addi %30, %arg4 : index |
| 54 |
%extracted = tensor.extract %6[%28, %c79, %31] : tensor<45x80x16xf32> |
| 55 |
linalg.yield %extracted : f32 |
| 56 |
} -> tensor<1x4xf32> |
| 57 |
return %25 : tensor<1x4xf32> |
| 58 |
} |
| 59 |
|
| 60 |
|
| 61 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_complex( |
| 62 |
// CHECK-SAME: %[[ARG0:.*]]: tensor<45x80x16xf32>, |
| 63 |
// CHECK-SAME: %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index, |
| 64 |
// CHECK-SAME: %[[ARG5:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 65 |
|
| 66 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 67 |
// CHECK-DAG: %[[PV:.*]] = ub.poison : f32 |
| 68 |
// CHECK-DAG: %[[C79:.*]] = arith.constant 79 : index |
| 69 |
// CHECK: %[[ADD1:.*]] = arith.addi %[[ARG1]], %[[ARG2]] : index |
| 70 |
// CHECK: %[[ADD2:.*]] = arith.addi %[[ARG3]], %[[ARG4]] : index |
| 71 |
|
| 72 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[ADD1]], %[[C79]], %[[ADD2]]], %[[PV]] {in_bounds = [true, true]} : tensor<45x80x16xf32>, vector<1x4xf32> |
| 73 |
// CHECK: %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG5]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32> |
| 74 |
// CHECK: return %[[WRITE]] : tensor<1x4xf32> |
| 75 |
// CHECK: } |
| 76 |
|
| 77 |
// ----- |
| 78 |
|
| 79 |
// The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Contiguous load. |
| 80 |
func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<80x16xf32>, %arg0: index, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 81 |
%c79 = arith.constant 79 : index |
| 82 |
%1 = linalg.generic { |
| 83 |
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>], |
| 84 |
iterator_types = ["parallel", "parallel"] |
| 85 |
} outs(%extracted_slice : tensor<1x4xf32>) { |
| 86 |
^bb0(%out: f32): |
| 87 |
%2 = linalg.index 1 : index |
| 88 |
%3 = affine.apply affine_map<(d0, d1) -> (d0 + d1)>(%2, %arg0) |
| 89 |
%extracted = tensor.extract %6[%c79, %3] : tensor<80x16xf32> |
| 90 |
linalg.yield %extracted : f32 |
| 91 |
} -> tensor<1x4xf32> |
| 92 |
return %1 : tensor<1x4xf32> |
| 93 |
} |
| 94 |
|
| 95 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous( |
| 96 |
// CHECK-SAME: %[[ARG0:.*]]: tensor<80x16xf32>, |
| 97 |
// CHECK-SAME: %[[ARG1:.*]]: index, |
| 98 |
// CHECK-SAME: %[[ARG2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 99 |
|
| 100 |
// CHECK-DAG: %[[PV:.*]] = ub.poison : f32 |
| 101 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 102 |
// CHECK-DAG: %[[C79:.*]] = arith.constant 79 : index |
| 103 |
|
| 104 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[C79]], %[[ARG1]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32> |
| 105 |
// CHECK: %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG2]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32> |
| 106 |
// CHECK: return %[[WRITE]] : tensor<1x4xf32> |
| 107 |
// CHECK: } |
| 108 |
|
| 109 |
// ----- |
| 110 |
|
| 111 |
func.func @vectorize_nd_tensor_extract_with_tensor_extract(%input_1: tensor<1x20xi32>, %input_2: tensor<257x24xf32>, %arg0 : index, %arg1 : index, %arg2 : index, %arg3 : index) -> tensor<1x1x4xf32> { |
| 112 |
%c0 = arith.constant 0 : index |
| 113 |
%c256 = arith.constant 256 : index |
| 114 |
%output = tensor.empty() : tensor<1x1x4xf32> |
| 115 |
%1 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} outs(%output : tensor<1x1x4xf32>) { |
| 116 |
^bb0(%out: f32): |
| 117 |
%13 = linalg.index 0 : index |
| 118 |
%14 = affine.apply affine_map<(d0, d1, d2) -> (d0 + d1 + d2)>(%arg0, %13, %arg2) |
| 119 |
%15 = linalg.index 2 : index |
| 120 |
%16 = linalg.index 1 : index |
| 121 |
%17 = affine.apply affine_map<(d0, d1, d2, d3) -> (d0 + d1 * 24 + d2 + d3)>(%arg1, %16, %15, %arg3) |
| 122 |
%extracted_0 = tensor.extract %input_1[%c0, %14] : tensor<1x20xi32> |
| 123 |
%18 = arith.index_cast %extracted_0 : i32 to index |
| 124 |
%19 = arith.maxsi %18, %c0 : index |
| 125 |
%20 = arith.minsi %19, %c256 : index |
| 126 |
%extracted_1 = tensor.extract %input_2[%20, %17] : tensor<257x24xf32> |
| 127 |
linalg.yield %extracted_1 : f32 |
| 128 |
} -> tensor<1x1x4xf32> |
| 129 |
return %1 : tensor<1x1x4xf32> |
| 130 |
} |
| 131 |
|
| 132 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_tensor_extract( |
| 133 |
// CHECK-SAME: %[[INPUT_1:.*]]: tensor<1x20xi32>, |
| 134 |
// CHECK-SAME: %[[INPUT_2:.*]]: tensor<257x24xf32>, |
| 135 |
// CHECK-SAME: %[[INPUT_3:.*]]: index, %[[INPUT_4:.*]]: index, %[[INPUT_5:.*]]: index, |
| 136 |
// CHECK: %[[EXTRACTED_0_IDX_0:.*]] = arith.constant 0 : index |
| 137 |
// CHECK: %[[SCALAR:.*]] = arith.addi %[[INPUT_3]], %[[INPUT_5]] : index |
| 138 |
// First `vector.transfer_read` from the generic Op - loop invariant scalar load. |
| 139 |
// CHECK: vector.transfer_read %[[INPUT_1]][%[[EXTRACTED_0_IDX_0]], %[[SCALAR]]] |
| 140 |
// CHECK-SAME: tensor<1x20xi32>, vector<i32> |
| 141 |
// The following `tensor.extract` from the generic Op s a contiguous load (all Ops used |
| 142 |
// for address calculation also satisfy the required conditions). |
| 143 |
// CHECK: vector.transfer_read %[[INPUT_2]][%{{.*}}, %{{.*}}, %{{.*}} {in_bounds = [true, true]} : tensor<257x24xf32>, vector<1x4xf32> |
| 144 |
|
| 145 |
// Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Contiguous load. |
| 146 |
func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16xf32>, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 147 |
%c16 = arith.constant 16 : index |
| 148 |
%1 = linalg.generic { |
| 149 |
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>], |
| 150 |
iterator_types = ["parallel", "parallel"] |
| 151 |
} outs(%extracted_slice : tensor<1x4xf32>) { |
| 152 |
^bb0(%out: f32): |
| 153 |
%2 = linalg.index 0 : index |
| 154 |
%3 = linalg.index 1 : index |
| 155 |
%4 = arith.maxsi %2, %c16 : index |
| 156 |
%extracted = tensor.extract %arg0[%4, %3] : tensor<80x16xf32> |
| 157 |
linalg.yield %extracted : f32 |
| 158 |
} -> tensor<1x4xf32> |
| 159 |
return %1 : tensor<1x4xf32> |
| 160 |
} |
| 161 |
|
| 162 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous( |
| 163 |
// CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, |
| 164 |
// CHECK-SAME: %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 165 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 166 |
// CHECK-DAG: %[[C16:.*]] = arith.constant 16 : index |
| 167 |
// CHECK-DAG: %[[PV:.*]] = ub.poison : f32 |
| 168 |
|
| 169 |
// CHECK: %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[C16]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32> |
| 170 |
// CHECK: %[[VAL_9:.*]] = vector.transfer_write %[[VAL_8]], %[[VAL_1]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32> |
| 171 |
// CHECK: return %[[VAL_9]] : tensor<1x4xf32> |
| 172 |
// CHECK: } |
| 173 |
|
| 174 |
// ----- |
| 175 |
|
| 176 |
//===----------------------------------------------------------------------===// |
| 177 |
// Gather load |
| 178 |
//===----------------------------------------------------------------------===// |
| 179 |
|
| 180 |
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2)> |
| 181 |
#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)> |
| 182 |
func.func @vectorize_1d_tensor_extract(%arg0: tensor<3xf32>, %arg1: tensor<4x3xi32>, %arg2: tensor<4x7x3x2xf32>) -> tensor<4x7x3x2xf32> { |
| 183 |
%1 = linalg.generic { |
| 184 |
indexing_maps = [#map0, #map1], |
| 185 |
iterator_types = ["parallel", "parallel", "parallel", "parallel"] |
| 186 |
} ins(%arg1 : tensor<4x3xi32>) outs(%arg2 : tensor<4x7x3x2xf32>) { |
| 187 |
^bb0(%arg3: i32, %arg4: f32): |
| 188 |
%2 = arith.index_cast %arg3 : i32 to index |
| 189 |
%3 = tensor.extract %arg0[%2] : tensor<3xf32> |
| 190 |
linalg.yield %3 : f32 |
| 191 |
} -> tensor<4x7x3x2xf32> |
| 192 |
return %1 : tensor<4x7x3x2xf32> |
| 193 |
} |
| 194 |
// CHECK-LABEL: func.func @vectorize_1d_tensor_extract |
| 195 |
// CHECK-SAME: %[[ARG0:.*]]: tensor<3xf32> |
| 196 |
// CHECK-SAME: %[[ARG1:.*]]: tensor<4x3xi32> |
| 197 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 198 |
// CHECK-DAG: %[[MASK:.*]] = arith.constant dense<true> : vector<4x7x3x2xi1> |
| 199 |
// CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32> |
| 200 |
// CHECK: %[[V0:.*]] = vector.transfer_read %[[ARG1]] |
| 201 |
// CHECK: %[[CAST:.*]] = arith.index_cast %[[V0]] |
| 202 |
// CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[CAST]] |
| 203 |
// CHECK: %[[INDICES:.*]] = vector.transpose %[[BROADCAST]] |
| 204 |
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]]] [%[[INDICES]]], %[[MASK]], %[[PASSTHRU]] |
| 205 |
// CHECK: vector.transfer_write %[[GATHER]] |
| 206 |
|
| 207 |
// ----- |
| 208 |
|
| 209 |
#map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2)> |
| 210 |
#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)> |
| 211 |
#map2 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)> |
| 212 |
func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>, %arg1: tensor<4x3xi32>, %arg2: tensor<4x3xi32>, %arg3: tensor<4x7x2xf32>, %arg4: tensor<4x7x3x2xf32>) -> tensor<4x7x3x2xf32> { |
| 213 |
%2 = linalg.generic { |
| 214 |
indexing_maps = [#map0, #map0, #map1, #map2], |
| 215 |
iterator_types = ["parallel", "parallel", "parallel", "parallel"] |
| 216 |
} ins(%arg1, %arg2, %arg3 : tensor<4x3xi32>, tensor<4x3xi32>, tensor<4x7x2xf32>) outs(%arg4 : tensor<4x7x3x2xf32>) { |
| 217 |
^bb0(%arg5: i32, %arg6: i32, %arg7: f32, %arg8: f32): |
| 218 |
%3 = arith.index_cast %arg5 : i32 to index |
| 219 |
%4 = arith.index_cast %arg6 : i32 to index |
| 220 |
%7 = tensor.extract %arg0[%3, %4] : tensor<3x3xf32> |
| 221 |
linalg.yield %7 : f32 |
| 222 |
} -> tensor<4x7x3x2xf32> |
| 223 |
return %2 : tensor<4x7x3x2xf32> |
| 224 |
} |
| 225 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_index_from_tensor |
| 226 |
// CHECK-SAME: %[[ARG0:.*]]: tensor<3x3xf32> |
| 227 |
// CHECK-SAME: %[[ARG1:arg1]]: tensor<4x3xi32> |
| 228 |
// CHECK-SAME: %[[ARG2:arg2]]: tensor<4x3xi32> |
| 229 |
// CHECK-SAME: %[[ARG3:.*]]: tensor<4x7x2xf32> |
| 230 |
// CHECK-SAME: %[[ARG4:.*]]: tensor<4x7x3x2xf32> |
| 231 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 232 |
// CHECK-DAG: %[[PV:.*]] = ub.poison : i32 |
| 233 |
// CHECK-DAG: %[[CST:.*]] = arith.constant dense<3> : vector<7x2x4x3xindex> |
| 234 |
// CHECK-DAG: %[[CST_1:.*]] = arith.constant dense<true> : vector<4x7x3x2xi1> |
| 235 |
// CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32> |
| 236 |
// CHECK: %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32> |
| 237 |
// CHECK: %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32> |
| 238 |
// CHECK: %[[CAST:.*]] = arith.index_cast %[[V0]] : vector<4x3xi32> to vector<4x3xindex> |
| 239 |
// CHECK: %[[B1:.*]] = vector.broadcast %[[CAST]] : vector<4x3xindex> to vector<7x2x4x3xindex> |
| 240 |
// CHECK: %[[CAST_1:.*]] = arith.index_cast %[[V1]] : vector<4x3xi32> to vector<4x3xindex> |
| 241 |
// CHECK: %[[B2:.*]] = vector.broadcast %[[CAST_1]] : vector<4x3xindex> to vector<7x2x4x3xindex> |
| 242 |
// CHECK: %[[MULI:.*]] = arith.muli %[[B1]], %[[CST]] : vector<7x2x4x3xindex> |
| 243 |
// CHECK: %[[ADDI:.*]] = arith.addi %[[B2]], %[[MULI]] : vector<7x2x4x3xindex> |
| 244 |
// CHECK: %[[T:.*]] = vector.transpose %[[ADDI]], [2, 0, 3, 1] : vector<7x2x4x3xindex> to vector<4x7x3x2xindex> |
| 245 |
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]]] [%[[T]]], %[[CST_1]], %[[PASSTHRU]] : tensor<3x3xf32>, vector<4x7x3x2xindex>, vector<4x7x3x2xi1>, vector<4x7x3x2xf32> into vector<4x7x3x2xf32> |
| 246 |
// CHECK: vector.transfer_write %[[GATHER]], %[[ARG4]][%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true, true]} : vector<4x7x3x2xf32>, tensor<4x7x3x2xf32> |
| 247 |
|
| 248 |
// ----- |
| 249 |
|
| 250 |
#map = affine_map<(d0, d1) -> (d0, d1)> |
| 251 |
#map1 = affine_map<(d0, d1, d2) -> (d0 + d1 + d2)> |
| 252 |
func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load(%arg0: tensor<8x128x768xf32>, %arg1 : index) -> tensor<8x1xf32> { |
| 253 |
%c0 = arith.constant 0 : index |
| 254 |
%0 = tensor.empty() : tensor<8x1xf32> |
| 255 |
%1 = linalg.generic { |
| 256 |
indexing_maps = [#map], |
| 257 |
iterator_types = ["parallel", "parallel"] |
| 258 |
} outs(%0 : tensor<8x1xf32>) { |
| 259 |
^bb0(%arg5: f32): |
| 260 |
%2 = linalg.index 0 : index |
| 261 |
%3 = linalg.index 1 : index |
| 262 |
%4 = affine.apply #map1(%arg1, %3, %arg1) |
| 263 |
%extracted = tensor.extract %arg0[%2, %c0, %4] : tensor<8x128x768xf32> |
| 264 |
linalg.yield %extracted : f32 |
| 265 |
} -> tensor<8x1xf32> |
| 266 |
return %1 : tensor<8x1xf32> |
| 267 |
} |
| 268 |
|
| 269 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load |
| 270 |
// CHECK-SAME: %[[ARG0:.*]]: tensor<8x128x768xf32> |
| 271 |
// CHECK-SAME: %[[ARG1:.*]]: index |
| 272 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 273 |
// CHECK-DAG: %[[CST:.*]] = arith.constant dense<768> : vector<1x8xindex> |
| 274 |
// CHECK-DAG: %[[CST_0:.*]] = arith.constant dense<128> : vector<1x8xindex> |
| 275 |
// CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<8x1xf32> |
| 276 |
// CHECK-DAG: %[[CST_2:.*]] = arith.constant dense<true> : vector<8x1xi1> |
| 277 |
// CHECK-DAG: %[[CST_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex> |
| 278 |
// CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<8x1xf32> |
| 279 |
// CHECK: %[[B1:.*]] = vector.broadcast %[[CST_3]] : vector<8xindex> to vector<1x8xindex> |
| 280 |
// CHECK: %[[ADDI_ARG1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : index |
| 281 |
// CHECK: %[[MULI_1:.*]] = arith.muli %[[B1]], %[[CST_0]] : vector<1x8xindex> |
| 282 |
// CHECK: %[[MULI_2:.*]] = arith.muli %[[MULI_1]], %[[CST]] : vector<1x8xindex> |
| 283 |
// CHECK: %[[T:.*]] = vector.transpose %[[MULI_2]], [1, 0] : vector<1x8xindex> to vector<8x1xindex> |
| 284 |
// CHECK: %[[B2:.*]] = vector.broadcast %[[ADDI_ARG1]] : index to vector<8x1xindex> |
| 285 |
// CHECK: %[[ADDI:.*]] = arith.addi %[[B2]], %[[T]] : vector<8x1xindex> |
| 286 |
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] [%[[ADDI]]], %[[CST_2]], %[[PASSTHRU]] : tensor<8x128x768xf32>, vector<8x1xindex>, vector<8x1xi1>, vector<8x1xf32> into vector<8x1xf32> |
| 287 |
// CHECK: vector.transfer_write %[[GATHER]], %[[EMPTY]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x1xf32>, tensor<8x1xf32> |
| 288 |
|
| 289 |
// ----- |
| 290 |
|
| 291 |
// Reading a 1D column vector (hence a candidate for a contiguous load), but given |
| 292 |
// %1, it's a gather load. |
| 293 |
|
| 294 |
#map = affine_map<(d0, d1) -> (d0, d1)> |
| 295 |
func.func @index_from_output_column_vector_gather_load(%src: tensor<8x128xf32>) -> tensor<8x1xf32> { |
| 296 |
%c0 = arith.constant 0 : index |
| 297 |
%0 = tensor.empty() : tensor<8x1xf32> |
| 298 |
%res = linalg.generic { |
| 299 |
indexing_maps = [#map], |
| 300 |
iterator_types = ["parallel", "parallel"] |
| 301 |
} outs(%0 : tensor<8x1xf32>) { |
| 302 |
^bb0(%arg1: f32): |
| 303 |
%1 = linalg.index 0 : index |
| 304 |
%extracted = tensor.extract %src[%1, %c0] : tensor<8x128xf32> |
| 305 |
linalg.yield %extracted : f32 |
| 306 |
} -> tensor<8x1xf32> |
| 307 |
return %res : tensor<8x1xf32> |
| 308 |
} |
| 309 |
|
| 310 |
// CHECK-LABEL: func.func @index_from_output_column_vector_gather_load( |
| 311 |
// CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> { |
| 312 |
// CHECK: %[[C128:.*]] = arith.constant dense<128> : vector<1x8xindex> |
| 313 |
// CHECK: %[[C0:.*]] = arith.constant 0 : index |
| 314 |
// CHECK: %[[PASS_THRU:.*]] = arith.constant dense<0.000000e+00> : vector<8x1xf32> |
| 315 |
// CHECK: %[[MASK:.*]] = arith.constant dense<true> : vector<8x1xi1> |
| 316 |
// CHECK: %[[IDX_VEC:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex> |
| 317 |
// CHECK: %[[OUT:.*]] = tensor.empty() : tensor<8x1xf32> |
| 318 |
// CHECK: %[[B:.*]] = vector.broadcast %[[IDX_VEC]] : vector<8xindex> to vector<1x8xindex> |
| 319 |
// CHECK: %[[MUL:.*]] = arith.muli %[[B]], %[[C128]] : vector<1x8xindex> |
| 320 |
// CHECK: %[[TR:.*]] = vector.transpose %[[MUL]], [1, 0] : vector<1x8xindex> to vector<8x1xindex> |
| 321 |
// CHECK: %[[GATHER:.*]] = vector.gather %[[SRC]]{{\[}}%[[C0]], %[[C0]]] {{\[}}%[[TR]]], %[[MASK]], %[[PASS_THRU]] : tensor<8x128xf32>, vector<8x1xindex>, vector<8x1xi1>, vector<8x1xf32> into vector<8x1xf32> |
| 322 |
// CHECK: %[[RES:.*]] = vector.transfer_write %[[GATHER]], %[[OUT]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x1xf32>, tensor<8x1xf32> |
| 323 |
// CHECK: return %[[RES]] : tensor<8x1xf32> |
| 324 |
|
| 325 |
// ----- |
| 326 |
|
| 327 |
// Same as above, but the access indices have been swapped and hence this _is_ |
| 328 |
// a contiguous load. Currently not supported and lowered as vector.gather |
| 329 |
// instead. |
| 330 |
// TODO: Make sure that this is lowered as a contiguous load. |
| 331 |
|
| 332 |
#map = affine_map<(d0, d1) -> (d0, d1)> |
| 333 |
func.func @index_from_output_column_vector_contiguous_load(%src: tensor<8x128xf32>) -> tensor<8x1xf32> { |
| 334 |
%c0 = arith.constant 0 : index |
| 335 |
%0 = tensor.empty() : tensor<8x1xf32> |
| 336 |
%res = linalg.generic { |
| 337 |
indexing_maps = [#map], |
| 338 |
iterator_types = ["parallel", "parallel"] |
| 339 |
} outs(%0 : tensor<8x1xf32>) { |
| 340 |
^bb0(%arg1: f32): |
| 341 |
%1 = linalg.index 0 : index |
| 342 |
%extracted = tensor.extract %src[%c0, %1] : tensor<8x128xf32> |
| 343 |
linalg.yield %extracted : f32 |
| 344 |
} -> tensor<8x1xf32> |
| 345 |
return %res : tensor<8x1xf32> |
| 346 |
} |
| 347 |
|
| 348 |
// CHECK-LABEL: func.func @index_from_output_column_vector_contiguous_load( |
| 349 |
// CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> { |
| 350 |
// CHECK: %[[C0:.*]] = arith.constant 0 : index |
| 351 |
// CHECK: %[[PASS_THRU:.*]] = arith.constant dense<0.000000e+00> : vector<8x1xf32> |
| 352 |
// CHECK: %[[MASK:.*]] = arith.constant dense<true> : vector<8x1xi1> |
| 353 |
// CHECK: %[[IDX_VEC:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex> |
| 354 |
// CHECK: %[[OUT:.*]] = tensor.empty() : tensor<8x1xf32> |
| 355 |
// CHECK: %[[B:.*]] = vector.broadcast %[[IDX_VEC]] : vector<8xindex> to vector<1x8xindex> |
| 356 |
// CHECK: %[[TR:.*]] = vector.transpose %[[B]], [1, 0] : vector<1x8xindex> to vector<8x1xindex> |
| 357 |
// CHECK: %[[GATHER:.*]] = vector.gather %[[SRC]]{{\[}}%[[C0]], %[[C0]]] {{\[}}%[[TR]]], %[[MASK]], %[[PASS_THRU]] : tensor<8x128xf32>, vector<8x1xindex>, vector<8x1xi1>, vector<8x1xf32> into vector<8x1xf32> |
| 358 |
// CHECK: %[[RES:.*]] = vector.transfer_write %[[GATHER]], %[[OUT]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x1xf32>, tensor<8x1xf32> |
| 359 |
// CHECK: return %[[RES]] : tensor<8x1xf32> |
| 360 |
|
| 361 |
// ----- |
| 362 |
|
| 363 |
#map = affine_map<(d0) -> (d0)> |
| 364 |
func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32>, %arg1: tensor<5xi32>) -> tensor<5xf32> { |
| 365 |
%c5 = arith.constant 5 : index |
| 366 |
%c0 = arith.constant 0 : index |
| 367 |
%0 = tensor.empty() : tensor<5xf32> |
| 368 |
%1 = linalg.generic {indexing_maps = [#map], iterator_types = ["parallel"]} outs(%0 : tensor<5xf32>) { |
| 369 |
^bb0(%out: f32): |
| 370 |
%2 = linalg.index 0 : index |
| 371 |
%extracted = tensor.extract %arg1[%2] : tensor<5xi32> |
| 372 |
%3 = arith.index_cast %extracted : i32 to index |
| 373 |
%4 = arith.maxsi %3, %c0 : index |
| 374 |
%5 = arith.minsi %4, %c5 : index |
| 375 |
%extracted_0 = tensor.extract %arg0[%5] : tensor<6xf32> |
| 376 |
linalg.yield %extracted_0 : f32 |
| 377 |
} -> tensor<5xf32> |
| 378 |
return %1 : tensor<5xf32> |
| 379 |
} |
| 380 |
|
| 381 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_contiguous_and_gather( |
| 382 |
// CHECK-SAME: %[[VAL_0:.*]]: tensor<6xf32> |
| 383 |
// CHECK-SAME: %[[VAL_1:.*]]: tensor<5xi32> |
| 384 |
// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 0 : index |
| 385 |
// CHECK-DAG: %[[VAL_3:.*]] = ub.poison : i32 |
| 386 |
// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<0> : vector<5xindex> |
| 387 |
// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<5> : vector<5xindex> |
| 388 |
// CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense<true> : vector<5xi1> |
| 389 |
// CHECK-DAG: %[[VAL_7:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32> |
| 390 |
// CHECK: %[[VAL_8:.*]] = tensor.empty() : tensor<5xf32> |
| 391 |
// CHECK: %[[VAL_9:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%{{.*}}], %[[VAL_3]] {in_bounds = [true]} : tensor<5xi32>, vector<5xi32> |
| 392 |
// CHECK: %[[VAL_10:.*]] = arith.index_cast %[[VAL_9]] : vector<5xi32> to vector<5xindex> |
| 393 |
// CHECK: %[[VAL_11:.*]] = arith.maxsi %[[VAL_10]], %[[VAL_4]] : vector<5xindex> |
| 394 |
// CHECK: %[[VAL_12:.*]] = arith.minsi %[[VAL_11]], %[[VAL_5]] : vector<5xindex> |
| 395 |
// CHECK: %[[VAL_13:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_2]]] {{\[}}%[[VAL_12]]], %[[VAL_6]], %[[VAL_7]] : tensor<6xf32>, vector<5xindex>, vector<5xi1>, vector<5xf32> into vector<5xf32> |
| 396 |
// CHECK: %[[VAL_14:.*]] = vector.transfer_write %[[VAL_13]], %[[VAL_8]]{{\[}}%[[VAL_2]]] {in_bounds = [true]} : vector<5xf32>, tensor<5xf32> |
| 397 |
// CHECK: return %[[VAL_14]] : tensor<5xf32> |
| 398 |
|
| 399 |
// The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Gather load. |
| 400 |
func.func @vectorize_nd_tensor_extract_with_affine_apply_gather(%6: tensor<80x16xf32>, %arg0: index, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 401 |
%c16 = arith.constant 16 : index |
| 402 |
%1 = linalg.generic { |
| 403 |
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>], |
| 404 |
iterator_types = ["parallel", "parallel"] |
| 405 |
} outs(%extracted_slice : tensor<1x4xf32>) { |
| 406 |
^bb0(%out: f32): |
| 407 |
%2 = linalg.index 1 : index |
| 408 |
%3 = affine.apply affine_map<(d0, d1) -> (d0 + d1)>(%2, %arg0) |
| 409 |
%extracted = tensor.extract %6[%3, %c16] : tensor<80x16xf32> |
| 410 |
linalg.yield %extracted : f32 |
| 411 |
} -> tensor<1x4xf32> |
| 412 |
return %1 : tensor<1x4xf32> |
| 413 |
} |
| 414 |
|
| 415 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_affine_apply_gather( |
| 416 |
// CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, |
| 417 |
// CHECK-SAME: %[[VAL_1:.*]]: index, |
| 418 |
// CHECK-SAME: %[[VAL_2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 419 |
// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> |
| 420 |
// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<true> : vector<1x4xi1> |
| 421 |
// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32> |
| 422 |
// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index |
| 423 |
// CHECK-DAG: %[[VAL_7:.*]] = arith.constant dense<16> : vector<1x4xindex> |
| 424 |
// CHECK: %[[VAL_8:.*]] = vector.broadcast %[[VAL_1]] : index to vector<4xindex> |
| 425 |
// CHECK: %[[VAL_9:.*]] = arith.addi %[[VAL_8]], %[[VAL_3]] : vector<4xindex> |
| 426 |
// CHECK: %[[VAL_10:.*]] = vector.broadcast %[[VAL_9]] : vector<4xindex> to vector<1x4xindex> |
| 427 |
// CHECK: %[[VAL_11:.*]] = arith.muli %[[VAL_10]], %[[VAL_7]] : vector<1x4xindex> |
| 428 |
// CHECK: %[[VAL_12:.*]] = arith.addi %[[VAL_11]], %[[VAL_7]] : vector<1x4xindex> |
| 429 |
// CHECK: %[[VAL_13:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {{\[}}%[[VAL_12]]], %[[VAL_4]], %[[VAL_5]] : tensor<80x16xf32>, vector<1x4xindex>, vector<1x4xi1>, vector<1x4xf32> into vector<1x4xf32> |
| 430 |
// CHECK: %[[VAL_14:.*]] = vector.transfer_write %[[VAL_13]], %[[VAL_2]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32> |
| 431 |
// CHECK: return %[[VAL_14]] : tensor<1x4xf32> |
| 432 |
// CHECK: } |
| 433 |
|
| 434 |
// Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Gather load. |
| 435 |
func.func @vectorize_nd_tensor_extract_with_maxsi_gather(%arg0: tensor<80x16xf32>, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 436 |
%c79 = arith.constant 79 : index |
| 437 |
%1 = linalg.generic { |
| 438 |
indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>], |
| 439 |
iterator_types = ["parallel", "parallel"] |
| 440 |
} outs(%extracted_slice : tensor<1x4xf32>) { |
| 441 |
^bb0(%out: f32): |
| 442 |
%2 = linalg.index 1 : index |
| 443 |
%3 = arith.maxsi %2, %c79 : index |
| 444 |
%extracted = tensor.extract %arg0[%3, %2] : tensor<80x16xf32> |
| 445 |
linalg.yield %extracted : f32 |
| 446 |
} -> tensor<1x4xf32> |
| 447 |
return %1 : tensor<1x4xf32> |
| 448 |
} |
| 449 |
|
| 450 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_maxsi_gather( |
| 451 |
// CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>, |
| 452 |
// CHECK-SAME: %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> { |
| 453 |
// CHECK-DAG: %[[VAL_2:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex> |
| 454 |
// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<1264> : vector<1x4xindex> |
| 455 |
// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<true> : vector<1x4xi1> |
| 456 |
// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32> |
| 457 |
// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index |
| 458 |
// CHECK: %[[VAL_7:.*]] = vector.broadcast %[[VAL_2]] : vector<4xindex> to vector<1x4xindex> |
| 459 |
// CHECK: %[[VAL_8:.*]] = arith.addi %[[VAL_7]], %[[VAL_3]] : vector<1x4xindex> |
| 460 |
// CHECK: %[[VAL_9:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {{\[}}%[[VAL_8]]], %[[VAL_4]], %[[VAL_5]] : tensor<80x16xf32>, vector<1x4xindex>, vector<1x4xi1>, vector<1x4xf32> into vector<1x4xf32> |
| 461 |
// CHECK: %[[VAL_10:.*]] = vector.transfer_write %[[VAL_9]], %[[VAL_1]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32> |
| 462 |
// CHECK: return %[[VAL_10]] : tensor<1x4xf32> |
| 463 |
// CHECK: } |
| 464 |
|
| 465 |
// ----- |
| 466 |
|
| 467 |
// The vectorizer assumes it's a gather load whenever using a block argument to calculate an index. |
| 468 |
#map = affine_map<(d0) -> (d0)> |
| 469 |
func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1: tensor<5xindex>) -> tensor<5xf32> { |
| 470 |
%0 = tensor.empty() : tensor<5xf32> |
| 471 |
%1 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel"]} ins(%arg1: tensor<5xindex>) outs(%0 : tensor<5xf32>) { |
| 472 |
^bb0(%in: index, %out: f32): |
| 473 |
%2 = linalg.index 0 : index |
| 474 |
%extracted_0 = tensor.extract %arg0[%in, %2] : tensor<5x6xf32> |
| 475 |
linalg.yield %extracted_0 : f32 |
| 476 |
} -> tensor<5xf32> |
| 477 |
return %1 : tensor<5xf32> |
| 478 |
} |
| 479 |
|
| 480 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_block_arg( |
| 481 |
// CHECK-SAME: %[[VAL_0:.*]]: tensor<5x6xf32>, |
| 482 |
// CHECK-SAME: %[[VAL_1:.*]]: tensor<5xindex>) -> tensor<5xf32> { |
| 483 |
// CHECK-DAG: %[[PAD:.*]] = ub.poison : index |
| 484 |
// CHECK-DAG: %[[VAL_2:.*]] = arith.constant 0 : index |
| 485 |
// CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4]> : vector<5xindex> |
| 486 |
// CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<true> : vector<5xi1> |
| 487 |
// CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32> |
| 488 |
// CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense<6> : vector<5xindex> |
| 489 |
// CHECK: %[[VAL_7:.*]] = tensor.empty() : tensor<5xf32> |
| 490 |
// CHECK: %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[PAD]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex> |
| 491 |
// CHECK: %[[VAL_9:.*]] = arith.muli %[[VAL_8]], %[[VAL_6]] : vector<5xindex> |
| 492 |
// CHECK: %[[VAL_10:.*]] = arith.addi %[[VAL_9]], %[[VAL_3]] : vector<5xindex> |
| 493 |
// CHECK: %[[VAL_11:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]]] {{\[}}%[[VAL_10]]], %[[VAL_4]], %[[VAL_5]] : tensor<5x6xf32>, vector<5xindex>, vector<5xi1>, vector<5xf32> into vector<5xf32> |
| 494 |
// CHECK: %[[VAL_12:.*]] = vector.transfer_write %[[VAL_11]], %[[VAL_7]]{{\[}}%[[VAL_2]]] {in_bounds = [true]} : vector<5xf32>, tensor<5xf32> |
| 495 |
// CHECK: return %[[VAL_12]] : tensor<5xf32> |
| 496 |
// CHECK: } |
| 497 |
|
| 498 |
// ----- |
| 499 |
|
| 500 |
#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> |
| 501 |
#map1 = affine_map<(d0, d1, d2) -> (d0 + d1 + d2)> |
| 502 |
func.func @vectorize_reverse_like_tensor_extract(%arg0: tensor<1x2x3xf32>, %arg1: tensor<1x1x3xf32>, %arg2: index) -> tensor<1x1x3xf32> { |
| 503 |
%c1 = arith.constant 1 : index |
| 504 |
%c0 = arith.constant 0 : index |
| 505 |
%c2 = arith.constant 2 : index |
| 506 |
%0 = linalg.generic {indexing_maps = [#map], iterator_types = ["parallel", "parallel", "parallel"]} outs(%arg1 : tensor<1x1x3xf32>) { |
| 507 |
^bb0(%out: f32): |
| 508 |
%1 = linalg.index 1 : index |
| 509 |
%2 = linalg.index 0 : index |
| 510 |
%3 = affine.apply #map1(%1, %2, %arg2) |
| 511 |
%4 = linalg.index 2 : index |
| 512 |
%5 = arith.subi %c2, %4 : index |
| 513 |
%extracted = tensor.extract %arg0[%c0, %3, %5] : tensor<1x2x3xf32> |
| 514 |
linalg.yield %extracted : f32 |
| 515 |
} -> tensor<1x1x3xf32> |
| 516 |
return %0 : tensor<1x1x3xf32> |
| 517 |
} |
| 518 |
// CHECK-LABEL: func.func @vectorize_reverse_like_tensor_extract |
| 519 |
// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]*]] |
| 520 |
// CHECK-SAME: %[[ARG1:[0-9a-zA-Z]*]] |
| 521 |
// CHECK-SAME: %[[ARG2:[0-9a-zA-Z]*]] |
| 522 |
// CHECK-DAG: %[[CST:.+]] = arith.constant dense<3> : vector<1x1x3xindex> |
| 523 |
// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index |
| 524 |
// CHECK-DAG: %[[MASK:.*]] = arith.constant dense<true> : vector<1x1x3xi1> |
| 525 |
// CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<1x1x3xf32> |
| 526 |
// CHECK-DAG: %[[INIT_IDX:.+]] = arith.constant dense<[2, 1, 0]> : vector<3xindex> |
| 527 |
// CHECK: %[[T0:.+]] = vector.broadcast %[[ARG2]] : index to vector<1x1x3xindex> |
| 528 |
// CHECK: %[[T1:.+]] = arith.muli %[[T0]], %[[CST]] : vector<1x1x3xindex> |
| 529 |
// CHECK: %[[T2:.+]] = vector.broadcast %[[INIT_IDX]] |
| 530 |
// CHECK: %[[T3:.+]] = arith.addi %[[T2]], %[[T1]] |
| 531 |
// CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] [%[[T3]]], %[[MASK]], %[[PASSTHRU]] |
| 532 |
// CHECK: vector.transfer_write %[[GATHER]] |
| 533 |
|
| 534 |
//===----------------------------------------------------------------------===// |
| 535 |
// Scalar load + broadcast |
| 536 |
//===----------------------------------------------------------------------===// |
| 537 |
|
| 538 |
// ----- |
| 539 |
|
| 540 |
#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> |
| 541 |
func.func @vectorize_nd_tensor_extract_scalar_broadcast(%src: tensor<3x3xf32>, %init: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> { |
| 542 |
%c0 = arith.constant 1 : index |
| 543 |
%c1 = arith.constant 2 : index |
| 544 |
|
| 545 |
%res = linalg.generic { |
| 546 |
indexing_maps = [#map], |
| 547 |
iterator_types = ["parallel", "parallel", "parallel"] |
| 548 |
} outs(%init : tensor<1x1x3xf32>) { |
| 549 |
^bb0(%arg4: f32): |
| 550 |
%1 = tensor.extract %src[%c0, %c1] : tensor<3x3xf32> |
| 551 |
linalg.yield %1 : f32 |
| 552 |
} -> tensor<1x1x3xf32> |
| 553 |
|
| 554 |
return %res : tensor<1x1x3xf32> |
| 555 |
} |
| 556 |
|
| 557 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_scalar_broadcast( |
| 558 |
// CHECK-SAME: %[[SRC:.*]]: tensor<3x3xf32>, |
| 559 |
// CHECK-SAME: %[[INIT:.*]]: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> { |
| 560 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 561 |
// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index |
| 562 |
// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index |
| 563 |
// CHECK-DAG: %[[PAD:.*]] = ub.poison : f32 |
| 564 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C1]], %[[C2]]], %[[PAD]] : tensor<3x3xf32>, vector<f32> |
| 565 |
// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1x1x3xf32> |
| 566 |
// CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32> |
| 567 |
|
| 568 |
// ----- |
| 569 |
|
| 570 |
#map = affine_map<() -> ()> |
| 571 |
func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>) -> tensor<f32> { |
| 572 |
%res = linalg.generic { |
| 573 |
indexing_maps = [#map], |
| 574 |
iterator_types = [] |
| 575 |
} outs(%init : tensor<f32>) { |
| 576 |
^bb0(%in: f32): |
| 577 |
%1 = tensor.extract %src[] : tensor<f32> |
| 578 |
linalg.yield %1 : f32 |
| 579 |
} -> tensor<f32> |
| 580 |
|
| 581 |
return %res : tensor<f32> |
| 582 |
} |
| 583 |
|
| 584 |
// CHECK-LABEL: func.func @extract_scalar_from_0d_into_0d( |
| 585 |
// CHECK-SAME: %[[SRC:.*]]: tensor<f32>, |
| 586 |
// CHECK-SAME: %[[INIT:.*]]: tensor<f32>) -> tensor<f32> { |
| 587 |
// CHECK: %[[PAD:.*]] = ub.poison : f32 |
| 588 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32> |
| 589 |
// CHECK: vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32> |
| 590 |
|
| 591 |
// ----- |
| 592 |
|
| 593 |
#map = affine_map<(n) -> (n)> |
| 594 |
func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32>) -> tensor<1xf32> { |
| 595 |
%res = linalg.generic { |
| 596 |
indexing_maps = [#map], |
| 597 |
iterator_types = ["parallel"] |
| 598 |
} outs(%init : tensor<1xf32>) { |
| 599 |
^bb0(%in: f32): |
| 600 |
%1 = tensor.extract %src[] : tensor<f32> |
| 601 |
linalg.yield %1 : f32 |
| 602 |
} -> tensor<1xf32> |
| 603 |
|
| 604 |
return %res : tensor<1xf32> |
| 605 |
} |
| 606 |
// CHECK-LABEL: func.func @extract_scalar_from_0d_into_1d( |
| 607 |
// CHECK-SAME: %[[SRC:.*]]: tensor<f32>, |
| 608 |
// CHECK-SAME: %[[INIT:.*]]: tensor<1xf32>) -> tensor<1xf32> { |
| 609 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 610 |
// CHECK-DAG: %[[PAD:.*]] = ub.poison : f32 |
| 611 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32> |
| 612 |
// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32> |
| 613 |
// CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32> |
| 614 |
|
| 615 |
// ----- |
| 616 |
|
| 617 |
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> |
| 618 |
func.func @vectorize_0d_tensor_extract(%src: tensor<f32>, %init: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> { |
| 619 |
%res = linalg.generic { |
| 620 |
indexing_maps = [#map1], |
| 621 |
iterator_types = ["parallel", "parallel", "parallel"] |
| 622 |
} outs(%init : tensor<1x1x3xf32>) { |
| 623 |
^bb0(%arg4: f32): |
| 624 |
%1 = tensor.extract %src[] : tensor<f32> |
| 625 |
linalg.yield %1 : f32 |
| 626 |
} -> tensor<1x1x3xf32> |
| 627 |
return %res : tensor<1x1x3xf32> |
| 628 |
} |
| 629 |
|
| 630 |
// CHECK-LABEL: func.func @vectorize_0d_tensor_extract( |
| 631 |
// CHECK-SAME: %[[SRC:.*]]: tensor<f32> |
| 632 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %{{.+}} : tensor<f32> |
| 633 |
// CHECK: vector.broadcast %[[READ]] : vector<f32> to vector<1x1x3xf32> |
| 634 |
|
| 635 |
// ----- |
| 636 |
|
| 637 |
func.func @scalar_read_with_broadcast_from_column_tensor(%init: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> { |
| 638 |
%c4 = arith.constant 4 : index |
| 639 |
%c0 = arith.constant 0 : index |
| 640 |
%src = arith.constant dense<[[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32> |
| 641 |
|
| 642 |
%res = linalg.generic { |
| 643 |
indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>], |
| 644 |
iterator_types = ["parallel", "parallel", "parallel"] |
| 645 |
} outs(%init : tensor<1x1x4xi32>) { |
| 646 |
|
| 647 |
^bb0(%out: i32): |
| 648 |
%idx = linalg.index 0 : index |
| 649 |
%extracted = tensor.extract %src[%idx, %c0] : tensor<15x1xi32> |
| 650 |
linalg.yield %extracted : i32 |
| 651 |
} -> tensor<1x1x4xi32> |
| 652 |
|
| 653 |
return %res : tensor<1x1x4xi32> |
| 654 |
} |
| 655 |
|
| 656 |
// CHECK-LABEL: func.func @scalar_read_with_broadcast_from_column_tensor |
| 657 |
// CHECK-SAME: %[[INIT:.*]]: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> { |
| 658 |
// CHECK-DAG: %[[PAD:.*]] = ub.poison : i32 |
| 659 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 660 |
// CHECK-DAG: %[[SRC:.*]] = arith.constant dense<{{\[\[}}0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32> |
| 661 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[C0]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32> |
| 662 |
// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<i32> to vector<1x1x4xi32> |
| 663 |
// CHECK: %[[RES:.*]] = vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x4xi32>, tensor<1x1x4xi32> |
| 664 |
|
| 665 |
// ----- |
| 666 |
|
| 667 |
// TODO: Currently this fails to vectorise when the indices are non-constant. |
| 668 |
|
| 669 |
#map = affine_map<(d0, d1, d2) -> (d0, d1, d2)> |
| 670 |
func.func @vectorize_nd_tensor_extract_transfer_read_basic_column( |
| 671 |
%src: tensor<3x3x3xf32>, |
| 672 |
%init: tensor<3x1x1xf32>) -> tensor<3x1x1xf32> { |
| 673 |
|
| 674 |
%c0 = arith.constant 0 : index |
| 675 |
|
| 676 |
%res = linalg.generic { |
| 677 |
indexing_maps = [#map], |
| 678 |
iterator_types = ["parallel", "parallel", "parallel"] |
| 679 |
} outs(%init : tensor<3x1x1xf32>) { |
| 680 |
^bb0(%out: f32): |
| 681 |
%1 = tensor.extract %src[%c0, %c0, %c0] : tensor<3x3x3xf32> |
| 682 |
linalg.yield %1 : f32 |
| 683 |
} -> tensor<3x1x1xf32> |
| 684 |
|
| 685 |
return %res : tensor<3x1x1xf32> |
| 686 |
} |
| 687 |
|
| 688 |
// CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_basic_column( |
| 689 |
// CHECK-SAME: %[[SRC:.*]]: tensor<3x3x3xf32>, |
| 690 |
// CHECK-SAME: %[[INIT:.*]]: tensor<3x1x1xf32>) |
| 691 |
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index |
| 692 |
// CHECK-DAG: %[[PV:.*]] = ub.poison : f32 |
| 693 |
// CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] : tensor<3x3x3xf32>, vector<f32> |
| 694 |
// CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<3x1x1xf32> |
| 695 |
// CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<3x1x1xf32>, tensor<3x1x1xf32> |