Debian Codegraph @

mlir/test/Dialect/Linalg/vectorization/extract-with-patterns.mlir

raw
1 // RUN: mlir-opt -split-input-file \
2 // RUN: -transform-preload-library='transform-library-paths=%p/../td/vectorize-with-patterns.mlir' \
3 // RUN: -transform-interpreter=entry-point=vectorize_with_patterns %s | FileCheck %s
4
5 //===----------------------------------------------------------------------===//
6 // Contiguous load
7 //===----------------------------------------------------------------------===//
8
9 #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
10 func.func @vectorize_nd_tensor_extract_transfer_read_basic(
11 %arg0: tensor<3x3x3xf32>,
12 %arg1: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
13
14 %res = linalg.generic {
15 indexing_maps = [#map],
16 iterator_types = ["parallel", "parallel", "parallel"]
17 } outs(%arg1 : tensor<1x1x3xf32>) {
18 ^bb0(%out: f32):
19 %1 = linalg.index 0 : index
20 %2 = linalg.index 1 : index
21 %3 = linalg.index 2 : index
22 %4 = tensor.extract %arg0[%1, %2, %3] : tensor<3x3x3xf32>
23 linalg.yield %4 : f32
24 } -> tensor<1x1x3xf32>
25
26 return %res : tensor<1x1x3xf32>
27 }
28
29 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_basic
30 // CHECK-SAME: %[[ARG0:.*]]: tensor<3x3x3xf32>
31 // CHECK-SAME: %[[ARG1:.*]]: tensor<1x1x3xf32>
32
33 // CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
34 // CHECK-DAG: %[[PV:.+]] = ub.poison : f32
35
36 // CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true, true]} : tensor<3x3x3xf32>, vector<1x1x3xf32>
37 // CHECK: vector.transfer_write %[[READ]], %[[ARG1]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32>
38
39 // -----
40
41 func.func @vectorize_nd_tensor_extract_transfer_read_complex(%6: tensor<45x80x16xf32>, %arg0: index, %arg2: index, %arg1: index, %arg4: index, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> {
42 %c79 = arith.constant 79 : index
43 %25 = linalg.generic {
44 indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>],
45 iterator_types = ["parallel", "parallel"]
46 } outs(%extracted_slice : tensor<1x4xf32>) {
47 ^bb0(%out: f32):
48 %26 = linalg.index 0 : index
49 %27 = arith.addi %arg0, %26 : index
50 %28 = arith.addi %27, %arg2 : index
51 %29 = linalg.index 1 : index
52 %30 = arith.addi %arg1, %29 : index
53 %31 = arith.addi %30, %arg4 : index
54 %extracted = tensor.extract %6[%28, %c79, %31] : tensor<45x80x16xf32>
55 linalg.yield %extracted : f32
56 } -> tensor<1x4xf32>
57 return %25 : tensor<1x4xf32>
58 }
59
60
61 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_complex(
62 // CHECK-SAME: %[[ARG0:.*]]: tensor<45x80x16xf32>,
63 // CHECK-SAME: %[[ARG1:.*]]: index, %[[ARG2:.*]]: index, %[[ARG3:.*]]: index, %[[ARG4:.*]]: index,
64 // CHECK-SAME: %[[ARG5:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
65
66 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
67 // CHECK-DAG: %[[PV:.*]] = ub.poison : f32
68 // CHECK-DAG: %[[C79:.*]] = arith.constant 79 : index
69 // CHECK: %[[ADD1:.*]] = arith.addi %[[ARG1]], %[[ARG2]] : index
70 // CHECK: %[[ADD2:.*]] = arith.addi %[[ARG3]], %[[ARG4]] : index
71
72 // CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[ADD1]], %[[C79]], %[[ADD2]]], %[[PV]] {in_bounds = [true, true]} : tensor<45x80x16xf32>, vector<1x4xf32>
73 // CHECK: %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG5]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
74 // CHECK: return %[[WRITE]] : tensor<1x4xf32>
75 // CHECK: }
76
77 // -----
78
79 // The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Contiguous load.
80 func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(%6: tensor<80x16xf32>, %arg0: index, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> {
81 %c79 = arith.constant 79 : index
82 %1 = linalg.generic {
83 indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>],
84 iterator_types = ["parallel", "parallel"]
85 } outs(%extracted_slice : tensor<1x4xf32>) {
86 ^bb0(%out: f32):
87 %2 = linalg.index 1 : index
88 %3 = affine.apply affine_map<(d0, d1) -> (d0 + d1)>(%2, %arg0)
89 %extracted = tensor.extract %6[%c79, %3] : tensor<80x16xf32>
90 linalg.yield %extracted : f32
91 } -> tensor<1x4xf32>
92 return %1 : tensor<1x4xf32>
93 }
94
95 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_affine_apply_contiguous(
96 // CHECK-SAME: %[[ARG0:.*]]: tensor<80x16xf32>,
97 // CHECK-SAME: %[[ARG1:.*]]: index,
98 // CHECK-SAME: %[[ARG2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
99
100 // CHECK-DAG: %[[PV:.*]] = ub.poison : f32
101 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
102 // CHECK-DAG: %[[C79:.*]] = arith.constant 79 : index
103
104 // CHECK: %[[READ:.*]] = vector.transfer_read %[[ARG0]]{{\[}}%[[C79]], %[[ARG1]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
105 // CHECK: %[[WRITE:.*]] = vector.transfer_write %[[READ]], %[[ARG2]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
106 // CHECK: return %[[WRITE]] : tensor<1x4xf32>
107 // CHECK: }
108
109 // -----
110
111 func.func @vectorize_nd_tensor_extract_with_tensor_extract(%input_1: tensor<1x20xi32>, %input_2: tensor<257x24xf32>, %arg0 : index, %arg1 : index, %arg2 : index, %arg3 : index) -> tensor<1x1x4xf32> {
112 %c0 = arith.constant 0 : index
113 %c256 = arith.constant 256 : index
114 %output = tensor.empty() : tensor<1x1x4xf32>
115 %1 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} outs(%output : tensor<1x1x4xf32>) {
116 ^bb0(%out: f32):
117 %13 = linalg.index 0 : index
118 %14 = affine.apply affine_map<(d0, d1, d2) -> (d0 + d1 + d2)>(%arg0, %13, %arg2)
119 %15 = linalg.index 2 : index
120 %16 = linalg.index 1 : index
121 %17 = affine.apply affine_map<(d0, d1, d2, d3) -> (d0 + d1 * 24 + d2 + d3)>(%arg1, %16, %15, %arg3)
122 %extracted_0 = tensor.extract %input_1[%c0, %14] : tensor<1x20xi32>
123 %18 = arith.index_cast %extracted_0 : i32 to index
124 %19 = arith.maxsi %18, %c0 : index
125 %20 = arith.minsi %19, %c256 : index
126 %extracted_1 = tensor.extract %input_2[%20, %17] : tensor<257x24xf32>
127 linalg.yield %extracted_1 : f32
128 } -> tensor<1x1x4xf32>
129 return %1 : tensor<1x1x4xf32>
130 }
131
132 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_tensor_extract(
133 // CHECK-SAME: %[[INPUT_1:.*]]: tensor<1x20xi32>,
134 // CHECK-SAME: %[[INPUT_2:.*]]: tensor<257x24xf32>,
135 // CHECK-SAME: %[[INPUT_3:.*]]: index, %[[INPUT_4:.*]]: index, %[[INPUT_5:.*]]: index,
136 // CHECK: %[[EXTRACTED_0_IDX_0:.*]] = arith.constant 0 : index
137 // CHECK: %[[SCALAR:.*]] = arith.addi %[[INPUT_3]], %[[INPUT_5]] : index
138 // First `vector.transfer_read` from the generic Op - loop invariant scalar load.
139 // CHECK: vector.transfer_read %[[INPUT_1]][%[[EXTRACTED_0_IDX_0]], %[[SCALAR]]]
140 // CHECK-SAME: tensor<1x20xi32>, vector<i32>
141 // The following `tensor.extract` from the generic Op s a contiguous load (all Ops used
142 // for address calculation also satisfy the required conditions).
143 // CHECK: vector.transfer_read %[[INPUT_2]][%{{.*}}, %{{.*}}, %{{.*}} {in_bounds = [true, true]} : tensor<257x24xf32>, vector<1x4xf32>
144
145 // Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Contiguous load.
146 func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(%arg0: tensor<80x16xf32>, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> {
147 %c16 = arith.constant 16 : index
148 %1 = linalg.generic {
149 indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>],
150 iterator_types = ["parallel", "parallel"]
151 } outs(%extracted_slice : tensor<1x4xf32>) {
152 ^bb0(%out: f32):
153 %2 = linalg.index 0 : index
154 %3 = linalg.index 1 : index
155 %4 = arith.maxsi %2, %c16 : index
156 %extracted = tensor.extract %arg0[%4, %3] : tensor<80x16xf32>
157 linalg.yield %extracted : f32
158 } -> tensor<1x4xf32>
159 return %1 : tensor<1x4xf32>
160 }
161
162 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_maxsi_contiguous(
163 // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>,
164 // CHECK-SAME: %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
165 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
166 // CHECK-DAG: %[[C16:.*]] = arith.constant 16 : index
167 // CHECK-DAG: %[[PV:.*]] = ub.poison : f32
168
169 // CHECK: %[[VAL_8:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[C16]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<80x16xf32>, vector<1x4xf32>
170 // CHECK: %[[VAL_9:.*]] = vector.transfer_write %[[VAL_8]], %[[VAL_1]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
171 // CHECK: return %[[VAL_9]] : tensor<1x4xf32>
172 // CHECK: }
173
174 // -----
175
176 //===----------------------------------------------------------------------===//
177 // Gather load
178 //===----------------------------------------------------------------------===//
179
180 #map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2)>
181 #map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
182 func.func @vectorize_1d_tensor_extract(%arg0: tensor<3xf32>, %arg1: tensor<4x3xi32>, %arg2: tensor<4x7x3x2xf32>) -> tensor<4x7x3x2xf32> {
183 %1 = linalg.generic {
184 indexing_maps = [#map0, #map1],
185 iterator_types = ["parallel", "parallel", "parallel", "parallel"]
186 } ins(%arg1 : tensor<4x3xi32>) outs(%arg2 : tensor<4x7x3x2xf32>) {
187 ^bb0(%arg3: i32, %arg4: f32):
188 %2 = arith.index_cast %arg3 : i32 to index
189 %3 = tensor.extract %arg0[%2] : tensor<3xf32>
190 linalg.yield %3 : f32
191 } -> tensor<4x7x3x2xf32>
192 return %1 : tensor<4x7x3x2xf32>
193 }
194 // CHECK-LABEL: func.func @vectorize_1d_tensor_extract
195 // CHECK-SAME: %[[ARG0:.*]]: tensor<3xf32>
196 // CHECK-SAME: %[[ARG1:.*]]: tensor<4x3xi32>
197 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
198 // CHECK-DAG: %[[MASK:.*]] = arith.constant dense<true> : vector<4x7x3x2xi1>
199 // CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32>
200 // CHECK: %[[V0:.*]] = vector.transfer_read %[[ARG1]]
201 // CHECK: %[[CAST:.*]] = arith.index_cast %[[V0]]
202 // CHECK: %[[BROADCAST:.*]] = vector.broadcast %[[CAST]]
203 // CHECK: %[[INDICES:.*]] = vector.transpose %[[BROADCAST]]
204 // CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]]] [%[[INDICES]]], %[[MASK]], %[[PASSTHRU]]
205 // CHECK: vector.transfer_write %[[GATHER]]
206
207 // -----
208
209 #map0 = affine_map<(d0, d1, d2, d3) -> (d0, d2)>
210 #map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
211 #map2 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
212 func.func @vectorize_nd_tensor_extract_index_from_tensor(%arg0: tensor<3x3xf32>, %arg1: tensor<4x3xi32>, %arg2: tensor<4x3xi32>, %arg3: tensor<4x7x2xf32>, %arg4: tensor<4x7x3x2xf32>) -> tensor<4x7x3x2xf32> {
213 %2 = linalg.generic {
214 indexing_maps = [#map0, #map0, #map1, #map2],
215 iterator_types = ["parallel", "parallel", "parallel", "parallel"]
216 } ins(%arg1, %arg2, %arg3 : tensor<4x3xi32>, tensor<4x3xi32>, tensor<4x7x2xf32>) outs(%arg4 : tensor<4x7x3x2xf32>) {
217 ^bb0(%arg5: i32, %arg6: i32, %arg7: f32, %arg8: f32):
218 %3 = arith.index_cast %arg5 : i32 to index
219 %4 = arith.index_cast %arg6 : i32 to index
220 %7 = tensor.extract %arg0[%3, %4] : tensor<3x3xf32>
221 linalg.yield %7 : f32
222 } -> tensor<4x7x3x2xf32>
223 return %2 : tensor<4x7x3x2xf32>
224 }
225 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_index_from_tensor
226 // CHECK-SAME: %[[ARG0:.*]]: tensor<3x3xf32>
227 // CHECK-SAME: %[[ARG1:arg1]]: tensor<4x3xi32>
228 // CHECK-SAME: %[[ARG2:arg2]]: tensor<4x3xi32>
229 // CHECK-SAME: %[[ARG3:.*]]: tensor<4x7x2xf32>
230 // CHECK-SAME: %[[ARG4:.*]]: tensor<4x7x3x2xf32>
231 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
232 // CHECK-DAG: %[[PV:.*]] = ub.poison : i32
233 // CHECK-DAG: %[[CST:.*]] = arith.constant dense<3> : vector<7x2x4x3xindex>
234 // CHECK-DAG: %[[CST_1:.*]] = arith.constant dense<true> : vector<4x7x3x2xi1>
235 // CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<4x7x3x2xf32>
236 // CHECK: %[[V0:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
237 // CHECK: %[[V1:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], %[[PV]] {in_bounds = [true, true]} : tensor<4x3xi32>, vector<4x3xi32>
238 // CHECK: %[[CAST:.*]] = arith.index_cast %[[V0]] : vector<4x3xi32> to vector<4x3xindex>
239 // CHECK: %[[B1:.*]] = vector.broadcast %[[CAST]] : vector<4x3xindex> to vector<7x2x4x3xindex>
240 // CHECK: %[[CAST_1:.*]] = arith.index_cast %[[V1]] : vector<4x3xi32> to vector<4x3xindex>
241 // CHECK: %[[B2:.*]] = vector.broadcast %[[CAST_1]] : vector<4x3xindex> to vector<7x2x4x3xindex>
242 // CHECK: %[[MULI:.*]] = arith.muli %[[B1]], %[[CST]] : vector<7x2x4x3xindex>
243 // CHECK: %[[ADDI:.*]] = arith.addi %[[B2]], %[[MULI]] : vector<7x2x4x3xindex>
244 // CHECK: %[[T:.*]] = vector.transpose %[[ADDI]], [2, 0, 3, 1] : vector<7x2x4x3xindex> to vector<4x7x3x2xindex>
245 // CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]]] [%[[T]]], %[[CST_1]], %[[PASSTHRU]] : tensor<3x3xf32>, vector<4x7x3x2xindex>, vector<4x7x3x2xi1>, vector<4x7x3x2xf32> into vector<4x7x3x2xf32>
246 // CHECK: vector.transfer_write %[[GATHER]], %[[ARG4]][%[[C0]], %[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true, true]} : vector<4x7x3x2xf32>, tensor<4x7x3x2xf32>
247
248 // -----
249
250 #map = affine_map<(d0, d1) -> (d0, d1)>
251 #map1 = affine_map<(d0, d1, d2) -> (d0 + d1 + d2)>
252 func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load(%arg0: tensor<8x128x768xf32>, %arg1 : index) -> tensor<8x1xf32> {
253 %c0 = arith.constant 0 : index
254 %0 = tensor.empty() : tensor<8x1xf32>
255 %1 = linalg.generic {
256 indexing_maps = [#map],
257 iterator_types = ["parallel", "parallel"]
258 } outs(%0 : tensor<8x1xf32>) {
259 ^bb0(%arg5: f32):
260 %2 = linalg.index 0 : index
261 %3 = linalg.index 1 : index
262 %4 = affine.apply #map1(%arg1, %3, %arg1)
263 %extracted = tensor.extract %arg0[%2, %c0, %4] : tensor<8x128x768xf32>
264 linalg.yield %extracted : f32
265 } -> tensor<8x1xf32>
266 return %1 : tensor<8x1xf32>
267 }
268
269 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_load_1d_column_vector_using_gather_load
270 // CHECK-SAME: %[[ARG0:.*]]: tensor<8x128x768xf32>
271 // CHECK-SAME: %[[ARG1:.*]]: index
272 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
273 // CHECK-DAG: %[[CST:.*]] = arith.constant dense<768> : vector<1x8xindex>
274 // CHECK-DAG: %[[CST_0:.*]] = arith.constant dense<128> : vector<1x8xindex>
275 // CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<8x1xf32>
276 // CHECK-DAG: %[[CST_2:.*]] = arith.constant dense<true> : vector<8x1xi1>
277 // CHECK-DAG: %[[CST_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex>
278 // CHECK: %[[EMPTY:.*]] = tensor.empty() : tensor<8x1xf32>
279 // CHECK: %[[B1:.*]] = vector.broadcast %[[CST_3]] : vector<8xindex> to vector<1x8xindex>
280 // CHECK: %[[ADDI_ARG1:.*]] = arith.addi %[[ARG1]], %[[ARG1]] : index
281 // CHECK: %[[MULI_1:.*]] = arith.muli %[[B1]], %[[CST_0]] : vector<1x8xindex>
282 // CHECK: %[[MULI_2:.*]] = arith.muli %[[MULI_1]], %[[CST]] : vector<1x8xindex>
283 // CHECK: %[[T:.*]] = vector.transpose %[[MULI_2]], [1, 0] : vector<1x8xindex> to vector<8x1xindex>
284 // CHECK: %[[B2:.*]] = vector.broadcast %[[ADDI_ARG1]] : index to vector<8x1xindex>
285 // CHECK: %[[ADDI:.*]] = arith.addi %[[B2]], %[[T]] : vector<8x1xindex>
286 // CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] [%[[ADDI]]], %[[CST_2]], %[[PASSTHRU]] : tensor<8x128x768xf32>, vector<8x1xindex>, vector<8x1xi1>, vector<8x1xf32> into vector<8x1xf32>
287 // CHECK: vector.transfer_write %[[GATHER]], %[[EMPTY]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x1xf32>, tensor<8x1xf32>
288
289 // -----
290
291 // Reading a 1D column vector (hence a candidate for a contiguous load), but given
292 // %1, it's a gather load.
293
294 #map = affine_map<(d0, d1) -> (d0, d1)>
295 func.func @index_from_output_column_vector_gather_load(%src: tensor<8x128xf32>) -> tensor<8x1xf32> {
296 %c0 = arith.constant 0 : index
297 %0 = tensor.empty() : tensor<8x1xf32>
298 %res = linalg.generic {
299 indexing_maps = [#map],
300 iterator_types = ["parallel", "parallel"]
301 } outs(%0 : tensor<8x1xf32>) {
302 ^bb0(%arg1: f32):
303 %1 = linalg.index 0 : index
304 %extracted = tensor.extract %src[%1, %c0] : tensor<8x128xf32>
305 linalg.yield %extracted : f32
306 } -> tensor<8x1xf32>
307 return %res : tensor<8x1xf32>
308 }
309
310 // CHECK-LABEL: func.func @index_from_output_column_vector_gather_load(
311 // CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> {
312 // CHECK: %[[C128:.*]] = arith.constant dense<128> : vector<1x8xindex>
313 // CHECK: %[[C0:.*]] = arith.constant 0 : index
314 // CHECK: %[[PASS_THRU:.*]] = arith.constant dense<0.000000e+00> : vector<8x1xf32>
315 // CHECK: %[[MASK:.*]] = arith.constant dense<true> : vector<8x1xi1>
316 // CHECK: %[[IDX_VEC:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex>
317 // CHECK: %[[OUT:.*]] = tensor.empty() : tensor<8x1xf32>
318 // CHECK: %[[B:.*]] = vector.broadcast %[[IDX_VEC]] : vector<8xindex> to vector<1x8xindex>
319 // CHECK: %[[MUL:.*]] = arith.muli %[[B]], %[[C128]] : vector<1x8xindex>
320 // CHECK: %[[TR:.*]] = vector.transpose %[[MUL]], [1, 0] : vector<1x8xindex> to vector<8x1xindex>
321 // CHECK: %[[GATHER:.*]] = vector.gather %[[SRC]]{{\[}}%[[C0]], %[[C0]]] {{\[}}%[[TR]]], %[[MASK]], %[[PASS_THRU]] : tensor<8x128xf32>, vector<8x1xindex>, vector<8x1xi1>, vector<8x1xf32> into vector<8x1xf32>
322 // CHECK: %[[RES:.*]] = vector.transfer_write %[[GATHER]], %[[OUT]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x1xf32>, tensor<8x1xf32>
323 // CHECK: return %[[RES]] : tensor<8x1xf32>
324
325 // -----
326
327 // Same as above, but the access indices have been swapped and hence this _is_
328 // a contiguous load. Currently not supported and lowered as vector.gather
329 // instead.
330 // TODO: Make sure that this is lowered as a contiguous load.
331
332 #map = affine_map<(d0, d1) -> (d0, d1)>
333 func.func @index_from_output_column_vector_contiguous_load(%src: tensor<8x128xf32>) -> tensor<8x1xf32> {
334 %c0 = arith.constant 0 : index
335 %0 = tensor.empty() : tensor<8x1xf32>
336 %res = linalg.generic {
337 indexing_maps = [#map],
338 iterator_types = ["parallel", "parallel"]
339 } outs(%0 : tensor<8x1xf32>) {
340 ^bb0(%arg1: f32):
341 %1 = linalg.index 0 : index
342 %extracted = tensor.extract %src[%c0, %1] : tensor<8x128xf32>
343 linalg.yield %extracted : f32
344 } -> tensor<8x1xf32>
345 return %res : tensor<8x1xf32>
346 }
347
348 // CHECK-LABEL: func.func @index_from_output_column_vector_contiguous_load(
349 // CHECK-SAME: %[[SRC:.*]]: tensor<8x128xf32>) -> tensor<8x1xf32> {
350 // CHECK: %[[C0:.*]] = arith.constant 0 : index
351 // CHECK: %[[PASS_THRU:.*]] = arith.constant dense<0.000000e+00> : vector<8x1xf32>
352 // CHECK: %[[MASK:.*]] = arith.constant dense<true> : vector<8x1xi1>
353 // CHECK: %[[IDX_VEC:.*]] = arith.constant dense<[0, 1, 2, 3, 4, 5, 6, 7]> : vector<8xindex>
354 // CHECK: %[[OUT:.*]] = tensor.empty() : tensor<8x1xf32>
355 // CHECK: %[[B:.*]] = vector.broadcast %[[IDX_VEC]] : vector<8xindex> to vector<1x8xindex>
356 // CHECK: %[[TR:.*]] = vector.transpose %[[B]], [1, 0] : vector<1x8xindex> to vector<8x1xindex>
357 // CHECK: %[[GATHER:.*]] = vector.gather %[[SRC]]{{\[}}%[[C0]], %[[C0]]] {{\[}}%[[TR]]], %[[MASK]], %[[PASS_THRU]] : tensor<8x128xf32>, vector<8x1xindex>, vector<8x1xi1>, vector<8x1xf32> into vector<8x1xf32>
358 // CHECK: %[[RES:.*]] = vector.transfer_write %[[GATHER]], %[[OUT]]{{\[}}%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x1xf32>, tensor<8x1xf32>
359 // CHECK: return %[[RES]] : tensor<8x1xf32>
360
361 // -----
362
363 #map = affine_map<(d0) -> (d0)>
364 func.func @vectorize_nd_tensor_extract_contiguous_and_gather(%arg0: tensor<6xf32>, %arg1: tensor<5xi32>) -> tensor<5xf32> {
365 %c5 = arith.constant 5 : index
366 %c0 = arith.constant 0 : index
367 %0 = tensor.empty() : tensor<5xf32>
368 %1 = linalg.generic {indexing_maps = [#map], iterator_types = ["parallel"]} outs(%0 : tensor<5xf32>) {
369 ^bb0(%out: f32):
370 %2 = linalg.index 0 : index
371 %extracted = tensor.extract %arg1[%2] : tensor<5xi32>
372 %3 = arith.index_cast %extracted : i32 to index
373 %4 = arith.maxsi %3, %c0 : index
374 %5 = arith.minsi %4, %c5 : index
375 %extracted_0 = tensor.extract %arg0[%5] : tensor<6xf32>
376 linalg.yield %extracted_0 : f32
377 } -> tensor<5xf32>
378 return %1 : tensor<5xf32>
379 }
380
381 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_contiguous_and_gather(
382 // CHECK-SAME: %[[VAL_0:.*]]: tensor<6xf32>
383 // CHECK-SAME: %[[VAL_1:.*]]: tensor<5xi32>
384 // CHECK-DAG: %[[VAL_2:.*]] = arith.constant 0 : index
385 // CHECK-DAG: %[[VAL_3:.*]] = ub.poison : i32
386 // CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<0> : vector<5xindex>
387 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<5> : vector<5xindex>
388 // CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense<true> : vector<5xi1>
389 // CHECK-DAG: %[[VAL_7:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32>
390 // CHECK: %[[VAL_8:.*]] = tensor.empty() : tensor<5xf32>
391 // CHECK: %[[VAL_9:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%{{.*}}], %[[VAL_3]] {in_bounds = [true]} : tensor<5xi32>, vector<5xi32>
392 // CHECK: %[[VAL_10:.*]] = arith.index_cast %[[VAL_9]] : vector<5xi32> to vector<5xindex>
393 // CHECK: %[[VAL_11:.*]] = arith.maxsi %[[VAL_10]], %[[VAL_4]] : vector<5xindex>
394 // CHECK: %[[VAL_12:.*]] = arith.minsi %[[VAL_11]], %[[VAL_5]] : vector<5xindex>
395 // CHECK: %[[VAL_13:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_2]]] {{\[}}%[[VAL_12]]], %[[VAL_6]], %[[VAL_7]] : tensor<6xf32>, vector<5xindex>, vector<5xi1>, vector<5xf32> into vector<5xf32>
396 // CHECK: %[[VAL_14:.*]] = vector.transfer_write %[[VAL_13]], %[[VAL_8]]{{\[}}%[[VAL_2]]] {in_bounds = [true]} : vector<5xf32>, tensor<5xf32>
397 // CHECK: return %[[VAL_14]] : tensor<5xf32>
398
399 // The vectorizer converts `affine.apply` so that the subsequent Ops can be vectorised based on the converted ops. Gather load.
400 func.func @vectorize_nd_tensor_extract_with_affine_apply_gather(%6: tensor<80x16xf32>, %arg0: index, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> {
401 %c16 = arith.constant 16 : index
402 %1 = linalg.generic {
403 indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>],
404 iterator_types = ["parallel", "parallel"]
405 } outs(%extracted_slice : tensor<1x4xf32>) {
406 ^bb0(%out: f32):
407 %2 = linalg.index 1 : index
408 %3 = affine.apply affine_map<(d0, d1) -> (d0 + d1)>(%2, %arg0)
409 %extracted = tensor.extract %6[%3, %c16] : tensor<80x16xf32>
410 linalg.yield %extracted : f32
411 } -> tensor<1x4xf32>
412 return %1 : tensor<1x4xf32>
413 }
414
415 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_affine_apply_gather(
416 // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>,
417 // CHECK-SAME: %[[VAL_1:.*]]: index,
418 // CHECK-SAME: %[[VAL_2:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
419 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex>
420 // CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<true> : vector<1x4xi1>
421 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32>
422 // CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index
423 // CHECK-DAG: %[[VAL_7:.*]] = arith.constant dense<16> : vector<1x4xindex>
424 // CHECK: %[[VAL_8:.*]] = vector.broadcast %[[VAL_1]] : index to vector<4xindex>
425 // CHECK: %[[VAL_9:.*]] = arith.addi %[[VAL_8]], %[[VAL_3]] : vector<4xindex>
426 // CHECK: %[[VAL_10:.*]] = vector.broadcast %[[VAL_9]] : vector<4xindex> to vector<1x4xindex>
427 // CHECK: %[[VAL_11:.*]] = arith.muli %[[VAL_10]], %[[VAL_7]] : vector<1x4xindex>
428 // CHECK: %[[VAL_12:.*]] = arith.addi %[[VAL_11]], %[[VAL_7]] : vector<1x4xindex>
429 // CHECK: %[[VAL_13:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {{\[}}%[[VAL_12]]], %[[VAL_4]], %[[VAL_5]] : tensor<80x16xf32>, vector<1x4xindex>, vector<1x4xi1>, vector<1x4xf32> into vector<1x4xf32>
430 // CHECK: %[[VAL_14:.*]] = vector.transfer_write %[[VAL_13]], %[[VAL_2]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
431 // CHECK: return %[[VAL_14]] : tensor<1x4xf32>
432 // CHECK: }
433
434 // Make sure that non-linear arithmetic operations (e.g. arith.maxsi) are allowed when calculating indices for load operations. Gather load.
435 func.func @vectorize_nd_tensor_extract_with_maxsi_gather(%arg0: tensor<80x16xf32>, %extracted_slice : tensor<1x4xf32>) -> tensor<1x4xf32> {
436 %c79 = arith.constant 79 : index
437 %1 = linalg.generic {
438 indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>],
439 iterator_types = ["parallel", "parallel"]
440 } outs(%extracted_slice : tensor<1x4xf32>) {
441 ^bb0(%out: f32):
442 %2 = linalg.index 1 : index
443 %3 = arith.maxsi %2, %c79 : index
444 %extracted = tensor.extract %arg0[%3, %2] : tensor<80x16xf32>
445 linalg.yield %extracted : f32
446 } -> tensor<1x4xf32>
447 return %1 : tensor<1x4xf32>
448 }
449
450 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_with_maxsi_gather(
451 // CHECK-SAME: %[[VAL_0:.*]]: tensor<80x16xf32>,
452 // CHECK-SAME: %[[VAL_1:.*]]: tensor<1x4xf32>) -> tensor<1x4xf32> {
453 // CHECK-DAG: %[[VAL_2:.*]] = arith.constant dense<[0, 1, 2, 3]> : vector<4xindex>
454 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<1264> : vector<1x4xindex>
455 // CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<true> : vector<1x4xi1>
456 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<1x4xf32>
457 // CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0 : index
458 // CHECK: %[[VAL_7:.*]] = vector.broadcast %[[VAL_2]] : vector<4xindex> to vector<1x4xindex>
459 // CHECK: %[[VAL_8:.*]] = arith.addi %[[VAL_7]], %[[VAL_3]] : vector<1x4xindex>
460 // CHECK: %[[VAL_9:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {{\[}}%[[VAL_8]]], %[[VAL_4]], %[[VAL_5]] : tensor<80x16xf32>, vector<1x4xindex>, vector<1x4xi1>, vector<1x4xf32> into vector<1x4xf32>
461 // CHECK: %[[VAL_10:.*]] = vector.transfer_write %[[VAL_9]], %[[VAL_1]]{{\[}}%[[VAL_6]], %[[VAL_6]]] {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
462 // CHECK: return %[[VAL_10]] : tensor<1x4xf32>
463 // CHECK: }
464
465 // -----
466
467 // The vectorizer assumes it's a gather load whenever using a block argument to calculate an index.
468 #map = affine_map<(d0) -> (d0)>
469 func.func @vectorize_nd_tensor_extract_block_arg(%arg0: tensor<5x6xf32>, %arg1: tensor<5xindex>) -> tensor<5xf32> {
470 %0 = tensor.empty() : tensor<5xf32>
471 %1 = linalg.generic {indexing_maps = [#map, #map], iterator_types = ["parallel"]} ins(%arg1: tensor<5xindex>) outs(%0 : tensor<5xf32>) {
472 ^bb0(%in: index, %out: f32):
473 %2 = linalg.index 0 : index
474 %extracted_0 = tensor.extract %arg0[%in, %2] : tensor<5x6xf32>
475 linalg.yield %extracted_0 : f32
476 } -> tensor<5xf32>
477 return %1 : tensor<5xf32>
478 }
479
480 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_block_arg(
481 // CHECK-SAME: %[[VAL_0:.*]]: tensor<5x6xf32>,
482 // CHECK-SAME: %[[VAL_1:.*]]: tensor<5xindex>) -> tensor<5xf32> {
483 // CHECK-DAG: %[[PAD:.*]] = ub.poison : index
484 // CHECK-DAG: %[[VAL_2:.*]] = arith.constant 0 : index
485 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant dense<[0, 1, 2, 3, 4]> : vector<5xindex>
486 // CHECK-DAG: %[[VAL_4:.*]] = arith.constant dense<true> : vector<5xi1>
487 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant dense<0.000000e+00> : vector<5xf32>
488 // CHECK-DAG: %[[VAL_6:.*]] = arith.constant dense<6> : vector<5xindex>
489 // CHECK: %[[VAL_7:.*]] = tensor.empty() : tensor<5xf32>
490 // CHECK: %[[VAL_8:.*]] = vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_2]]], %[[PAD]] {in_bounds = [true]} : tensor<5xindex>, vector<5xindex>
491 // CHECK: %[[VAL_9:.*]] = arith.muli %[[VAL_8]], %[[VAL_6]] : vector<5xindex>
492 // CHECK: %[[VAL_10:.*]] = arith.addi %[[VAL_9]], %[[VAL_3]] : vector<5xindex>
493 // CHECK: %[[VAL_11:.*]] = vector.gather %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]]] {{\[}}%[[VAL_10]]], %[[VAL_4]], %[[VAL_5]] : tensor<5x6xf32>, vector<5xindex>, vector<5xi1>, vector<5xf32> into vector<5xf32>
494 // CHECK: %[[VAL_12:.*]] = vector.transfer_write %[[VAL_11]], %[[VAL_7]]{{\[}}%[[VAL_2]]] {in_bounds = [true]} : vector<5xf32>, tensor<5xf32>
495 // CHECK: return %[[VAL_12]] : tensor<5xf32>
496 // CHECK: }
497
498 // -----
499
500 #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
501 #map1 = affine_map<(d0, d1, d2) -> (d0 + d1 + d2)>
502 func.func @vectorize_reverse_like_tensor_extract(%arg0: tensor<1x2x3xf32>, %arg1: tensor<1x1x3xf32>, %arg2: index) -> tensor<1x1x3xf32> {
503 %c1 = arith.constant 1 : index
504 %c0 = arith.constant 0 : index
505 %c2 = arith.constant 2 : index
506 %0 = linalg.generic {indexing_maps = [#map], iterator_types = ["parallel", "parallel", "parallel"]} outs(%arg1 : tensor<1x1x3xf32>) {
507 ^bb0(%out: f32):
508 %1 = linalg.index 1 : index
509 %2 = linalg.index 0 : index
510 %3 = affine.apply #map1(%1, %2, %arg2)
511 %4 = linalg.index 2 : index
512 %5 = arith.subi %c2, %4 : index
513 %extracted = tensor.extract %arg0[%c0, %3, %5] : tensor<1x2x3xf32>
514 linalg.yield %extracted : f32
515 } -> tensor<1x1x3xf32>
516 return %0 : tensor<1x1x3xf32>
517 }
518 // CHECK-LABEL: func.func @vectorize_reverse_like_tensor_extract
519 // CHECK-SAME: %[[ARG0:[0-9a-zA-Z]*]]
520 // CHECK-SAME: %[[ARG1:[0-9a-zA-Z]*]]
521 // CHECK-SAME: %[[ARG2:[0-9a-zA-Z]*]]
522 // CHECK-DAG: %[[CST:.+]] = arith.constant dense<3> : vector<1x1x3xindex>
523 // CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index
524 // CHECK-DAG: %[[MASK:.*]] = arith.constant dense<true> : vector<1x1x3xi1>
525 // CHECK-DAG: %[[PASSTHRU:.*]] = arith.constant dense<0.000000e+00> : vector<1x1x3xf32>
526 // CHECK-DAG: %[[INIT_IDX:.+]] = arith.constant dense<[2, 1, 0]> : vector<3xindex>
527 // CHECK: %[[T0:.+]] = vector.broadcast %[[ARG2]] : index to vector<1x1x3xindex>
528 // CHECK: %[[T1:.+]] = arith.muli %[[T0]], %[[CST]] : vector<1x1x3xindex>
529 // CHECK: %[[T2:.+]] = vector.broadcast %[[INIT_IDX]]
530 // CHECK: %[[T3:.+]] = arith.addi %[[T2]], %[[T1]]
531 // CHECK: %[[GATHER:.*]] = vector.gather %[[ARG0]][%[[C0]], %[[C0]], %[[C0]]] [%[[T3]]], %[[MASK]], %[[PASSTHRU]]
532 // CHECK: vector.transfer_write %[[GATHER]]
533
534 //===----------------------------------------------------------------------===//
535 // Scalar load + broadcast
536 //===----------------------------------------------------------------------===//
537
538 // -----
539
540 #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
541 func.func @vectorize_nd_tensor_extract_scalar_broadcast(%src: tensor<3x3xf32>, %init: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
542 %c0 = arith.constant 1 : index
543 %c1 = arith.constant 2 : index
544
545 %res = linalg.generic {
546 indexing_maps = [#map],
547 iterator_types = ["parallel", "parallel", "parallel"]
548 } outs(%init : tensor<1x1x3xf32>) {
549 ^bb0(%arg4: f32):
550 %1 = tensor.extract %src[%c0, %c1] : tensor<3x3xf32>
551 linalg.yield %1 : f32
552 } -> tensor<1x1x3xf32>
553
554 return %res : tensor<1x1x3xf32>
555 }
556
557 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_scalar_broadcast(
558 // CHECK-SAME: %[[SRC:.*]]: tensor<3x3xf32>,
559 // CHECK-SAME: %[[INIT:.*]]: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
560 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
561 // CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
562 // CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
563 // CHECK-DAG: %[[PAD:.*]] = ub.poison : f32
564 // CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C1]], %[[C2]]], %[[PAD]] : tensor<3x3xf32>, vector<f32>
565 // CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1x1x3xf32>
566 // CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x3xf32>, tensor<1x1x3xf32>
567
568 // -----
569
570 #map = affine_map<() -> ()>
571 func.func @extract_scalar_from_0d_into_0d(%src: tensor<f32>, %init: tensor<f32>) -> tensor<f32> {
572 %res = linalg.generic {
573 indexing_maps = [#map],
574 iterator_types = []
575 } outs(%init : tensor<f32>) {
576 ^bb0(%in: f32):
577 %1 = tensor.extract %src[] : tensor<f32>
578 linalg.yield %1 : f32
579 } -> tensor<f32>
580
581 return %res : tensor<f32>
582 }
583
584 // CHECK-LABEL: func.func @extract_scalar_from_0d_into_0d(
585 // CHECK-SAME: %[[SRC:.*]]: tensor<f32>,
586 // CHECK-SAME: %[[INIT:.*]]: tensor<f32>) -> tensor<f32> {
587 // CHECK: %[[PAD:.*]] = ub.poison : f32
588 // CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
589 // CHECK: vector.transfer_write %[[READ]], %[[INIT]][] : vector<f32>, tensor<f32>
590
591 // -----
592
593 #map = affine_map<(n) -> (n)>
594 func.func @extract_scalar_from_0d_into_1d(%src: tensor<f32>, %init: tensor<1xf32>) -> tensor<1xf32> {
595 %res = linalg.generic {
596 indexing_maps = [#map],
597 iterator_types = ["parallel"]
598 } outs(%init : tensor<1xf32>) {
599 ^bb0(%in: f32):
600 %1 = tensor.extract %src[] : tensor<f32>
601 linalg.yield %1 : f32
602 } -> tensor<1xf32>
603
604 return %res : tensor<1xf32>
605 }
606 // CHECK-LABEL: func.func @extract_scalar_from_0d_into_1d(
607 // CHECK-SAME: %[[SRC:.*]]: tensor<f32>,
608 // CHECK-SAME: %[[INIT:.*]]: tensor<1xf32>) -> tensor<1xf32> {
609 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
610 // CHECK-DAG: %[[PAD:.*]] = ub.poison : f32
611 // CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %[[PAD]] : tensor<f32>, vector<f32>
612 // CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<1xf32>
613 // CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]]] {in_bounds = [true]} : vector<1xf32>, tensor<1xf32>
614
615 // -----
616
617 #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
618 func.func @vectorize_0d_tensor_extract(%src: tensor<f32>, %init: tensor<1x1x3xf32>) -> tensor<1x1x3xf32> {
619 %res = linalg.generic {
620 indexing_maps = [#map1],
621 iterator_types = ["parallel", "parallel", "parallel"]
622 } outs(%init : tensor<1x1x3xf32>) {
623 ^bb0(%arg4: f32):
624 %1 = tensor.extract %src[] : tensor<f32>
625 linalg.yield %1 : f32
626 } -> tensor<1x1x3xf32>
627 return %res : tensor<1x1x3xf32>
628 }
629
630 // CHECK-LABEL: func.func @vectorize_0d_tensor_extract(
631 // CHECK-SAME: %[[SRC:.*]]: tensor<f32>
632 // CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][], %{{.+}} : tensor<f32>
633 // CHECK: vector.broadcast %[[READ]] : vector<f32> to vector<1x1x3xf32>
634
635 // -----
636
637 func.func @scalar_read_with_broadcast_from_column_tensor(%init: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
638 %c4 = arith.constant 4 : index
639 %c0 = arith.constant 0 : index
640 %src = arith.constant dense<[[0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
641
642 %res = linalg.generic {
643 indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>],
644 iterator_types = ["parallel", "parallel", "parallel"]
645 } outs(%init : tensor<1x1x4xi32>) {
646
647 ^bb0(%out: i32):
648 %idx = linalg.index 0 : index
649 %extracted = tensor.extract %src[%idx, %c0] : tensor<15x1xi32>
650 linalg.yield %extracted : i32
651 } -> tensor<1x1x4xi32>
652
653 return %res : tensor<1x1x4xi32>
654 }
655
656 // CHECK-LABEL: func.func @scalar_read_with_broadcast_from_column_tensor
657 // CHECK-SAME: %[[INIT:.*]]: tensor<1x1x4xi32>) -> tensor<1x1x4xi32> {
658 // CHECK-DAG: %[[PAD:.*]] = ub.poison : i32
659 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
660 // CHECK-DAG: %[[SRC:.*]] = arith.constant dense<{{\[\[}}0], [1], [2], [3], [4], [5], [6], [7], [8], [9], [10], [11], [12], [13], [14]]> : tensor<15x1xi32>
661 // CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]]{{\[}}%[[C0]], %[[C0]]], %[[PAD]] : tensor<15x1xi32>, vector<i32>
662 // CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<i32> to vector<1x1x4xi32>
663 // CHECK: %[[RES:.*]] = vector.transfer_write %[[READ_BCAST]], %[[INIT]][%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<1x1x4xi32>, tensor<1x1x4xi32>
664
665 // -----
666
667 // TODO: Currently this fails to vectorise when the indices are non-constant.
668
669 #map = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
670 func.func @vectorize_nd_tensor_extract_transfer_read_basic_column(
671 %src: tensor<3x3x3xf32>,
672 %init: tensor<3x1x1xf32>) -> tensor<3x1x1xf32> {
673
674 %c0 = arith.constant 0 : index
675
676 %res = linalg.generic {
677 indexing_maps = [#map],
678 iterator_types = ["parallel", "parallel", "parallel"]
679 } outs(%init : tensor<3x1x1xf32>) {
680 ^bb0(%out: f32):
681 %1 = tensor.extract %src[%c0, %c0, %c0] : tensor<3x3x3xf32>
682 linalg.yield %1 : f32
683 } -> tensor<3x1x1xf32>
684
685 return %res : tensor<3x1x1xf32>
686 }
687
688 // CHECK-LABEL: func.func @vectorize_nd_tensor_extract_transfer_read_basic_column(
689 // CHECK-SAME: %[[SRC:.*]]: tensor<3x3x3xf32>,
690 // CHECK-SAME: %[[INIT:.*]]: tensor<3x1x1xf32>)
691 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
692 // CHECK-DAG: %[[PV:.*]] = ub.poison : f32
693 // CHECK: %[[READ:.*]] = vector.transfer_read %[[SRC]][%[[C0]], %[[C0]], %[[C0]]], %[[PV]] : tensor<3x3x3xf32>, vector<f32>
694 // CHECK: %[[READ_BCAST:.*]] = vector.broadcast %[[READ]] : vector<f32> to vector<3x1x1xf32>
695 // CHECK: vector.transfer_write %[[READ_BCAST]], %[[INIT]]{{\[}}%[[C0]], %[[C0]], %[[C0]]] {in_bounds = [true, true, true]} : vector<3x1x1xf32>, tensor<3x1x1xf32>