Debian Codegraph @

mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir

raw
1 // RUN: mlir-opt %s -transform-interpreter -split-input-file | FileCheck %s
2
3 ///----------------------------------------------------------------------------------------
4 /// Tests for linalg.generic
5 ///----------------------------------------------------------------------------------------
6
7 func.func @vectorize_dynamic_identity(%arg0: tensor<?xf32>,
8 %arg1: tensor<?xf32>,
9 %arg2: tensor<?xf32>) -> tensor<?xf32> {
10 %0 = linalg.generic { indexing_maps = [affine_map<(d0) -> (d0)>,
11 affine_map<(d0) -> (d0)>,
12 affine_map<(d0) -> (d0)>],
13 iterator_types = ["parallel"] }
14 ins(%arg0, %arg1 : tensor<?xf32>, tensor<?xf32>)
15 outs(%arg2 : tensor<?xf32>) {
16 ^bb(%in0: f32, %in1: f32, %out: f32) :
17 %0 = arith.addf %in0, %in1 : f32
18 linalg.yield %0 : f32
19 } -> tensor<?xf32>
20 return %0 : tensor<?xf32>
21 }
22
23 // CHECK-LABEL: @vectorize_dynamic_identity
24 // CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
25 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?xf32>
26 // CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_4]] : vector<4xi1>
27 // CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
28 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
29 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
30 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<4xf32>
31 // CHECK: %[[VAL_14:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %{{.*}} {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
32
33 module attributes {transform.with_named_sequence} {
34 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
35 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
36 transform.structured.vectorize %0 vector_sizes [4] : !transform.any_op
37 transform.yield
38 }
39 }
40
41 // -----
42
43 func.func @vectorize_dynamic_identity_scalable(%arg0: tensor<?xf32>,
44 %arg1: tensor<?xf32>,
45 %arg2: tensor<?xf32>) -> tensor<?xf32> {
46 %0 = linalg.generic { indexing_maps = [affine_map<(d0) -> (d0)>,
47 affine_map<(d0) -> (d0)>,
48 affine_map<(d0) -> (d0)>],
49 iterator_types = ["parallel"] }
50 ins(%arg0, %arg1 : tensor<?xf32>, tensor<?xf32>)
51 outs(%arg2 : tensor<?xf32>) {
52 ^bb(%in0: f32, %in1: f32, %out: f32) :
53 %0 = arith.addf %in0, %in1 : f32
54 linalg.yield %0 : f32
55 } -> tensor<?xf32>
56 return %0 : tensor<?xf32>
57 }
58
59 // CHECK-LABEL: @vectorize_dynamic_identity_scalable
60 // CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
61 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?xf32>
62 // CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_4]] : vector<[4]xi1>
63 // CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
64 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
65 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
66 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<[4]xf32>
67 // CHECK: %[[VAL_14:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %{{.*}} {in_bounds = [true]} : vector<[4]xf32>, tensor<?xf32> } : vector<[4]xi1> -> tensor<?xf32>
68
69 module attributes {transform.with_named_sequence} {
70 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
71 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
72 transform.structured.vectorize %0 vector_sizes [[4]] : !transform.any_op
73 transform.yield
74 }
75 }
76
77 // -----
78
79 func.func @vectorize_dynamic_identity_with_constant(%arg0: tensor<?xf32>,
80 %arg1: tensor<?xf32>,
81 %arg2: tensor<?xf32>) -> tensor<?xf32> {
82 %c4 = arith.constant 4 : index
83 %0 = linalg.generic { indexing_maps = [affine_map<(d0) -> (d0)>,
84 affine_map<(d0) -> (d0)>,
85 affine_map<(d0) -> (d0)>],
86 iterator_types = ["parallel"] }
87 ins(%arg0, %arg1 : tensor<?xf32>, tensor<?xf32>)
88 outs(%arg2 : tensor<?xf32>) {
89 ^bb(%in0: f32, %in1: f32, %out: f32) :
90 %0 = arith.addf %in0, %in1 : f32
91 linalg.yield %0 : f32
92 } -> tensor<?xf32>
93 return %0 : tensor<?xf32>
94 }
95
96 // CHECK-LABEL: @vectorize_dynamic_identity_with_constant
97 // CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
98 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?xf32>
99 // CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_4]] : vector<4xi1>
100 // CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
101 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
102 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
103 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<4xf32>
104 // CHECK: %[[VAL_14:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %{{.*}} {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
105
106 module attributes {transform.with_named_sequence} {
107 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
108 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
109 %size = transform.structured.match ops{["arith.constant"]} in %arg1 : (!transform.any_op) -> !transform.any_op
110 transform.structured.vectorize %0 vector_sizes [%size] : !transform.any_op, !transform.any_op
111 transform.yield
112 }
113 }
114
115 // -----
116
117 func.func @vectorize_dynamic_identity_with_param(%arg0: tensor<?xf32>,
118 %arg1: tensor<?xf32>,
119 %arg2: tensor<?xf32>) -> tensor<?xf32> {
120 %0 = linalg.generic { indexing_maps = [affine_map<(d0) -> (d0)>,
121 affine_map<(d0) -> (d0)>,
122 affine_map<(d0) -> (d0)>],
123 iterator_types = ["parallel"] }
124 ins(%arg0, %arg1 : tensor<?xf32>, tensor<?xf32>)
125 outs(%arg2 : tensor<?xf32>) {
126 ^bb(%in0: f32, %in1: f32, %out: f32) :
127 %0 = arith.addf %in0, %in1 : f32
128 linalg.yield %0 : f32
129 } -> tensor<?xf32>
130 return %0 : tensor<?xf32>
131 }
132
133 // CHECK-LABEL: @vectorize_dynamic_identity_with_param
134 // CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
135 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?xf32>
136 // CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_4]] : vector<4xi1>
137 // CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
138 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
139 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
140 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<4xf32>
141 // CHECK: %[[VAL_14:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %{{.*}} {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
142
143 module attributes {transform.with_named_sequence} {
144 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
145 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
146 %vector_size = transform.param.constant 4 : i64 -> !transform.param<i64>
147 transform.structured.vectorize %0 vector_sizes [%vector_size] : !transform.any_op, !transform.param<i64>
148 transform.yield
149 }
150 }
151
152 // -----
153
154 func.func @vectorize_dynamic_1d_broadcast(%arg0: tensor<?xf32>,
155 %arg1: tensor<?xf32>,
156 %arg2: tensor<?xf32>) -> tensor<?xf32> {
157 %0 = linalg.generic { indexing_maps = [affine_map<(d0) -> (0)>,
158 affine_map<(d0) -> (d0)>,
159 affine_map<(d0) -> (d0)>],
160 iterator_types = ["parallel"] }
161 ins(%arg0, %arg1 : tensor<?xf32>, tensor<?xf32>)
162 outs(%arg2 : tensor<?xf32>) {
163 ^bb(%in0: f32, %in1: f32, %out: f32) :
164 %0 = arith.addf %in0, %in1 : f32
165 linalg.yield %0 : f32
166 } -> tensor<?xf32>
167 return %0 : tensor<?xf32>
168 }
169
170 // CHECK-LABEL: @vectorize_dynamic_1d_broadcast
171 // CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
172 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?xf32>
173 // CHECK: %[[VAL_7:.*]] = vector.transfer_read %{{.*}} {permutation_map = #{{.*}}} : tensor<?xf32>, vector<4xf32>
174 // CHECK: %[[VAL_9:.*]] = vector.create_mask %[[VAL_4]] : vector<4xi1>
175 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
176 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
177 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_7]], %[[VAL_10]] : vector<4xf32>
178 // CHECK: %[[VAL_14:.*]] = vector.mask %{{.*}} { vector.transfer_write %[[VAL_13]], {{.*}} {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
179
180 module attributes {transform.with_named_sequence} {
181 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
182 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
183 transform.structured.vectorize %0 vector_sizes [4] : !transform.any_op
184 transform.yield
185 }
186 }
187
188 // -----
189
190 #map = affine_map<(d0, d1) -> (d0, d1)>
191 #map1 = affine_map<(d0, d1) -> (d0, 0)>
192
193 func.func @dynamic_generic_with_reduction_and_broadcast(%arg0: tensor<?x?xf32>, %init: tensor<?x?xf32>) -> (tensor<?x?xf32>) {
194 %0 = linalg.generic { indexing_maps = [#map, #map1],
195 iterator_types = ["parallel", "reduction"]}
196 ins(%arg0 : tensor<?x?xf32>)
197 outs(%init : tensor<?x?xf32>) {
198 ^bb0(%in: f32, %out: f32):
199 %1 = arith.addf %in, %out : f32
200 linalg.yield %1 : f32
201 } -> tensor<?x?xf32>
202 return %0 : tensor<?x?xf32>
203 }
204 // CHECK: #[[$MAP:.+]] = affine_map<(d0, d1) -> (d0)>
205
206 // CHECK-LABEL: func.func @dynamic_generic_with_reduction_and_broadcast(
207 // CHECK-SAME: %[[VAL_0:.*]]: tensor<?x?xf32>,
208 // CHECK-SAME: %[[VAL_1:.*]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
209 // CHECK: %[[VAL_2:.*]] = arith.constant 0 : index
210 // CHECK: %[[VAL_3:.*]] = tensor.dim %[[VAL_0]], %[[VAL_2]] : tensor<?x?xf32>
211 // CHECK: %[[VAL_4:.*]] = arith.constant 1 : index
212 // CHECK: %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf32>
213 // CHECK: %[[VAL_6:.*]] = arith.constant 0 : index
214 // CHECK: %[[VAL_7:.*]] = ub.poison : f32
215 // CHECK: %[[VAL_8:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]] : vector<4x4xi1>
216 // CHECK: %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_7]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x4xf32> } : vector<4x4xi1> -> vector<4x4xf32>
217 // CHECK: %[[VAL_10:.*]] = ub.poison : f32
218 // CHECK: %[[VAL_11:.*]] = vector.create_mask %[[VAL_3]] : vector<4xi1>
219 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_11]] { vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_10]] {in_bounds = [true], permutation_map = #[[$MAP]]} : tensor<?x?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
220 // CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.multi_reduction <add>, %[[VAL_9]], %[[VAL_12]] [1] : vector<4x4xf32> to vector<4xf32> } : vector<4x4xi1> -> vector<4xf32>
221 // CHECK: %[[VAL_14:.*]] = arith.constant 0 : index
222 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_11]] { vector.transfer_write %[[VAL_13]], %[[VAL_1]]{{\[}}%[[VAL_14]], %[[VAL_14]]] {in_bounds = [true], permutation_map = #[[$MAP]]} : vector<4xf32>, tensor<?x?xf32> } : vector<4xi1> -> tensor<?x?xf32>
223 // CHECK: return %[[VAL_15]] : tensor<?x?xf32>
224
225 module attributes {transform.with_named_sequence} {
226 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
227 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
228 transform.structured.vectorize %0 vector_sizes [4, 4] : !transform.any_op
229 transform.yield
230 }
231 }
232
233 // -----
234
235 func.func @vectorize_dynamic_2d_transpose(%arg0: tensor<?x?xf32>,
236 %arg1: tensor<?x?xf32>,
237 %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
238 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d1, d0)>,
239 affine_map<(d0, d1) -> (d0, d1)>,
240 affine_map<(d0, d1) -> (d0, d1)>],
241 iterator_types = ["parallel", "parallel"] }
242 ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
243 outs(%arg2 : tensor<?x?xf32>) {
244 ^bb(%in0: f32, %in1: f32, %out: f32) :
245 %0 = arith.addf %in0, %in1 : f32
246 linalg.yield %0 : f32
247 } -> tensor<?x?xf32>
248 return %0 : tensor<?x?xf32>
249 }
250
251 // CHECK-LABEL: @vectorize_dynamic_2d_transpose
252 // CHECK: %[[VAL_3:.*]] = arith.constant 1 : index
253 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?x?xf32>
254 // CHECK: %[[VAL_5:.*]] = arith.constant 0 : index
255 // CHECK: %[[VAL_6:.*]] = tensor.dim %{{.*}}, %[[VAL_5]] : tensor<?x?xf32>
256 // CHECK: %[[VAL_9:.*]] = vector.create_mask %[[VAL_6]], %[[VAL_4]] : vector<8x4xi1>
257 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : tensor<?x?xf32>, vector<4x8xf32> } : vector<8x4xi1> -> vector<4x8xf32>
258 // CHECK: %[[VAL_12:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]] : vector<4x8xi1>
259 // CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
260 // CHECK: %[[VAL_14:.*]] = ub.poison : f32
261 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
262 // CHECK: %[[VAL_16:.*]] = arith.addf %[[VAL_10]], %[[VAL_13]] : vector<4x8xf32>
263 // CHECK: %[[VAL_17:.*]] = vector.mask %[[VAL_12]] { vector.transfer_write %[[VAL_16]], %{{.*}} {in_bounds = [true, true]} : vector<4x8xf32>, tensor<?x?xf32> } : vector<4x8xi1> -> tensor<?x?xf32>
264
265 module attributes {transform.with_named_sequence} {
266 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
267 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
268 transform.structured.vectorize %0 vector_sizes [4, 8] : !transform.any_op
269 transform.yield
270 }
271 }
272
273 // -----
274
275 func.func @vectorize_dynamic_generic_2d_broadcast(%arg0: tensor<?x?xf32>,
276 %arg1: tensor<?x?xf32>,
277 %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
278 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (0, d1)>,
279 affine_map<(d0, d1) -> (d0, d1)>,
280 affine_map<(d0, d1) -> (d0, d1)>],
281 iterator_types = ["parallel", "parallel"] }
282 ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>)
283 outs(%arg2 : tensor<?x?xf32>) {
284 ^bb(%in0: f32, %in1: f32, %out: f32) :
285 %0 = arith.addf %in0, %in1 : f32
286 linalg.yield %0 : f32
287 } -> tensor<?x?xf32>
288 return %0 : tensor<?x?xf32>
289 }
290
291 // CHECK-LABEL: @vectorize_dynamic_generic_2d_broadcast
292 // CHECK: %[[VAL_3:.*]] = arith.constant 0 : index
293 // CHECK: %[[VAL_4:.*]] = tensor.dim %{{.*}}, %[[VAL_3]] : tensor<?x?xf32>
294 // CHECK: %[[VAL_5:.*]] = arith.constant 1 : index
295 // CHECK: %[[VAL_6:.*]] = tensor.dim %{{.*}}, %[[VAL_5]] : tensor<?x?xf32>
296 // CHECK: %[[VAL_9:.*]] = vector.create_mask %[[VAL_6]] : vector<8xi1>
297 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_9]] { vector.transfer_read %{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : tensor<?x?xf32>, vector<4x8xf32> } : vector<8xi1> -> vector<4x8xf32>
298 // CHECK: %[[VAL_12:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]] : vector<4x8xi1>
299 // CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
300 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_12]] { vector.transfer_read %{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
301 // CHECK: %[[VAL_16:.*]] = arith.addf %[[VAL_10]], %[[VAL_13]] : vector<4x8xf32>
302 // CHECK: %[[VAL_18:.*]] = vector.mask %[[VAL_12]] { vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<4x8xf32>, tensor<?x?xf32> } : vector<4x8xi1> -> tensor<?x?xf32>
303
304 module attributes {transform.with_named_sequence} {
305 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
306 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
307 transform.structured.vectorize %0 vector_sizes [4, 8] : !transform.any_op
308 transform.yield
309 }
310 }
311
312 // -----
313
314 func.func @vectorize_dynamic_reduction_2d(%arg0: tensor<?x?xf32>,
315 %arg1: tensor<?xf32>) -> tensor<?xf32> {
316 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
317 affine_map<(d0, d1) -> (d0)>],
318 iterator_types = ["parallel", "reduction"] }
319 ins(%arg0 : tensor<?x?xf32>)
320 outs(%arg1 : tensor<?xf32>) {
321 ^bb(%in: f32, %out: f32) :
322 %0 = arith.addf %in, %out : f32
323 linalg.yield %0 : f32
324 } -> tensor<?xf32>
325 return %0 : tensor<?xf32>
326 }
327
328 module attributes {transform.with_named_sequence} {
329 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
330 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
331 transform.structured.vectorize %0 vector_sizes [4, 8] : !transform.any_op
332 transform.yield
333 }
334 }
335
336 // CHECK-LABEL: @vectorize_dynamic_reduction_2d(
337 // CHECK-SAME: %[[VAL_0:.*]]: tensor<?x?xf32>,
338 // CHECK-SAME: %[[VAL_1:.*]]: tensor<?xf32>) -> tensor<?xf32> {
339 // CHECK: %[[VAL_2:.*]] = arith.constant 0 : index
340 // CHECK: %[[VAL_3:.*]] = tensor.dim %[[VAL_0]], %[[VAL_2]] : tensor<?x?xf32>
341 // CHECK: %[[VAL_4:.*]] = arith.constant 1 : index
342 // CHECK: %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf32>
343 // CHECK: %[[VAL_8:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]] : vector<4x8xi1>
344 // CHECK: %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]]{{.*}} {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x8xf32> } : vector<4x8xi1> -> vector<4x8xf32>
345 // CHECK: %[[VAL_11:.*]] = vector.create_mask %[[VAL_3]] : vector<4xi1>
346 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_11]] { vector.transfer_read %[[VAL_1]]{{.*}} {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
347 // CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.multi_reduction <add>, %[[VAL_9]], %[[VAL_12]] [1] : vector<4x8xf32> to vector<4xf32> } : vector<4x8xi1> -> vector<4xf32>
348 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_11]] { vector.transfer_write %[[VAL_13]], %[[VAL_1]]{{.*}} {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
349 // CHECK: return %[[VAL_15]] : tensor<?xf32>
350 // CHECK: }
351
352 // -----
353
354 func.func @vectorize_dynamic_reduction_2d_scalable(%arg0: tensor<?x?xf32>,
355 %arg1: tensor<?xf32>) -> tensor<?xf32> {
356 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
357 affine_map<(d0, d1) -> (d0)>],
358 iterator_types = ["parallel", "reduction"] }
359 ins(%arg0 : tensor<?x?xf32>)
360 outs(%arg1 : tensor<?xf32>) {
361 ^bb(%in: f32, %out: f32) :
362 %0 = arith.addf %in, %out : f32
363 linalg.yield %0 : f32
364 } -> tensor<?xf32>
365 return %0 : tensor<?xf32>
366 }
367
368 // CHECK-LABEL: func.func @vectorize_dynamic_reduction_2d_scalable(
369 // CHECK-SAME: %[[ARG_0:.*]]: tensor<?x?xf32>, %[[ARG_1:.*]]: tensor<?xf32>) -> tensor<?xf32> {
370 // CHECK: %[[C0_IDX:.*]] = arith.constant 0 : index
371 // CHECK: %[[DIM_A0_0:.*]] = tensor.dim %[[ARG_0]], %[[C0_IDX]] : tensor<?x?xf32>
372 // CHECK: %[[C1_IDX:.*]] = arith.constant 1 : index
373 // CHECK: %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_IDX]] : tensor<?x?xf32>
374 // CHECK: %[[C0_IDX:.*]] = arith.constant 0 : index
375 // CHECK: %[[PV:.*]] = ub.poison : f32
376 // CHECK: %[[MASK_2D:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<4x[8]xi1>
377 // CHECK: %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2D]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]], %[[C0_IDX]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[8]xf32> } : vector<4x[8]xi1> -> vector<4x[8]xf32>
378 // CHECK: %[[PV:.*]] = ub.poison : f32
379 // CHECK: %[[MASK_1D:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<4xi1>
380 // CHECK: %[[VEC_RD_1:.*]] = vector.mask %[[MASK_1D]] { vector.transfer_read %[[ARG_1]][%[[C0_IDX]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
381 // CHECK: %[[REDUCE:.*]] = vector.mask %[[MASK_2D]] { vector.multi_reduction <add>, %[[VEC_RD_0]], %[[VEC_RD_1]] [1] : vector<4x[8]xf32> to vector<4xf32> } : vector<4x[8]xi1> -> vector<4xf32>
382 // CHECK: %[[C0_IDX:.*]] = arith.constant 0 : index
383 // CHECK: %{{.*}} = vector.mask %[[MASK_1D]] { vector.transfer_write %[[REDUCE]], %[[ARG_1]][%[[C0_IDX]]] {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
384
385 module attributes {transform.with_named_sequence} {
386 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
387 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
388 transform.structured.vectorize %0 vector_sizes [4, [8]] : !transform.any_op
389 transform.yield
390 }
391 }
392
393 // -----
394
395 func.func @vectorize_dynamic_reduction_scalable_1d(%arg0: tensor<?xf32>,
396 %arg1: tensor<f32>) -> tensor<f32> {
397
398 %0 = linalg.reduce ins(%arg0 : tensor<?xf32>) outs(%arg1 : tensor<f32>) dimensions = [0]
399 (%in: f32, %init: f32) {
400 %0 = arith.addf %in, %init : f32
401 linalg.yield %0 : f32
402 }
403 return %0 : tensor<f32>
404 }
405
406 // CHECK-LABEL: func.func @vectorize_dynamic_reduction_scalable_1d(
407 // CHECK-SAME: %[[ARG_0:.*]]: tensor<?xf32>, %[[ARG_1:.*]]: tensor<f32>) -> tensor<f32> {
408 // CHECK: %[[C0_IDX:.*]] = arith.constant 0 : index
409 // CHECK: %[[DIM_A0_0:.*]] = tensor.dim %[[ARG_0]], %[[C0_IDX]] : tensor<?xf32>
410 // CHECK: %[[C0_IDX:.*]] = arith.constant 0 : index
411 // CHECK: %[[PV:.*]] = ub.poison : f32
412 // CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<[4]xi1>
413 // CHECK: %[[VEC_RD_0:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[ARG_0]][%[[C0_IDX]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
414 // CHECK: %[[PV:.*]] = ub.poison : f32
415 // CHECK: %[[VEC_RD_1:.*]] = vector.transfer_read %[[ARG_1]][], %[[PV]] : tensor<f32>, vector<f32>
416 // CHECK: %[[ACC_f32:.*]] = vector.extract %[[VEC_RD_1]][] : f32 from vector<f32>
417 // CHECK: %[[REDUCE:.*]] = vector.mask %[[MASK]] { vector.multi_reduction <add>, %[[VEC_RD_0]], %[[ACC_f32]] [0] : vector<[4]xf32> to f32 } : vector<[4]xi1> -> f32
418 // CHECK: %[[VEC_f32:.*]] = vector.broadcast %[[REDUCE]] : f32 to vector<f32>
419 // CHECK: %{{.*}} = vector.transfer_write %[[VEC_f32]], %[[ARG_1]][] : vector<f32>, tensor<f32>
420
421 module attributes {transform.with_named_sequence} {
422 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
423 %0 = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!transform.any_op) -> !transform.any_op
424 transform.structured.vectorize %0 vector_sizes [[4]] : !transform.any_op
425 transform.yield
426 }
427 }
428
429 // -----
430
431 func.func @vectorize_dynamic_transpose_reduction(%arg0: tensor<?x?x?xf32>,
432 %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
433 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
434 affine_map<(d0, d1, d2) -> (d2, d1)>],
435 iterator_types = ["reduction", "parallel", "parallel"] }
436 ins(%arg0 : tensor<?x?x?xf32>)
437 outs(%arg1 : tensor<?x?xf32>) {
438 ^bb(%in: f32, %out: f32) :
439 %0 = arith.addf %in, %out : f32
440 linalg.yield %0 : f32
441 } -> tensor<?x?xf32>
442 return %0 : tensor<?x?xf32>
443 }
444
445 module attributes {transform.with_named_sequence} {
446 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
447 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
448 transform.structured.vectorize %0 vector_sizes [4, 8, 16] : !transform.any_op
449 transform.yield
450 }
451 }
452
453 // CHECK-LABEL: @vectorize_dynamic_transpose_reduction(
454 // CHECK-SAME: %[[VAL_0:.*]]: tensor<?x?x?xf32>,
455 // CHECK-SAME: %[[VAL_1:.*]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
456 // CHECK: %[[VAL_2:.*]] = arith.constant 0 : index
457 // CHECK: %[[VAL_3:.*]] = tensor.dim %[[VAL_0]], %[[VAL_2]] : tensor<?x?x?xf32>
458 // CHECK: %[[VAL_4:.*]] = arith.constant 1 : index
459 // CHECK: %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?x?xf32>
460 // CHECK: %[[VAL_6:.*]] = arith.constant 2 : index
461 // CHECK: %[[VAL_7:.*]] = tensor.dim %[[VAL_0]], %[[VAL_6]] : tensor<?x?x?xf32>
462 // CHECK: %[[VAL_10:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]], %[[VAL_7]] : vector<4x8x16xi1>
463 // CHECK: %[[VAL_11:.*]] = vector.mask %[[VAL_10]] { vector.transfer_read %[[VAL_0]]{{.*}} {in_bounds = [true, true, true]} : tensor<?x?x?xf32>, vector<4x8x16xf32> } : vector<4x8x16xi1> -> vector<4x8x16xf32>
464 // CHECK: %[[VAL_13:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_5]] : vector<16x8xi1>
465 // CHECK: %[[VAL_14:.*]] = vector.mask %[[VAL_13]] { vector.transfer_read %[[VAL_1]]{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : tensor<?x?xf32>, vector<8x16xf32> } : vector<16x8xi1> -> vector<8x16xf32>
466 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_10]] { vector.multi_reduction <add>, %[[VAL_11]], %[[VAL_14]] [0] : vector<4x8x16xf32> to vector<8x16xf32> } : vector<4x8x16xi1> -> vector<8x16xf32>
467 // CHECK: %[[VAL_17:.*]] = vector.mask %[[VAL_13]] { vector.transfer_write %[[VAL_15]], %{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : vector<8x16xf32>, tensor<?x?xf32> } : vector<16x8xi1> -> tensor<?x?xf32>
468
469 // -----
470
471 func.func @vectorize_dynamic_transpose_reduction_with_params(%arg0: tensor<?x?x?xf32>,
472 %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
473 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>,
474 affine_map<(d0, d1, d2) -> (d2, d1)>],
475 iterator_types = ["reduction", "parallel", "parallel"] }
476 ins(%arg0 : tensor<?x?x?xf32>)
477 outs(%arg1 : tensor<?x?xf32>) {
478 ^bb(%in: f32, %out: f32) :
479 %0 = arith.addf %in, %out : f32
480 linalg.yield %0 : f32
481 } -> tensor<?x?xf32>
482 return %0 : tensor<?x?xf32>
483 }
484
485 module attributes {transform.with_named_sequence} {
486 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
487 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
488 %vector_size_0 = transform.param.constant 4 : i64 -> !transform.param<i64>
489 %vector_size_2 = transform.param.constant 16 : i64 -> !transform.param<i64>
490 transform.structured.vectorize %0 vector_sizes
491 [%vector_size_0, 8, %vector_size_2] : !transform.any_op, !transform.param<i64>, !transform.param<i64>
492 transform.yield
493 }
494 }
495
496 // CHECK-LABEL: @vectorize_dynamic_transpose_reduction_with_params(
497 // CHECK-SAME: %[[VAL_0:.*]]: tensor<?x?x?xf32>,
498 // CHECK-SAME: %[[VAL_1:.*]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
499 // CHECK: %[[VAL_2:.*]] = arith.constant 0 : index
500 // CHECK: %[[VAL_3:.*]] = tensor.dim %[[VAL_0]], %[[VAL_2]] : tensor<?x?x?xf32>
501 // CHECK: %[[VAL_4:.*]] = arith.constant 1 : index
502 // CHECK: %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?x?xf32>
503 // CHECK: %[[VAL_6:.*]] = arith.constant 2 : index
504 // CHECK: %[[VAL_7:.*]] = tensor.dim %[[VAL_0]], %[[VAL_6]] : tensor<?x?x?xf32>
505 // CHECK: %[[VAL_10:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]], %[[VAL_7]] : vector<4x8x16xi1>
506 // CHECK: %[[VAL_11:.*]] = vector.mask %[[VAL_10]] { vector.transfer_read %[[VAL_0]]{{.*}} {in_bounds = [true, true, true]} : tensor<?x?x?xf32>, vector<4x8x16xf32> } : vector<4x8x16xi1> -> vector<4x8x16xf32>
507 // CHECK: %[[VAL_13:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_5]] : vector<16x8xi1>
508 // CHECK: %[[VAL_14:.*]] = vector.mask %[[VAL_13]] { vector.transfer_read %[[VAL_1]]{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : tensor<?x?xf32>, vector<8x16xf32> } : vector<16x8xi1> -> vector<8x16xf32>
509 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_10]] { vector.multi_reduction <add>, %[[VAL_11]], %[[VAL_14]] [0] : vector<4x8x16xf32> to vector<8x16xf32> } : vector<4x8x16xi1> -> vector<8x16xf32>
510 // CHECK: %[[VAL_17:.*]] = vector.mask %[[VAL_13]] { vector.transfer_write %[[VAL_15]], %{{.*}} {in_bounds = [true, true], permutation_map = #{{.*}}} : vector<8x16xf32>, tensor<?x?xf32> } : vector<16x8xi1> -> tensor<?x?xf32>
511
512 // -----
513
514 func.func @vectorize_partial_dynamic_identity(%arg0: tensor<8x?xf32>,
515 %arg1: tensor<8x?xf32>,
516 %arg2: tensor<8x?xf32>) -> tensor<8x?xf32> {
517 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
518 affine_map<(d0, d1) -> (d0, d1)>,
519 affine_map<(d0, d1) -> (d0, d1)>],
520 iterator_types = ["parallel", "parallel"] }
521 ins(%arg0, %arg1 : tensor<8x?xf32>, tensor<8x?xf32>)
522 outs(%arg2 : tensor<8x?xf32>) {
523 ^bb(%in0: f32, %in1: f32, %out: f32) :
524 %0 = arith.addf %in0, %in1 : f32
525 linalg.yield %0 : f32
526 } -> tensor<8x?xf32>
527 return %0 : tensor<8x?xf32>
528 }
529
530 // CHECK-LABEL: func.func @vectorize_partial_dynamic_identity(
531 // CHECK-SAME: %[[VAL_0:.*]]: tensor<8x?xf32>, %[[VAL_1:.*]]: tensor<8x?xf32>, %[[VAL_2:.*]]: tensor<8x?xf32>) -> tensor<8x?xf32> {
532 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant 1 : index
533 // CHECK-DAG: %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32>
534 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 0 : index
535 // CHECK-DAG: %[[VAL_6:.*]] = ub.poison : f32
536 // CHECK-DAG: %[[VAL_7:.*]] = arith.constant 8 : index
537 // CHECK: %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x32xi1>
538 // CHECK: %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
539 // CHECK: %[[VAL_10:.*]] = ub.poison : f32
540 // CHECK: %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
541 // CHECK: %[[VAL_12:.*]] = ub.poison : f32
542 // CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
543 // CHECK: %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x32xf32>
544 // CHECK: %[[VAL_15:.*]] = arith.constant 0 : index
545 // CHECK: %[[VAL_16:.*]] = vector.mask %[[VAL_8]] { vector.transfer_write %[[VAL_14]], %[[VAL_2]][%[[VAL_15]], %[[VAL_15]]] {in_bounds = [true, true]} : vector<8x32xf32>, tensor<8x?xf32> } : vector<8x32xi1> -> tensor<8x?xf32>
546
547
548 module attributes {transform.with_named_sequence} {
549 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
550 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
551 transform.structured.vectorize %0 vector_sizes [8, 32] : !transform.any_op
552 transform.yield
553 }
554 }
555
556 // -----
557
558 func.func @vectorize_partial_dynamic_identity_scalable(%arg0: tensor<8x?xf32>,
559 %arg1: tensor<8x?xf32>,
560 %arg2: tensor<8x?xf32>) -> tensor<8x?xf32> {
561 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
562 affine_map<(d0, d1) -> (d0, d1)>,
563 affine_map<(d0, d1) -> (d0, d1)>],
564 iterator_types = ["parallel", "parallel"] }
565 ins(%arg0, %arg1 : tensor<8x?xf32>, tensor<8x?xf32>)
566 outs(%arg2 : tensor<8x?xf32>) {
567 ^bb(%in0: f32, %in1: f32, %out: f32) :
568 %0 = arith.addf %in0, %in1 : f32
569 linalg.yield %0 : f32
570 } -> tensor<8x?xf32>
571 return %0 : tensor<8x?xf32>
572 }
573
574 // CHECK-LABEL: func.func @vectorize_partial_dynamic_identity_scalable
575 // CHECK-SAME: %[[VAL_0:.*]]: tensor<8x?xf32>, %[[VAL_1:.*]]: tensor<8x?xf32>, %[[VAL_2:.*]]: tensor<8x?xf32>) -> tensor<8x?xf32> {
576 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant 1 : index
577 // CHECK-DAG: %[[VAL_4:.*]] = tensor.dim %[[VAL_0]], %[[VAL_3]] : tensor<8x?xf32>
578 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 0 : index
579 // CHECK-DAG: %[[VAL_6:.*]] = ub.poison : f32
580 // CHECK-DAG: %[[VAL_7:.*]] = arith.constant 8 : index
581 // CHECK: %[[VAL_8:.*]] = vector.create_mask %[[VAL_7]], %[[VAL_4]] : vector<8x[32]xi1>
582 // CHECK: %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_6]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
583 // CHECK: %[[VAL_10:.*]] = ub.poison : f32
584 // CHECK: %[[VAL_11:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_1]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_10]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
585 // CHECK: %[[VAL_12:.*]] = ub.poison : f32
586 // CHECK: %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_2]][%[[VAL_5]], %[[VAL_5]]], %[[VAL_12]] {in_bounds = [true, true]} : tensor<8x?xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
587 // CHECK: %[[VAL_14:.*]] = arith.addf %[[VAL_9]], %[[VAL_11]] : vector<8x[32]xf32>
588 // CHECK: %[[VAL_15:.*]] = arith.constant 0 : index
589 // CHECK: %[[VAL_16:.*]] = vector.mask %[[VAL_8]] { vector.transfer_write %[[VAL_14]], %[[VAL_2]][%[[VAL_15]], %[[VAL_15]]] {in_bounds = [true, true]} : vector<8x[32]xf32>, tensor<8x?xf32> } : vector<8x[32]xi1> -> tensor<8x?xf32>
590
591
592 module attributes {transform.with_named_sequence} {
593 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
594 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
595 transform.structured.vectorize %0 vector_sizes [8, [32]] : !transform.any_op
596 transform.yield
597 }
598 }
599
600 // -----
601
602 func.func @do_not_generate_masks(%arg0: tensor<8x32xf32>,
603 %arg1: tensor<8x32xf32>,
604 %arg2: tensor<8x32xf32>) -> tensor<8x32xf32> {
605 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
606 affine_map<(d0, d1) -> (d0, d1)>,
607 affine_map<(d0, d1) -> (d0, d1)>],
608 iterator_types = ["parallel", "parallel"] }
609 ins(%arg0, %arg1 : tensor<8x32xf32>, tensor<8x32xf32>)
610 outs(%arg2 : tensor<8x32xf32>) {
611 ^bb(%in0: f32, %in1: f32, %out: f32) :
612 %0 = arith.addf %in0, %in1 : f32
613 linalg.yield %0 : f32
614 } -> tensor<8x32xf32>
615 return %0 : tensor<8x32xf32>
616 }
617
618 // CHECK-LABEL: func.func @do_not_generate_masks
619 // CHECK-NOT: vector.mask
620
621 module attributes {transform.with_named_sequence} {
622 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
623 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
624 transform.structured.vectorize %0 vector_sizes [8, 32] : !transform.any_op
625 transform.yield
626 }
627 }
628
629 // -----
630
631 func.func @vectorize_static_shape_with_mask(%arg0: tensor<8x30xf32>,
632 %arg1: tensor<8x30xf32>,
633 %arg2: tensor<8x30xf32>) -> tensor<8x30xf32> {
634 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
635 affine_map<(d0, d1) -> (d0, d1)>,
636 affine_map<(d0, d1) -> (d0, d1)>],
637 iterator_types = ["parallel", "parallel"] }
638 ins(%arg0, %arg1 : tensor<8x30xf32>, tensor<8x30xf32>)
639 outs(%arg2 : tensor<8x30xf32>) {
640 ^bb(%in0: f32, %in1: f32, %out: f32) :
641 %0 = arith.addf %in0, %in1 : f32
642 linalg.yield %0 : f32
643 } -> tensor<8x30xf32>
644 return %0 : tensor<8x30xf32>
645 }
646
647 // CHECK-LABEL: func.func @vectorize_static_shape_with_mask(
648 // CHECK-SAME: %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> {
649 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : index
650 // CHECK-DAG: %[[VAL_4:.*]] = ub.poison : f32
651 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 8 : index
652 // CHECK-DAG: %[[VAL_6:.*]] = arith.constant 30 : index
653 // CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x32xi1>
654 // CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
655 // CHECK: %[[VAL_9:.*]] = ub.poison : f32
656 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
657 // CHECK: %[[VAL_11:.*]] = ub.poison : f32
658 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x32xf32> } : vector<8x32xi1> -> vector<8x32xf32>
659 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x32xf32>
660 // CHECK: %[[VAL_14:.*]] = arith.constant 0 : index
661 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %[[VAL_13]], %[[VAL_2]][%[[VAL_14]], %[[VAL_14]]] {in_bounds = [true, true]} : vector<8x32xf32>, tensor<8x30xf32> } : vector<8x32xi1> -> tensor<8x30xf32>
662
663 module attributes {transform.with_named_sequence} {
664 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
665 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
666 transform.structured.vectorize %0 vector_sizes [8, 32] : !transform.any_op
667 transform.yield
668 }
669 }
670
671 // -----
672
673 func.func @vectorize_static_shape_with_mask_scalable(%arg0: tensor<8x30xf32>,
674 %arg1: tensor<8x30xf32>,
675 %arg2: tensor<8x30xf32>) -> tensor<8x30xf32> {
676 %0 = linalg.generic { indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
677 affine_map<(d0, d1) -> (d0, d1)>,
678 affine_map<(d0, d1) -> (d0, d1)>],
679 iterator_types = ["parallel", "parallel"] }
680 ins(%arg0, %arg1 : tensor<8x30xf32>, tensor<8x30xf32>)
681 outs(%arg2 : tensor<8x30xf32>) {
682 ^bb(%in0: f32, %in1: f32, %out: f32) :
683 %0 = arith.addf %in0, %in1 : f32
684 linalg.yield %0 : f32
685 } -> tensor<8x30xf32>
686 return %0 : tensor<8x30xf32>
687 }
688
689 // CHECK-LABEL: func.func @vectorize_static_shape_with_mask_scalable(
690 // CHECK-SAME: %[[VAL_0:.*]]: tensor<8x30xf32>, %[[VAL_1:.*]]: tensor<8x30xf32>, %[[VAL_2:.*]]: tensor<8x30xf32>) -> tensor<8x30xf32> {
691 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : index
692 // CHECK-DAG: %[[VAL_4:.*]] = ub.poison : f32
693 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 8 : index
694 // CHECK-DAG: %[[VAL_6:.*]] = arith.constant 30 : index
695 // CHECK: %[[VAL_7:.*]] = vector.create_mask %[[VAL_5]], %[[VAL_6]] : vector<8x[32]xi1>
696 // CHECK: %[[VAL_8:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_0]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_4]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
697 // CHECK: %[[VAL_9:.*]] = ub.poison : f32
698 // CHECK: %[[VAL_10:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_1]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_9]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
699 // CHECK: %[[VAL_11:.*]] = ub.poison : f32
700 // CHECK: %[[VAL_12:.*]] = vector.mask %[[VAL_7]] { vector.transfer_read %[[VAL_2]][%[[VAL_3]], %[[VAL_3]]], %[[VAL_11]] {in_bounds = [true, true]} : tensor<8x30xf32>, vector<8x[32]xf32> } : vector<8x[32]xi1> -> vector<8x[32]xf32>
701 // CHECK: %[[VAL_13:.*]] = arith.addf %[[VAL_8]], %[[VAL_10]] : vector<8x[32]xf32>
702 // CHECK: %[[VAL_14:.*]] = arith.constant 0 : index
703 // CHECK: %[[VAL_15:.*]] = vector.mask %[[VAL_7]] { vector.transfer_write %[[VAL_13]], %[[VAL_2]][%[[VAL_14]], %[[VAL_14]]] {in_bounds = [true, true]} : vector<8x[32]xf32>, tensor<8x30xf32> } : vector<8x[32]xi1> -> tensor<8x30xf32>
704
705 module attributes {transform.with_named_sequence} {
706 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
707 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
708 transform.structured.vectorize %0 vector_sizes [8, [32]] : !transform.any_op
709 transform.yield
710 }
711 }
712
713 // -----
714
715 ///----------------------------------------------------------------------------------------
716 /// Tests for linalg.matvec
717 ///----------------------------------------------------------------------------------------
718
719 // Scalable _reduction_ dimension.
720
721 func.func @vectorize_dynamic_matvec_trailing_reduction_dim(%arg0: tensor<?x?xf32>,
722 %arg1: tensor<?xf32>,
723 %arg2: tensor<?xf32>) {
724 linalg.matvec ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?xf32>)
725 outs(%arg2 : tensor<?xf32>) -> tensor<?xf32>
726 return
727 }
728
729 // CHECK-LABEL: func.func @vectorize_dynamic_matvec_trailing_reduction_dim(
730 // CHECK-SAME: %[[ARG_0:.*]]: tensor<?x?xf32>, %[[ARG_1:.*]]: tensor<?xf32>, %[[ARG_2:.*]]: tensor<?xf32>) {
731 // CHECK: %[[C0_idx:.*]] = arith.constant 0 : index
732 // CHECK: %[[DIM_A0_0:.*]] = tensor.dim %[[ARG_0]], %[[C0_idx]] : tensor<?x?xf32>
733 // CHECK: %[[C1_idx:.*]] = arith.constant 1 : index
734 // CHECK: %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_idx]] : tensor<?x?xf32>
735 // CHECK: %[[C0_idx:.*]] = arith.constant 0 : index
736 // CHECK: %[[PV:.*]] = ub.poison : f32
737 // CHECK: %[[MASK_2d:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<4x[4]xi1>
738 // CHECK: %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x[4]xf32> } : vector<4x[4]xi1> -> vector<4x[4]xf32>
739 // CHECK: %[[PV:.*]] = ub.poison : f32
740 // CHECK: %[[MASK_d1:.*]] = vector.create_mask %[[DIM_A0_1]] : vector<[4]xi1>
741 // CHECK: %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[PV]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<4x[4]xf32> } : vector<[4]xi1> -> vector<4x[4]xf32>
742 // CHECK: %[[PV:.*]] = ub.poison : f32
743 // CHECK: %[[MASK_d2:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<4xi1>
744 // CHECK: %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
745 // CHECK: %[[MUL:.*]] = arith.mulf %[[VEC_RD_0:.*]], %[[VEC_RD_1:.*]] : vector<4x[4]xf32>
746 // CHECK: %[[REDUCE:.*]] = vector.mask %[[MASK_2d]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_RD_2]] [1] : vector<4x[4]xf32> to vector<4xf32> } : vector<4x[4]xi1> -> vector<4xf32>
747 // CHECK: %[[C0_idx:.*]] = arith.constant 0 : index
748 // CHECK: %{{.*}} = vector.mask %[[MASK_d2]] { vector.transfer_write %[[REDUCE]], %[[ARG_2]][%[[C0_idx]]] {in_bounds = [true]} : vector<4xf32>, tensor<?xf32> } : vector<4xi1> -> tensor<?xf32>
749
750 module attributes {transform.with_named_sequence} {
751 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
752 %0 = transform.structured.match ops{["linalg.matvec"]} in %arg1 : (!transform.any_op) -> !transform.any_op
753 transform.structured.vectorize %0 vector_sizes [4, [4]] : !transform.any_op
754 transform.yield
755 }
756 }
757
758 // -----
759
760 // Scalable _parallel_ dimension.
761
762 func.func @vectorize_dynamic_matvec_trailing_reduction_dim(%arg0: tensor<?x?xf32>,
763 %arg1: tensor<?xf32>,
764 %arg2:
765 tensor<?xf32>) ->
766 tensor<?xf32>{
767 %0 = linalg.matvec ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?xf32>)
768 outs(%arg2 : tensor<?xf32>) -> tensor<?xf32>
769 return %0 : tensor<?xf32>
770 }
771
772 // CHECK-LABEL: func.func @vectorize_dynamic_matvec_trailing_reduction_dim(
773 // CHECK-SAME: %[[ARG_0:.*]]: tensor<?x?xf32>, %[[ARG_1:.*]]: tensor<?xf32>, %[[ARG_2:.*]]: tensor<?xf32>) -> tensor<?xf32> {
774 // CHECK: %[[C0_idx:.*]] = arith.constant 0 : index
775 // CHECK: %[[DIM_A0_0:.*]] = tensor.dim %[[ARG_0]], %[[C0_idx]] : tensor<?x?xf32>
776 // CHECK: %[[C1_idx:.*]] = arith.constant 1 : index
777 // CHECK: %[[DIM_A0_1:.*]] = tensor.dim %[[ARG_0]], %[[C1_idx]] : tensor<?x?xf32>
778 // CHECK: %[[C0_idx:.*]] = arith.constant 0 : index
779 // CHECK: %[[PV:.*]] = ub.poison : f32
780 // CHECK: %[[MASK_2d:.*]] = vector.create_mask %[[DIM_A0_0]], %[[DIM_A0_1]] : vector<[4]x4xi1>
781 // CHECK: %[[VEC_RD_0:.*]] = vector.mask %[[MASK_2d]] { vector.transfer_read %[[ARG_0]][%[[C0_idx]], %[[C0_idx]]], %[[PV]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<[4]x4xf32> } : vector<[4]x4xi1> -> vector<[4]x4xf32>
782 // CHECK: %[[PV:.*]] = ub.poison : f32
783 // CHECK: %[[MASK_d1:.*]] = vector.create_mask %[[DIM_A0_1]] : vector<4xi1>
784 // CHECK: %[[VEC_RD_1:.*]] = vector.mask %[[MASK_d1]] { vector.transfer_read %[[ARG_1]][%[[C0_idx]]], %[[PV]] {in_bounds = [true, true], permutation_map = #map} : tensor<?xf32>, vector<[4]x4xf32> } : vector<4xi1> -> vector<[4]x4xf32>
785 // CHECK: %[[PV:.*]] = ub.poison : f32
786 // CHECK: %[[MASK_d2:.*]] = vector.create_mask %[[DIM_A0_0]] : vector<[4]xi1>
787 // CHECK: %[[VEC_RD_2:.*]] = vector.mask %[[MASK_d2]] { vector.transfer_read %[[ARG_2]][%[[C0_idx]]], %[[PV]] {in_bounds = [true]} : tensor<?xf32>, vector<[4]xf32> } : vector<[4]xi1> -> vector<[4]xf32>
788 // CHECK: %[[MUL:.*]] = arith.mulf %[[VEC_RD_0:.*]], %[[VEC_RD_1:.*]] : vector<[4]x4xf32>
789 // CHECK: %[[REDUCE:.*]] = vector.mask %[[MASK_2d]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_RD_2]] [1] : vector<[4]x4xf32> to vector<[4]xf32> } : vector<[4]x4xi1> -> vector<[4]xf32>
790 // CHECK: %[[C0_idx:.*]] = arith.constant 0 : index
791 // CHECK: %{{.*}} = vector.mask %[[MASK_d2]] { vector.transfer_write %[[REDUCE]], %[[ARG_2]][%[[C0_idx]]] {in_bounds = [true]} : vector<[4]xf32>, tensor<?xf32> } : vector<[4]xi1> -> tensor<?xf32>
792
793 module attributes {transform.with_named_sequence} {
794 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
795 %0 = transform.structured.match ops{["linalg.matvec"]} in %arg1 : (!transform.any_op) -> !transform.any_op
796 transform.structured.vectorize %0 vector_sizes [[4], 4] : !transform.any_op
797 transform.yield
798 }
799 }
800
801 // -----
802
803 ///----------------------------------------------------------------------------------------
804 /// Tests for linalg.index
805 ///----------------------------------------------------------------------------------------
806
807 #map = affine_map<(d0) -> (d0)>
808 func.func @vectorize_linalg_index_scalable(%dest: tensor<?xindex>) -> tensor<?xindex> {
809 %0 = linalg.generic {
810 indexing_maps = [#map],
811 iterator_types = ["parallel"]
812 } outs(%dest : tensor<?xindex>) {
813 ^bb0(%in: index):
814 %1 = linalg.index 0 : index
815 linalg.yield %1: index
816 } -> tensor<?xindex>
817 return %0 : tensor<?xindex>
818 }
819
820 // CHECK-LABEL: func.func @vectorize_linalg_index_scalable(
821 // CHECK-SAME: %[[DEST:.*]]: tensor<?xindex>) -> tensor<?xindex> {
822 // CHECK: %[[C0:.*]] = arith.constant 0 : index
823 // CHECK: %[[D0:.*]] = tensor.dim %[[DEST]], %[[C0]] : tensor<?xindex>
824 // CHECK: %[[C0_1:.*]] = arith.constant 0 : index
825 // CHECK: %[[PV:.*]] = ub.poison : index
826 // CHECK: %[[MASK:.*]] = vector.create_mask %[[D0]] : vector<[4]xi1>
827 // TODO: This xfer_read is not used - avoid creating it.
828 // CHECK: %[[READ:.*]] = vector.mask %[[MASK]] { vector.transfer_read %[[DEST]]{{\[}}%[[C0_1]]], %[[PV]] {in_bounds = [true]} : tensor<?xindex>, vector<[4]xindex> } : vector<[4]xi1> -> vector<[4]xindex>
829 // CHECK: %[[STEP:.*]] = vector.step : vector<[4]xindex>
830 // CHECK: %[[C0_3:.*]] = arith.constant 0 : index
831 // CHECK: %[[WRITE:.*]] = vector.mask %[[MASK]] { vector.transfer_write %[[STEP]], %[[DEST]]{{\[}}%[[C0_3]]] {in_bounds = [true]} : vector<[4]xindex>, tensor<?xindex> } : vector<[4]xi1> -> tensor<?xindex>
832 // CHECK: return %[[WRITE]] : tensor<?xindex>
833
834 module attributes {transform.with_named_sequence} {
835 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
836 %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
837 transform.structured.vectorize %0 vector_sizes [[4]] : !transform.any_op
838
839 transform.yield
840 }
841 }
842
843 ///----------------------------------------------------------------------------------------
844 /// Tests for other Ops
845 ///----------------------------------------------------------------------------------------
846
847 // -----
848
849 func.func @vectorize_dynamic_fill(%A : tensor<?x?xf32>, %arg0 : f32) -> tensor<?x?xf32> {
850 %0 = linalg.fill ins(%arg0 : f32) outs(%A : tensor<?x?xf32>) -> tensor<?x?xf32>
851 return %0 : tensor<?x?xf32>
852 }
853
854 // CHECK-LABEL: func.func @vectorize_dynamic_fill
855 // CHECK: %[[DIM0:.*]] = tensor.dim
856 // CHECK: %[[DIM1:.*]] = tensor.dim
857 // CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM0]], %[[DIM1]] : vector<8x16xi1>
858 // CHECK: %[[BCAST:.*]] = vector.broadcast %{{.*}} : f32 to vector<8x16xf32>
859 // CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[BCAST]], {{.*}} {in_bounds = [true, true]} : vector<8x16xf32>, tensor<?x?xf32> } : vector<8x16xi1>
860
861 module attributes {transform.with_named_sequence} {
862 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
863 %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
864 transform.structured.vectorize %0 vector_sizes [8, 16] : !transform.any_op
865 transform.yield
866 }
867 }
868
869 // -----
870
871 // NOTE: Often, non-trailing scalable sizes are problematic - there are no
872 // "scalable" arrays of vectors at the LLVM level (multi-dim vectors are
873 // decomposed into arrays of aggregates). However, the trailing dim in this
874 // case is 1 and that can be folded away later.
875
876 // NOTE: This is similar to the example above, but the trailing dim was set to
877 // 1 to make it foldable + vectorizable.
878
879 func.func @vectorize_dynamic_fill_scalable(%A : tensor<?x?xf32>, %arg0 : f32) -> tensor<?x?xf32> {
880 %0 = linalg.fill ins(%arg0 : f32) outs(%A : tensor<?x?xf32>) -> tensor<?x?xf32>
881 return %0 : tensor<?x?xf32>
882 }
883
884 // CHECK-LABEL: func.func @vectorize_dynamic_fill_scalable
885 // CHECK: %[[DIM0:.*]] = tensor.dim
886 // CHECK: %[[DIM1:.*]] = tensor.dim
887 // CHECK: %[[MASK:.*]] = vector.create_mask %[[DIM0]], %[[DIM1]] : vector<[8]x1xi1>
888 // CHECK: %[[BCAST:.*]] = vector.broadcast %{{.*}} : f32 to vector<[8]x1xf32>
889 // CHECK: vector.mask %[[MASK]] { vector.transfer_write %[[BCAST]], {{.*}} {in_bounds = [true, true]} : vector<[8]x1xf32>, tensor<?x?xf32> } : vector<[8]x1xi1>
890
891 module attributes {transform.with_named_sequence} {
892 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
893 %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
894 transform.structured.vectorize %0 vector_sizes [[8], 1] : !transform.any_op
895 transform.yield
896 }
897 }
898
899 // -----
900
901 // CHECK: #[[MAP:.*]] = affine_map<(d0, d1) -> (d1, d0)>
902 // CHECK: func @test_masked_vectorize_linalg_transpose
903 func.func @test_masked_vectorize_linalg_transpose(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
904 // CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
905 // CHECK-DAG: %[[D0:.*]] = tensor.dim %arg0, %[[C0]]
906 // CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
907 // CHECK-DAG: %[[D1:.*]] = tensor.dim %arg0, %[[C1]]
908 // CHECK: %[[MASK0:.*]] = vector.create_mask %[[D0]], %[[D1]]
909 // CHECK: %[[LOAD:.*]] = vector.mask %[[MASK0]] { vector.transfer_read %arg0{{.+}} permutation_map = #[[MAP]]{{.+}} }
910 // CHECK-SAME: vector<4x2xi1> -> vector<2x4xf32>
911 // CHECK: %[[MASK1:.*]] = vector.create_mask %[[D1]], %[[D0]]
912 // CHECK: %[[WRITE:.*]] = vector.mask %[[MASK1]] { vector.transfer_write %[[LOAD]], %arg1{{.+}} }
913 // CHECK-SAME: vector<2x4xi1> -> tensor<?x?xf32>
914 // CHECK: return %[[WRITE]]
915 %0 = linalg.transpose ins(%arg0 : tensor<?x?xf32>) outs(%arg1 : tensor<?x?xf32>) permutation = [1, 0]
916 return %0 : tensor<?x?xf32>
917 }
918
919 module attributes {transform.with_named_sequence} {
920 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
921 %0 = transform.structured.match ops{["linalg.transpose"]} in %arg1 : (!transform.any_op) -> !transform.any_op
922 transform.structured.vectorize %0 vector_sizes [2, 4] : !transform.any_op
923 transform.yield
924 }
925 }
926
927 // -----
928
929 // CHECK-LABEL: func @test_masked_vectorize_linalg_copy
930 func.func @test_masked_vectorize_linalg_copy(%A : memref<?x?xf32>, %B : memref<?x?xf32>) {
931 // CHECK: %[[c0:.*]] = arith.constant 0 : index
932 // CHECK: %[[d0:.*]] = memref.dim %{{.*}}, %[[c0]] : memref<?x?xf32>
933 // CHECK: %[[c1:.*]] = arith.constant 1 : index
934 // CHECK: %[[d1:.*]] = memref.dim %{{.*}}, %[[c1]] : memref<?x?xf32>
935 // CHECK: %[[mask:.*]] = vector.create_mask %[[d0]], %[[d1]] : vector<2x4xi1>
936 // CHECK: vector.mask %[[mask]] {{.*}} vector.transfer_read %{{.*}} {in_bounds = [true, true]} : memref<?x?xf32>, vector<2x4xf32> } : vector<2x4xi1> -> vector<2x4xf32>
937 // CHECK: vector.mask %[[mask]] {{.*}} vector.transfer_write %{{.*}} {in_bounds = [true, true]} : vector<2x4xf32>, memref<?x?xf32> } : vector<2x4xi1>
938 linalg.copy ins(%A : memref<?x?xf32>) outs(%B : memref<?x?xf32>)
939 return
940 }
941
942 module attributes {transform.with_named_sequence} {
943 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
944 %0 = transform.structured.match ops{["linalg.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
945 transform.structured.vectorize %0 vector_sizes [2, 4] : !transform.any_op
946 transform.yield
947 }
948 }
949
950
951 // -----
952
953 // Input identical as the test in vectorization-with-patterns.mlir. Output is
954 // different - vector sizes are inferred (rather than user-specified) and hence
955 // masking was used.
956
957 func.func @test_vectorize_pack(%arg0: tensor<32x8x16xf32>, %arg1: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
958 %pack = linalg.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x8x16xf32> -> tensor<4x1x32x16x2xf32>
959 return %pack : tensor<4x1x32x16x2xf32>
960 }
961 // CHECK-DAG: %[[cst:.*]] = arith.constant 0.000000e+00 : f32
962 // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
963 // CHECK: %[[read:.*]] = vector.transfer_read %{{.*}}[%[[c0]], %[[c0]], %[[c0]]], %[[cst]]
964 // CHECK-SAME: {in_bounds = [true, true, true]} : tensor<32x8x16xf32>, vector<32x8x16xf32>
965 // CHECK: %[[shape_cast:.*]] = vector.shape_cast %[[read]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
966 // CHECK: %[[transpose:.*]] = vector.transpose %[[shape_cast]], [1, 3, 0, 4, 2] : vector<32x4x2x1x16xf32> to vector<4x1x32x16x2xf32>
967 // CHECK-DAG: %[[c0_1:.*]] = arith.constant 0 : index
968 // CHECK-DAG: %[[empty:.*]] = tensor.empty() : tensor<4x1x32x16x2xf32>
969 // CHECK: %[[write:.*]] = vector.transfer_write %[[transpose]], %[[empty]][%[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]]]
970 // CHECK-SAME: {in_bounds = [true, true, true, true, true]} : vector<4x1x32x16x2xf32>, tensor<4x1x32x16x2xf32>
971 // CHECK: return %[[write]] : tensor<4x1x32x16x2xf32>
972
973 module attributes {transform.with_named_sequence} {
974 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
975 %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
976 transform.structured.vectorize %0 vector_sizes [4, 1, 32] : !transform.any_op
977 transform.yield
978 }
979 }
980
981 // -----
982
983 // Input identical as the test in vectorization-with-patterns.mlir. Output is
984 // different - vector sizes are inferred (rather than user-specified) and hence
985 // masking was used.
986
987 func.func @test_vectorize_padded_pack(%arg0: tensor<32x7x15xf32>, %arg1: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
988 %pad = arith.constant 0.000000e+00 : f32
989 %pack = linalg.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x7x15xf32> -> tensor<32x4x1x16x2xf32>
990 return %pack : tensor<32x4x1x16x2xf32>
991 }
992 // CHECK-DAG: %[[cst:.*]] = arith.constant 0.000000e+00 : f32
993 // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
994 // CHECK-DAG: %[[c32:.*]] = arith.constant 32 : index
995 // CHECK-DAG: %[[c7:.*]] = arith.constant 7 : index
996 // CHECK-DAG: %[[c15:.*]] = arith.constant 15 : index
997 // CHECK: %[[mask:.*]] = vector.create_mask %[[c32]], %[[c7]], %[[c15]] : vector<32x8x16xi1>
998 // CHECK: %[[masked_read:.*]] = vector.mask %[[mask]] {
999 // CHECK-SAME: vector.transfer_read %{{.*}}[%[[c0]], %[[c0]], %[[c0]]], %[[cst]]
1000 // CHECK-SAME: {in_bounds = [true, true, true]} : tensor<32x7x15xf32>, vector<32x8x16xf32>
1001 // CHECK-SAME: } : vector<32x8x16xi1> -> vector<32x8x16xf32>
1002 // CHECK: %[[shape_cast:.*]] = vector.shape_cast %[[masked_read]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
1003 // CHECK: %[[transpose:.*]] = vector.transpose %[[shape_cast]], [0, 1, 3, 4, 2] : vector<32x4x2x1x16xf32> to vector<32x4x1x16x2xf32>
1004 // CHECK-DAG: %[[c0_1:.*]] = arith.constant 0 : index
1005 // CHECK-DAG: %[[empty:.*]] = tensor.empty() : tensor<32x4x1x16x2xf32>
1006 // CHECK: %[[write:.*]] = vector.transfer_write %[[transpose]], %[[empty]][%[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]]]
1007 // CHECK-SAME: {in_bounds = [true, true, true, true, true]} : vector<32x4x1x16x2xf32>, tensor<32x4x1x16x2xf32>
1008 // CHECK: return %[[write]] : tensor<32x4x1x16x2xf32>
1009
1010 module attributes {transform.with_named_sequence} {
1011 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1012 %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1013 transform.structured.vectorize %0 vector_sizes [32, 4, 1] : !transform.any_op
1014 transform.yield
1015 }
1016 }
1017
1018 // -----
1019
1020 func.func @test_vectorize_dynamic_pack(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?x16x2xf32>) -> tensor<?x?x16x2xf32> {
1021 %pack = linalg.pack %arg0 inner_dims_pos = [1, 0] inner_tiles = [16, 2] into %arg1 : tensor<?x?xf32> -> tensor<?x?x16x2xf32>
1022 return %pack : tensor<?x?x16x2xf32>
1023 }
1024 // CHECK-DAG: %[[cst:.*]] = arith.constant 0.000000e+00 : f32
1025 // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
1026 // CHECK-DAG: %[[c1:.*]] = arith.constant 1 : index
1027 // CHECK-DAG: %[[d0:.*]] = tensor.dim {{.*}} %[[c0]] : tensor<?x?x16x2xf32>
1028 // CHECK-DAG: %[[d1:.*]] = tensor.dim {{.*}} %[[c1]] : tensor<?x?x16x2xf32>
1029 // CHECK-DAG: %[[c0_1:.*]] = arith.constant 0 : index
1030 // CHECK-DAG: %[[c0_0:.*]] = arith.constant 0 : index
1031 // CHECK-DAG: %[[c1_0:.*]] = arith.constant 1 : index
1032 // CHECK-DAG: %[[d0_0:.*]] = tensor.dim {{.*}} %[[c0_0]] : tensor<?x?xf32>
1033 // CHECK-DAG: %[[d1_0:.*]] = tensor.dim {{.*}} %[[c1_0]] : tensor<?x?xf32>
1034 // CHECK: %[[mask:.*]] = vector.create_mask %[[d0_0]], %[[d1_0]] : vector<8x16xi1>
1035 // CHECK: %[[masked_read:.*]] = vector.mask %[[mask]] {
1036 // CHECK-SAME: vector.transfer_read %{{.*}}[%[[c0_1]], %[[c0_1]]], %[[cst]]
1037 // CHECK-SAME: {in_bounds = [true, true]} : tensor<?x?xf32>, vector<8x16xf32>
1038 // CHECK-SAME: } : vector<8x16xi1> -> vector<8x16xf32>
1039 // CHECK: %[[shape_cast:.*]] = vector.shape_cast %[[masked_read]] : vector<8x16xf32> to vector<4x2x1x16xf32>
1040 // CHECK: %[[transpose:.*]] = vector.transpose %[[shape_cast]], [0, 2, 3, 1] : vector<4x2x1x16xf32> to vector<4x1x16x2xf32>
1041 // CHECK-DAG: %[[c0_2:.*]] = arith.constant 0 : index
1042 // CHECK-DAG: %[[c16:.*]] = arith.constant 16 : index
1043 // CHECK-DAG: %[[c2:.*]] = arith.constant 2 : index
1044 // CHECK-DAG: %[[empty:.*]] = tensor.empty(%[[d0]], %[[d1]]) : tensor<?x?x16x2xf32>
1045 // CHECK-DAG: %[[d2:.*]] = tensor.dim %[[empty]], {{.*}} : tensor<?x?x16x2xf32>
1046 // CHECK-DAG: %[[d3:.*]] = tensor.dim %[[empty]], {{.*}} : tensor<?x?x16x2xf32>
1047 // CHECK: %[[mask_0:.*]] = vector.create_mask %[[d2]], %[[d3]], %[[c16]], %[[c2]] : vector<4x1x16x2xi1>
1048 // CHECK: %[[masked_write:.*]] = vector.mask %[[mask_0]] {
1049 // CHECK-SAME: vector.transfer_write %[[transpose]], %[[empty]][%[[c0_2]], %[[c0_2]], %[[c0_2]], %[[c0_2]]]
1050 // CHECK-SAME: {in_bounds = [true, true, true, true]} : vector<4x1x16x2xf32>, tensor<?x?x16x2xf32>
1051 // CHECK: return %[[masked_write]] : tensor<?x?x16x2xf32>
1052
1053 module attributes {transform.with_named_sequence} {
1054 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1055 %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1056 transform.structured.vectorize %0 vector_sizes [4, 1] : !transform.any_op
1057 transform.yield
1058 }
1059 }
1060
1061 // -----
1062
1063 func.func @matmul(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
1064 linalg.matmul ins(%A, %B: memref<?x?xf32>, memref<?x?xf32>)
1065 outs(%C: memref<?x?xf32>)
1066 return
1067 }
1068
1069 // CHECK-LABEL: func.func @matmul(
1070 // CHECK-SAME: %[[A:.*]]: memref<?x?xf32>, %[[B:.*]]: memref<?x?xf32>, %[[C:.*]]: memref<?x?xf32>) {
1071 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : index
1072 // CHECK-DAG: %[[VAL_4:.*]] = memref.dim %[[A]], %[[VAL_3]] : memref<?x?xf32>
1073 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 1 : index
1074 // CHECK-DAG: %[[VAL_6:.*]] = memref.dim %[[B]], %[[VAL_5]] : memref<?x?xf32>
1075 // CHECK-DAG: %[[VAL_7:.*]] = arith.constant 1 : index
1076 // CHECK-DAG: %[[VAL_8:.*]] = memref.dim %[[A]], %[[VAL_7]] : memref<?x?xf32>
1077 // CHECK: %[[MASK_A:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_8]] : vector<8x4xi1>
1078 // CHECK: %[[LOAD_A:.*]] = vector.mask %[[MASK_A]] { vector.transfer_read %[[A]]{{\[}}%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true, true, true], permutation_map = #{{.*}}} : memref<?x?xf32>, vector<8x16x4xf32> } : vector<8x4xi1> -> vector<8x16x4xf32>
1079 // CHECK: %[[MASK_B:.*]] = vector.create_mask %[[VAL_8]], %[[VAL_6]] : vector<4x16xi1>
1080 // CHECK: %[[LOAD_B:.*]] = vector.mask %[[MASK_B]] { vector.transfer_read %[[B]]{{\[}}%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true, true, true], permutation_map = #{{.*}}} : memref<?x?xf32>, vector<8x16x4xf32> } : vector<4x16xi1> -> vector<8x16x4xf32>
1081 // CHECK: %[[MASK_C:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]] : vector<8x16xi1>
1082 // CHECK: %[[LOAD_C:.*]] = vector.mask %[[MASK_C]] { vector.transfer_read %[[C]]{{\[}}%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true, true]} : memref<?x?xf32>, vector<8x16xf32> } : vector<8x16xi1> -> vector<8x16xf32>
1083 // CHECK: %[[MULF:.*]] = arith.mulf %[[LOAD_A]], %[[LOAD_B]] : vector<8x16x4xf32>
1084 // CHECK: %[[MASK_MULIT_RED:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]], %[[VAL_8]] : vector<8x16x4xi1>
1085 // CHECK: %[[MULTI_RED:.*]] = vector.mask %[[MASK_MULIT_RED]] { vector.multi_reduction <add>, %[[MULF]], %[[LOAD_C]] [2] : vector<8x16x4xf32> to vector<8x16xf32> } : vector<8x16x4xi1> -> vector<8x16xf32>
1086 // CHECK: %[[C2:.*]] = arith.constant 0 : index
1087 // CHECK: vector.mask %[[MASK_C]] { vector.transfer_write %[[MULTI_RED]], %[[C]]{{\[}}%[[C2]], %[[C2]]] {in_bounds = [true, true]} : vector<8x16xf32>, memref<?x?xf32> } : vector<8x16xi1>
1088
1089 module attributes {transform.with_named_sequence} {
1090 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
1091 %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
1092 transform.structured.vectorize %matmul vector_sizes [8, 16, 4] : !transform.any_op
1093 transform.yield
1094 }
1095 }
1096
1097 // -----
1098
1099 func.func @mmt4d(%A: memref<16x16x8x1xf32>, %B: memref<16x16x8x1xf32>, %C_in: memref<16x16x8x8xf32>) {
1100 linalg.mmt4d ins(%A, %B: memref<16x16x8x1xf32>, memref<16x16x8x1xf32>)
1101 outs(%C_in: memref<16x16x8x8xf32>)
1102 return
1103 }
1104
1105 // CHECK-LABEL: func.func @mmt4d(
1106 // CHECK-SAME: %[[A:.*]]: memref<16x16x8x1xf32>, %[[B:.*]]: memref<16x16x8x1xf32>, %[[C:.*]]: memref<16x16x8x8xf32>) {
1107 // CHECK: %[[VEC_A:.*]] = vector.transfer_read %[[A]]{{.*}} : memref<16x16x8x1xf32>, vector<16x16x16x8x8x1xf32>
1108 // CHECK: %[[VEC_B:.*]] = vector.transfer_read %[[B]]{{.*}} : memref<16x16x8x1xf32>, vector<16x16x16x8x8x1xf32>
1109 // CHECK: %[[VEC_C:.*]] = vector.transfer_read %[[C]]{{.*}} : memref<16x16x8x8xf32>, vector<16x16x8x8xf32>
1110 // CHECK: %[[MUL:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<16x16x16x8x8x1xf32>
1111 // CHECK: %[[RED:.*]] = vector.multi_reduction <add>, %[[MUL]], %[[VEC_C]] [2, 5] : vector<16x16x16x8x8x1xf32> to vector<16x16x8x8xf32>
1112 // CHECK: vector.transfer_write %[[RED]], %[[C]]{{.*}} : vector<16x16x8x8xf32>, memref<16x16x8x8xf32>
1113
1114 module attributes {transform.with_named_sequence} {
1115 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
1116 %mmt4d = transform.structured.match ops{["linalg.mmt4d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
1117 transform.structured.vectorize %mmt4d : !transform.any_op
1118 transform.yield
1119 }
1120 }
1121
1122 // -----
1123
1124 func.func @matmul_scalable(%A: memref<?x?xf32>, %B: memref<?x?xf32>, %C: memref<?x?xf32>) {
1125 linalg.matmul ins(%A, %B: memref<?x?xf32>, memref<?x?xf32>)
1126 outs(%C: memref<?x?xf32>)
1127 return
1128 }
1129
1130 // CHECK-LABEL: func.func @matmul_scalable(
1131 // CHECK-SAME: %[[A:.*]]: memref<?x?xf32>, %[[B:.*]]: memref<?x?xf32>, %[[C:.*]]: memref<?x?xf32>) {
1132 // CHECK-DAG: %[[VAL_3:.*]] = arith.constant 0 : index
1133 // CHECK-DAG: %[[VAL_4:.*]] = memref.dim %[[A]], %[[VAL_3]] : memref<?x?xf32>
1134 // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 1 : index
1135 // CHECK-DAG: %[[VAL_6:.*]] = memref.dim %[[B]], %[[VAL_5]] : memref<?x?xf32>
1136 // CHECK-DAG: %[[VAL_7:.*]] = arith.constant 1 : index
1137 // CHECK-DAG: %[[VAL_8:.*]] = memref.dim %[[A]], %[[VAL_7]] : memref<?x?xf32>
1138 // CHECK: %[[MASK_A:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_8]] : vector<8x4xi1>
1139 // CHECK: %[[LOAD_A:.*]] = vector.mask %[[MASK_A]] { vector.transfer_read %[[A]]{{\[}}%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true, true, true], permutation_map = #{{.*}}} : memref<?x?xf32>, vector<8x[16]x4xf32> } : vector<8x4xi1> -> vector<8x[16]x4xf32>
1140 // CHECK: %[[MASK_B:.*]] = vector.create_mask %[[VAL_8]], %[[VAL_6]] : vector<4x[16]xi1>
1141 // CHECK: %[[LOAD_B:.*]] = vector.mask %[[MASK_B]] { vector.transfer_read %[[B]]{{\[}}%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true, true, true], permutation_map = #{{.*}}} : memref<?x?xf32>, vector<8x[16]x4xf32> } : vector<4x[16]xi1> -> vector<8x[16]x4xf32>
1142 // CHECK: %[[MASK_C:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]] : vector<8x[16]xi1>
1143 // CHECK: %[[LOAD_C:.*]] = vector.mask %[[MASK_C]] { vector.transfer_read %[[C]]{{\[}}%{{.*}}, %{{.*}}], %{{.*}} {in_bounds = [true, true]} : memref<?x?xf32>, vector<8x[16]xf32> } : vector<8x[16]xi1> -> vector<8x[16]xf32>
1144 // CHECK: %[[MULF:.*]] = arith.mulf %[[LOAD_A]], %[[LOAD_B]] : vector<8x[16]x4xf32>
1145 // CHECK: %[[MASK_MULIT_RED:.*]] = vector.create_mask %[[VAL_4]], %[[VAL_6]], %[[VAL_8]] : vector<8x[16]x4xi1>
1146 // CHECK: %[[MULTI_RED:.*]] = vector.mask %[[MASK_MULIT_RED]] { vector.multi_reduction <add>, %[[MULF]], %[[LOAD_C]] [2] : vector<8x[16]x4xf32> to vector<8x[16]xf32> } : vector<8x[16]x4xi1> -> vector<8x[16]xf32>
1147 // CHECK: %[[C2:.*]] = arith.constant 0 : index
1148 // CHECK: vector.mask %[[MASK_C]] { vector.transfer_write %[[MULTI_RED]], %[[C]]{{\[}}%[[C2]], %[[C2]]] {in_bounds = [true, true]} : vector<8x[16]xf32>, memref<?x?xf32> } : vector<8x[16]xi1>
1149
1150 module attributes {transform.with_named_sequence} {
1151 transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
1152 %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
1153 transform.structured.vectorize %matmul vector_sizes [8, [16], 4] : !transform.any_op
1154 transform.yield
1155 }
1156 }
1157
1158 // -----
1159
1160 // CHECK-LABEL: func @test_vectorize_dynamic_shapes_unpack
1161 func.func @test_vectorize_dynamic_shapes_unpack(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?x16x2xf32>) -> tensor<?x?xf32> {
1162 // CHECK: %[[C0:.*]] = arith.constant 0
1163 // CHECK: %[[DIM:.*]] = tensor.dim %arg0, %[[C0]] : tensor<?x?xf32>
1164 // CHECK: %[[C1:.*]] = arith.constant 1 : index
1165 // CHECK: %[[DIM0:.*]] = tensor.dim %arg0, %[[C1]] : tensor<?x?xf32>
1166 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
1167 // CHECK: %[[C01:.*]] = arith.constant 0
1168 // CHECK: %[[C02:.*]] = arith.constant 0
1169 // CHECK: %[[DIM4:.*]] = tensor.dim %arg1, %[[C02]] : tensor<?x?x16x2xf32>
1170 // CHECK: %[[CNST14:.*]] = arith.constant 1
1171 // CHECK: %[[DIM6:.*]] = tensor.dim %arg1, %[[CNST14]] : tensor<?x?x16x2xf32>
1172 // CHECK: %[[CNST16:.*]] = arith.constant 16 : index
1173 // CHECK: %[[CNST2:.*]] = arith.constant 2 : index
1174 // CHECK: %[[readMsk0:.*]] = vector.create_mask %[[DIM4]], %[[DIM6]], %[[CNST16]], %[[CNST2]] : vector<2x1x16x2xi1>
1175 // CHECK: %[[read0:.*]] = vector.mask %[[readMsk0]] {{.*}} vector.transfer_read %{{.*}} : tensor<?x?x16x2xf32>, vector<2x1x16x2xf32> } : vector<2x1x16x2xi1> -> vector<2x1x16x2xf32>
1176 // CHECK: %[[trans0:.*]] = vector.transpose %[[read0]], [0, 3, 1, 2] : vector<2x1x16x2xf32> to vector<2x2x1x16xf32>
1177 // CHECK: %[[sc0:.*]] = vector.shape_cast %[[trans0]] : vector<2x2x1x16xf32> to vector<4x16xf32>
1178 // CHECK: %[[empt0:.*]] = tensor.empty
1179 // CHECK: %[[writeMsk0:.*]] = vector.create_mask {{.*}} : vector<4x16xi1>
1180 // CHECK: %[[write0:.*]] = vector.mask %[[writeMsk0:.*]] {{.*}} vector.transfer_write %[[sc0]], %[[empt0]]
1181 // CHECK: return %[[write0]]
1182 %ret = linalg.unpack %arg1 inner_dims_pos = [1, 0] inner_tiles = [16, 2] into %arg0 : tensor<?x?x16x2xf32> -> tensor<?x?xf32>
1183 return %ret : tensor<?x?xf32>
1184 }
1185 module attributes {transform.with_named_sequence} {
1186 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1187 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1188 transform.structured.vectorize %0 vector_sizes [4, 16] : !transform.any_op
1189 transform.yield
1190 }
1191 }
1192
1193 // -----
1194
1195 // CHECK-LABEL: func @test_vectorize_unpack
1196 func.func @test_vectorize_unpack(%source: tensor<8x8x32x16xf32>, %dest: tensor<256x128xf32>) -> tensor<256x128xf32> {
1197 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
1198 // CHECK: %[[C0:.*]]= arith.constant 0 : index
1199 // CHECK: %[[C8:.*]] = arith.constant 8 : index
1200 // CHECK: %[[C80:.*]] = arith.constant 8 : index
1201 // CHECK: %[[C32:.*]] = arith.constant 32 : index
1202 // CHECK: %[[C16:.*]] = arith.constant 16 : index
1203 // CHECK: %[[MSK0:.*]] = vector.create_mask %[[C8]], %[[C80]], %[[C32]], %[[C16]] : vector<16x8x32x16xi1>
1204 // CHECK: %[[READ0:.*]] = vector.mask %[[MSK0]] {{.*}} : vector<16x8x32x16xi1> -> vector<16x8x32x16xf32>
1205 // CHECK: %[[TRANSP0:.*]] = vector.transpose %[[READ0]], [0, 2, 1, 3] : vector<16x8x32x16xf32> to vector<16x32x8x16xf32>
1206 // CHECK: %[[SHAPC:.*]] = vector.shape_cast %[[TRANSP0]] : vector<16x32x8x16xf32> to vector<512x128xf32>
1207 // CHECK: %[[EMPT:.*]] = tensor.empty() : tensor<256x128xf32>
1208 // CHECK: %[[C01:.*]] = arith.constant 0 : index
1209 // CHECK: %[[C256:.*]] = arith.constant 256 : index
1210 // CHECK: %[[C128:.*]] = arith.constant 128 : index
1211 // CHECK: %[[WRITEMSK:.*]] = vector.create_mask %[[C256]], %[[C128]] : vector<512x128xi1>
1212 // CHECK: %[[WRIT:.*]] = vector.mask %[[WRITEMSK]] {{.*}} : vector<512x128xi1> -> tensor<256x128xf32>
1213 // CHECK: return %[[WRIT]] : tensor<256x128xf32>
1214 %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [32, 16] into %dest : tensor<8x8x32x16xf32> -> tensor<256x128xf32>
1215 return %0 : tensor<256x128xf32>
1216 }
1217 module attributes {transform.with_named_sequence} {
1218 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1219 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1220 transform.structured.vectorize %0 vector_sizes [512, 128] : !transform.any_op
1221 transform.yield
1222 }
1223 }
1224
1225 // -----
1226
1227 // CHECK-LABEL: func @test_vectorize_unpack_no_masks
1228 func.func @test_vectorize_unpack_no_masks(%source: tensor<8x8x32x16xf32>, %dest: tensor<256x128xf32>) -> tensor<256x128xf32> {
1229 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
1230 // CHECK: %[[C0:.*]] = arith.constant 0 : index
1231 // CHECK: %[[READ:.*]] = vector.transfer_read {{.*}} : tensor<8x8x32x16xf32>, vector<8x8x32x16xf32>
1232 // CHECK: %[[TRANSP:.*]] = vector.transpose %[[READ]], [0, 2, 1, 3] : vector<8x8x32x16xf32> to vector<8x32x8x16xf32>
1233 // CHECK: %[[SHAPC:.*]] = vector.shape_cast %[[TRANSP]] : vector<8x32x8x16xf32> to vector<256x128xf32>
1234 // CHECK: %[[EMPT:.*]] = tensor.empty() : tensor<256x128xf32>
1235 // CHECK: %[[C00:.*]] = arith.constant 0 : index
1236 // CHECK: %[[WRIT:.*]] = vector.transfer_write %[[SHAPC]], {{.*}} : vector<256x128xf32>, tensor<256x128xf32>
1237 // CHECK: return %[[WRIT]] : tensor<256x128xf32>
1238 %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [32, 16] into %dest : tensor<8x8x32x16xf32> -> tensor<256x128xf32>
1239 return %0 : tensor<256x128xf32>
1240 }
1241 module attributes {transform.with_named_sequence} {
1242 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1243 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1244 transform.structured.vectorize %0 vector_sizes [256, 128] : !transform.any_op
1245 transform.yield
1246 }
1247 }
1248
1249 // -----
1250
1251 // CHECK-LABEL: test_vectorize_unpack_with_outer_perm
1252 func.func @test_vectorize_unpack_with_outer_perm(%source: tensor<8x8x32x16xf32>, %dest: tensor<256x128xf32>) -> tensor<256x128xf32> {
1253 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
1254 // CHECK: %[[C0:.*]] = arith.constant 0 : index
1255 // CHECK: %[[READ:.*]] = vector.transfer_read {{.*}} : tensor<8x8x32x16xf32>, vector<8x8x32x16xf32>
1256 // CHECK: %[[TRANSP:.*]] = vector.transpose %[[READ]], [1, 2, 0, 3] : vector<8x8x32x16xf32> to vector<8x32x8x16xf32>
1257 // CHECK: %[[SHAPC:.*]] = vector.shape_cast %[[TRANSP]] : vector<8x32x8x16xf32> to vector<256x128xf32>
1258 // CHECK: %[[EMPT:.*]] = tensor.empty() : tensor<256x128xf32>
1259 // CHECK: %[[C00:.*]] = arith.constant 0 : index
1260 // CHECK: %[[WRIT:.*]] = vector.transfer_write %[[SHAPC]], {{.*}} : vector<256x128xf32>, tensor<256x128xf32>
1261 // CHECK: return %[[WRIT]] : tensor<256x128xf32>
1262 %0 = linalg.unpack %source outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [32, 16] into %dest : tensor<8x8x32x16xf32> -> tensor<256x128xf32>
1263 return %0 : tensor<256x128xf32>
1264 }
1265 module attributes {transform.with_named_sequence} {
1266 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1267 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1268 transform.structured.vectorize %0 vector_sizes [256, 128] : !transform.any_op
1269 transform.yield
1270 }
1271 }
1272
1273 // -----
1274
1275 // CHECK-LABEL: test_vectorize_pack_no_vector_sizes
1276 func.func @test_vectorize_pack_no_vector_sizes(%arg0: tensor<64x4xf32>, %arg1: tensor<2x4x16x2xf32>) -> tensor<2x4x16x2xf32> {
1277 %pack = linalg.pack %arg0 outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [16, 2] into %arg1 : tensor<64x4xf32> -> tensor<2x4x16x2xf32>
1278 return %pack : tensor<2x4x16x2xf32>
1279 }
1280 // CHECK-DAG: %[[cst:.*]] = arith.constant 0.000000e+00 : f32
1281 // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
1282 // CHECK: %[[read:.*]] = vector.transfer_read %{{.*}}[%[[c0]], %[[c0]]], %[[cst]]
1283 // CHECK-SAME: {in_bounds = [true, true]} : tensor<64x4xf32>, vector<64x4xf32>
1284 // CHECK: %[[shape_cast:.*]] = vector.shape_cast %[[read]] : vector<64x4xf32> to vector<4x16x2x2xf32>
1285 // CHECK: %[[transpose:.*]] = vector.transpose %[[shape_cast]], [2, 0, 1, 3] : vector<4x16x2x2xf32> to vector<2x4x16x2xf32>
1286 // CHECK-DAG: %[[c0_1:.*]] = arith.constant 0 : index
1287 // CHECK-DAG: %[[empty:.*]] = tensor.empty() : tensor<2x4x16x2xf32>
1288 // CHECK: %[[write:.*]] = vector.transfer_write %[[transpose]], %[[empty]][%[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]]]
1289 // CHECK-SAME: {in_bounds = [true, true, true, true]} : vector<2x4x16x2xf32>, tensor<2x4x16x2xf32>
1290 // CHECK: return %[[write]] : tensor<2x4x16x2xf32>
1291
1292 module attributes {transform.with_named_sequence} {
1293 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1294 %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1295 transform.structured.vectorize %0 : !transform.any_op
1296 transform.yield
1297 }
1298 }
1299
1300 // -----
1301
1302 // CHECK-LABEL: test_vectorize_padded_pack_no_vector_sizes
1303 func.func @test_vectorize_padded_pack_no_vector_sizes(%arg0: tensor<32x7x15xf32>, %arg1: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
1304 %pad = arith.constant 0.000000e+00 : f32
1305 %pack = linalg.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x7x15xf32> -> tensor<32x4x1x16x2xf32>
1306 return %pack : tensor<32x4x1x16x2xf32>
1307 }
1308 // CHECK-DAG: %[[cst:.*]] = arith.constant 0.000000e+00 : f32
1309 // CHECK-DAG: %[[c0:.*]] = arith.constant 0 : index
1310 // CHECK: %[[transfer_read:.*]] = vector.transfer_read %{{.*}}[%[[c0]], %[[c0]], %[[c0]]], %[[cst]]
1311 // CHECK-SAME: {in_bounds = [true, false, false]} : tensor<32x7x15xf32>, vector<32x8x16xf32>
1312 // CHECK: %[[shape_cast:.*]] = vector.shape_cast %[[transfer_read]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
1313 // CHECK: %[[transpose:.*]] = vector.transpose %[[shape_cast]], [0, 1, 3, 4, 2] : vector<32x4x2x1x16xf32> to vector<32x4x1x16x2xf32>
1314 // CHECK-DAG: %[[c0_1:.*]] = arith.constant 0 : index
1315 // CHECK-DAG: %[[empty:.*]] = tensor.empty() : tensor<32x4x1x16x2xf32>
1316 // CHECK: %[[write:.*]] = vector.transfer_write %[[transpose]], %[[empty]][%[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]], %[[c0_1]]]
1317 // CHECK-SAME: {in_bounds = [true, true, true, true, true]} : vector<32x4x1x16x2xf32>, tensor<32x4x1x16x2xf32>
1318 // CHECK: return %[[write]] : tensor<32x4x1x16x2xf32>
1319
1320 module attributes {transform.with_named_sequence} {
1321 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1322 %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1323 transform.structured.vectorize %0 : !transform.any_op
1324 transform.yield
1325 }
1326 }
1327
1328 // -----
1329
1330 func.func @test_vectorize_unpack_no_vector_sizes(%source: tensor<8x8x32x16xf32>, %dest: tensor<256x128xf32>) -> tensor<256x128xf32> {
1331 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
1332 // CHECK: %[[C0:.*]] = arith.constant 0 : index
1333 // CHECK: %[[READ:.*]] = vector.transfer_read {{.*}} : tensor<8x8x32x16xf32>, vector<8x8x32x16xf32>
1334 // CHECK: %[[TRANSP:.*]] = vector.transpose %[[READ]], [0, 2, 1, 3] : vector<8x8x32x16xf32> to vector<8x32x8x16xf32>
1335 // CHECK: %[[SHAPC:.*]] = vector.shape_cast %[[TRANSP]] : vector<8x32x8x16xf32> to vector<256x128xf32>
1336 // CHECK: %[[EMPT:.*]] = tensor.empty() : tensor<256x128xf32>
1337 // CHECK: %[[C00:.*]] = arith.constant 0 : index
1338 // CHECK: %[[WRIT:.*]] = vector.transfer_write %[[SHAPC]], {{.*}} : vector<256x128xf32>, tensor<256x128xf32>
1339 // CHECK: return %[[WRIT]] : tensor<256x128xf32>
1340 %0 = linalg.unpack %source inner_dims_pos = [0, 1] inner_tiles = [32, 16] into %dest : tensor<8x8x32x16xf32> -> tensor<256x128xf32>
1341 return %0 : tensor<256x128xf32>
1342 }
1343 module attributes {transform.with_named_sequence} {
1344 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1345 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1346 transform.structured.vectorize %0 : !transform.any_op
1347 transform.yield
1348 }
1349 }
1350
1351 // -----
1352
1353 func.func @test_vectorize_unpack_no_vector_sizes_slice_output(%source: tensor<8x4x16x16xf32>, %dest: tensor<64x127xf32>) -> tensor<64x127xf32> {
1354 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
1355 // CHECK: %[[C0:.*]] = arith.constant 0 : index
1356 // CHECK: %[[READ:.*]] = vector.transfer_read {{.*}} : tensor<8x4x16x16xf32>, vector<8x4x16x16xf32>
1357 // CHECK: %[[TRANSP:.*]] = vector.transpose %[[READ]], [1, 2, 0, 3] : vector<8x4x16x16xf32> to vector<4x16x8x16xf32>
1358 // CHECK: %[[SHAPC:.*]] = vector.shape_cast %[[TRANSP]] : vector<4x16x8x16xf32> to vector<64x128xf32>
1359 // CHECK: %[[EMPT:.*]] = tensor.empty() : tensor<64x127xf32>
1360 // CHECK: %[[C00:.*]] = arith.constant 0 : index
1361 // CHECK: %[[WRIT:.*]] = vector.transfer_write %[[SHAPC]], %[[EMPT]]{{\[}}%[[C00]], %[[C00]]]
1362 // CHECK-SAME: {in_bounds = [true, false]} : vector<64x128xf32>, tensor<64x127xf32>
1363 // CHECK: return %[[WRIT]] : tensor<64x127xf32>
1364 %0 = linalg.unpack %source outer_dims_perm = [1, 0] inner_dims_pos = [0, 1] inner_tiles = [16, 16] into %dest : tensor<8x4x16x16xf32> -> tensor<64x127xf32>
1365 return %0 : tensor<64x127xf32>
1366 }
1367 module attributes {transform.with_named_sequence} {
1368 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1369 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1370 transform.structured.vectorize %0 : !transform.any_op
1371 transform.yield
1372 }
1373 }
1374
1375 // -----
1376
1377 func.func @test_vectorize_unpack_no_vector_sizes_permute(%source: tensor<4x7x4xf32>, %dest: tensor<7x16xf32>) -> tensor<7x16xf32> {
1378 %0 = linalg.unpack %source outer_dims_perm=[1, 0] inner_dims_pos = [1] inner_tiles = [4] into %dest : tensor<4x7x4xf32> -> tensor<7x16xf32>
1379 return %0 : tensor<7x16xf32>
1380 }
1381 // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00 : f32
1382 // CHECK: %[[C0:.*]] = arith.constant 0 : index
1383 // CHECK: %[[READ:.*]] = vector.transfer_read {{.*}} : tensor<4x7x4xf32>, vector<4x7x4xf32>
1384 // CHECK: %[[TRANSP:.*]] = vector.transpose %[[READ]], [1, 0, 2] : vector<4x7x4xf32> to vector<7x4x4xf32>
1385 // CHECK: %[[SHAPC:.*]] = vector.shape_cast %[[TRANSP]] : vector<7x4x4xf32> to vector<7x16xf32>
1386 // CHECK: %[[EMPT:.*]] = tensor.empty() : tensor<7x16xf32>
1387 // CHECK: %[[C00:.*]] = arith.constant 0 : index
1388 // CHECK: %[[WRIT:.*]] = vector.transfer_write %[[SHAPC]], {{.*}} : vector<7x16xf32>, tensor<7x16xf32>
1389 // CHECK: return %[[WRIT]] : tensor<7x16xf32>
1390 module attributes {transform.with_named_sequence} {
1391 transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
1392 %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
1393 transform.structured.vectorize %0 : !transform.any_op
1394 transform.yield
1395 }
1396 }