Debian Codegraph @

llvm/test/Transforms/VectorCombine/AArch64/load-extractelement-scalarization.ll

raw
1 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2 ; RUN: opt -passes=vector-combine -mtriple=arm64-apple-darwinos -S %s | FileCheck --check-prefixes=CHECK,LIMIT-DEFAULT %s
3 ; RUN: opt -passes=vector-combine -mtriple=arm64-apple-darwinos -vector-combine-max-scan-instrs=2 -S %s | FileCheck --check-prefixes=CHECK,LIMIT2 %s
4
5 define i32 @load_extract_idx_0(ptr %x) {
6 ; CHECK-LABEL: @load_extract_idx_0(
7 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 3
8 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4
9 ; CHECK-NEXT: ret i32 [[R]]
10 ;
11 %lv = load <4 x i32>, ptr %x
12 %r = extractelement <4 x i32> %lv, i32 3
13 ret i32 %r
14 }
15
16 define i32 @vscale_load_extract_idx_0(ptr %x) {
17 ; CHECK-LABEL: @vscale_load_extract_idx_0(
18 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1:%.*]], align 16
19 ; CHECK-NEXT: ret i32 [[R]]
20 ;
21 %lv = load <vscale x 4 x i32>, ptr %x
22 %r = extractelement <vscale x 4 x i32> %lv, i32 0
23 ret i32 %r
24 }
25
26 ; If the original load had a smaller alignment than the scalar type, the
27 ; smaller alignment should be used.
28 define i32 @load_extract_idx_0_small_alignment(ptr %x) {
29 ; CHECK-LABEL: @load_extract_idx_0_small_alignment(
30 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 3
31 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 2
32 ; CHECK-NEXT: ret i32 [[R]]
33 ;
34 %lv = load <4 x i32>, ptr %x, align 2
35 %r = extractelement <4 x i32> %lv, i32 3
36 ret i32 %r
37 }
38
39 define i32 @load_extract_idx_1(ptr %x) {
40 ; CHECK-LABEL: @load_extract_idx_1(
41 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 1
42 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4
43 ; CHECK-NEXT: ret i32 [[R]]
44 ;
45 %lv = load <4 x i32>, ptr %x
46 %r = extractelement <4 x i32> %lv, i32 1
47 ret i32 %r
48 }
49
50 define i32 @load_extract_idx_2(ptr %x) {
51 ; CHECK-LABEL: @load_extract_idx_2(
52 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 2
53 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
54 ; CHECK-NEXT: ret i32 [[R]]
55 ;
56 %lv = load <4 x i32>, ptr %x
57 %r = extractelement <4 x i32> %lv, i32 2
58 ret i32 %r
59 }
60
61 define i32 @vscale_load_extract_idx_2(ptr %x) {
62 ; CHECK-LABEL: @vscale_load_extract_idx_2(
63 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <vscale x 4 x i32>, ptr [[X:%.*]], i32 0, i32 2
64 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
65 ; CHECK-NEXT: ret i32 [[R]]
66 ;
67 %lv = load <vscale x 4 x i32>, ptr %x
68 %r = extractelement <vscale x 4 x i32> %lv, i32 2
69 ret i32 %r
70 }
71
72 define i32 @load_extract_idx_3(ptr %x) {
73 ; CHECK-LABEL: @load_extract_idx_3(
74 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 3
75 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 4
76 ; CHECK-NEXT: ret i32 [[R]]
77 ;
78 %lv = load <4 x i32>, ptr %x
79 %r = extractelement <4 x i32> %lv, i32 3
80 ret i32 %r
81 }
82
83 ; Out-of-bounds index for extractelement, should not be converted to narrow
84 ; load, because it would introduce a dereference of a poison pointer.
85 define i32 @load_extract_idx_4(ptr %x) {
86 ; CHECK-LABEL: @load_extract_idx_4(
87 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
88 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 4
89 ; CHECK-NEXT: ret i32 [[R]]
90 ;
91 %lv = load <4 x i32>, ptr %x
92 %r = extractelement <4 x i32> %lv, i32 4
93 ret i32 %r
94 }
95
96 define i32 @vscale_load_extract_idx_4(ptr %x) {
97 ; CHECK-LABEL: @vscale_load_extract_idx_4(
98 ; CHECK-NEXT: [[LV:%.*]] = load <vscale x 4 x i32>, ptr [[X:%.*]], align 16
99 ; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[LV]], i32 4
100 ; CHECK-NEXT: ret i32 [[R]]
101 ;
102 %lv = load <vscale x 4 x i32>, ptr %x
103 %r = extractelement <vscale x 4 x i32> %lv, i32 4
104 ret i32 %r
105 }
106
107 define i32 @load_extract_idx_var_i64(ptr %x, i64 %idx) {
108 ; CHECK-LABEL: @load_extract_idx_var_i64(
109 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
110 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX:%.*]]
111 ; CHECK-NEXT: ret i32 [[R]]
112 ;
113 %lv = load <4 x i32>, ptr %x
114 %r = extractelement <4 x i32> %lv, i64 %idx
115 ret i32 %r
116 }
117
118 declare void @maythrow() readnone
119
120 define i32 @load_extract_idx_var_i64_known_valid_by_assume(ptr %x, i64 %idx) {
121 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_assume(
122 ; CHECK-NEXT: entry:
123 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 4
124 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
125 ; CHECK-NEXT: call void @maythrow()
126 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX]]
127 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
128 ; CHECK-NEXT: ret i32 [[R]]
129 ;
130 entry:
131 %cmp = icmp ult i64 %idx, 4
132 call void @llvm.assume(i1 %cmp)
133 %lv = load <4 x i32>, ptr %x
134 call void @maythrow()
135 %r = extractelement <4 x i32> %lv, i64 %idx
136 ret i32 %r
137 }
138
139 define i32 @vscale_load_extract_idx_var_i64_known_valid_by_assume(ptr %x, i64 %idx) {
140 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_known_valid_by_assume(
141 ; CHECK-NEXT: entry:
142 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 4
143 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
144 ; CHECK-NEXT: call void @maythrow()
145 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <vscale x 4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX]]
146 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
147 ; CHECK-NEXT: ret i32 [[R]]
148 ;
149 entry:
150 %cmp = icmp ult i64 %idx, 4
151 call void @llvm.assume(i1 %cmp)
152 %lv = load <vscale x 4 x i32>, ptr %x
153 call void @maythrow()
154 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx
155 ret i32 %r
156 }
157
158 declare i1 @cond()
159
160 define i32 @load_extract_idx_var_i64_known_valid_by_assume_in_dominating_block(ptr %x, i64 %idx, i1 %c.1) {
161 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_assume_in_dominating_block(
162 ; CHECK-NEXT: entry:
163 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 4
164 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
165 ; CHECK-NEXT: br i1 [[C_1:%.*]], label [[LOOP:%.*]], label [[EXIT:%.*]]
166 ; CHECK: loop:
167 ; CHECK-NEXT: call void @maythrow()
168 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX]]
169 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
170 ; CHECK-NEXT: [[C_2:%.*]] = call i1 @cond()
171 ; CHECK-NEXT: br i1 [[C_2]], label [[LOOP]], label [[EXIT]]
172 ; CHECK: exit:
173 ; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[R]], [[LOOP]] ], [ 0, [[ENTRY:%.*]] ]
174 ; CHECK-NEXT: ret i32 [[P]]
175 ;
176 entry:
177 %cmp = icmp ult i64 %idx, 4
178 call void @llvm.assume(i1 %cmp)
179 br i1 %c.1, label %loop, label %exit
180
181 loop:
182 %lv = load <4 x i32>, ptr %x
183 call void @maythrow()
184 %r = extractelement <4 x i32> %lv, i64 %idx
185 %c.2 = call i1 @cond()
186 br i1 %c.2, label %loop, label %exit
187
188 exit:
189 %p = phi i32 [ %r, %loop ], [ 0, %entry ]
190 ret i32 %p
191 }
192
193 define i32 @load_extract_idx_var_i64_known_valid_by_assume_in_non_dominating_block(ptr %x, i64 %idx, i1 %c.1, i1 %c.2) {
194 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_assume_in_non_dominating_block(
195 ; CHECK-NEXT: entry:
196 ; CHECK-NEXT: br i1 [[C_1:%.*]], label [[ASSUME_CHECK:%.*]], label [[LOOP:%.*]]
197 ; CHECK: assume_check:
198 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 4
199 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
200 ; CHECK-NEXT: br i1 [[C_2:%.*]], label [[LOOP]], label [[EXIT:%.*]]
201 ; CHECK: loop:
202 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
203 ; CHECK-NEXT: call void @maythrow()
204 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX]]
205 ; CHECK-NEXT: [[C_3:%.*]] = call i1 @cond()
206 ; CHECK-NEXT: br i1 [[C_3]], label [[LOOP]], label [[EXIT]]
207 ; CHECK: exit:
208 ; CHECK-NEXT: [[P:%.*]] = phi i32 [ [[R]], [[LOOP]] ], [ 0, [[ASSUME_CHECK]] ]
209 ; CHECK-NEXT: ret i32 0
210 ;
211 entry:
212 br i1 %c.1, label %assume_check, label %loop
213
214 assume_check:
215 %cmp = icmp ult i64 %idx, 4
216 call void @llvm.assume(i1 %cmp)
217 br i1 %c.2, label %loop, label %exit
218
219 loop:
220 %lv = load <4 x i32>, ptr %x
221 call void @maythrow()
222 %r = extractelement <4 x i32> %lv, i64 %idx
223 %c.3 = call i1 @cond()
224 br i1 %c.3, label %loop, label %exit
225
226 exit:
227 %p = phi i32 [ %r, %loop ], [ 0, %assume_check ]
228 ret i32 0
229 }
230
231 define i32 @load_extract_idx_var_i64_not_known_valid_by_assume_after_load(ptr %x, i64 %idx) {
232 ; CHECK-LABEL: @load_extract_idx_var_i64_not_known_valid_by_assume_after_load(
233 ; CHECK-NEXT: entry:
234 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 4
235 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
236 ; CHECK-NEXT: call void @maythrow()
237 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
238 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX]]
239 ; CHECK-NEXT: ret i32 [[R]]
240 ;
241 entry:
242 %cmp = icmp ult i64 %idx, 4
243 %lv = load <4 x i32>, ptr %x
244 call void @maythrow()
245 call void @llvm.assume(i1 %cmp)
246 %r = extractelement <4 x i32> %lv, i64 %idx
247 ret i32 %r
248 }
249
250 define i32 @load_extract_idx_var_i64_not_known_valid_by_assume(ptr %x, i64 %idx) {
251 ; CHECK-LABEL: @load_extract_idx_var_i64_not_known_valid_by_assume(
252 ; CHECK-NEXT: entry:
253 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 5
254 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
255 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
256 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX]]
257 ; CHECK-NEXT: ret i32 [[R]]
258 ;
259 entry:
260 %cmp = icmp ult i64 %idx, 5
261 call void @llvm.assume(i1 %cmp)
262 %lv = load <4 x i32>, ptr %x
263 %r = extractelement <4 x i32> %lv, i64 %idx
264 ret i32 %r
265 }
266
267 define i32 @vscale_load_extract_idx_var_i64_not_known_valid_by_assume_0(ptr %x, i64 %idx) {
268 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_not_known_valid_by_assume_0(
269 ; CHECK-NEXT: entry:
270 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], 5
271 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
272 ; CHECK-NEXT: [[LV:%.*]] = load <vscale x 4 x i32>, ptr [[X:%.*]], align 16
273 ; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[LV]], i64 [[IDX]]
274 ; CHECK-NEXT: ret i32 [[R]]
275 ;
276 entry:
277 %cmp = icmp ult i64 %idx, 5
278 call void @llvm.assume(i1 %cmp)
279 %lv = load <vscale x 4 x i32>, ptr %x
280 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx
281 ret i32 %r
282 }
283
284 define i32 @vscale_load_extract_idx_var_i64_not_known_valid_by_assume_1(ptr %x, i64 %idx) {
285 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_not_known_valid_by_assume_1(
286 ; CHECK-NEXT: entry:
287 ; CHECK-NEXT: [[VS:%.*]] = call i64 @llvm.vscale.i64()
288 ; CHECK-NEXT: [[VM:%.*]] = mul i64 [[VS]], 4
289 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IDX:%.*]], [[VM]]
290 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
291 ; CHECK-NEXT: [[LV:%.*]] = load <vscale x 4 x i32>, ptr [[X:%.*]], align 16
292 ; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[LV]], i64 [[IDX]]
293 ; CHECK-NEXT: ret i32 [[R]]
294 ;
295 entry:
296 %vs = call i64 @llvm.vscale.i64()
297 %vm = mul i64 %vs, 4
298 %cmp = icmp ult i64 %idx, %vm
299 call void @llvm.assume(i1 %cmp)
300 %lv = load <vscale x 4 x i32>, ptr %x
301 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx
302 ret i32 %r
303 }
304
305 declare i64 @llvm.vscale.i64()
306 declare void @llvm.assume(i1)
307
308 define i32 @load_extract_idx_var_i64_known_valid_by_and(ptr %x, i64 %idx) {
309 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_and(
310 ; CHECK-NEXT: entry:
311 ; CHECK-NEXT: [[IDX_FROZEN:%.*]] = freeze i64 [[IDX:%.*]]
312 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX_FROZEN]], 3
313 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
314 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
315 ; CHECK-NEXT: ret i32 [[R]]
316 ;
317 entry:
318 %idx.clamped = and i64 %idx, 3
319 %lv = load <4 x i32>, ptr %x
320 %r = extractelement <4 x i32> %lv, i64 %idx.clamped
321 ret i32 %r
322 }
323
324 define i32 @vscale_load_extract_idx_var_i64_known_valid_by_and(ptr %x, i64 %idx) {
325 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_known_valid_by_and(
326 ; CHECK-NEXT: entry:
327 ; CHECK-NEXT: [[IDX_FROZEN:%.*]] = freeze i64 [[IDX:%.*]]
328 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX_FROZEN]], 3
329 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <vscale x 4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
330 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
331 ; CHECK-NEXT: ret i32 [[R]]
332 ;
333 entry:
334 %idx.clamped = and i64 %idx, 3
335 %lv = load <vscale x 4 x i32>, ptr %x
336 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx.clamped
337 ret i32 %r
338 }
339
340 define i32 @load_extract_idx_var_i64_known_valid_by_and_noundef(ptr %x, i64 noundef %idx) {
341 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_and_noundef(
342 ; CHECK-NEXT: entry:
343 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX:%.*]], 3
344 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
345 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
346 ; CHECK-NEXT: ret i32 [[R]]
347 ;
348 entry:
349 %idx.clamped = and i64 %idx, 3
350 %lv = load <4 x i32>, ptr %x
351 %r = extractelement <4 x i32> %lv, i64 %idx.clamped
352 ret i32 %r
353 }
354
355 define i32 @load_extract_idx_var_i64_not_known_valid_by_and(ptr %x, i64 %idx) {
356 ; CHECK-LABEL: @load_extract_idx_var_i64_not_known_valid_by_and(
357 ; CHECK-NEXT: entry:
358 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX:%.*]], 4
359 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
360 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_CLAMPED]]
361 ; CHECK-NEXT: ret i32 [[R]]
362 ;
363 entry:
364 %idx.clamped = and i64 %idx, 4
365 %lv = load <4 x i32>, ptr %x
366 %r = extractelement <4 x i32> %lv, i64 %idx.clamped
367 ret i32 %r
368 }
369
370 define i32 @vscale_load_extract_idx_var_i64_not_known_valid_by_and(ptr %x, i64 %idx) {
371 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_not_known_valid_by_and(
372 ; CHECK-NEXT: entry:
373 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX:%.*]], 4
374 ; CHECK-NEXT: [[LV:%.*]] = load <vscale x 4 x i32>, ptr [[X:%.*]], align 16
375 ; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[LV]], i64 [[IDX_CLAMPED]]
376 ; CHECK-NEXT: ret i32 [[R]]
377 ;
378 entry:
379 %idx.clamped = and i64 %idx, 4
380 %lv = load <vscale x 4 x i32>, ptr %x
381 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx.clamped
382 ret i32 %r
383 }
384
385 define i32 @load_extract_idx_var_i64_known_valid_by_urem(ptr %x, i64 %idx) {
386 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_urem(
387 ; CHECK-NEXT: entry:
388 ; CHECK-NEXT: [[IDX_FROZEN:%.*]] = freeze i64 [[IDX:%.*]]
389 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = urem i64 [[IDX_FROZEN]], 4
390 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
391 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
392 ; CHECK-NEXT: ret i32 [[R]]
393 ;
394 entry:
395 %idx.clamped = urem i64 %idx, 4
396 %lv = load <4 x i32>, ptr %x
397 %r = extractelement <4 x i32> %lv, i64 %idx.clamped
398 ret i32 %r
399 }
400
401 define i32 @vscale_load_extract_idx_var_i64_known_valid_by_urem(ptr %x, i64 %idx) {
402 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_known_valid_by_urem(
403 ; CHECK-NEXT: entry:
404 ; CHECK-NEXT: [[IDX_FROZEN:%.*]] = freeze i64 [[IDX:%.*]]
405 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = urem i64 [[IDX_FROZEN]], 4
406 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <vscale x 4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
407 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
408 ; CHECK-NEXT: ret i32 [[R]]
409 ;
410 entry:
411 %idx.clamped = urem i64 %idx, 4
412 %lv = load <vscale x 4 x i32>, ptr %x
413 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx.clamped
414 ret i32 %r
415 }
416
417 define i32 @load_extract_idx_var_i64_known_valid_by_urem_noundef(ptr %x, i64 noundef %idx) {
418 ; CHECK-LABEL: @load_extract_idx_var_i64_known_valid_by_urem_noundef(
419 ; CHECK-NEXT: entry:
420 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = urem i64 [[IDX:%.*]], 4
421 ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
422 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP0]], align 4
423 ; CHECK-NEXT: ret i32 [[R]]
424 ;
425 entry:
426 %idx.clamped = urem i64 %idx, 4
427 %lv = load <4 x i32>, ptr %x
428 %r = extractelement <4 x i32> %lv, i64 %idx.clamped
429 ret i32 %r
430 }
431
432 define i32 @load_extract_idx_var_i64_not_known_valid_by_urem(ptr %x, i64 %idx) {
433 ; CHECK-LABEL: @load_extract_idx_var_i64_not_known_valid_by_urem(
434 ; CHECK-NEXT: entry:
435 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = urem i64 [[IDX:%.*]], 5
436 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
437 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_CLAMPED]]
438 ; CHECK-NEXT: ret i32 [[R]]
439 ;
440 entry:
441 %idx.clamped = urem i64 %idx, 5
442 %lv = load <4 x i32>, ptr %x
443 %r = extractelement <4 x i32> %lv, i64 %idx.clamped
444 ret i32 %r
445 }
446
447 define i32 @vscale_load_extract_idx_var_i64_not_known_valid_by_urem(ptr %x, i64 %idx) {
448 ; CHECK-LABEL: @vscale_load_extract_idx_var_i64_not_known_valid_by_urem(
449 ; CHECK-NEXT: entry:
450 ; CHECK-NEXT: [[IDX_CLAMPED:%.*]] = urem i64 [[IDX:%.*]], 5
451 ; CHECK-NEXT: [[LV:%.*]] = load <vscale x 4 x i32>, ptr [[X:%.*]], align 16
452 ; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[LV]], i64 [[IDX_CLAMPED]]
453 ; CHECK-NEXT: ret i32 [[R]]
454 ;
455 entry:
456 %idx.clamped = urem i64 %idx, 5
457 %lv = load <vscale x 4 x i32>, ptr %x
458 %r = extractelement <vscale x 4 x i32> %lv, i64 %idx.clamped
459 ret i32 %r
460 }
461
462 define i32 @load_extract_idx_var_i32(ptr %x, i32 %idx) {
463 ; CHECK-LABEL: @load_extract_idx_var_i32(
464 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
465 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 [[IDX:%.*]]
466 ; CHECK-NEXT: ret i32 [[R]]
467 ;
468 %lv = load <4 x i32>, ptr %x
469 %r = extractelement <4 x i32> %lv, i32 %idx
470 ret i32 %r
471 }
472
473 declare void @clobber()
474
475 define i32 @load_extract_clobber_call_before(ptr %x) {
476 ; CHECK-LABEL: @load_extract_clobber_call_before(
477 ; CHECK-NEXT: call void @clobber()
478 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 2
479 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
480 ; CHECK-NEXT: ret i32 [[R]]
481 ;
482 call void @clobber()
483 %lv = load <4 x i32>, ptr %x
484 %r = extractelement <4 x i32> %lv, i32 2
485 ret i32 %r
486 }
487
488 define i32 @load_extract_clobber_call_between(ptr %x) {
489 ; CHECK-LABEL: @load_extract_clobber_call_between(
490 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
491 ; CHECK-NEXT: call void @clobber()
492 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 2
493 ; CHECK-NEXT: ret i32 [[R]]
494 ;
495 %lv = load <4 x i32>, ptr %x
496 call void @clobber()
497 %r = extractelement <4 x i32> %lv, i32 2
498 ret i32 %r
499 }
500
501 define i32 @load_extract_clobber_call_after(ptr %x) {
502 ; CHECK-LABEL: @load_extract_clobber_call_after(
503 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 2
504 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
505 ; CHECK-NEXT: call void @clobber()
506 ; CHECK-NEXT: ret i32 [[R]]
507 ;
508 %lv = load <4 x i32>, ptr %x
509 %r = extractelement <4 x i32> %lv, i32 2
510 call void @clobber()
511 ret i32 %r
512 }
513
514 define i32 @load_extract_clobber_store_before(ptr %x, ptr %y) {
515 ; CHECK-LABEL: @load_extract_clobber_store_before(
516 ; CHECK-NEXT: store i8 0, ptr [[Y:%.*]], align 1
517 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 2
518 ; CHECK-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
519 ; CHECK-NEXT: ret i32 [[R]]
520 ;
521 store i8 0, ptr %y
522 %lv = load <4 x i32>, ptr %x
523 %r = extractelement <4 x i32> %lv, i32 2
524 ret i32 %r
525 }
526
527 define i32 @load_extract_clobber_store_between(ptr %x, ptr %y) {
528 ; CHECK-LABEL: @load_extract_clobber_store_between(
529 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
530 ; CHECK-NEXT: store i8 0, ptr [[Y:%.*]], align 1
531 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 2
532 ; CHECK-NEXT: ret i32 [[R]]
533 ;
534 %lv = load <4 x i32>, ptr %x
535 store i8 0, ptr %y
536 %r = extractelement <4 x i32> %lv, i32 2
537 ret i32 %r
538 }
539
540 define i32 @load_extract_clobber_store_between_limit(ptr %x, ptr %y, <8 x i32> %z) {
541 ; CHECK-LABEL: @load_extract_clobber_store_between_limit(
542 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
543 ; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <8 x i32> [[Z1:%.*]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
544 ; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[Z1]], [[SHIFT]]
545 ; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
546 ; CHECK-NEXT: [[TMP2:%.*]] = add <8 x i32> [[TMP1]], [[SHIFT1]]
547 ; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
548 ; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[SHIFT2]]
549 ; CHECK-NEXT: [[SHIFT3:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
550 ; CHECK-NEXT: [[Z:%.*]] = add <8 x i32> [[TMP3]], [[SHIFT3]]
551 ; CHECK-NEXT: [[Z_0:%.*]] = extractelement <8 x i32> [[Z]], i32 0
552 ; CHECK-NEXT: store i8 0, ptr [[Y:%.*]], align 1
553 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 2
554 ; CHECK-NEXT: [[ADD_4:%.*]] = add i32 [[Z_0]], [[R]]
555 ; CHECK-NEXT: ret i32 [[ADD_4]]
556 ;
557 %lv = load <4 x i32>, ptr %x
558 %z.0 = extractelement <8 x i32> %z, i32 0
559 %z.1 = extractelement <8 x i32> %z, i32 1
560 %add.0 = add i32 %z.0, %z.1
561 %z.2 = extractelement <8 x i32> %z, i32 2
562 %add.1 = add i32 %add.0, %z.2
563 %z.3 = extractelement <8 x i32> %z, i32 3
564 %add.2 = add i32 %add.1, %z.3
565 %z.4 = extractelement <8 x i32> %z, i32 4
566 %add.3 = add i32 %add.2, %z.4
567 store i8 0, ptr %y
568 %r = extractelement <4 x i32> %lv, i32 2
569 %add.4 = add i32 %add.3, %r
570 ret i32 %add.4
571 }
572
573 define i32 @load_extract_clobber_store_after_limit(ptr %x, ptr %y, <8 x i32> %z) {
574 ; LIMIT-DEFAULT-LABEL: @load_extract_clobber_store_after_limit(
575 ; LIMIT-DEFAULT-NEXT: [[SHIFT:%.*]] = shufflevector <8 x i32> [[Z1:%.*]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
576 ; LIMIT-DEFAULT-NEXT: [[TMP4:%.*]] = add <8 x i32> [[Z1]], [[SHIFT]]
577 ; LIMIT-DEFAULT-NEXT: [[SHIFT1:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
578 ; LIMIT-DEFAULT-NEXT: [[TMP2:%.*]] = add <8 x i32> [[TMP4]], [[SHIFT1]]
579 ; LIMIT-DEFAULT-NEXT: [[SHIFT2:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
580 ; LIMIT-DEFAULT-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[SHIFT2]]
581 ; LIMIT-DEFAULT-NEXT: [[SHIFT3:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
582 ; LIMIT-DEFAULT-NEXT: [[Z:%.*]] = add <8 x i32> [[TMP3]], [[SHIFT3]]
583 ; LIMIT-DEFAULT-NEXT: [[Z_0:%.*]] = extractelement <8 x i32> [[Z]], i32 0
584 ; LIMIT-DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr inbounds <4 x i32>, ptr [[X:%.*]], i32 0, i32 2
585 ; LIMIT-DEFAULT-NEXT: [[R:%.*]] = load i32, ptr [[TMP1]], align 8
586 ; LIMIT-DEFAULT-NEXT: store i8 0, ptr [[Y:%.*]], align 1
587 ; LIMIT-DEFAULT-NEXT: [[ADD_4:%.*]] = add i32 [[Z_0]], [[R]]
588 ; LIMIT-DEFAULT-NEXT: ret i32 [[ADD_4]]
589 ;
590 ; LIMIT2-LABEL: @load_extract_clobber_store_after_limit(
591 ; LIMIT2-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
592 ; LIMIT2-NEXT: [[SHIFT:%.*]] = shufflevector <8 x i32> [[Z1:%.*]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
593 ; LIMIT2-NEXT: [[TMP1:%.*]] = add <8 x i32> [[Z1]], [[SHIFT]]
594 ; LIMIT2-NEXT: [[SHIFT1:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
595 ; LIMIT2-NEXT: [[TMP2:%.*]] = add <8 x i32> [[TMP1]], [[SHIFT1]]
596 ; LIMIT2-NEXT: [[SHIFT2:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
597 ; LIMIT2-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[SHIFT2]]
598 ; LIMIT2-NEXT: [[SHIFT3:%.*]] = shufflevector <8 x i32> [[Z1]], <8 x i32> poison, <8 x i32> <i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
599 ; LIMIT2-NEXT: [[Z:%.*]] = add <8 x i32> [[TMP3]], [[SHIFT3]]
600 ; LIMIT2-NEXT: [[Z_0:%.*]] = extractelement <8 x i32> [[Z]], i32 0
601 ; LIMIT2-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 2
602 ; LIMIT2-NEXT: store i8 0, ptr [[Y:%.*]], align 1
603 ; LIMIT2-NEXT: [[ADD_4:%.*]] = add i32 [[Z_0]], [[R]]
604 ; LIMIT2-NEXT: ret i32 [[ADD_4]]
605 ;
606 %lv = load <4 x i32>, ptr %x
607 %z.0 = extractelement <8 x i32> %z, i32 0
608 %z.1 = extractelement <8 x i32> %z, i32 1
609 %add.0 = add i32 %z.0, %z.1
610 %z.2 = extractelement <8 x i32> %z, i32 2
611 %add.1 = add i32 %add.0, %z.2
612 %z.3 = extractelement <8 x i32> %z, i32 3
613 %add.2 = add i32 %add.1, %z.3
614 %z.4 = extractelement <8 x i32> %z, i32 4
615 %add.3 = add i32 %add.2, %z.4
616 %r = extractelement <4 x i32> %lv, i32 2
617 store i8 0, ptr %y
618 %add.4 = add i32 %add.3, %r
619 ret i32 %add.4
620 }
621
622 declare void @use.v4i32(<4 x i32>)
623
624 define i32 @load_extract_idx_different_bbs(ptr %x, i1 %c) {
625 ; CHECK-LABEL: @load_extract_idx_different_bbs(
626 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
627 ; CHECK-NEXT: br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
628 ; CHECK: then:
629 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[LV]], i32 1
630 ; CHECK-NEXT: ret i32 [[R]]
631 ; CHECK: else:
632 ; CHECK-NEXT: call void @use.v4i32(<4 x i32> [[LV]])
633 ; CHECK-NEXT: ret i32 20
634 ;
635 %lv = load <4 x i32>, ptr %x
636 br i1 %c, label %then, label %else
637
638 then:
639 %r = extractelement <4 x i32> %lv, i32 1
640 ret i32 %r
641
642 else:
643 call void @use.v4i32(<4 x i32> %lv)
644 ret i32 20
645 }
646
647 define i31 @load_with_non_power_of_2_element_type(ptr %x) {
648 ; CHECK-LABEL: @load_with_non_power_of_2_element_type(
649 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i31>, ptr [[X:%.*]], align 16
650 ; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i31> [[LV]], i32 1
651 ; CHECK-NEXT: ret i31 [[R]]
652 ;
653 %lv = load <4 x i31>, ptr %x
654 %r = extractelement <4 x i31> %lv, i32 1
655 ret i31 %r
656 }
657
658 define i1 @load_with_non_power_of_2_element_type_2(ptr %x) {
659 ; CHECK-LABEL: @load_with_non_power_of_2_element_type_2(
660 ; CHECK-NEXT: [[LV:%.*]] = load <8 x i1>, ptr [[X:%.*]], align 1
661 ; CHECK-NEXT: [[R:%.*]] = extractelement <8 x i1> [[LV]], i32 1
662 ; CHECK-NEXT: ret i1 [[R]]
663 ;
664 %lv = load <8 x i1>, ptr %x
665 %r = extractelement <8 x i1> %lv, i32 1
666 ret i1 %r
667 }
668
669 ; Scalarizing the load for multiple constant indices may not be profitable.
670 define i32 @load_multiple_extracts_with_constant_idx(ptr %x) {
671 ; CHECK-LABEL: @load_multiple_extracts_with_constant_idx(
672 ; CHECK-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1:%.*]], align 16
673 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <4 x i32>, ptr [[TMP1]], i32 0, i32 1
674 ; CHECK-NEXT: [[E_1:%.*]] = load i32, ptr [[TMP2]], align 4
675 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
676 ; CHECK-NEXT: ret i32 [[RES]]
677 ;
678 %lv = load <4 x i32>, ptr %x
679 %e.0 = extractelement <4 x i32> %lv, i32 0
680 %e.1 = extractelement <4 x i32> %lv, i32 1
681 %res = add i32 %e.0, %e.1
682 ret i32 %res
683 }
684
685 ; Scalarizing the load for multiple extracts is profitable in this case,
686 ; because the vector large vector requires 2 vector registers.
687 define i32 @load_multiple_extracts_with_constant_idx_profitable(ptr %x) {
688 ; CHECK-LABEL: @load_multiple_extracts_with_constant_idx_profitable(
689 ; CHECK-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1:%.*]], align 16
690 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <8 x i32>, ptr [[TMP1]], i32 0, i32 6
691 ; CHECK-NEXT: [[E_1:%.*]] = load i32, ptr [[TMP2]], align 8
692 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
693 ; CHECK-NEXT: ret i32 [[RES]]
694 ;
695 %lv = load <8 x i32>, ptr %x, align 16
696 %e.0 = extractelement <8 x i32> %lv, i32 0
697 %e.1 = extractelement <8 x i32> %lv, i32 6
698 %res = add i32 %e.0, %e.1
699 ret i32 %res
700 }
701
702 ; Scalarizing may or may not be profitable, depending on the target.
703 define i32 @load_multiple_2_with_variable_indices(ptr %x, i64 %idx.0, i64 %idx.1) {
704 ; CHECK-LABEL: @load_multiple_2_with_variable_indices(
705 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
706 ; CHECK-NEXT: [[E_0:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_0:%.*]]
707 ; CHECK-NEXT: [[E_1:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_1:%.*]]
708 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
709 ; CHECK-NEXT: ret i32 [[RES]]
710 ;
711 %lv = load <4 x i32>, ptr %x
712 %e.0 = extractelement <4 x i32> %lv, i64 %idx.0
713 %e.1 = extractelement <4 x i32> %lv, i64 %idx.1
714 %res = add i32 %e.0, %e.1
715 ret i32 %res
716 }
717
718 define i32 @load_4_extracts_with_variable_indices_short_vector(ptr %x, i64 %idx.0, i64 %idx.1, i64 %idx.2, i64 %idx.3) {
719 ; CHECK-LABEL: @load_4_extracts_with_variable_indices_short_vector(
720 ; CHECK-NEXT: [[LV:%.*]] = load <4 x i32>, ptr [[X:%.*]], align 16
721 ; CHECK-NEXT: [[E_0:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_0:%.*]]
722 ; CHECK-NEXT: [[E_1:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_1:%.*]]
723 ; CHECK-NEXT: [[E_2:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_2:%.*]]
724 ; CHECK-NEXT: [[E_3:%.*]] = extractelement <4 x i32> [[LV]], i64 [[IDX_3:%.*]]
725 ; CHECK-NEXT: [[RES_0:%.*]] = add i32 [[E_0]], [[E_1]]
726 ; CHECK-NEXT: [[RES_1:%.*]] = add i32 [[RES_0]], [[E_2]]
727 ; CHECK-NEXT: [[RES_2:%.*]] = add i32 [[RES_1]], [[E_3]]
728 ; CHECK-NEXT: ret i32 [[RES_2]]
729 ;
730 %lv = load <4 x i32>, ptr %x
731 %e.0 = extractelement <4 x i32> %lv, i64 %idx.0
732 %e.1 = extractelement <4 x i32> %lv, i64 %idx.1
733 %e.2 = extractelement <4 x i32> %lv, i64 %idx.2
734 %e.3 = extractelement <4 x i32> %lv, i64 %idx.3
735 %res.0 = add i32 %e.0, %e.1
736 %res.1 = add i32 %res.0, %e.2
737 %res.2 = add i32 %res.1, %e.3
738 ret i32 %res.2
739 }
740
741 define i32 @load_multiple_extracts_with_variable_indices_large_vector_only_first_valid(ptr %x, i64 %idx.0, i64 %idx.1) {
742 ; CHECK-LABEL: @load_multiple_extracts_with_variable_indices_large_vector_only_first_valid(
743 ; CHECK-NEXT: [[CMP_IDX_0:%.*]] = icmp ult i64 [[IDX_0:%.*]], 16
744 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_IDX_0]])
745 ; CHECK-NEXT: [[LV:%.*]] = load <16 x i32>, ptr [[X:%.*]], align 64
746 ; CHECK-NEXT: [[E_0:%.*]] = extractelement <16 x i32> [[LV]], i64 [[IDX_0]]
747 ; CHECK-NEXT: [[E_1:%.*]] = extractelement <16 x i32> [[LV]], i64 [[IDX_1:%.*]]
748 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
749 ; CHECK-NEXT: ret i32 [[RES]]
750 ;
751 %cmp.idx.0 = icmp ult i64 %idx.0, 16
752 call void @llvm.assume(i1 %cmp.idx.0)
753
754 %lv = load <16 x i32>, ptr %x
755 %e.0 = extractelement <16 x i32> %lv, i64 %idx.0
756 %e.1 = extractelement <16 x i32> %lv, i64 %idx.1
757 %res = add i32 %e.0, %e.1
758 ret i32 %res
759 }
760
761 define i32 @load_multiple_extracts_with_variable_indices_large_vector_only_all_valid(ptr %x, i64 %idx.0, i64 %idx.1) {
762 ; CHECK-LABEL: @load_multiple_extracts_with_variable_indices_large_vector_only_all_valid(
763 ; CHECK-NEXT: [[CMP_IDX_0:%.*]] = icmp ult i64 [[IDX_0:%.*]], 16
764 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_IDX_0]])
765 ; CHECK-NEXT: [[CMP_IDX_1:%.*]] = icmp ult i64 [[IDX_1:%.*]], 16
766 ; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_IDX_1]])
767 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_0]]
768 ; CHECK-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1]], align 4
769 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X]], i32 0, i64 [[IDX_1]]
770 ; CHECK-NEXT: [[E_1:%.*]] = load i32, ptr [[TMP2]], align 4
771 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
772 ; CHECK-NEXT: ret i32 [[RES]]
773 ;
774 %cmp.idx.0 = icmp ult i64 %idx.0, 16
775 call void @llvm.assume(i1 %cmp.idx.0)
776 %cmp.idx.1 = icmp ult i64 %idx.1, 16
777 call void @llvm.assume(i1 %cmp.idx.1)
778
779 %lv = load <16 x i32>, ptr %x
780 %e.0 = extractelement <16 x i32> %lv, i64 %idx.0
781 %e.1 = extractelement <16 x i32> %lv, i64 %idx.1
782 %res = add i32 %e.0, %e.1
783 ret i32 %res
784 }
785
786 define i32 @load_multiple_extracts_with_variable_indices_large_vector_only_first_valid_by_and(ptr %x, i64 %idx.0, i64 %idx.1) {
787 ; CHECK-LABEL: @load_multiple_extracts_with_variable_indices_large_vector_only_first_valid_by_and(
788 ; CHECK-NEXT: [[IDX_0_CLAMPED:%.*]] = and i64 [[IDX_0:%.*]], 15
789 ; CHECK-NEXT: [[LV:%.*]] = load <16 x i32>, ptr [[X:%.*]], align 64
790 ; CHECK-NEXT: [[E_0:%.*]] = extractelement <16 x i32> [[LV]], i64 [[IDX_0_CLAMPED]]
791 ; CHECK-NEXT: [[E_1:%.*]] = extractelement <16 x i32> [[LV]], i64 [[IDX_1:%.*]]
792 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
793 ; CHECK-NEXT: ret i32 [[RES]]
794 ;
795 %idx.0.clamped = and i64 %idx.0, 15
796
797 %lv = load <16 x i32>, ptr %x
798 %e.0 = extractelement <16 x i32> %lv, i64 %idx.0.clamped
799 %e.1 = extractelement <16 x i32> %lv, i64 %idx.1
800 %res = add i32 %e.0, %e.1
801 ret i32 %res
802 }
803
804 define i32 @load_multiple_extracts_with_variable_indices_large_vector_all_valid_by_and(ptr %x, i64 %idx.0, i64 %idx.1) {
805 ; CHECK-LABEL: @load_multiple_extracts_with_variable_indices_large_vector_all_valid_by_and(
806 ; CHECK-NEXT: [[IDX_0_FROZEN:%.*]] = freeze i64 [[IDX_0:%.*]]
807 ; CHECK-NEXT: [[IDX_0_CLAMPED:%.*]] = and i64 [[IDX_0_FROZEN]], 15
808 ; CHECK-NEXT: [[IDX_1_FROZEN:%.*]] = freeze i64 [[IDX_1:%.*]]
809 ; CHECK-NEXT: [[IDX_1_CLAMPED:%.*]] = and i64 [[IDX_1_FROZEN]], 15
810 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_0_CLAMPED]]
811 ; CHECK-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1]], align 4
812 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X]], i32 0, i64 [[IDX_1_CLAMPED]]
813 ; CHECK-NEXT: [[E_1:%.*]] = load i32, ptr [[TMP2]], align 4
814 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
815 ; CHECK-NEXT: ret i32 [[RES]]
816 ;
817 %idx.0.clamped = and i64 %idx.0, 15
818 %idx.1.clamped = and i64 %idx.1, 15
819
820 %lv = load <16 x i32>, ptr %x
821 %e.0 = extractelement <16 x i32> %lv, i64 %idx.0.clamped
822 %e.1 = extractelement <16 x i32> %lv, i64 %idx.1.clamped
823 %res = add i32 %e.0, %e.1
824 ret i32 %res
825 }
826
827 define i32 @load_multiple_extracts_with_unique_variable_indices_large_vector_valid_by_and(ptr %x, ptr %y, i64 %idx) {
828 ; LIMIT-DEFAULT-LABEL: @load_multiple_extracts_with_unique_variable_indices_large_vector_valid_by_and(
829 ; LIMIT-DEFAULT-NEXT: [[IDX_FROZEN:%.*]] = freeze i64 [[IDX:%.*]]
830 ; LIMIT-DEFAULT-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX_FROZEN]], 15
831 ; LIMIT-DEFAULT-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
832 ; LIMIT-DEFAULT-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1]], align 4
833 ; LIMIT-DEFAULT-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i32>, ptr [[Y:%.*]], i32 0, i64 [[IDX_CLAMPED]]
834 ; LIMIT-DEFAULT-NEXT: [[E_1:%.*]] = load i32, ptr [[TMP2]], align 4
835 ; LIMIT-DEFAULT-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
836 ; LIMIT-DEFAULT-NEXT: ret i32 [[RES]]
837 ;
838 ; LIMIT2-LABEL: @load_multiple_extracts_with_unique_variable_indices_large_vector_valid_by_and(
839 ; LIMIT2-NEXT: [[IDX_FROZEN:%.*]] = freeze i64 [[IDX:%.*]]
840 ; LIMIT2-NEXT: [[IDX_CLAMPED:%.*]] = and i64 [[IDX_FROZEN]], 15
841 ; LIMIT2-NEXT: [[LY:%.*]] = load <16 x i32>, ptr [[Y:%.*]], align 64
842 ; LIMIT2-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_CLAMPED]]
843 ; LIMIT2-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1]], align 4
844 ; LIMIT2-NEXT: [[E_1:%.*]] = extractelement <16 x i32> [[LY]], i64 [[IDX_CLAMPED]]
845 ; LIMIT2-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
846 ; LIMIT2-NEXT: ret i32 [[RES]]
847 ;
848 %idx.clamped = and i64 %idx, 15
849
850 %lx = load <16 x i32>, ptr %x
851 %ly = load <16 x i32>, ptr %y
852 %e.0 = extractelement <16 x i32> %lx, i64 %idx.clamped
853 %e.1 = extractelement <16 x i32> %ly, i64 %idx.clamped
854 %res = add i32 %e.0, %e.1
855 ret i32 %res
856 }
857
858 define i32 @load_multiple_extracts_with_variable_indices_large_vector_all_valid_by_and_some_noundef(ptr %x, i64 %idx.0, i64 noundef %idx.1) {
859 ; CHECK-LABEL: @load_multiple_extracts_with_variable_indices_large_vector_all_valid_by_and_some_noundef(
860 ; CHECK-NEXT: [[IDX_0_FROZEN:%.*]] = freeze i64 [[IDX_0:%.*]]
861 ; CHECK-NEXT: [[IDX_0_CLAMPED:%.*]] = and i64 [[IDX_0_FROZEN]], 15
862 ; CHECK-NEXT: [[IDX_1_CLAMPED:%.*]] = and i64 [[IDX_1:%.*]], 15
863 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X:%.*]], i32 0, i64 [[IDX_0_CLAMPED]]
864 ; CHECK-NEXT: [[E_0:%.*]] = load i32, ptr [[TMP1]], align 4
865 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds <16 x i32>, ptr [[X]], i32 0, i64 [[IDX_1_CLAMPED]]
866 ; CHECK-NEXT: [[E_1:%.*]] = load i32, ptr [[TMP2]], align 4
867 ; CHECK-NEXT: [[RES:%.*]] = add i32 [[E_0]], [[E_1]]
868 ; CHECK-NEXT: ret i32 [[RES]]
869 ;
870 %idx.0.clamped = and i64 %idx.0, 15
871 %idx.1.clamped = and i64 %idx.1, 15
872
873 %lv = load <16 x i32>, ptr %x
874 %e.0 = extractelement <16 x i32> %lv, i64 %idx.0.clamped
875 %e.1 = extractelement <16 x i32> %lv, i64 %idx.1.clamped
876 %res = add i32 %e.0, %e.1
877 ret i32 %res
878 }
879
880 ; Test case from PR51992.
881 define i8 @load_extract_safe_due_to_branch_on_poison(<8 x i8> %in, ptr %src) {
882 ; CHECK-LABEL: @load_extract_safe_due_to_branch_on_poison(
883 ; CHECK-NEXT: entry:
884 ; CHECK-NEXT: [[EXT_IDX:%.*]] = extractelement <8 x i8> [[IN:%.*]], i32 0
885 ; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[EXT_IDX]], 99
886 ; CHECK-NEXT: br i1 [[CMP]], label [[THEN:%.*]], label [[EXIT:%.*]]
887 ; CHECK: then:
888 ; CHECK-NEXT: br label [[EXIT]]
889 ; CHECK: exit:
890 ; CHECK-NEXT: ret i8 0
891 ;
892 entry:
893 %ext.idx = extractelement <8 x i8> %in, i32 0
894 %ext.idx.i32 = zext i8 %ext.idx to i32
895 %cmp = icmp ult i8 %ext.idx, 99
896 br i1 %cmp, label %then, label %exit
897
898 then:
899 %load = load <16 x i8>, ptr %src, align 16
900 %and = and i32 %ext.idx.i32, 15
901 %ext = extractelement <16 x i8> %load, i32 %and
902 br label %exit
903
904 exit:
905 %p = phi i8 [ 0, %entry ], [ %ext, %then ]
906 ret i8 0
907 }
908
909 declare void @use(...)
910
911 ; Make sure we don't assert.
912 define void @pr69820(ptr %p, i32 %arg) {
913 ; CHECK-LABEL: @pr69820(
914 ; CHECK-NEXT: [[V:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
915 ; CHECK-NEXT: [[AND:%.*]] = and i32 [[ARG:%.*]], 3
916 ; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x float> [[V]], i32 [[AND]]
917 ; CHECK-NEXT: call void @use(<4 x float> [[V]], float [[EXT]])
918 ; CHECK-NEXT: ret void
919 ;
920 %v = load <4 x float>, ptr %p, align 16
921 %and = and i32 %arg, 3
922 %ext = extractelement <4 x float> %v, i32 %and
923 call void @use(<4 x float> %v, float %ext)
924 ret void
925
926 ; uselistorder directives
927 uselistorder <4 x float> %v, { 1, 0 }
928 }