| 1 |
; RUN: opt < %s -passes=loop-interchange -cache-line-size=64 \ |
| 2 |
; RUN: -pass-remarks-output=%t -disable-output -loop-interchange-profitabilities=vectorize |
| 3 |
; RUN: FileCheck -input-file %t %s |
| 4 |
|
| 5 |
@A = dso_local global [256 x [256 x float]] zeroinitializer |
| 6 |
@B = dso_local global [256 x [256 x float]] zeroinitializer |
| 7 |
@C = dso_local global [256 x [256 x float]] zeroinitializer |
| 8 |
@D = global [256 x [256 x [256 x float]]] zeroinitializer |
| 9 |
@E = global [256 x [256 x [256 x float]]] zeroinitializer |
| 10 |
|
| 11 |
; Check that the below loops are exchanged for vectorization. |
| 12 |
; |
| 13 |
; for (int i = 0; i < 256; i++) { |
| 14 |
; for (int j = 1; j < 256; j++) { |
| 15 |
; A[i][j] = A[i][j-1] + B[i][j]; |
| 16 |
; C[i][j] += 1; |
| 17 |
; } |
| 18 |
; } |
| 19 |
; |
| 20 |
|
| 21 |
; CHECK: --- !Passed |
| 22 |
; CHECK-NEXT: Pass: loop-interchange |
| 23 |
; CHECK-NEXT: Name: Interchanged |
| 24 |
; CHECK-NEXT: Function: interchange_necessary_for_vectorization |
| 25 |
; CHECK-NEXT: Args: |
| 26 |
; CHECK-NEXT: - String: Loop interchanged with enclosing loop. |
| 27 |
define void @interchange_necessary_for_vectorization() { |
| 28 |
entry: |
| 29 |
br label %for.i.header |
| 30 |
|
| 31 |
for.i.header: |
| 32 |
%i = phi i64 [ 1, %entry ], [ %i.next, %for.i.inc ] |
| 33 |
br label %for.j.body |
| 34 |
|
| 35 |
for.j.body: |
| 36 |
%j = phi i64 [ 1, %for.i.header ], [ %j.next, %for.j.body ] |
| 37 |
%j.dec = add nsw i64 %j, -1 |
| 38 |
%a.load.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @A, i64 0, i64 %i, i64 %j.dec |
| 39 |
%b.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @B, i64 0, i64 %i, i64 %j |
| 40 |
%c.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @C, i64 0, i64 %i, i64 %j |
| 41 |
%a = load float, ptr %a.load.index, align 4 |
| 42 |
%b = load float, ptr %b.index, align 4 |
| 43 |
%c = load float, ptr %c.index, align 4 |
| 44 |
%add.0 = fadd float %a, %b |
| 45 |
%a.store.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @A, i64 0, i64 %i, i64 %j |
| 46 |
store float %add.0, ptr %a.store.index, align 4 |
| 47 |
%add.1 = fadd float %c, 1.0 |
| 48 |
store float %add.1, ptr %c.index, align 4 |
| 49 |
%j.next = add nuw nsw i64 %j, 1 |
| 50 |
%cmp.j = icmp eq i64 %j.next, 256 |
| 51 |
br i1 %cmp.j, label %for.i.inc, label %for.j.body |
| 52 |
|
| 53 |
for.i.inc: |
| 54 |
%i.next = add nuw nsw i64 %i, 1 |
| 55 |
%cmp.i = icmp eq i64 %i.next, 256 |
| 56 |
br i1 %cmp.i, label %exit, label %for.i.header |
| 57 |
|
| 58 |
exit: |
| 59 |
ret void |
| 60 |
} |
| 61 |
|
| 62 |
; Check that the following innermost loop can be vectorized so that |
| 63 |
; interchanging is unnecessary. |
| 64 |
; |
| 65 |
; for (int i = 0; i < 256; i++) |
| 66 |
; for (int j = 1; j < 256; j++) |
| 67 |
; A[i][j-1] = A[i][j] + B[i][j]; |
| 68 |
; |
| 69 |
|
| 70 |
; CHECK: --- !Missed |
| 71 |
; CHECK-NEXT: Pass: loop-interchange |
| 72 |
; CHECK-NEXT: Name: InterchangeNotProfitable |
| 73 |
; CHECK-NEXT: Function: interchange_unnecesasry_for_vectorization |
| 74 |
; CHECK-NEXT: Args: |
| 75 |
; CHECK-NEXT: - String: Insufficient information to calculate the cost of loop for interchange. |
| 76 |
define void @interchange_unnecesasry_for_vectorization() { |
| 77 |
entry: |
| 78 |
br label %for.i.header |
| 79 |
|
| 80 |
for.i.header: |
| 81 |
%i = phi i64 [ 1, %entry ], [ %i.next, %for.i.inc ] |
| 82 |
br label %for.j.body |
| 83 |
|
| 84 |
for.j.body: |
| 85 |
%j = phi i64 [ 1, %for.i.header ], [ %j.next, %for.j.body ] |
| 86 |
%j.dec = add nsw i64 %j, -1 |
| 87 |
%a.load.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @A, i64 0, i64 %i, i64 %j |
| 88 |
%b.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @B, i64 0, i64 %i, i64 %j |
| 89 |
%a = load float, ptr %a.load.index, align 4 |
| 90 |
%b = load float, ptr %b.index, align 4 |
| 91 |
%add = fadd float %a, %b |
| 92 |
%a.store.index = getelementptr nuw inbounds [256 x [256 x float]], ptr @A, i64 0, i64 %i, i64 %j.dec |
| 93 |
store float %add, ptr %a.store.index, align 4 |
| 94 |
%j.next = add nuw nsw i64 %j, 1 |
| 95 |
%cmp.j = icmp eq i64 %j.next, 256 |
| 96 |
br i1 %cmp.j, label %for.i.inc, label %for.j.body |
| 97 |
|
| 98 |
for.i.inc: |
| 99 |
%i.next = add nuw nsw i64 %i, 1 |
| 100 |
%cmp.i = icmp eq i64 %i.next, 256 |
| 101 |
br i1 %cmp.i, label %exit, label %for.i.header |
| 102 |
|
| 103 |
exit: |
| 104 |
ret void |
| 105 |
} |
| 106 |
|
| 107 |
; Check that the below loops are exchanged to allow innermost loop |
| 108 |
; vectorization. We cannot vectorize the j-loop because it has a lexically |
| 109 |
; backward dependency, but the i-loop can be vectorized because all the |
| 110 |
; loop-carried dependencies are lexically forward. LoopVectorize currently only |
| 111 |
; vectorizes innermost loop, hence move the i-loop to that position. |
| 112 |
; |
| 113 |
; for (int i = 0; i < 255; i++) { |
| 114 |
; for (int j = 1; j < 256; j++) { |
| 115 |
; A[i][j] = A[i][j-1] + B[i][j]; |
| 116 |
; C[i][j] += C[i+1][j]; |
| 117 |
; } |
| 118 |
; } |
| 119 |
; |
| 120 |
|
| 121 |
; CHECK: --- !Passed |
| 122 |
; CHECK-NEXT: Pass: loop-interchange |
| 123 |
; CHECK-NEXT: Name: Interchanged |
| 124 |
; CHECK-NEXT: Function: interchange_necessary_for_vectorization2 |
| 125 |
; CHECK-NEXT: Args: |
| 126 |
; CHECK-NEXT: - String: Loop interchanged with enclosing loop. |
| 127 |
define void @interchange_necessary_for_vectorization2() { |
| 128 |
entry: |
| 129 |
br label %for.i.header |
| 130 |
|
| 131 |
for.i.header: |
| 132 |
%i = phi i64 [ 1, %entry ], [ %i.next, %for.i.inc ] |
| 133 |
%i.inc = add nuw nsw i64 %i, 1 |
| 134 |
br label %for.j.body |
| 135 |
|
| 136 |
for.j.body: |
| 137 |
%j = phi i64 [ 1, %for.i.header ], [ %j.next, %for.j.body ] |
| 138 |
%j.dec = add nsw i64 %j, -1 |
| 139 |
%a.load.index = getelementptr inbounds [256 x [256 x float]], ptr @A, i64 0, i64 %i, i64 %j.dec |
| 140 |
%b.index = getelementptr inbounds [256 x [256 x float]], ptr @B, i64 0, i64 %i, i64 %j |
| 141 |
%c.load.index = getelementptr inbounds [256 x [256 x float]], ptr @C, i64 0, i64 %i.inc, i64 %j |
| 142 |
%c.store.index = getelementptr inbounds [256 x [256 x float]], ptr @C, i64 0, i64 %i, i64 %j |
| 143 |
%a = load float, ptr %a.load.index |
| 144 |
%b = load float, ptr %b.index |
| 145 |
%c0 = load float, ptr %c.load.index |
| 146 |
%c1 = load float, ptr %c.store.index |
| 147 |
%add.0 = fadd float %a, %b |
| 148 |
%a.store.index = getelementptr inbounds [256 x [256 x float]], ptr @A, i64 0, i64 %i, i64 %j |
| 149 |
store float %add.0, ptr %a.store.index |
| 150 |
%add.1 = fadd float %c0, %c1 |
| 151 |
store float %add.1, ptr %c.store.index |
| 152 |
%j.next = add nuw nsw i64 %j, 1 |
| 153 |
%cmp.j = icmp eq i64 %j.next, 256 |
| 154 |
br i1 %cmp.j, label %for.i.inc, label %for.j.body |
| 155 |
|
| 156 |
for.i.inc: |
| 157 |
%i.next = add nuw nsw i64 %i, 1 |
| 158 |
%cmp.i = icmp eq i64 %i.next, 255 |
| 159 |
br i1 %cmp.i, label %exit, label %for.i.header |
| 160 |
|
| 161 |
exit: |
| 162 |
ret void |
| 163 |
} |
| 164 |
|
| 165 |
; Check that no interchange is performed for the following loop. Interchanging |
| 166 |
; the j-loop and k-loop makes the innermost loop vectorizble, since the j-loop |
| 167 |
; has only forward dependencies. However, at the moment, a loop body consisting |
| 168 |
; of multiple BBs is handled pesimistically. Hence the j-loop isn't moved to |
| 169 |
; the innermost place. |
| 170 |
; |
| 171 |
; for (int i = 0; i < 255; i++) { |
| 172 |
; for (int j = 0; j < 255; j++) { |
| 173 |
; for (int k = 0; k < 128; k++) { |
| 174 |
; E[i][j][k] = D[i+1][j+1][2*k]; |
| 175 |
; if (cond) |
| 176 |
; D[i][j][k+1] = 1.0; |
| 177 |
; } |
| 178 |
; } |
| 179 |
|
| 180 |
; CHECK: --- !Missed |
| 181 |
; CHECK-NEXT: Pass: loop-interchange |
| 182 |
; CHECK-NEXT: Name: InterchangeNotProfitable |
| 183 |
; CHECK-NEXT: Function: multiple_BBs_in_loop |
| 184 |
; CHECK-NEXT: Args: |
| 185 |
; CHECK-NEXT: - String: Interchanging loops is not considered to improve cache locality nor vectorization. |
| 186 |
; CHECK: --- !Missed |
| 187 |
; CHECK-NEXT: Pass: loop-interchange |
| 188 |
; CHECK-NEXT: Name: InterchangeNotProfitable |
| 189 |
; CHECK-NEXT: Function: multiple_BBs_in_loop |
| 190 |
; CHECK-NEXT: Args: |
| 191 |
; CHECK-NEXT: - String: Interchanging loops is not considered to improve cache locality nor vectorization. |
| 192 |
define void @multiple_BBs_in_loop() { |
| 193 |
entry: |
| 194 |
br label %for.i.header |
| 195 |
|
| 196 |
for.i.header: |
| 197 |
%i = phi i64 [ 0, %entry ], [ %i.inc, %for.i.inc ] |
| 198 |
%i.inc = add nuw nsw i64 %i, 1 |
| 199 |
br label %for.j.header |
| 200 |
|
| 201 |
for.j.header: |
| 202 |
%j = phi i64 [ 0, %for.i.header ], [ %j.inc, %for.j.inc ] |
| 203 |
%j.inc = add nuw nsw i64 %j, 1 |
| 204 |
br label %for.k.body |
| 205 |
|
| 206 |
for.k.body: |
| 207 |
%k = phi i64 [ 0, %for.j.header ], [ %k.inc, %for.k.inc ] |
| 208 |
%k.inc = add nuw nsw i64 %k, 1 |
| 209 |
%k.2 = mul nuw nsw i64 %k, 2 |
| 210 |
%d.index = getelementptr inbounds [256 x [256 x [256 x float]]], ptr @D, i64 0, i64 %i.inc, i64 %j.inc, i64 %k.2 |
| 211 |
%e.index = getelementptr inbounds [256 x [256 x [256 x float]]], ptr @E, i64 0, i64 %i, i64 %j, i64 %k |
| 212 |
%d.load = load float, ptr %d.index |
| 213 |
store float %d.load, ptr %e.index |
| 214 |
%cond = freeze i1 undef |
| 215 |
br i1 %cond, label %if.then, label %for.k.inc |
| 216 |
|
| 217 |
if.then: |
| 218 |
%d.index2 = getelementptr inbounds [256 x [256 x [256 x float]]], ptr @D, i64 0, i64 %i, i64 %j, i64 %k.inc |
| 219 |
store float 1.0, ptr %d.index2 |
| 220 |
br label %for.k.inc |
| 221 |
|
| 222 |
for.k.inc: |
| 223 |
%cmp.k = icmp eq i64 %k.inc, 128 |
| 224 |
br i1 %cmp.k, label %for.j.inc, label %for.k.body |
| 225 |
|
| 226 |
for.j.inc: |
| 227 |
%cmp.j = icmp eq i64 %j.inc, 255 |
| 228 |
br i1 %cmp.j, label %for.i.inc, label %for.j.header |
| 229 |
|
| 230 |
for.i.inc: |
| 231 |
%cmp.i = icmp eq i64 %i.inc, 255 |
| 232 |
br i1 %cmp.i, label %exit, label %for.i.header |
| 233 |
|
| 234 |
exit: |
| 235 |
ret void |
| 236 |
} |