Debian Codegraph @

shared/source/built_ins/kernels/copy_buffer_to_buffer.builtin_kernel

raw
1 /*
2 * Copyright (C) 2020-2025 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8 R"===(
9
10 #define ALIGNED4(ptr) __builtin_assume(((size_t)ptr&0b11) == 0)
11
12 __kernel void CopyBufferToBufferBytes(
13 const __global uchar* pSrc,
14 __global uchar* pDst,
15 offset_t srcOffsetInBytes,
16 offset_t dstOffsetInBytes,
17 offset_t bytesToRead )
18 {
19 ALIGNED4(pSrc);
20 ALIGNED4(pDst);
21 pSrc += ( srcOffsetInBytes + get_global_id(0) );
22 pDst += ( dstOffsetInBytes + get_global_id(0) );
23 pDst[ 0 ] = pSrc[ 0 ];
24 }
25
26 __kernel void CopyBufferToBufferLeftLeftover(
27 const __global uchar* pSrc,
28 __global uchar* pDst,
29 offset_t srcOffsetInBytes,
30 offset_t dstOffsetInBytes)
31 {
32 ALIGNED4(pSrc);
33 ALIGNED4(pDst);
34 idx_t gid = get_global_id(0);
35 pDst[ gid + dstOffsetInBytes ] = pSrc[ gid + srcOffsetInBytes ];
36 }
37
38 __kernel void CopyBufferToBufferMiddle(
39 const __global uint* pSrc,
40 __global uint* pDst,
41 offset_t srcOffsetInBytes,
42 offset_t dstOffsetInBytes)
43 {
44 ALIGNED4(pSrc);
45 ALIGNED4(pDst);
46 idx_t gid = get_global_id(0);
47 pDst += dstOffsetInBytes >> 2;
48 pSrc += srcOffsetInBytes >> 2;
49 uint4 loaded = vload4(gid, pSrc);
50 vstore4(loaded, gid, pDst);
51 }
52
53 __kernel void CopyBufferToBufferMiddleMisaligned(
54 __global const uint* pSrc,
55 __global uint* pDst,
56 offset_t srcOffsetInBytes,
57 offset_t dstOffsetInBytes,
58 uint misalignmentInBits)
59 {
60 ALIGNED4(pSrc);
61 ALIGNED4(pDst);
62 idx_t gid = get_global_id(0);
63 pDst += dstOffsetInBytes >> 2;
64 pSrc += srcOffsetInBytes >> 2;
65 const uint4 src0 = vload4(gid, pSrc);
66 const uint4 src1 = vload4((gid + 1), pSrc);
67
68 uint4 result;
69 result.x = (src0.x >> misalignmentInBits) | (src0.y << (32 - misalignmentInBits));
70 result.y = (src0.y >> misalignmentInBits) | (src0.z << (32 - misalignmentInBits));
71 result.z = (src0.z >> misalignmentInBits) | (src0.w << (32 - misalignmentInBits));
72 result.w = (src0.w >> misalignmentInBits) | (src1.x << (32 - misalignmentInBits));
73 vstore4(result, gid, pDst);
74 }
75
76 __kernel void CopyBufferToBufferRightLeftover(
77 const __global uchar* pSrc,
78 __global uchar* pDst,
79 offset_t srcOffsetInBytes,
80 offset_t dstOffsetInBytes)
81 {
82 ALIGNED4(pSrc);
83 ALIGNED4(pDst);
84 idx_t gid = get_global_id(0);
85 pDst[ gid + dstOffsetInBytes ] = pSrc[ gid + srcOffsetInBytes ];
86 }
87
88 __kernel void copyBufferToBufferBytesSingle(__global uchar *dst, const __global uchar *src) {
89 ALIGNED4(dst);
90 ALIGNED4(src);
91 idx_t gid = get_global_id(0);
92 dst[gid] = (uchar)(src[gid]);
93 }
94
95 __kernel void CopyBufferToBufferSideRegion(
96 __global uchar* pDst,
97 const __global uchar* pSrc,
98 idx_t len,
99 offset_t dstSshOffset, // Offset needed in case ptr has been adjusted for SSH alignment
100 offset_t srcSshOffset // Offset needed in case ptr has been adjusted for SSH alignment
101 )
102 {
103 ALIGNED4(pSrc);
104 ALIGNED4(pDst);
105 idx_t gid = get_global_id(0);
106 __global uchar* pDstWithOffset = (__global uchar*)((__global uchar*)pDst + dstSshOffset);
107 __global uchar* pSrcWithOffset = (__global uchar*)((__global uchar*)pSrc + srcSshOffset);
108 if (gid < len) {
109 pDstWithOffset[ gid ] = pSrcWithOffset[ gid ];
110 }
111 }
112
113 __kernel void CopyBufferToBufferMiddleRegion(
114 __global uint* pDst,
115 const __global uint* pSrc,
116 idx_t elems,
117 offset_t dstSshOffset, // Offset needed in case ptr has been adjusted for SSH alignment
118 offset_t srcSshOffset // Offset needed in case ptr has been adjusted for SSH alignment
119 )
120 {
121 ALIGNED4(pSrc);
122 ALIGNED4(pDst);
123 idx_t gid = get_global_id(0);
124 __global uint* pDstWithOffset = (__global uint*)((__global uchar*)pDst + dstSshOffset);
125 __global uint* pSrcWithOffset = (__global uint*)((__global uchar*)pSrc + srcSshOffset);
126 if (gid < elems) {
127 uint4 loaded = vload4(gid, pSrcWithOffset);
128 vstore4(loaded, gid, pDstWithOffset);
129 }
130 }
131
132 )==="