/* * Copyright (C) 2020-2025 Intel Corporation * * SPDX-License-Identifier: MIT * */ R"===( #define ALIGNED4(ptr) __builtin_assume(((size_t)ptr&0b11) == 0) __kernel void CopyBufferToBufferBytes( const __global uchar* pSrc, __global uchar* pDst, offset_t srcOffsetInBytes, offset_t dstOffsetInBytes, offset_t bytesToRead ) { ALIGNED4(pSrc); ALIGNED4(pDst); pSrc += ( srcOffsetInBytes + get_global_id(0) ); pDst += ( dstOffsetInBytes + get_global_id(0) ); pDst[ 0 ] = pSrc[ 0 ]; } __kernel void CopyBufferToBufferLeftLeftover( const __global uchar* pSrc, __global uchar* pDst, offset_t srcOffsetInBytes, offset_t dstOffsetInBytes) { ALIGNED4(pSrc); ALIGNED4(pDst); idx_t gid = get_global_id(0); pDst[ gid + dstOffsetInBytes ] = pSrc[ gid + srcOffsetInBytes ]; } __kernel void CopyBufferToBufferMiddle( const __global uint* pSrc, __global uint* pDst, offset_t srcOffsetInBytes, offset_t dstOffsetInBytes) { ALIGNED4(pSrc); ALIGNED4(pDst); idx_t gid = get_global_id(0); pDst += dstOffsetInBytes >> 2; pSrc += srcOffsetInBytes >> 2; uint4 loaded = vload4(gid, pSrc); vstore4(loaded, gid, pDst); } __kernel void CopyBufferToBufferMiddleMisaligned( __global const uint* pSrc, __global uint* pDst, offset_t srcOffsetInBytes, offset_t dstOffsetInBytes, uint misalignmentInBits) { ALIGNED4(pSrc); ALIGNED4(pDst); idx_t gid = get_global_id(0); pDst += dstOffsetInBytes >> 2; pSrc += srcOffsetInBytes >> 2; const uint4 src0 = vload4(gid, pSrc); const uint4 src1 = vload4((gid + 1), pSrc); uint4 result; result.x = (src0.x >> misalignmentInBits) | (src0.y << (32 - misalignmentInBits)); result.y = (src0.y >> misalignmentInBits) | (src0.z << (32 - misalignmentInBits)); result.z = (src0.z >> misalignmentInBits) | (src0.w << (32 - misalignmentInBits)); result.w = (src0.w >> misalignmentInBits) | (src1.x << (32 - misalignmentInBits)); vstore4(result, gid, pDst); } __kernel void CopyBufferToBufferRightLeftover( const __global uchar* pSrc, __global uchar* pDst, offset_t srcOffsetInBytes, offset_t dstOffsetInBytes) { ALIGNED4(pSrc); ALIGNED4(pDst); idx_t gid = get_global_id(0); pDst[ gid + dstOffsetInBytes ] = pSrc[ gid + srcOffsetInBytes ]; } __kernel void copyBufferToBufferBytesSingle(__global uchar *dst, const __global uchar *src) { ALIGNED4(dst); ALIGNED4(src); idx_t gid = get_global_id(0); dst[gid] = (uchar)(src[gid]); } __kernel void CopyBufferToBufferSideRegion( __global uchar* pDst, const __global uchar* pSrc, idx_t len, offset_t dstSshOffset, // Offset needed in case ptr has been adjusted for SSH alignment offset_t srcSshOffset // Offset needed in case ptr has been adjusted for SSH alignment ) { ALIGNED4(pSrc); ALIGNED4(pDst); idx_t gid = get_global_id(0); __global uchar* pDstWithOffset = (__global uchar*)((__global uchar*)pDst + dstSshOffset); __global uchar* pSrcWithOffset = (__global uchar*)((__global uchar*)pSrc + srcSshOffset); if (gid < len) { pDstWithOffset[ gid ] = pSrcWithOffset[ gid ]; } } __kernel void CopyBufferToBufferMiddleRegion( __global uint* pDst, const __global uint* pSrc, idx_t elems, offset_t dstSshOffset, // Offset needed in case ptr has been adjusted for SSH alignment offset_t srcSshOffset // Offset needed in case ptr has been adjusted for SSH alignment ) { ALIGNED4(pSrc); ALIGNED4(pDst); idx_t gid = get_global_id(0); __global uint* pDstWithOffset = (__global uint*)((__global uchar*)pDst + dstSshOffset); __global uint* pSrcWithOffset = (__global uint*)((__global uchar*)pSrc + srcSshOffset); if (gid < elems) { uint4 loaded = vload4(gid, pSrcWithOffset); vstore4(loaded, gid, pDstWithOffset); } } )==="