CUDA: Why are calculations are not hidden by memory latency? - optimization

Question

I have created a program to test GPU RAM bandwidth and have got several strange effects on it.
The one is that straigtforward implementation is not purely memory bounded. If I unroll the
for (int *p = pStart; p < pEnd; p += shift)
sum += *p;
loop by adding
p += shift; sum += *p;
p += shift; sum += *p;
p += shift; sum += *p;
into the loop body, the speed raises by 20%, from 183 to 222 GB/s on GeForce 2060 RTX (theoretical bandwidth 264 GB/s).
Shouldn't such things should be hidden by memory latency?
Or in such program in an ideal case all warps only should await for data from memory and every additional calculation a warp done between such waits make memory bus underloaded?
I analyzed executables with NVidia Nsight Compute 2021.3.0, it reports less compute throughput and higher memory throughput. Cache usage is negligible. This all is logical.
The results in schedulers statistics are more interesting and I don't understand the meaning of these numbers. Could someone explain it please? I am not a newbie in CUDA, I understand that warps can move through pipeline or stall because of awaiting data from RAM, but I don't understand well what it means by eligible warps or warp cycles per issued instruction (in unrolled version each warp only did some instruction once per 114 clock cycles of GPU in average?)
Base is without unrolling, main values - with. So e.g. with unrolling we have 7.88 active warps per scheduler, without - 7.96 (isn't the more - the better?)
Full code (look at __global__ void gpuReadMemory):
#include <assert.h>
#include <conio.h>
#include <stdio.h>
#include <cuda.h>
#define VC_EXTRALEAN
#include <windows.h>
#define CUDA_CHECK(err) __cudaSafeCall(err, __FILE__, __LINE__)
inline void __cudaSafeCall(cudaError err, const char *file, const int line)
{
if (err != cudaSuccess)
{
fprintf(stderr, "%s(%i): CUDA error %d (%s)\n",
file, line, int(err), cudaGetErrorString(err));
throw "CUDA error";
}
}
int getMultiprocessorCount()
{
int num;
CUDA_CHECK(cudaDeviceGetAttribute(&num, cudaDevAttrMultiProcessorCount, 0));
return num;
}
__global__ void gpuWriteMemory(int *gpuArr, int dataSizeInMBs,
int packetShift, int passCount, int *gpuDebugArr)
{
int *pStart = gpuArr + ((long long)packetShift *
(blockDim.y * blockIdx.x + threadIdx.y)) / sizeof(int) + threadIdx.x;
int *pEnd = gpuArr + (((long long)dataSizeInMBs) << 20) / sizeof(int);
int shift = gridDim.x * blockDim.y * packetShift / sizeof(int);
for (int passInd = 0; passInd < passCount; passInd++)
for (int *p = pStart; p < pEnd; p += shift)
*p = blockIdx.x * 10000000 + threadIdx.y * 1000 + threadIdx.x;
}
__global__ void gpuReadMemory(int *gpuArr, int dataSizeInMBs,
int packetShift, int passCount, int *gpuDebugArr)
{
int *pStart = gpuArr + ((long long)packetShift *
(blockDim.y * blockIdx.x + threadIdx.y)) / sizeof(int) + threadIdx.x;
int *pEnd = gpuArr + (((long long)dataSizeInMBs) << 20) / sizeof(int);
int shift = gridDim.x * blockDim.y * packetShift / sizeof(int);
int sum = 0;
int accessCount = 0;
for (int passInd = 0; passInd < passCount; passInd++)
{
#pragma unroll // - doesn't have effect
for (int *p = pStart; p < pEnd; p += shift)
{
sum += *p;
p += shift; sum += *p;
p += shift; sum += *p;
p += shift; sum += *p;
}
*pStart = sum; // Without it bandwidth reported is 3 times bigger than theoretical
}
// Suspiciously fast code:
//for (int passInd = 0; passInd < passCount; passInd++)
// 0x0000000500999d10 IADD3 R8, R8, 0x1, RZ
// 0x0000000500999d20 BSSY B0, 0x500999de0
// for (int *p = pStart; p < pEnd; p += shift)
// 0x0000000500999d30 ISETP.GE.AND.EX P0, PT, R7, UR6, PT, P0
// for (int passInd = 0; passInd < passCount; passInd++)
// 0x0000000500999d40 ISETP.GE.AND P1, PT, R8, c[0x0][0x170], PT
// for (int *p = pStart; p < pEnd; p += shift)
// 0x0000000500999d50 #P0 BRA 0x500999dd0
// 0x0000000500999d60 IMAD.MOV.U32 R3, RZ, RZ, R2
// 0x0000000500999d70 IMAD.MOV.U32 R5, RZ, RZ, R4
// 0x0000000500999d80 LEA R3, P0, R0, R3, 0x2
// 0x0000000500999d90 LEA.HI.X R5, R0, R5, RZ, 0x2, P0
// 0x0000000500999da0 ISETP.GE.U32.AND P0, PT, R3, UR4, PT
// 0x0000000500999db0 ISETP.GE.U32.AND.EX P0, PT, R5, UR5, PT, P0
// 0x0000000500999dc0 #!P0 BRA 0x500999d80
// 0x0000000500999dd0 BSYNC B0
// Normal code:
//for (int *p = pStart; p < pEnd; p += shift)
// 0x0000000500999da0 IMAD.MOV.U32 R8, RZ, RZ, R6
// sum += *p;
//0x0000000500999db0 IMAD.MOV.U32 R4, RZ, RZ, R8
// 0x0000000500999dc0 LDG.E.SYS R4, [R4] // Reading from memory
// for (int *p = pStart; p < pEnd; p += shift)
// 0x0000000500999dd0 IADD3 R11, P0, R11, UR5, RZ
// 0x0000000500999de0 IADD3 R8, P1, R8, UR5, RZ
// 0x0000000500999df0 IADD3.X R13, R13, UR4, RZ, P0, !PT
// 0x0000000500999e00 ISETP.GE.U32.AND P0, PT, R11, R2, PT
// 0x0000000500999e10 IADD3.X R5, R5, UR4, RZ, P1, !PT
// 0x0000000500999e20 ISETP.GE.U32.AND.EX P0, PT, R13, R3, PT, P0
// sum += *p;
//0x0000000500999e30 IMAD.IADD R9, R4, 0x1, R9
// for (int *p = pStart; p < pEnd; p += shift)
// 0x0000000500999e40 #!P0 BRA 0x500999db0
// 0x0000000500999e50 BSYNC B0
// *pStart = sum; // Lowers bandwidth being reported on GT 520M from 33 to 9.7 GB/s
//0x0000000500999e60 STG.E.SYS [R6], R9
}
class CMemorySpeedTester
{
public:
CMemorySpeedTester()
{
m_multiprocessorCount = getMultiprocessorCount();
CUDA_CHECK(cudaMalloc((void**)&gpuArr, ((long long)dataSizeInMBs) << 20));
CUDA_CHECK(cudaMalloc((void**)&gpuDebugArr, debugArrLen * sizeof(int)));
CUDA_CHECK(cudaMemset(gpuArr, -1, ((long long)dataSizeInMBs) << 20));
debugArr = (int*)malloc(debugArrLen * sizeof(int));
debugArr2D = (int (*)[1000])debugArr;
QueryPerformanceFrequency(&timerFreq);
}
void testBandwidth()
{
int threadPerBlock = 256;
int blockCount = m_multiprocessorCount * 24;
dim3 blocks(blockCount);
dim3 threads1(32, threadPerBlock / 32);
gpuWriteMemory<<<blocks, threads1>>>(gpuArr, dataSizeInMBs,
32 * sizeof(int), 1, gpuDebugArr);
CUDA_CHECK(cudaDeviceSynchronize());
for (int passCount = 10; passCount <= 100; passCount *= 10)
{
int threadPerPacket = 32;
for (int packetShiftMult = 1; packetShiftMult <= 16; packetShiftMult *= 16)
{
int packetShift = threadPerPacket * sizeof(int) * packetShiftMult;
dim3 threads(threadPerPacket, threadPerBlock / threadPerPacket);
QueryPerformanceCounter(&t0);
gpuReadMemory<<<blocks, threads>>>(gpuArr, dataSizeInMBs,
packetShift, passCount, gpuDebugArr);
CUDA_CHECK(cudaDeviceSynchronize());
QueryPerformanceCounter(&t);
double dt = double(t.QuadPart - t0.QuadPart) / timerFreq.QuadPart;
printf(" %2d th./packet, packet shift %4d, %d pass(es): %.3f ms, %.2f GB/s\n",
threadPerPacket, packetShift, passCount,
dt * 1000, (double)(dataSizeInMBs) / packetShiftMult / (1 << 10) * passCount / dt);
}
}
}
void copyToHostDebugArr()
{
CUDA_CHECK(cudaMemcpy(debugArr, gpuDebugArr, debugArrLen * sizeof(int), cudaMemcpyDeviceToHost));
CUDA_CHECK(cudaDeviceSynchronize());
}
protected:
static const int dataSizeInMBs = 800;
static const int debugArrLen = 4000000;
int m_multiprocessorCount;
int *gpuArr, *gpuDebugArr;
int *debugArr;
int (*debugArr2D)[1000];
LARGE_INTEGER timerFreq, t, t0;
};
int main(int argc, char **argv)
{
printf("Started\n");
CMemorySpeedTester runner;
for (int runInd = 0; runInd < 5; runInd++)
{
printf("%d.\n", runInd);
runner.testBandwidth();
}
printf("Finished. Press Enter...");
getch();
}
Windows 10 x64, Visual Studio 2019, CUDA Toolkit 10.2, also can be compiled and reproduced on GeForce GT 520M, Windows 7 x64, Visual Studio 2010 and CUDA 7.5.