WARNING: C local variables in VC++ Embedded assembly

 0 Issues

Below code snippet is a function to brighten a bitmap. When it runs, a "memory access violation" will happen.

void assembly_BLUR(BITMAP* bitmap, INT brighten, BYTE* buffer) 
{
INT width = bitmap->bmWidth;
INT height = bitmap->bmHeight;
INT bitsPerPixel = bitmap->bmBitsPixel;

__asm
{
// save registers
push eax
push ebx
push ecx
push edx
push edi
push ebp
push esi
mov eax, width // eax = width
mul  height // eax = eax * height
mov ebx, bitsPerPixel
shr ebx, 3 // ebx = ebx >> 3 (i.e. ebx=ebx/8), bytes per pixel
mul ebx // eax = eax * ebx, (eax = width * height * bytesPerPixel)
mov ebx, buffer 
xor ecx, ecx // ecx = 0
start:

cmp eax, ecx 
jle end // jump to label "end" if eax <= ecx
mov edx, [ebx] // edx = *ebx
mov esi, edx
mov edi, edx
mov ebp, edx
and edx, 255  // edx = edx & 0x000000FF
shr esi, 8
and esi, 255
shr edi, 16
and edi, 255
shr ebp, 24
and ebp, 255

add edx, brighten// brighten
add esi, brighten
add edi, brighten
add ebp, brighten
cmp edx, 255
jle nonsaturate
mov edx, 255 //if value is greater than 255 saturate to 255
nonsaturate :
cmp esi, 255
jle nonsaturate2
mov esi, 255
nonsaturate2 :
cmp edi, 255
jle nonsaturate3
mov edi, 255
nonsaturate3 :
cmp ebp, 255
jle nonsaturate4
mov ebp, 255
nonsaturate4 :
shl edi, 16
shl esi, 8
shl ebp, 24
add edi, esi
add edx, edi
add edx, ebp
mov [ebx], edx  // *(buffer + ecx ) = edx
add ebx, 4 // RGBA model? 
add ecx, 4
jmp start
end:
//restore registers
pop esi
pop ebp
pop edi
pop edx
pop ecx
pop ebx
pop eax
}
}

1 Root cause

"brighten" is a local auto variable that is compiled as a stack memory area and addressed by (EBP + offset). But before it's referenced the EBP was already changed by "mov ebp, edx".Then (EBP+offset) is no longer valid anymore. 

So here is the rule for embedded assembly code, Don't reference local auto variables after EBP was changed.

2 Solution

The easiest solution is to save local variables value into a "static variable" before changing EBP. Because static variables are allocated in the data section and addressed by memory address directly without touching EBP, we can reference them anytime without being interfered with by EBP.

void assembly_BLUR(BITMAP* bitmap, INT brighten, BYTE* buffer) 

{

INT width = bitmap->bmWidth;

INT height = bitmap->bmHeight;

INT bitsPerPixel = bitmap->bmBitsPixel;

// static variable saved in data section (not stack), 

// so no EBP needed to reference it. 

static int staticBrighten = 0;

staticBrighten = brighten;

......

mov ebp, edx
and edx, 255  // edx = edx & 0x000000FF
shr esi, 8
and esi, 255
shr edi, 16
and edi, 255
shr ebp, 24
and ebp, 255

add edx, staticBrighten // brighten
add esi, staticBrighten 
add edi, staticBrighten 
add ebp, staticBrighten 

......