morgan/FreeRDP
1
0
Fork 0
mirror of https://github.com/morgan9e/FreeRDP synced 2026-04-15 00:44:19 +09:00
Code Issues Packages Projects Releases Wiki Activity

[codec,planar] fix datatypes in plane encode

Browse Source 
do not assume char is signed, use int8_t instead.
This commit is contained in:
akallabeth
2025-09-22 12:15:33 +02:00
parent e423476e95
commit 3ac6c06e17
1 changed files with 29 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
libfreerdp/codec/planar.c
View File

@@ -113,7 +113,7 @@ static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
UINT32 DstFormat)
{
WINPR_ASSERT(planar);
if (planar->bgr && alpha)
{
switch (DstFormat)
@@ -375,8 +375,6 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD
WINPR_ASSERT(nWidth <= INT32_MAX);
WINPR_ASSERT(nDstStep <= INT32_MAX);
previousScanline = NULL;
if (vFlip)
{
beg = (INT32)nHeight - 1;
@@ -500,8 +498,8 @@ static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstSte
BOOL vFlip)
{
INT32 beg = 0;
INT32 end = 0;
INT32 inc = 0;
INT32 end = (INT32)nHeight;
INT32 inc = 1;
WINPR_ASSERT(nHeight <= INT32_MAX);
WINPR_ASSERT(nWidth <= INT32_MAX);
@@ -513,12 +511,6 @@ static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstSte
end = -1;
inc = -1;
}
else
{
beg = 0;
end = (INT32)nHeight;
inc = 1;
}
for (INT32 y = beg; y != end; y += inc)
{
@@ -1183,12 +1175,10 @@ static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RES
BYTE* WINPR_RESTRICT pOutBuffer,
UINT32 outBufferSize)
{
const BYTE* pInput = NULL;
BYTE* pOutput = NULL;
const BYTE* pInput = pInBuffer;
BYTE* pOutput = pOutBuffer;
BYTE controlByte = 0;
UINT32 nBytesToWrite = 0;
pInput = pInBuffer;
pOutput = pOutBuffer;
if (!cRawBytes && !nRunLength)
return 0;
@@ -1302,20 +1292,13 @@ static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RE
UINT32 outBufferSize)
{
BYTE symbol = 0;
const BYTE* pInput = NULL;
BYTE* pOutput = NULL;
const BYTE* pInput = pInBuffer;
BYTE* pOutput = pOutBuffer;
const BYTE* pBytes = NULL;
UINT32 cRawBytes = 0;
UINT32 nRunLength = 0;
UINT32 bSymbolMatch = 0;
UINT32 nBytesWritten = 0;
UINT32 nTotalBytesWritten = 0;
symbol = 0;
cRawBytes = 0;
nRunLength = 0;
pInput = pInBuffer;
pOutput = pOutBuffer;
nTotalBytesWritten = 0;
if (!outBufferSize)
return 0;
@@ -1325,7 +1308,7 @@ static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RE
if (!inBufferSize)
break;
bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
const UINT32 bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
symbol = *pInput;
pInput++;
inBufferSize--;
@@ -1380,25 +1363,18 @@ BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane
UINT32 height, BYTE* WINPR_RESTRICT outPlane,
UINT32* WINPR_RESTRICT dstSize)
{
UINT32 index = 0;
const BYTE* pInput = NULL;
BYTE* pOutput = NULL;
UINT32 outBufferSize = 0;
UINT32 nBytesWritten = 0;
UINT32 nTotalBytesWritten = 0;
if (!outPlane)
return FALSE;
index = 0;
pInput = inPlane;
pOutput = outPlane;
outBufferSize = *dstSize;
nTotalBytesWritten = 0;
UINT32 index = 0;
const BYTE* pInput = inPlane;
BYTE* pOutput = outPlane;
UINT32 outBufferSize = *dstSize;
UINT32 nTotalBytesWritten = 0;
while (outBufferSize)
while (outBufferSize > 0)
{
nBytesWritten =
const UINT32 nBytesWritten =
freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
@@ -1474,35 +1450,33 @@ static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRIC
BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
UINT32 height, BYTE* WINPR_RESTRICT outPlane)
{
BYTE* outPtr = NULL;
const BYTE* srcPtr = NULL;
const BYTE* prevLinePtr = NULL;
if (!outPlane)
{
if (width * height == 0)
return NULL;
if (!(outPlane = (BYTE*)calloc(height, width)))
outPlane = (BYTE*)calloc(height, width);
if (!outPlane)
return NULL;
}
// first line is copied as is
CopyMemory(outPlane, inPlane, width);
outPtr = outPlane + width;
srcPtr = inPlane + width;
prevLinePtr = inPlane;
for (UINT32 y = 1; y < height; y++)
{
for (UINT32 x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
const size_t off = 1ull * width * y;
BYTE* outPtr = &outPlane[off];
const BYTE* srcPtr = &inPlane[off];
const BYTE* prevLinePtr = &inPlane[off - width];
for (UINT32 x = 0; x < width; x++)
{
const INT32 delta = *srcPtr - *prevLinePtr;
const int s2c1i = (delta >= 0) ? delta : (~((BYTE)(-delta)) + 1);
const char s2c1 = WINPR_CXX_COMPAT_CAST(char, s2c1i & 0xff);
const uint32_t s2c = (s2c1 >= 0) ? ((UINT32)s2c1 << 1)
: (((UINT32)(~((BYTE)(s2c1 & 0xFF)) + 1) << 1) - 1);
*outPtr = (BYTE)s2c;
const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
const uint32_t s2c =
(s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
outPtr[x] = (BYTE)s2c;
}
}
@@ -1532,7 +1506,6 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT conte
{
UINT32 size = 0;
BYTE* dstp = NULL;
UINT32 planeSize = 0;
UINT32 dstSizes[4] = { 0 };
BYTE FormatHeader = 0;
@@ -1542,7 +1515,7 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT conte
if (context->AllowSkipAlpha)
FormatHeader |= PLANAR_FORMAT_HEADER_NA;
planeSize = width * height;
const UINT32 planeSize = width * height;
if (!context->AllowSkipAlpha)
format = planar_invert_format(context, TRUE, format);