From cc934795e4eebdbfb337dbc0661323ad8b8f7295 Mon Sep 17 00:00:00 2001 From: akallabeth Date: Thu, 19 Dec 2024 14:45:06 +0100 Subject: [PATCH] [warnings] fix integer casting * use asserting casts to detect overflows or sign conversions * code cleanup for issues uncovered by casts --- include/freerdp/codec/planar.h | 21 +- libfreerdp/cache/bitmap.c | 38 +- libfreerdp/cache/glyph.c | 31 +- libfreerdp/cache/offscreen.c | 8 +- libfreerdp/codec/bitmap.c | 19 +- libfreerdp/codec/h264_ffmpeg.c | 6 +- libfreerdp/codec/h264_openh264.c | 22 +- libfreerdp/codec/include/bitmap.h | 39 +- libfreerdp/codec/interleaved.c | 26 +- libfreerdp/codec/ncrush.c | 65 ++-- libfreerdp/codec/nsc.c | 23 +- libfreerdp/codec/nsc_encode.c | 24 +- libfreerdp/codec/nsc_types.h | 7 +- libfreerdp/codec/planar.c | 218 +++++------ libfreerdp/codec/progressive.c | 94 +++-- libfreerdp/codec/region.c | 29 +- libfreerdp/codec/rfx.c | 54 +-- libfreerdp/codec/rfx_bitstream.h | 179 +++++---- libfreerdp/codec/rfx_dwt.c | 64 ++-- libfreerdp/codec/rfx_encode.c | 58 ++- libfreerdp/codec/rfx_quantization.c | 9 +- libfreerdp/codec/rfx_rlgr.c | 210 +++++----- libfreerdp/codec/sse/nsc_sse2.c | 189 +++++---- libfreerdp/codec/sse/rfx_sse2.c | 238 +++++------- libfreerdp/codec/test/TestFreeRDPCodecCopy.c | 4 +- .../codec/test/TestFreeRDPCodecPlanar.c | 7 +- .../codec/test/TestFreeRDPCodecProgressive.c | 32 +- libfreerdp/codec/test/TestFreeRDPRegion.c | 39 +- libfreerdp/codec/xcrush.c | 43 ++- libfreerdp/codec/yuv.c | 13 +- libfreerdp/codec/zgfx.c | 28 +- libfreerdp/common/assistance.c | 16 +- libfreerdp/common/settings.c | 21 +- libfreerdp/core/aad.c | 8 +- libfreerdp/core/activation.c | 11 +- libfreerdp/core/capabilities.c | 55 ++- libfreerdp/core/childsession.c | 7 +- libfreerdp/core/client.c | 8 +- libfreerdp/core/client.h | 4 +- libfreerdp/core/connection.c | 4 +- libfreerdp/core/connection.h | 2 +- libfreerdp/core/credssp_auth.c | 4 +- libfreerdp/core/credssp_auth.h | 2 +- libfreerdp/core/freerdp.c | 9 +- libfreerdp/core/gateway/arm.c | 16 +- libfreerdp/core/gateway/http.c | 18 +- libfreerdp/core/gateway/ncacn_http.c | 7 +- libfreerdp/core/gateway/rdg.c | 43 ++- libfreerdp/core/gateway/rpc.c | 10 +- libfreerdp/core/gateway/rpc.h | 2 +- libfreerdp/core/gateway/rpc_bind.c | 26 +- libfreerdp/core/gateway/rpc_client.c | 9 +- libfreerdp/core/gateway/rts.c | 3 +- libfreerdp/core/gateway/tsg.c | 4 +- libfreerdp/core/gateway/websocket.c | 2 +- libfreerdp/core/gateway/wst.c | 8 +- libfreerdp/core/gcc.c | 65 ++-- libfreerdp/core/input.c | 2 +- libfreerdp/core/listener.c | 15 +- libfreerdp/core/message.c | 8 +- libfreerdp/core/multitransport.c | 2 +- libfreerdp/core/nego.c | 6 +- libfreerdp/core/nla.c | 18 +- libfreerdp/core/nla.h | 2 +- libfreerdp/core/orders.c | 74 ++-- libfreerdp/core/orders.h | 2 +- libfreerdp/core/peer.c | 27 +- libfreerdp/core/proxy.c | 2 +- libfreerdp/core/rdp.c | 65 ++-- libfreerdp/core/server.c | 37 +- libfreerdp/core/streamdump.c | 6 +- libfreerdp/core/surface.c | 15 +- libfreerdp/core/tcp.c | 46 +-- libfreerdp/core/test/TestSettings.c | 24 +- libfreerdp/core/timezone.c | 6 +- libfreerdp/core/transport.c | 24 +- libfreerdp/core/update.c | 162 +++++--- libfreerdp/core/window.c | 4 +- libfreerdp/crypto/ber.c | 11 +- libfreerdp/crypto/certificate.c | 10 +- libfreerdp/crypto/crypto.c | 10 +- libfreerdp/crypto/der.c | 10 +- libfreerdp/crypto/er.c | 13 +- libfreerdp/crypto/per.c | 9 +- libfreerdp/crypto/privatekey.c | 4 +- libfreerdp/crypto/tls.c | 4 +- libfreerdp/crypto/x509_utils.c | 50 +-- libfreerdp/gdi/bitmap.c | 32 +- libfreerdp/gdi/brush.h | 4 +- libfreerdp/gdi/drawing.c | 2 +- libfreerdp/gdi/drawing.h | 2 +- libfreerdp/gdi/gdi.c | 95 +++-- libfreerdp/gdi/gdi.h | 15 +- libfreerdp/gdi/gfx.c | 26 +- libfreerdp/gdi/graphics.c | 6 +- libfreerdp/gdi/line.c | 10 +- libfreerdp/gdi/line.h | 4 +- libfreerdp/gdi/shape.c | 33 +- libfreerdp/gdi/test/TestGdiRect.c | 21 +- libfreerdp/gdi/test/helpers.c | 26 +- libfreerdp/primitives/prim_YCoCg.c | 4 +- libfreerdp/primitives/prim_YUV.c | 83 ++-- libfreerdp/primitives/prim_colors.c | 109 ++++-- libfreerdp/primitives/prim_copy.c | 17 +- libfreerdp/primitives/prim_internal.h | 16 + libfreerdp/primitives/prim_shift.c | 25 +- libfreerdp/primitives/prim_sign.c | 2 +- libfreerdp/primitives/prim_templates.h | 254 ++++++------- libfreerdp/primitives/sse/prim_YCoCg_ssse3.c | 70 ++-- libfreerdp/primitives/sse/prim_YUV_ssse3.c | 56 +-- .../primitives/sse/prim_alphaComp_sse3.c | 27 +- libfreerdp/primitives/sse/prim_colors_sse2.c | 359 ++++++++---------- libfreerdp/primitives/sse/prim_copy_avx2.c | 19 +- libfreerdp/primitives/sse/prim_copy_sse4_1.c | 4 +- libfreerdp/primitives/sse/prim_set_sse2.c | 4 +- libfreerdp/primitives/sse/prim_shift_sse3.c | 32 +- libfreerdp/primitives/sse/prim_sign_ssse3.c | 4 +- .../primitives/test/TestPrimitivesAndOr.c | 2 +- .../primitives/test/TestPrimitivesShift.c | 14 +- .../primitives/test/TestPrimitivesYCbCr.c | 25 +- .../primitives/test/TestPrimitivesYCoCg.c | 10 +- .../primitives/test/TestPrimitivesYUV.c | 4 +- libfreerdp/primitives/test/prim_test.c | 8 +- libfreerdp/utils/http.c | 2 +- libfreerdp/utils/passphrase.c | 4 +- libfreerdp/utils/smartcard_call.c | 2 +- libfreerdp/utils/smartcard_pack.c | 6 +- 127 files changed, 2356 insertions(+), 1951 deletions(-) diff --git a/include/freerdp/codec/planar.h b/include/freerdp/codec/planar.h index dd25e43ff..fc51f44ec 100644 --- a/include/freerdp/codec/planar.h +++ b/include/freerdp/codec/planar.h @@ -22,6 +22,7 @@ #ifndef FREERDP_CODEC_PLANAR_H #define FREERDP_CODEC_PLANAR_H +#include #include #include @@ -32,17 +33,25 @@ #define PLANAR_FORMAT_HEADER_NA (1 << 5) #define PLANAR_FORMAT_HEADER_CLL_MASK 0x07 -#define PLANAR_CONTROL_BYTE(_nRunLength, _cRawBytes) \ - (_nRunLength & 0x0F) | ((_cRawBytes & 0x0F) << 4) - -#define PLANAR_CONTROL_BYTE_RUN_LENGTH(_controlByte) (_controlByte & 0x0F) -#define PLANAR_CONTROL_BYTE_RAW_BYTES(_controlByte) ((_controlByte >> 4) & 0x0F) - #ifdef __cplusplus extern "C" { #endif + static inline BYTE PLANAR_CONTROL_BYTE(UINT32 nRunLength, UINT32 cRawBytes) + { + return WINPR_ASSERTING_INT_CAST(UINT8, ((nRunLength & 0x0F) | ((cRawBytes & 0x0F) << 4))); + } + + static inline BYTE PLANAR_CONTROL_BYTE_RUN_LENGTH(UINT32 controlByte) + { + return (controlByte & 0x0F); + } + static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte) + { + return ((controlByte >> 4) & 0x0F); + } + typedef struct S_BITMAP_PLANAR_CONTEXT BITMAP_PLANAR_CONTEXT; FREERDP_API BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT context, diff --git a/libfreerdp/cache/bitmap.c b/libfreerdp/cache/bitmap.c index 6b6463ca1..2b0992a2d 100644 --- a/libfreerdp/cache/bitmap.c +++ b/libfreerdp/cache/bitmap.c @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -65,7 +66,6 @@ static BOOL update_gdi_memblt(rdpContext* context, MEMBLT_ORDER* memblt) static BOOL update_gdi_mem3blt(rdpContext* context, MEM3BLT_ORDER* mem3blt) { - BYTE style = 0; rdpBitmap* bitmap = NULL; rdpCache* cache = context->cache; rdpBrush* brush = &mem3blt->brush; @@ -80,7 +80,7 @@ static BOOL update_gdi_mem3blt(rdpContext* context, MEM3BLT_ORDER* mem3blt) if (!bitmap) return TRUE; - style = brush->style; + const BYTE style = WINPR_ASSERTING_INT_CAST(UINT8, brush->style); if (brush->style & CACHED_BRUSH) { @@ -108,26 +108,26 @@ static BOOL update_gdi_cache_bitmap(rdpContext* context, const CACHE_BITMAP_ORDE if (!bitmap) return FALSE; - Bitmap_SetDimensions(bitmap, cacheBitmap->bitmapWidth, cacheBitmap->bitmapHeight); + if (!Bitmap_SetDimensions(bitmap, WINPR_ASSERTING_INT_CAST(UINT16, cacheBitmap->bitmapWidth), + WINPR_ASSERTING_INT_CAST(UINT16, cacheBitmap->bitmapHeight))) + goto fail; if (!bitmap->Decompress(context, bitmap, cacheBitmap->bitmapDataStream, cacheBitmap->bitmapWidth, cacheBitmap->bitmapHeight, cacheBitmap->bitmapBpp, cacheBitmap->bitmapLength, cacheBitmap->compressed, RDP_CODEC_ID_NONE)) - { - Bitmap_Free(context, bitmap); - return FALSE; - } + goto fail; if (!bitmap->New(context, bitmap)) - { - Bitmap_Free(context, bitmap); - return FALSE; - } + goto fail; prevBitmap = bitmap_cache_get(cache->bitmap, cacheBitmap->cacheId, cacheBitmap->cacheIndex); Bitmap_Free(context, prevBitmap); return bitmap_cache_put(cache->bitmap, cacheBitmap->cacheId, cacheBitmap->cacheIndex, bitmap); + +fail: + Bitmap_Free(context, bitmap); + return FALSE; } static BOOL update_gdi_cache_bitmap_v2(rdpContext* context, CACHE_BITMAP_V2_ORDER* cacheBitmapV2) @@ -150,7 +150,9 @@ static BOOL update_gdi_cache_bitmap_v2(rdpContext* context, CACHE_BITMAP_V2_ORDE if ((ColorDepth == 15) && (cacheBitmapV2->bitmapBpp == 16)) cacheBitmapV2->bitmapBpp = ColorDepth; - Bitmap_SetDimensions(bitmap, cacheBitmapV2->bitmapWidth, cacheBitmapV2->bitmapHeight); + if (!Bitmap_SetDimensions(bitmap, WINPR_ASSERTING_INT_CAST(UINT16, cacheBitmapV2->bitmapWidth), + WINPR_ASSERTING_INT_CAST(UINT16, cacheBitmapV2->bitmapHeight))) + goto fail; if (!bitmap->Decompress(context, bitmap, cacheBitmapV2->bitmapDataStream, cacheBitmapV2->bitmapWidth, cacheBitmapV2->bitmapHeight, @@ -192,7 +194,10 @@ static BOOL update_gdi_cache_bitmap_v3(rdpContext* context, CACHE_BITMAP_V3_ORDE cacheBitmapV3->bpp = ColorDepth; compressed = (bitmapData->codecID != RDP_CODEC_ID_NONE); - Bitmap_SetDimensions(bitmap, bitmapData->width, bitmapData->height); + + if (!Bitmap_SetDimensions(bitmap, WINPR_ASSERTING_INT_CAST(UINT16, bitmapData->width), + WINPR_ASSERTING_INT_CAST(UINT16, bitmapData->height))) + goto fail; if (!bitmap->Decompress(context, bitmap, bitmapData->data, bitmapData->width, bitmapData->height, bitmapData->bpp, bitmapData->length, compressed, @@ -317,15 +322,16 @@ static int bitmap_cache_save_persistent(rdpBitmapCache* bitmapCache) BITMAP_V2_CELL* cell = &bitmapCache->cells[i]; for (UINT32 j = 0; j < cell->number + 1 && cell->entries; j++) { - PERSISTENT_CACHE_ENTRY cacheEntry; + PERSISTENT_CACHE_ENTRY cacheEntry = { 0 }; rdpBitmap* bitmap = cell->entries[j]; if (!bitmap || !bitmap->key64) continue; cacheEntry.key64 = bitmap->key64; - cacheEntry.width = bitmap->width; - cacheEntry.height = bitmap->height; + + cacheEntry.width = WINPR_ASSERTING_INT_CAST(UINT16, bitmap->width); + cacheEntry.height = WINPR_ASSERTING_INT_CAST(UINT16, bitmap->height); const UINT64 size = 4ULL * bitmap->width * bitmap->height; if (size > UINT32_MAX) continue; diff --git a/libfreerdp/cache/glyph.c b/libfreerdp/cache/glyph.c index d0935c13f..54d91eeea 100644 --- a/libfreerdp/cache/glyph.c +++ b/libfreerdp/cache/glyph.c @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -62,10 +63,10 @@ static UINT32 update_glyph_offset(const BYTE* data, size_t length, UINT32 index, } if (flAccel & SO_VERTICAL) - *y += offset; + *y += WINPR_ASSERTING_INT_CAST(int32_t, offset); if (flAccel & SO_HORIZONTAL) - *x += offset; + *x += WINPR_ASSERTING_INT_CAST(int32_t, offset); } return index; @@ -109,8 +110,8 @@ static BOOL update_process_glyph(rdpContext* context, const BYTE* data, UINT32 c if ((dx <= (bound->x + bound->width)) && (dy <= (bound->y + bound->height))) { - INT32 dw = glyph->cx - sx; - INT32 dh = glyph->cy - sy; + INT32 dw = WINPR_ASSERTING_INT_CAST(int32_t, glyph->cx) - sx; + INT32 dh = WINPR_ASSERTING_INT_CAST(int32_t, glyph->cy) - sy; if ((dw + dx) > (bound->x + bound->width)) dw = (bound->x + bound->width) - (dw + dx); @@ -126,7 +127,7 @@ static BOOL update_process_glyph(rdpContext* context, const BYTE* data, UINT32 c } if (flAccel & SO_CHAR_INC_EQUAL_BM_BASE) - *x += glyph->cx; + *x += WINPR_ASSERTING_INT_CAST(int32_t, glyph->cx); return TRUE; } @@ -195,7 +196,8 @@ static BOOL update_process_glyph_fragments(rdpContext* context, const BYTE* data if (bkHeight < 0) bkHeight = 0; - if (opX + opWidth > (INT64)freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth)) + const UINT32 w = freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth); + if (opX + opWidth > (INT64)w) { /** * Some Microsoft servers send erroneous high values close to the @@ -206,10 +208,10 @@ static BOOL update_process_glyph_fragments(rdpContext* context, const BYTE* data * a RDP session to Windows XP Professional SP3. * This workaround prevents resulting problems in the UI callbacks. */ - opWidth = freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth) - opX; + opWidth = WINPR_ASSERTING_INT_CAST(int, w) - opX; } - if (bkX + bkWidth > (INT64)freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth)) + if (bkX + bkWidth > (INT64)w) { /** * Some Microsoft servers send erroneous high values close to the @@ -220,13 +222,13 @@ static BOOL update_process_glyph_fragments(rdpContext* context, const BYTE* data * a RDP session to Windows XP Professional SP3. * This workaround prevents resulting problems in the UI callbacks. */ - bkWidth = freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth) - bkX; + bkWidth = WINPR_ASSERTING_INT_CAST(int, w) - bkX; } - bound.x = bkX; - bound.y = bkY; - bound.width = bkWidth; - bound.height = bkHeight; + bound.x = WINPR_ASSERTING_INT_CAST(INT16, bkX); + bound.y = WINPR_ASSERTING_INT_CAST(INT16, bkY); + bound.width = WINPR_ASSERTING_INT_CAST(INT16, bkWidth); + bound.height = WINPR_ASSERTING_INT_CAST(INT16, bkHeight); if (!glyph->BeginDraw(context, opX, opY, opWidth, opHeight, bgcolor, fgcolor, fOpRedundant)) return FALSE; @@ -311,7 +313,8 @@ static BOOL update_gdi_glyph_index(rdpContext* context, GLYPH_INDEX_ORDER* glyph context, glyphIndex->data, glyphIndex->cbData, glyphIndex->cacheId, glyphIndex->ulCharInc, glyphIndex->flAccel, glyphIndex->backColor, glyphIndex->foreColor, glyphIndex->x, glyphIndex->y, glyphIndex->bkLeft, glyphIndex->bkTop, bkWidth, bkHeight, glyphIndex->opLeft, - glyphIndex->opTop, opWidth, opHeight, glyphIndex->fOpRedundant); + glyphIndex->opTop, opWidth, opHeight, + WINPR_ASSERTING_INT_CAST(int32_t, glyphIndex->fOpRedundant)); } static BOOL update_gdi_fast_index(rdpContext* context, const FAST_INDEX_ORDER* fastIndex) diff --git a/libfreerdp/cache/offscreen.c b/libfreerdp/cache/offscreen.c index 36e997a06..91fa38f32 100644 --- a/libfreerdp/cache/offscreen.c +++ b/libfreerdp/cache/offscreen.c @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -64,7 +65,12 @@ update_gdi_create_offscreen_bitmap(rdpContext* context, if (!bitmap) return FALSE; - Bitmap_SetDimensions(bitmap, createOffscreenBitmap->cx, createOffscreenBitmap->cy); + if (!Bitmap_SetDimensions(bitmap, WINPR_ASSERTING_INT_CAST(UINT16, createOffscreenBitmap->cx), + WINPR_ASSERTING_INT_CAST(UINT16, createOffscreenBitmap->cy))) + { + Bitmap_Free(context, bitmap); + return FALSE; + } if (!bitmap->New(context, bitmap)) { diff --git a/libfreerdp/codec/bitmap.c b/libfreerdp/codec/bitmap.c index bd99837b7..5047fa1c2 100644 --- a/libfreerdp/codec/bitmap.c +++ b/libfreerdp/codec/bitmap.c @@ -17,6 +17,9 @@ * limitations under the License. */ +#include +#include + #include #include @@ -25,7 +28,7 @@ static INLINE UINT16 GETPIXEL16(const void* WINPR_RESTRICT d, UINT32 x, UINT32 y, UINT32 w) { const BYTE* WINPR_RESTRICT src = (const BYTE*)d + ((y * w + x) * sizeof(UINT16)); - return (UINT16)(((UINT16)src[1] << 8) | (UINT16)src[0]); + return WINPR_ASSERTING_INT_CAST(UINT16, ((UINT16)src[1] << 8) | (UINT16)src[0]); } static INLINE UINT32 GETPIXEL32(const void* WINPR_RESTRICT d, UINT32 x, UINT32 y, UINT32 w) @@ -380,7 +383,7 @@ static INLINE UINT16 out_mix_count_3(UINT16 in_count, wStream* WINPR_RESTRICT in /*****************************************************************************/ /* fom */ static INLINE UINT16 out_from_count_2(UINT16 in_count, wStream* WINPR_RESTRICT in_s, - const char* WINPR_RESTRICT in_mask, size_t in_mask_len) + const int8_t* WINPR_RESTRICT in_mask, size_t in_mask_len) { if (in_count > 0) { @@ -412,7 +415,7 @@ static INLINE UINT16 out_from_count_2(UINT16 in_count, wStream* WINPR_RESTRICT i /*****************************************************************************/ /* fill or mix (fom) */ static INLINE UINT16 out_from_count_3(UINT16 in_count, wStream* WINPR_RESTRICT in_s, - const char* WINPR_RESTRICT in_mask, size_t in_mask_len) + const int8_t* WINPR_RESTRICT in_mask, size_t in_mask_len) { if (in_count > 0) { @@ -466,7 +469,7 @@ static INLINE SSIZE_T freerdp_bitmap_compress_24(const void* WINPR_RESTRICT srcD UINT32 byte_limit, UINT32 start_line, wStream* WINPR_RESTRICT temp_s, UINT32 e) { - char fom_mask[8192] = { 0 }; /* good for up to 64K bitmap */ + int8_t fom_mask[8192] = { 0 }; /* good for up to 64K bitmap */ SSIZE_T lines_sent = 0; UINT16 count = 0; UINT16 color_count = 0; @@ -630,7 +633,8 @@ static INLINE SSIZE_T freerdp_bitmap_compress_24(const void* WINPR_RESTRICT srcD if (pixel == (ypixel ^ mix)) { - fom_mask[fom_mask_len - 1] |= (1 << (fom_count % 8)); + fom_mask[fom_mask_len - 1] |= + WINPR_ASSERTING_INT_CAST(int8_t, (1 << (fom_count % 8))); } fom_count++; @@ -772,7 +776,7 @@ static INLINE SSIZE_T freerdp_bitmap_compress_16(const void* WINPR_RESTRICT srcD UINT32 bpp, UINT32 byte_limit, UINT32 start_line, wStream* WINPR_RESTRICT temp_s, UINT32 e) { - char fom_mask[8192] = { 0 }; /* good for up to 64K bitmap */ + int8_t fom_mask[8192] = { 0 }; /* good for up to 64K bitmap */ SSIZE_T lines_sent = 0; UINT16 count = 0; UINT16 color_count = 0; @@ -936,7 +940,8 @@ static INLINE SSIZE_T freerdp_bitmap_compress_16(const void* WINPR_RESTRICT srcD if (pixel == (ypixel ^ mix)) { - fom_mask[fom_mask_len - 1] |= (1 << (fom_count % 8)); + fom_mask[fom_mask_len - 1] |= + WINPR_ASSERTING_INT_CAST(int8_t, (1 << (fom_count % 8))); } fom_count++; diff --git a/libfreerdp/codec/h264_ffmpeg.c b/libfreerdp/codec/h264_ffmpeg.c index dd856e957..d762b4292 100644 --- a/libfreerdp/codec/h264_ffmpeg.c +++ b/libfreerdp/codec/h264_ffmpeg.c @@ -169,9 +169,11 @@ static BOOL libavcodec_create_encoder(H264_CONTEXT* WINPR_RESTRICT h264) sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height); sys->codecEncoderContext->delay = 0; #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100) - sys->codecEncoderContext->framerate = (AVRational){ h264->FrameRate, 1 }; + sys->codecEncoderContext->framerate = + (AVRational){ WINPR_ASSERTING_INT_CAST(int, h264->FrameRate), 1 }; #endif - sys->codecEncoderContext->time_base = (AVRational){ 1, h264->FrameRate }; + sys->codecEncoderContext->time_base = + (AVRational){ 1, WINPR_ASSERTING_INT_CAST(int, h264->FrameRate) }; av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN); av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN); sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER; diff --git a/libfreerdp/codec/h264_openh264.c b/libfreerdp/codec/h264_openh264.c index d6baac779..b64b89690 100644 --- a/libfreerdp/codec/h264_openh264.c +++ b/libfreerdp/codec/h264_openh264.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -69,9 +70,9 @@ static const char* openh264_library_names[] = { }; #endif -static void openh264_trace_callback(H264_CONTEXT* WINPR_RESTRICT h264, int level, - const char* WINPR_RESTRICT message) +static void openh264_trace_callback(void* ctx, int level, const char* message) { + H264_CONTEXT* h264 = ctx; if (h264) WLog_Print(h264->log, WLOG_TRACE, "%d - %s", level, message); } @@ -109,8 +110,9 @@ static int openh264_decompress(H264_CONTEXT* WINPR_RESTRICT h264, pYUVData[2] = NULL; WINPR_ASSERT(sys->pDecoder); - state = - (*sys->pDecoder)->DecodeFrame2(sys->pDecoder, pSrcData, SrcSize, pYUVData, &sBufferInfo); + state = (*sys->pDecoder) + ->DecodeFrame2(sys->pDecoder, pSrcData, WINPR_ASSERTING_INT_CAST(int, SrcSize), + pYUVData, &sBufferInfo); if (sBufferInfo.iBufferStatus != 1) { @@ -248,9 +250,9 @@ static int openh264_compress(H264_CONTEXT* WINPR_RESTRICT h264, } sys->EncParamExt.iUsageType = usageType; - sys->EncParamExt.iPicWidth = (int)h264->width; - sys->EncParamExt.iPicHeight = (int)h264->height; - sys->EncParamExt.fMaxFrameRate = (int)h264->FrameRate; + sys->EncParamExt.iPicWidth = WINPR_ASSERTING_INT_CAST(int, h264->width); + sys->EncParamExt.iPicHeight = WINPR_ASSERTING_INT_CAST(int, h264->height); + sys->EncParamExt.fMaxFrameRate = WINPR_ASSERTING_INT_CAST(int, h264->FrameRate); sys->EncParamExt.iMaxBitrate = UNSPECIFIED_BIT_RATE; sys->EncParamExt.bEnableDenoise = 0; sys->EncParamExt.bEnableLongTermReference = 0; @@ -518,7 +520,7 @@ static BOOL openh264_init(H264_CONTEXT* h264) #if (OPENH264_MAJOR == 1) && (OPENH264_MINOR <= 5) static EVideoFormatType videoFormat = videoFormatI420; #endif - static WelsTraceCallback traceCallback = (WelsTraceCallback)openh264_trace_callback; + static WelsTraceCallback traceCallback = openh264_trace_callback; WINPR_ASSERT(h264); @@ -619,7 +621,7 @@ static BOOL openh264_init(H264_CONTEXT* h264) status = (*sys->pDecoder) - ->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK_CONTEXT, &h264); + ->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK_CONTEXT, h264); if (status != 0) { @@ -632,7 +634,7 @@ static BOOL openh264_init(H264_CONTEXT* h264) status = (*sys->pDecoder) - ->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK, &traceCallback); + ->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK, traceCallback); if (status != 0) { diff --git a/libfreerdp/codec/include/bitmap.h b/libfreerdp/codec/include/bitmap.h index c51f61352..add8ec80b 100644 --- a/libfreerdp/codec/include/bitmap.h +++ b/libfreerdp/codec/include/bitmap.h @@ -19,16 +19,20 @@ * limitations under the License. */ +#include +#include +#include + /* do not compile the file directly */ /** * Write a foreground/background image to a destination buffer. */ -static INLINE BYTE* WRITEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, +static inline BYTE* WRITEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, const BYTE* WINPR_RESTRICT pbDestEnd, UINT32 rowDelta, - BYTE bitmask, PIXEL fgPel, INT32 cBits) + BYTE bitmask, PIXEL fgPel, UINT32 cBits) { - PIXEL xorPixel; + PIXEL xorPixel = 0; BYTE mask = 0x01; if (cBits > 8) @@ -41,7 +45,7 @@ static INLINE BYTE* WRITEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, return NULL; UNROLL(cBits, { - UINT32 data; + PIXEL data = 0; DESTREADPIXEL(xorPixel, pbDest - rowDelta); if (bitmask & mask) @@ -50,7 +54,7 @@ static INLINE BYTE* WRITEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, data = xorPixel; DESTWRITEPIXEL(pbDest, data); - mask = mask << 1; + mask = WINPR_ASSERTING_INT_CAST(BYTE, (mask << 1) & 0xFF); }); return pbDest; } @@ -59,7 +63,7 @@ static INLINE BYTE* WRITEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, * Write a foreground/background image to a destination buffer * for the first line of compressed data. */ -static INLINE BYTE* WRITEFIRSTLINEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, +static inline BYTE* WRITEFIRSTLINEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, const BYTE* WINPR_RESTRICT pbDestEnd, BYTE bitmask, PIXEL fgPel, UINT32 cBits) { @@ -75,7 +79,7 @@ static INLINE BYTE* WRITEFIRSTLINEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, return NULL; UNROLL(cBits, { - UINT32 data; + PIXEL data; if (bitmask & mask) data = fgPel; @@ -83,7 +87,7 @@ static INLINE BYTE* WRITEFIRSTLINEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, data = BLACK_PIXEL; DESTWRITEPIXEL(pbDest, data); - mask = mask << 1; + mask = WINPR_ASSERTING_INT_CAST(BYTE, (mask << 1) & 0xFF); }); return pbDest; } @@ -91,7 +95,7 @@ static INLINE BYTE* WRITEFIRSTLINEFGBGIMAGE(BYTE* WINPR_RESTRICT pbDest, /** * Decompress an RLE compressed bitmap. */ -static INLINE BOOL RLEDECOMPRESS(const BYTE* WINPR_RESTRICT pbSrcBuffer, UINT32 cbSrcBuffer, +static inline BOOL RLEDECOMPRESS(const BYTE* WINPR_RESTRICT pbSrcBuffer, UINT32 cbSrcBuffer, BYTE* WINPR_RESTRICT pbDestBuffer, UINT32 rowDelta, UINT32 width, UINT32 height) { @@ -99,17 +103,16 @@ static INLINE BOOL RLEDECOMPRESS(const BYTE* WINPR_RESTRICT pbSrcBuffer, UINT32 char sbuffer[128] = { 0 }; #endif const BYTE* pbSrc = pbSrcBuffer; - const BYTE* pbEnd; - const BYTE* pbDestEnd; BYTE* pbDest = pbDestBuffer; - PIXEL temp; + PIXEL temp = 0; PIXEL fgPel = WHITE_PIXEL; BOOL fInsertFgPel = FALSE; BOOL fFirstLine = TRUE; - BYTE bitmask; - PIXEL pixelA, pixelB; - UINT32 runLength; - UINT32 code; + BYTE bitmask = 0; + PIXEL pixelA = 0; + PIXEL pixelB = 0; + UINT32 runLength = 0; + UINT32 code = 0; UINT32 advance = 0; RLEEXTRA @@ -127,8 +130,8 @@ static INLINE BOOL RLEDECOMPRESS(const BYTE* WINPR_RESTRICT pbSrcBuffer, UINT32 return FALSE; } - pbEnd = pbSrcBuffer + cbSrcBuffer; - pbDestEnd = pbDestBuffer + rowDelta * height; + const BYTE* pbEnd = &pbSrcBuffer[cbSrcBuffer]; + const BYTE* pbDestEnd = &pbDestBuffer[1ULL * rowDelta * height]; while (pbSrc < pbEnd) { diff --git a/libfreerdp/codec/interleaved.c b/libfreerdp/codec/interleaved.c index 59f3da32f..804a9253f 100644 --- a/libfreerdp/codec/interleaved.c +++ b/libfreerdp/codec/interleaved.c @@ -22,6 +22,7 @@ */ #include +#include #include #include @@ -288,7 +289,7 @@ static UINT ExtractRunLengthMegaMega(const BYTE* pbOrderHdr, const BYTE* pbEnd, return 0; } - runLength = ((UINT16)pbOrderHdr[1]) | (((UINT16)pbOrderHdr[2]) << 8); + runLength = ((UINT16)pbOrderHdr[1]) | ((((UINT16)pbOrderHdr[2]) << 8) & 0xFF00); (*advance) += 2; return runLength; @@ -466,11 +467,11 @@ static INLINE void write_pixel_16(BYTE* _buf, UINT16 _pix) (_buf) += 2; \ } while (0) #define DESTREADPIXEL(_pix, _buf) _pix = ((UINT16*)(_buf))[0] -#define SRCREADPIXEL(_pix, _buf) \ - do \ - { \ - (_pix) = (_buf)[0] | ((_buf)[1] << 8); \ - (_buf) += 2; \ +#define SRCREADPIXEL(_pix, _buf) \ + do \ + { \ + (_pix) = WINPR_ASSERTING_INT_CAST(UINT16, (_buf)[0] | (((_buf)[1] << 8) & 0xFF00)); \ + (_buf) += 2; \ } while (0) #define WRITEFGBGIMAGE WriteFgBgImage16to16 #define WRITEFIRSTLINEFGBGIMAGE WriteFirstLineFgBgImage16to16 @@ -499,12 +500,13 @@ static INLINE void write_pixel_16(BYTE* _buf, UINT16 _pix) write_pixel_24(_buf, _pix); \ (_buf) += 3; \ } while (0) -#define DESTREADPIXEL(_pix, _buf) _pix = (_buf)[0] | ((_buf)[1] << 8) | ((_buf)[2] << 16) -#define SRCREADPIXEL(_pix, _buf) \ - do \ - { \ - (_pix) = (_buf)[0] | ((_buf)[1] << 8) | ((_buf)[2] << 16); \ - (_buf) += 3; \ +#define DESTREADPIXEL(_pix, _buf) \ + _pix = (_buf)[0] | (((_buf)[1] << 8) & 0xFF00) | (((_buf)[2] << 16) & 0xFF0000) +#define SRCREADPIXEL(_pix, _buf) \ + do \ + { \ + (_pix) = (_buf)[0] | (((_buf)[1] << 8) & 0xFF00) | (((_buf)[2] << 16) & 0xFF0000); \ + (_buf) += 3; \ } while (0) #define WRITEFGBGIMAGE WriteFgBgImage24to24 diff --git a/libfreerdp/codec/ncrush.c b/libfreerdp/codec/ncrush.c index a81102298..bda9a981f 100644 --- a/libfreerdp/codec/ncrush.c +++ b/libfreerdp/codec/ncrush.c @@ -19,10 +19,10 @@ * limitations under the License. */ -#include - #include +#include +#include #include #include #include @@ -1958,15 +1958,17 @@ static INLINE BOOL NCrushFetchBits(const BYTE** SrcPtr, const BYTE** SrcEnd, INT } else { - *bits += *(*SrcPtr)++ << *nbits; + const UINT32 src = *(*SrcPtr)++; + const UINT32 val = (src << *nbits); + *bits += val; *nbits += 8; } } else { - UINT16 tmp = *(*SrcPtr)++; - tmp |= (*(*SrcPtr)++) << 8; - *bits += tmp << *nbits; + UINT32 tmp = *(*SrcPtr)++; + tmp |= ((*(*SrcPtr)++) << 8) & 0xFF00; + *bits += (tmp << *nbits) & UINT32_MAX; *nbits += 16; } } @@ -2090,7 +2092,7 @@ int ncrush_decompress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSi IndexLEC = HuffTableLEC[MaskedBits] & 0xFFF; BitLength = HuffTableLEC[MaskedBits] >> 12; bits >>= BitLength; - nbits -= BitLength; + nbits -= WINPR_ASSERTING_INT_CAST(int32_t, BitLength); if (!NCrushFetchBits(&SrcPtr, &SrcEnd, &nbits, &bits)) return -1; @@ -2130,7 +2132,7 @@ int ncrush_decompress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSi LengthOfMatch = HuffTableLOM[MaskedBits] & 0xFFF; BitLength = HuffTableLOM[MaskedBits] >> 12; bits >>= BitLength; - nbits -= BitLength; + nbits -= WINPR_ASSERTING_INT_CAST(int32_t, BitLength); } if (!NCrushFetchBits(&SrcPtr, &SrcEnd, &nbits, &bits)) @@ -2154,7 +2156,7 @@ int ncrush_decompress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSi const UINT16 Mask = get_word(&HuffTableMask[idx]); const UINT32 MaskedBits = bits & Mask; bits >>= LengthOfMatchBits; - nbits -= LengthOfMatchBits; + nbits -= WINPR_ASSERTING_INT_CAST(int32_t, LengthOfMatchBits); LengthOfMatchBase += MaskedBits; if (!NCrushFetchBits(&SrcPtr, &SrcEnd, &nbits, &bits)) @@ -2192,7 +2194,7 @@ int ncrush_decompress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSi CopyOffset = tmp - 1; } bits >>= CopyOffsetBits; - nbits -= CopyOffsetBits; + nbits -= WINPR_ASSERTING_INT_CAST(int32_t, CopyOffsetBits); if (!NCrushFetchBits(&SrcPtr, &SrcEnd, &nbits, &bits)) return -1; @@ -2206,7 +2208,7 @@ int ncrush_decompress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSi LengthOfMatch = HuffTableLOM[MaskedBits] & 0xFFF; BitLength = HuffTableLOM[MaskedBits] >> 12; bits >>= BitLength; - nbits -= BitLength; + nbits -= WINPR_ASSERTING_INT_CAST(int32_t, BitLength); } if (!NCrushFetchBits(&SrcPtr, &SrcEnd, &nbits, &bits)) return -1; @@ -2229,7 +2231,7 @@ int ncrush_decompress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSi const UINT16 Mask = get_word(&HuffTableMask[idx]); const UINT32 MaskedBits = bits & Mask; bits >>= LengthOfMatchBits; - nbits -= LengthOfMatchBits; + nbits -= WINPR_ASSERTING_INT_CAST(int32_t, LengthOfMatchBits); LengthOfMatchBase += MaskedBits; if (!NCrushFetchBits(&SrcPtr, &SrcEnd, &nbits, &bits)) @@ -2321,7 +2323,6 @@ static int ncrush_hash_table_add(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, U UINT32 HistoryOffset) { const BYTE* SrcPtr = pSrcData; - UINT32 Hash = 0; UINT32 Offset = HistoryOffset; UINT32 EndOffset = Offset + SrcSize - 8; @@ -2331,8 +2332,8 @@ static int ncrush_hash_table_add(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, U while (Offset < EndOffset) { - Hash = ncrush->HashTable[get_word(SrcPtr)]; - ncrush->HashTable[get_word(SrcPtr)] = Offset; + const UINT16 Hash = ncrush->HashTable[get_word(SrcPtr)]; + ncrush->HashTable[get_word(SrcPtr)] = WINPR_ASSERTING_INT_CAST(UINT16, Offset); ncrush->MatchTable[Offset] = Hash; SrcPtr++; Offset++; @@ -2500,18 +2501,18 @@ static int ncrush_move_encoder_windows(NCRUSH_CONTEXT* ncrush, BYTE* HistoryPtr) for (size_t i = 0; i < ARRAYSIZE(ncrush->HashTable); i++) { - INT32 NewHash = ncrush->HashTable[i] - HistoryOffset; - ncrush->HashTable[i] = (NewHash <= 0) ? 0 : NewHash; + const INT32 NewHash = ncrush->HashTable[i] - HistoryOffset; + ncrush->HashTable[i] = (NewHash <= 0) ? 0 : WINPR_ASSERTING_INT_CAST(UINT16, NewHash); } const size_t match_half = ARRAYSIZE(ncrush->MatchTable) / 2; for (size_t j = 0; j < match_half; j++) { - if (HistoryOffset + j >= ARRAYSIZE(ncrush->MatchTable)) + if ((size_t)HistoryOffset + j >= ARRAYSIZE(ncrush->MatchTable)) continue; - INT32 NewMatch = ncrush->MatchTable[HistoryOffset + j] - HistoryOffset; - ncrush->MatchTable[j] = (NewMatch <= 0) ? 0 : NewMatch; + const INT32 NewMatch = ncrush->MatchTable[(size_t)HistoryOffset + j] - HistoryOffset; + ncrush->MatchTable[j] = (NewMatch <= 0) ? 0 : WINPR_ASSERTING_INT_CAST(UINT16, NewMatch); } ZeroMemory(&ncrush->MatchTable[match_half], match_half * sizeof(UINT16)); @@ -2526,7 +2527,6 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSize BYTE* DstPtr = NULL; UINT32 bits = 0; UINT32 offset = 0; - UINT16 Mask = 0; UINT32 MaskedBits = 0; UINT32 accumulator = 0; const BYTE* SrcEndPtr = NULL; @@ -2639,7 +2639,8 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSize int rc = 0; MatchOffset = 0; - rc = ncrush_find_best_match(ncrush, HistoryOffset, &MatchOffset); + rc = ncrush_find_best_match(ncrush, WINPR_ASSERTING_INT_CAST(UINT16, HistoryOffset), + &MatchOffset); if (rc < 0) return -1005; @@ -2778,7 +2779,7 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSize return -1009; NCrushWriteBits(&DstPtr, &accumulator, &offset, CodeLEC, BitLength); - Mask = ((1 << CopyOffsetBits) - 1); + UINT16 Mask = WINPR_ASSERTING_INT_CAST(UINT16, (1 << CopyOffsetBits) - 1); MaskedBits = CopyOffset & Mask; NCrushWriteBits(&DstPtr, &accumulator, &offset, MaskedBits, CopyOffsetBits); @@ -2798,7 +2799,7 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSize if (IndexCO >= ARRAYSIZE(HuffCodeLOM)) return -1; NCrushWriteBits(&DstPtr, &accumulator, &offset, HuffCodeLOM[IndexCO], BitLength); - Mask = ((1 << IndexLOM) - 1); + Mask = WINPR_ASSERTING_INT_CAST(UINT16, (1 << IndexLOM) - 1); MaskedBits = (MatchLength - 2) & Mask; NCrushWriteBits(&DstPtr, &accumulator, &offset, MaskedBits, IndexLOM); @@ -2840,7 +2841,7 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, const BYTE* pSrcData, UINT32 SrcSize if (IndexCO >= ARRAYSIZE(HuffCodeLOM)) return -1; NCrushWriteBits(&DstPtr, &accumulator, &offset, HuffCodeLOM[IndexCO], BitLength); - Mask = ((1 << IndexLOM) - 1); + const UINT16 Mask = WINPR_ASSERTING_INT_CAST(UINT16, (1 << IndexLOM) - 1); MaskedBits = (MatchLength - 2) & Mask; NCrushWriteBits(&DstPtr, &accumulator, &offset, MaskedBits, IndexLOM); @@ -2937,9 +2938,9 @@ static int ncrush_generate_tables(NCRUSH_CONTEXT* context) WINPR_ASSERT(28 < ARRAYSIZE(LOMBitsLUT)); UINT32 cnt = 0; - for (int i = 0; i < 28; i++) + for (BYTE i = 0; i < 28; i++) { - for (int j = 0; j < 1 << LOMBitsLUT[i]; j++) + for (UINT32 j = 0; j < 1 << LOMBitsLUT[i]; j++) { size_t l = (cnt++) + 2ull; context->HuffTableLOM[l] = i; @@ -2962,9 +2963,9 @@ static int ncrush_generate_tables(NCRUSH_CONTEXT* context) } size_t k = 0; - for (int i = 0; i < 16; i++) + for (BYTE i = 0; i < 16; i++) { - for (int j = 0; j < 1 << CopyOffsetBitsLUT[i]; j++) + for (size_t j = 0; j < 1 << CopyOffsetBitsLUT[i]; j++) { size_t l = k++ + 2ull; context->HuffTableCopyOffset[l] = i; @@ -2973,9 +2974,11 @@ static int ncrush_generate_tables(NCRUSH_CONTEXT* context) k /= 128; - for (int i = 16; i < 32; i++) + for (BYTE i = 16; i < 32; i++) { - for (int j = 0; j < 1 << (CopyOffsetBitsLUT[i] - 7); j++) + const UINT32 lut = CopyOffsetBitsLUT[i]; + WINPR_ASSERT(lut >= 7); + for (size_t j = 0; j < 1 << (lut - 7); j++) { size_t l = k++ + 2 + 256ull; context->HuffTableCopyOffset[l] = i; diff --git a/libfreerdp/codec/nsc.c b/libfreerdp/codec/nsc.c index f7a5e920f..3e7f39eee 100644 --- a/libfreerdp/codec/nsc.c +++ b/libfreerdp/codec/nsc.c @@ -26,6 +26,8 @@ #include #include +#include +#include #include #include @@ -41,17 +43,16 @@ static BOOL nsc_decode(NSC_CONTEXT* WINPR_RESTRICT context) { - UINT16 rw = 0; - BYTE shift = 0; - BYTE* bmpdata = NULL; size_t pos = 0; if (!context) return FALSE; - rw = ROUND_UP_TO(context->width, 8); - shift = context->ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */ - bmpdata = context->BitmapData; + const UINT16 rw = ROUND_UP_TO(context->width, 8); + WINPR_ASSERT(context->ColorLossLevel >= 1); + const BYTE shift = WINPR_ASSERTING_INT_CAST(BYTE, context->ColorLossLevel - + 1); /* colorloss recovery + YCoCg shift */ + BYTE* bmpdata = context->BitmapData; if (!bmpdata) return FALSE; @@ -81,9 +82,9 @@ static BOOL nsc_decode(NSC_CONTEXT* WINPR_RESTRICT context) INT16 y_val = (INT16)*yplane; INT16 co_val = (INT16)(INT8)(((INT16)*coplane) << shift); INT16 cg_val = (INT16)(INT8)(((INT16)*cgplane) << shift); - INT16 r_val = y_val + co_val - cg_val; - INT16 g_val = y_val + cg_val; - INT16 b_val = y_val - co_val - cg_val; + INT16 r_val = WINPR_ASSERTING_INT_CAST(int16_t, y_val + co_val - cg_val); + INT16 g_val = WINPR_ASSERTING_INT_CAST(int16_t, y_val + cg_val); + INT16 b_val = WINPR_ASSERTING_INT_CAST(int16_t, y_val - co_val - cg_val); if (pos + 4 > context->BitmapDataLength) return FALSE; @@ -474,8 +475,8 @@ BOOL nsc_process_message(NSC_CONTEXT* WINPR_RESTRICT context, UINT16 bpp, UINT32 return FALSE; } - context->width = width; - context->height = height; + context->width = WINPR_ASSERTING_INT_CAST(UINT16, width); + context->height = WINPR_ASSERTING_INT_CAST(UINT16, height); ret = nsc_context_initialize(context, s); if (!ret) diff --git a/libfreerdp/codec/nsc_encode.c b/libfreerdp/codec/nsc_encode.c index c6c28ed02..26645695b 100644 --- a/libfreerdp/codec/nsc_encode.c +++ b/libfreerdp/codec/nsc_encode.c @@ -25,6 +25,8 @@ #include #include +#include +#include #include #include @@ -114,8 +116,6 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* WINPR_RESTRICT context, const BYTE* WINPR_RESTRICT data, UINT32 scanline) { size_t y = 0; - UINT16 rw = 0; - BYTE ccl = 0; const BYTE* src = NULL; BYTE* yplane = NULL; BYTE* coplane = NULL; @@ -125,11 +125,10 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* WINPR_RESTRICT context, INT16 g_val = 0; INT16 b_val = 0; BYTE a_val = 0; - UINT32 tempWidth = 0; - tempWidth = ROUND_UP_TO(context->width, 8); - rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width); - ccl = context->ColorLossLevel; + UINT16 tempWidth = ROUND_UP_TO(context->width, 8); + const UINT16 rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width); + const BYTE ccl = WINPR_ASSERTING_INT_CAST(BYTE, context->ColorLossLevel); for (; y < context->height; y++) { @@ -359,7 +358,8 @@ static UINT32 nsc_rle_encode(const BYTE* WINPR_RESTRICT in, BYTE* WINPR_RESTRICT { *out++ = *in; *out++ = *in; - *out++ = runlength - 2; + WINPR_ASSERT(runlength >= 2); + *out++ = WINPR_ASSERTING_INT_CAST(BYTE, runlength - 2); runlength = 1; planeSize += 3; } @@ -491,8 +491,8 @@ BOOL nsc_compose_message(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RES if (!context || !s || !data) return FALSE; - context->width = width; - context->height = height; + context->width = WINPR_ASSERTING_INT_CAST(UINT16, width); + context->height = WINPR_ASSERTING_INT_CAST(UINT16, height); if (!nsc_context_initialize_encode(context)) return FALSE; @@ -516,8 +516,10 @@ BOOL nsc_compose_message(NSC_CONTEXT* WINPR_RESTRICT context, wStream* WINPR_RES message.OrangeChromaPlaneByteCount = context->PlaneByteCount[1]; message.GreenChromaPlaneByteCount = context->PlaneByteCount[2]; message.AlphaPlaneByteCount = context->PlaneByteCount[3]; - message.ColorLossLevel = context->ColorLossLevel; - message.ChromaSubsamplingLevel = context->ChromaSubsamplingLevel; + + message.ColorLossLevel = WINPR_ASSERTING_INT_CAST(BYTE, context->ColorLossLevel); + message.ChromaSubsamplingLevel = + WINPR_ASSERTING_INT_CAST(BYTE, context->ChromaSubsamplingLevel); return nsc_write_message(context, s, &message); } diff --git a/libfreerdp/codec/nsc_types.h b/libfreerdp/codec/nsc_types.h index f598fe00b..4cb7dea3f 100644 --- a/libfreerdp/codec/nsc_types.h +++ b/libfreerdp/codec/nsc_types.h @@ -23,6 +23,8 @@ #include +#include +#include #include #include #include @@ -31,7 +33,10 @@ #include #define ROUND_UP_TO(_b, _n) (_b + ((~(_b & (_n - 1)) + 0x1) & (_n - 1))) -#define MINMAX(_v, _l, _h) ((_v) < (_l) ? (_l) : ((_v) > (_h) ? (_h) : (_v))) +#define MINMAX(_v, _l, _h) \ + ((_v) < (_l) ? WINPR_ASSERTING_INT_CAST(BYTE, (_l)) \ + : ((_v) > (_h) ? WINPR_ASSERTING_INT_CAST(BYTE, (_h)) \ + : WINPR_ASSERTING_INT_CAST(BYTE, (_v)))) typedef struct { diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c index 32f193695..70aa7a899 100644 --- a/libfreerdp/codec/planar.c +++ b/libfreerdp/codec/planar.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -170,7 +171,6 @@ static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, U UINT32 nWidth, UINT32 nHeight) { UINT32 used = 0; - BYTE controlByte = 0; WINPR_ASSERT(pSrcData); @@ -178,9 +178,6 @@ static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, U { for (UINT32 x = 0; x < nWidth;) { - int cRawBytes = 0; - int nRunLength = 0; - if (used >= SrcSize) { WLog_ERR(TAG, "planar plane used %" PRIu32 " exceeds SrcSize %" PRIu32, used, @@ -188,9 +185,9 @@ static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, U return -1; } - controlByte = pSrcData[used++]; - nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); - cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); + const uint8_t controlByte = pSrcData[used++]; + uint32_t nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); + uint32_t cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); if (nRunLength == 1) { @@ -230,33 +227,34 @@ static INLINE INT32 planar_skip_plane_rle(const BYTE* WINPR_RESTRICT pSrcData, U return (INT32)used; } +static inline UINT8 clamp(INT16 val) +{ + if (val < 0) + return 0; + if (val > 0xFF) + return 0xFF; + return (UINT8)val; +} + static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, BYTE* WINPR_RESTRICT pDstData, UINT32 nWidth, UINT32 nHeight) { - UINT32 pixel = 0; - UINT32 cRawBytes = 0; - UINT32 nRunLength = 0; - INT32 deltaValue = 0; - BYTE controlByte = 0; - BYTE* currentScanline = NULL; BYTE* previousScanline = NULL; const BYTE* srcp = pSrcData; WINPR_ASSERT(nHeight <= INT32_MAX); WINPR_ASSERT(nWidth <= INT32_MAX); - previousScanline = NULL; - - for (INT32 y = 0; y < (INT32)nHeight; y++) + for (UINT32 y = 0; y < nHeight; y++) { - BYTE* dstp = &pDstData[(1ULL * (y) * (INT32)nWidth)]; - pixel = 0; - currentScanline = dstp; + BYTE* dstp = &pDstData[1ULL * (y)*nWidth]; + BYTE pixel = 0; + BYTE* currentScanline = dstp; - for (INT32 x = 0; x < (INT32)nWidth;) + for (UINT32 x = 0; x < nWidth;) { - controlByte = *srcp; + BYTE controlByte = *srcp; srcp++; if ((srcp - pSrcData) > SrcSize * 1ll) @@ -265,8 +263,8 @@ static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT return -1; } - nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); - cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); + UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); + UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); if (nRunLength == 1) { @@ -309,25 +307,26 @@ static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT else { /* delta values relative to previous scanline */ + INT16 p = 0; while (cRawBytes > 0) { - deltaValue = *srcp; + UINT8 deltaValue = *srcp; srcp++; if (deltaValue & 1) { deltaValue = deltaValue >> 1; deltaValue = deltaValue + 1; - pixel = -deltaValue; + p = WINPR_ASSERTING_INT_CAST(int16_t, -1 * (int16_t)deltaValue); } else { deltaValue = deltaValue >> 1; - pixel = deltaValue; + p = WINPR_ASSERTING_INT_CAST(INT16, deltaValue); } - deltaValue = previousScanline[x] + pixel; - *dstp = deltaValue; + const INT16 delta = WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + p); + *dstp = clamp(delta); dstp++; x++; cRawBytes--; @@ -335,8 +334,9 @@ static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* WINPR_RESTRICT while (nRunLength > 0) { - deltaValue = previousScanline[x] + pixel; - *dstp = deltaValue; + const INT16 deltaValue = + WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[x] + p); + *dstp = clamp(deltaValue); dstp++; x++; nRunLength--; @@ -355,15 +355,9 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel, BOOL vFlip) { - UINT32 pixel = 0; - UINT32 cRawBytes = 0; - UINT32 nRunLength = 0; - INT32 deltaValue = 0; INT32 beg = 0; INT32 end = 0; INT32 inc = 0; - BYTE controlByte = 0; - BYTE* currentScanline = NULL; BYTE* previousScanline = NULL; const BYTE* srcp = pSrcData; @@ -388,13 +382,14 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD for (INT32 y = beg; y != end; y += inc) { - BYTE* dstp = &pDstData[((nYDst + y) * nDstStep) + (nXDst * 4) + nChannel]; - pixel = 0; - currentScanline = dstp; + const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL; + BYTE* dstp = &pDstData[off]; + BYTE* currentScanline = dstp; + INT16 pixel = 0; for (INT32 x = 0; x < (INT32)nWidth;) { - controlByte = *srcp; + const BYTE controlByte = *srcp; srcp++; if ((srcp - pSrcData) > SrcSize * 1ll) @@ -403,8 +398,8 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD return -1; } - nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); - cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); + UINT32 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte); + UINT32 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte); if (nRunLength == 1) { @@ -430,7 +425,7 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD { pixel = *srcp; srcp++; - *dstp = pixel; + *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel); dstp += 4; x++; cRawBytes--; @@ -438,7 +433,7 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD while (nRunLength > 0) { - *dstp = pixel; + *dstp = WINPR_ASSERTING_INT_CAST(BYTE, pixel); dstp += 4; x++; nRunLength--; @@ -449,14 +444,14 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD /* delta values relative to previous scanline */ while (cRawBytes > 0) { - deltaValue = *srcp; + BYTE deltaValue = *srcp; srcp++; if (deltaValue & 1) { deltaValue = deltaValue >> 1; deltaValue = deltaValue + 1; - pixel = -deltaValue; + pixel = WINPR_ASSERTING_INT_CAST(int16_t, -deltaValue); } else { @@ -464,8 +459,9 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD pixel = deltaValue; } - deltaValue = previousScanline[4LL * x] + pixel; - *dstp = deltaValue; + const INT16 delta = + WINPR_ASSERTING_INT_CAST(int16_t, previousScanline[4LL * x] + pixel); + *dstp = clamp(delta); dstp += 4; x++; cRawBytes--; @@ -473,8 +469,9 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD while (nRunLength > 0) { - deltaValue = previousScanline[4LL * x] + pixel; - *dstp = deltaValue; + const INT16 deltaValue = + WINPR_ASSERTING_INT_CAST(int16_t, pixel + previousScanline[4LL * x]); + *dstp = clamp(deltaValue); dstp += 4; x++; nRunLength--; @@ -515,7 +512,8 @@ static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstSte for (INT32 y = beg; y != end; y += inc) { - BYTE* dstp = &pDstData[((nYDst + y) * nDstStep) + (nXDst * 4) + nChannel]; + const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (4LL * nXDst) + nChannel * 1LL; + BYTE* dstp = &pDstData[off]; for (INT32 x = 0; x < (INT32)nWidth; ++x) { @@ -606,14 +604,14 @@ static INLINE BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcD if (vFlip) { - beg = nHeight - 1; + beg = WINPR_ASSERTING_INT_CAST(int32_t, nHeight - 1); end = -1; inc = -1; } else { beg = 0; - end = nHeight; + end = WINPR_ASSERTING_INT_CAST(int32_t, nHeight); inc = 1; } @@ -639,14 +637,15 @@ static INLINE BOOL planar_decompress_planes_raw(const BYTE* WINPR_RESTRICT pSrcD { BYTE* pRGB = NULL; - if (y > (INT64)nHeight) + if (y > WINPR_ASSERTING_INT_CAST(INT64, nHeight)) { WLog_ERR(TAG, "planar plane destination Y %" PRId32 " exceeds height %" PRIu32, y, nHeight); return FALSE; } - pRGB = &pDstData[((nYDst + y) * nDstStep) + (nXDst * bpp)]; + const intptr_t off = ((1LL * nYDst + y) * nDstStep) + (1LL * nXDst * bpp); + pRGB = &pDstData[off]; if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA)) return FALSE; @@ -698,22 +697,12 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth, UINT32 nDstHeight, BOOL vFlip) { - BOOL cs = 0; - BOOL rle = 0; - UINT32 cll = 0; - BOOL alpha = 0; BOOL useAlpha = FALSE; INT32 status = 0; - const BYTE* srcp = NULL; - UINT32 subSize = 0; - UINT32 subWidth = 0; - UINT32 subHeight = 0; - UINT32 planeSize = 0; INT32 rleSizes[4] = { 0, 0, 0, 0 }; - UINT32 rawSizes[4]; - UINT32 rawWidths[4]; - UINT32 rawHeights[4]; - BYTE FormatHeader = 0; + UINT32 rawSizes[4] = { 0 }; + UINT32 rawWidths[4] = { 0 }; + UINT32 rawHeights[4] = { 0 }; const BYTE* planes[4] = { 0 }; const UINT32 w = MIN(nSrcWidth, nDstWidth); const UINT32 h = MIN(nSrcHeight, nDstHeight); @@ -725,7 +714,7 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, if (nDstStep <= 0) nDstStep = nDstWidth * FreeRDPGetBytesPerPixel(DstFormat); - srcp = pSrcData; + const BYTE* srcp = pSrcData; if (!pSrcData) { @@ -739,11 +728,11 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, return FALSE; } - FormatHeader = *srcp++; - cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK); - cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE; - rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE; - alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE; + const BYTE FormatHeader = *srcp++; + const BYTE cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK); + const BYTE cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE; + const BYTE rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE; + const BYTE alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE; DstFormat = planar_invert_format(planar, alpha, DstFormat); @@ -759,10 +748,10 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, return FALSE; /* Chroma subsampling requires YCoCg */ } - subWidth = (nSrcWidth / 2) + (nSrcWidth % 2); - subHeight = (nSrcHeight / 2) + (nSrcHeight % 2); - planeSize = nSrcWidth * nSrcHeight; - subSize = subWidth * subHeight; + const UINT32 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2); + const UINT32 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2); + const UINT32 planeSize = nSrcWidth * nSrcHeight; + const UINT32 subSize = subWidth * subHeight; if (!cs) { @@ -795,7 +784,7 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, rawHeights[3] = nSrcHeight; } - const size_t diff = srcp - pSrcData; + const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(srcp - pSrcData)); if (SrcSize < diff) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff); @@ -866,7 +855,7 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, else planes[0] = srcp; - const size_t diff0 = (planes[0] - pSrcData); + const size_t diff0 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[0] - pSrcData)); if (SrcSize < diff0) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff0); @@ -880,7 +869,7 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, planes[1] = planes[0] + rleSizes[0]; - const size_t diff1 = (planes[1] - pSrcData); + const size_t diff1 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[1] - pSrcData)); if (SrcSize < diff1) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff1); @@ -893,7 +882,7 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, return FALSE; planes[2] = planes[1] + rleSizes[1]; - const size_t diff2 = (planes[2] - pSrcData); + const size_t diff2 = WINPR_ASSERTING_INT_CAST(size_t, (intptr_t)(planes[2] - pSrcData)); if (SrcSize < diff2) { WLog_ERR(TAG, "Size mismatch %" PRIu32 " < %" PRIuz, SrcSize, diff); @@ -943,23 +932,23 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, } else /* RLE */ { - status = - planar_decompress_plane_rle(planes[0], rleSizes[0], pTempData, nTempStep, nXDst, - nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */ + status = planar_decompress_plane_rle( + planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), pTempData, nTempStep, + nXDst, nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */ if (status < 0) return FALSE; - status = planar_decompress_plane_rle(planes[1], rleSizes[1], pTempData, nTempStep, - nXDst, nYDst, nSrcWidth, nSrcHeight, 1, - vFlip); /* GreenPlane */ + status = planar_decompress_plane_rle( + planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), pTempData, nTempStep, + nXDst, nYDst, nSrcWidth, nSrcHeight, 1, vFlip); /* GreenPlane */ if (status < 0) return FALSE; - status = - planar_decompress_plane_rle(planes[2], rleSizes[2], pTempData, nTempStep, nXDst, - nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */ + status = planar_decompress_plane_rle( + planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), pTempData, nTempStep, + nXDst, nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */ if (status < 0) return FALSE; @@ -968,9 +957,9 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, if (useAlpha) { - status = planar_decompress_plane_rle(planes[3], rleSizes[3], pTempData, nTempStep, - nXDst, nYDst, nSrcWidth, nSrcHeight, 3, - vFlip); /* AlphaPlane */ + status = planar_decompress_plane_rle( + planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), pTempData, + nTempStep, nXDst, nYDst, nSrcWidth, nSrcHeight, 3, vFlip); /* AlphaPlane */ } else status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth, @@ -1023,9 +1012,9 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */ if (useAlpha) { - status = - planar_decompress_plane_rle_only(planes[3], rleSizes[3], rleBuffer[3], - rawWidths[3], rawHeights[3]); /* AlphaPlane */ + status = planar_decompress_plane_rle_only( + planes[3], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[3]), rleBuffer[3], + rawWidths[3], rawHeights[3]); /* AlphaPlane */ if (status < 0) return FALSE; @@ -1034,22 +1023,23 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, if (alpha) srcp += rleSizes[3]; - status = planar_decompress_plane_rle_only(planes[0], rleSizes[0], rleBuffer[0], - rawWidths[0], rawHeights[0]); /* LumaPlane */ + status = planar_decompress_plane_rle_only( + planes[0], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[0]), rleBuffer[0], + rawWidths[0], rawHeights[0]); /* LumaPlane */ if (status < 0) return FALSE; - status = - planar_decompress_plane_rle_only(planes[1], rleSizes[1], rleBuffer[1], rawWidths[1], - rawHeights[1]); /* OrangeChromaPlane */ + status = planar_decompress_plane_rle_only( + planes[1], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[1]), rleBuffer[1], + rawWidths[1], rawHeights[1]); /* OrangeChromaPlane */ if (status < 0) return FALSE; - status = - planar_decompress_plane_rle_only(planes[2], rleSizes[2], rleBuffer[2], rawWidths[2], - rawHeights[2]); /* GreenChromaPlane */ + status = planar_decompress_plane_rle_only( + planes[2], WINPR_ASSERTING_INT_CAST(uint32_t, rleSizes[2]), rleBuffer[2], + rawWidths[2], rawHeights[2]); /* GreenChromaPlane */ if (status < 0) return FALSE; @@ -1099,8 +1089,9 @@ BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, } WINPR_ASSERT(prims->YCoCgToRGB_8u_AC4R); - int rc = prims->YCoCgToRGB_8u_AC4R(pTempData, nTempStep, dst, DstFormat, nDstStep, w, h, - cll, useAlpha); + int rc = prims->YCoCgToRGB_8u_AC4R( + pTempData, WINPR_ASSERTING_INT_CAST(int32_t, nTempStep), dst, DstFormat, + WINPR_ASSERTING_INT_CAST(int32_t, nDstStep), w, h, cll, useAlpha); if (rc != PRIMITIVES_SUCCESS) { WLog_ERR(TAG, "YCoCgToRGB_8u_AC4R failed with %d", rc); @@ -1459,7 +1450,6 @@ static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRIC BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width, UINT32 height, BYTE* WINPR_RESTRICT outPlane) { - char s2c = 0; BYTE* outPtr = NULL; const BYTE* srcPtr = NULL; const BYTE* prevLinePtr = NULL; @@ -1484,10 +1474,12 @@ BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlan for (UINT32 x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++) { INT32 delta = *srcPtr - *prevLinePtr; - s2c = (delta >= 0) ? (char)delta : (char)(~((BYTE)(-delta)) + 1); - s2c = (s2c >= 0) ? (char)((UINT32)s2c << 1) - : (char)(((UINT32)(~((BYTE)s2c) + 1) << 1) - 1); - *outPtr = (BYTE)s2c; + BYTE s2c = WINPR_ASSERTING_INT_CAST( + BYTE, (delta >= 0) ? delta : (~((BYTE)(-delta)) + 1) & 0xFF); + s2c = WINPR_ASSERTING_INT_CAST( + BYTE, + (s2c >= 0) ? (s2c << 1) & 0xFF : (((UINT32)(~((BYTE)s2c) + 1) << 1) - 1) & 0xFF); + *outPtr = s2c; } } @@ -1548,7 +1540,7 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT conte return NULL; { - int offset = 0; + uint32_t offset = 0; FormatHeader |= PLANAR_FORMAT_HEADER_RLE; context->rlePlanes[0] = &context->rlePlanesBuffer[offset]; offset += dstSizes[0]; diff --git a/libfreerdp/codec/progressive.c b/libfreerdp/codec/progressive.c index 529ab8ac7..ce09c1496 100644 --- a/libfreerdp/codec/progressive.c +++ b/libfreerdp/codec/progressive.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -624,8 +625,8 @@ static INLINE void progressive_rfx_idwt_x(const INT16* WINPR_RESTRICT pLowBand, INT16* pX = pDstBand; H0 = *pH++; L0 = *pL++; - X0 = L0 - H0; - X2 = L0 - H0; + X0 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - H0); + X2 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - H0); for (size_t j = 0; j < (nHighCount - 1); j++) { @@ -633,8 +634,8 @@ static INLINE void progressive_rfx_idwt_x(const INT16* WINPR_RESTRICT pLowBand, pH++; L0 = *pL; pL++; - X2 = L0 - ((H0 + H1) / 2); - X1 = ((X0 + X2) / 2) + (2 * H0); + X2 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - ((H0 + H1) / 2)); + X1 = WINPR_ASSERTING_INT_CAST(int16_t, ((X0 + X2) / 2) + (2 * H0)); pX[0] = X0; pX[1] = X1; pX += 2; @@ -647,15 +648,15 @@ static INLINE void progressive_rfx_idwt_x(const INT16* WINPR_RESTRICT pLowBand, if (nLowCount <= nHighCount) { pX[0] = X2; - pX[1] = X2 + (2 * H0); + pX[1] = WINPR_ASSERTING_INT_CAST(int16_t, X2 + (2 * H0)); } else { L0 = *pL; pL++; - X0 = L0 - H0; + X0 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - H0); pX[0] = X2; - pX[1] = ((X0 + X2) / 2) + (2 * H0); + pX[1] = WINPR_ASSERTING_INT_CAST(int16_t, ((X0 + X2) / 2) + (2 * H0)); pX[2] = X0; } } @@ -663,13 +664,13 @@ static INLINE void progressive_rfx_idwt_x(const INT16* WINPR_RESTRICT pLowBand, { L0 = *pL; pL++; - X0 = L0 - (H0 / 2); + X0 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - (H0 / 2)); pX[0] = X2; - pX[1] = ((X0 + X2) / 2) + (2 * H0); + pX[1] = WINPR_ASSERTING_INT_CAST(int16_t, ((X0 + X2) / 2) + (2 * H0)); pX[2] = X0; L0 = *pL; pL++; - pX[3] = (X0 + L0) / 2; + pX[3] = WINPR_ASSERTING_INT_CAST(int16_t, (X0 + L0) / 2); } pLowBand += nLowStep; @@ -699,8 +700,8 @@ static INLINE void progressive_rfx_idwt_y(const INT16* WINPR_RESTRICT pLowBand, pH += nHighStep; L0 = *pL; pL += nLowStep; - X0 = L0 - H0; - X2 = L0 - H0; + X0 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - H0); + X2 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - H0); for (size_t j = 0; j < (nHighCount - 1); j++) { @@ -708,8 +709,8 @@ static INLINE void progressive_rfx_idwt_y(const INT16* WINPR_RESTRICT pLowBand, pH += nHighStep; L0 = *pL; pL += nLowStep; - X2 = L0 - ((H0 + H1) / 2); - X1 = ((X0 + X2) / 2) + (2 * H0); + X2 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - ((H0 + H1) / 2)); + X1 = WINPR_ASSERTING_INT_CAST(int16_t, ((X0 + X2) / 2) + (2 * H0)); *pX = X0; pX += nDstStep; *pX = X1; @@ -724,15 +725,15 @@ static INLINE void progressive_rfx_idwt_y(const INT16* WINPR_RESTRICT pLowBand, { *pX = X2; pX += nDstStep; - *pX = X2 + (2 * H0); + *pX = WINPR_ASSERTING_INT_CAST(int16_t, X2 + (2 * H0)); } else { L0 = *pL; - X0 = L0 - H0; + X0 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - H0); *pX = X2; pX += nDstStep; - *pX = ((X0 + X2) / 2) + (2 * H0); + *pX = WINPR_ASSERTING_INT_CAST(int16_t, ((X0 + X2) / 2) + (2 * H0)); pX += nDstStep; *pX = X0; } @@ -741,15 +742,15 @@ static INLINE void progressive_rfx_idwt_y(const INT16* WINPR_RESTRICT pLowBand, { L0 = *pL; pL += nLowStep; - X0 = L0 - (H0 / 2); + X0 = WINPR_ASSERTING_INT_CAST(int16_t, L0 - (H0 / 2)); *pX = X2; pX += nDstStep; - *pX = ((X0 + X2) / 2) + (2 * H0); + *pX = WINPR_ASSERTING_INT_CAST(int16_t, ((X0 + X2) / 2) + (2 * H0)); pX += nDstStep; *pX = X0; pX += nDstStep; L0 = *pL; - *pX = (X0 + L0) / 2; + *pX = WINPR_ASSERTING_INT_CAST(int16_t, (X0 + L0) / 2); } pLowBand++; @@ -1101,7 +1102,8 @@ static INLINE INT16 progressive_rfx_srl_read(RFX_PROGRESSIVE_UPGRADE_STATE* WINP if (k) { bs->mask = ((1 << k) - 1); - state->nz = ((bs->accumulator >> (32u - k)) & bs->mask); + state->nz = + WINPR_ASSERTING_INT_CAST(int16_t, ((bs->accumulator >> (32u - k)) & bs->mask)); BitStream_Shift(bs, k); } @@ -1194,7 +1196,8 @@ static INLINE int progressive_rfx_upgrade_block(RFX_PROGRESSIVE_UPGRADE_STATE* W raw->mask = ((1 << numBits) - 1); input = (INT16)((raw->accumulator >> (32 - numBits)) & raw->mask); BitStream_Shift(raw, numBits); - buffer[index] += (input << shift); + buffer[index] += + WINPR_ASSERTING_INT_CAST(INT16, (INT16)(((UINT32)input << shift) & 0xFFFF)); } return 1; @@ -1224,7 +1227,8 @@ static INLINE int progressive_rfx_upgrade_block(RFX_PROGRESSIVE_UPGRADE_STATE* W sign[index] = input; } - buffer[index] += (INT16)((UINT32)input << shift); + const int32_t val = input << shift; + buffer[index] += WINPR_ASSERTING_INT_CAST(INT16, val); } return 1; @@ -1791,7 +1795,8 @@ static INLINE SSIZE_T progressive_process_tiles( } SubmitThreadpoolWork(progressive->work_objects[idx]); - close_cnt = idx + 1; + + close_cnt = WINPR_ASSERTING_INT_CAST(UINT16, idx + 1); } else { @@ -2055,7 +2060,7 @@ static INLINE SSIZE_T progressive_wb_read_region_header( return -1014; } - const SSIZE_T rc = Stream_GetRemainingLength(s); + const SSIZE_T rc = WINPR_ASSERTING_INT_CAST(SSIZE_T, Stream_GetRemainingLength(s)); const SSIZE_T expect = region->numRects * 8ll + region->numQuant * 5ll + region->numProgQuant * 16ll + region->tileDataSize; SSIZE_T len = rc; @@ -2112,7 +2117,7 @@ static INLINE SSIZE_T progressive_wb_skip_region(PROGRESSIVE_CONTEXT* WINPR_REST if (rc < 0) return rc; - if (!Stream_SafeSeek(s, rc)) + if (!Stream_SafeSeek(s, WINPR_ASSERTING_INT_CAST(size_t, rc))) return -1111; return rc; @@ -2201,8 +2206,8 @@ static INLINE SSIZE_T progressive_wb_region(PROGRESSIVE_CONTEXT* WINPR_RESTRICT region->numQuant, region->numProgQuant); #endif - boxLeft = surface->gridWidth; - boxTop = surface->gridHeight; + boxLeft = WINPR_ASSERTING_INT_CAST(UINT16, surface->gridWidth); + boxTop = WINPR_ASSERTING_INT_CAST(UINT16, surface->gridHeight); boxRight = 0; boxBottom = 0; @@ -2236,7 +2241,7 @@ static INLINE SSIZE_T progressive_wb_region(PROGRESSIVE_CONTEXT* WINPR_RESTRICT const SSIZE_T res = progressive_process_tiles(progressive, s, region, surface, context); if (res < 0) return -1; - return (size_t)rc; + return rc; } static INLINE SSIZE_T progressive_parse_block(PROGRESSIVE_CONTEXT* WINPR_RESTRICT progressive, @@ -2344,8 +2349,11 @@ static INLINE BOOL update_tiles(PROGRESSIVE_CONTEXT* WINPR_RESTRICT progressive, RFX_PROGRESSIVE_TILE* tile = surface->tiles[index]; WINPR_ASSERT(tile); - updateRect.left = nXDst + tile->x; - updateRect.top = nYDst + tile->y; + const UINT32 dl = nXDst + tile->x; + updateRect.left = WINPR_ASSERTING_INT_CAST(UINT16, dl); + + const UINT32 dt = nYDst + tile->y; + updateRect.top = WINPR_ASSERTING_INT_CAST(UINT16, dt); updateRect.right = updateRect.left + 64; updateRect.bottom = updateRect.top + 64; @@ -2478,8 +2486,6 @@ int progressive_compress(PROGRESSIVE_CONTEXT* WINPR_RESTRICT progressive, int res = -6; wStream* s = NULL; UINT32 numRects = 0; - UINT32 x = 0; - UINT32 y = 0; RFX_RECT* rects = NULL; RFX_MESSAGE* message = NULL; @@ -2516,7 +2522,10 @@ int progressive_compress(PROGRESSIVE_CONTEXT* WINPR_RESTRICT progressive, numRects *= (Height + 63) / 64; } else - numRects = region16_n_rects(invalidRegion); + { + const int nr = region16_n_rects(invalidRegion); + numRects = WINPR_ASSERTING_INT_CAST(uint32_t, nr); + } if (numRects == 0) return 0; @@ -2540,15 +2549,19 @@ int progressive_compress(PROGRESSIVE_CONTEXT* WINPR_RESTRICT progressive, } else { - x = 0; - y = 0; + UINT16 x = 0; + UINT16 y = 0; + for (UINT32 i = 0; i < numRects; i++) { RFX_RECT* r = &rects[i]; r->x = x; r->y = y; - r->width = MIN(64, Width - x); - r->height = MIN(64, Height - y); + + WINPR_ASSERT(Width >= x); + WINPR_ASSERT(Height >= y); + r->width = MIN(64, WINPR_ASSERTING_INT_CAST(UINT16, Width - x)); + r->height = MIN(64, WINPR_ASSERTING_INT_CAST(UINT16, Height - y)); if (x + 64 >= Width) { @@ -2568,8 +2581,9 @@ int progressive_compress(PROGRESSIVE_CONTEXT* WINPR_RESTRICT progressive, Stream_SetPosition(s, 0); progressive->rfx_context->mode = RLGR1; - progressive->rfx_context->width = Width; - progressive->rfx_context->height = Height; + + progressive->rfx_context->width = WINPR_ASSERTING_INT_CAST(UINT16, Width); + progressive->rfx_context->height = WINPR_ASSERTING_INT_CAST(UINT16, Height); rfx_context_set_pixel_format(progressive->rfx_context, SrcFormat); message = rfx_encode_message(progressive->rfx_context, rects, numRects, pSrcData, Width, Height, ScanLine); diff --git a/libfreerdp/codec/region.c b/libfreerdp/codec/region.c index 35ad4551f..1983715e2 100644 --- a/libfreerdp/codec/region.c +++ b/libfreerdp/codec/region.c @@ -194,13 +194,14 @@ void region16_clear(REGION16* region) static INLINE REGION16_DATA* allocateRegion(long nbItems) { - long allocSize = sizeof(REGION16_DATA) + (nbItems * sizeof(RECTANGLE_16)); + size_t allocSize = + sizeof(REGION16_DATA) + (WINPR_ASSERTING_INT_CAST(size_t, nbItems) * sizeof(RECTANGLE_16)); REGION16_DATA* ret = (REGION16_DATA*)malloc(allocSize); if (!ret) return ret; - ret->size = allocSize; + ret->size = WINPR_ASSERTING_INT_CAST(long, allocSize); ret->nbRects = nbItems; return ret; } @@ -229,7 +230,7 @@ BOOL region16_copy(REGION16* dst, const REGION16* src) if (!dst->data) return FALSE; - CopyMemory(dst->data, src->data, src->data->size); + CopyMemory(dst->data, src->data, WINPR_ASSERTING_INT_CAST(size_t, src->data->size)); } return TRUE; @@ -456,7 +457,8 @@ static BOOL region16_simplify_bands(REGION16* region) /* override band2, we don't move band1 pointer as the band after band2 * may be merged too */ endBand = band2 + bandItems; - const size_t toMove = (endPtr - endBand) * sizeof(RECTANGLE_16); + const size_t toMove = + WINPR_ASSERTING_INT_CAST(size_t, (endPtr - endBand)) * sizeof(RECTANGLE_16); if (toMove) MoveMemory(band2, endBand, toMove); @@ -472,7 +474,8 @@ static BOOL region16_simplify_bands(REGION16* region) if (finalNbRects != nbRects) { - size_t allocSize = sizeof(REGION16_DATA) + (finalNbRects * sizeof(RECTANGLE_16)); + size_t allocSize = sizeof(REGION16_DATA) + + (WINPR_ASSERTING_INT_CAST(size_t, finalNbRects) * sizeof(RECTANGLE_16)); REGION16_DATA* data = realloc(region->data, allocSize); if (!data) free(region->data); @@ -485,7 +488,7 @@ static BOOL region16_simplify_bands(REGION16* region) } region->data->nbRects = finalNbRects; - region->data->size = allocSize; + region->data->size = WINPR_ASSERTING_INT_CAST(long, allocSize); } return TRUE; @@ -526,7 +529,7 @@ BOOL region16_union_rect(REGION16* dst, const REGION16* src, const RECTANGLE_16* return TRUE; } - newItems = allocateRegion((1ULL + 4ULL * region16_n_rects(src))); + newItems = allocateRegion((1L + 4L * region16_n_rects(src))); if (!newItems) return FALSE; @@ -673,10 +676,13 @@ BOOL region16_union_rect(REGION16* dst, const REGION16* src, const RECTANGLE_16* dstExtents->left = MIN(rect->left, srcExtents->left); dstExtents->bottom = MAX(rect->bottom, srcExtents->bottom); dstExtents->right = MAX(rect->right, srcExtents->right); - newItems->size = sizeof(REGION16_DATA) + (usedRects * sizeof(RECTANGLE_16)); - tmpItems = realloc(newItems, newItems->size); + newItems->size = + WINPR_ASSERTING_INT_CAST(long, sizeof(REGION16_DATA) + (usedRects * sizeof(RECTANGLE_16))); + if (newItems->size != 0) + tmpItems = realloc(newItems, WINPR_ASSERTING_INT_CAST(size_t, newItems->size)); if (!tmpItems) free(newItems); + newItems = tmpItems; dst->data = newItems; @@ -792,12 +798,13 @@ BOOL region16_intersect_rect(REGION16* dst, const REGION16* src, const RECTANGLE } newItems->nbRects = usedRects; - newItems->size = sizeof(REGION16_DATA) + (usedRects * sizeof(RECTANGLE_16)); + newItems->size = + WINPR_ASSERTING_INT_CAST(long, sizeof(REGION16_DATA) + (usedRects * sizeof(RECTANGLE_16))); if ((dst->data->size > 0) && (dst->data != &empty_region)) free(dst->data); - dst->data = realloc(newItems, newItems->size); + dst->data = realloc(newItems, WINPR_ASSERTING_INT_CAST(size_t, newItems->size)); if (!dst->data) { diff --git a/libfreerdp/codec/rfx.c b/libfreerdp/codec/rfx.c index 36a547417..06937cb89 100644 --- a/libfreerdp/codec/rfx.c +++ b/libfreerdp/codec/rfx.c @@ -21,11 +21,12 @@ #include -#include #include #include #include +#include +#include #include #include #include @@ -416,7 +417,8 @@ void rfx_context_set_pixel_format(RFX_CONTEXT* WINPR_RESTRICT context, UINT32 pi { WINPR_ASSERT(context); context->pixel_format = pixel_format; - context->bits_per_pixel = FreeRDPGetBitsPerPixel(pixel_format); + const UINT32 bpp = FreeRDPGetBitsPerPixel(pixel_format); + context->bits_per_pixel = WINPR_ASSERTING_INT_CAST(UINT8, bpp); } UINT32 rfx_context_get_pixel_format(RFX_CONTEXT* WINPR_RESTRICT context) @@ -443,8 +445,8 @@ BOOL rfx_context_reset(RFX_CONTEXT* WINPR_RESTRICT context, UINT32 width, UINT32 if (!context) return FALSE; - context->width = width; - context->height = height; + context->width = WINPR_ASSERTING_INT_CAST(UINT16, width); + context->height = WINPR_ASSERTING_INT_CAST(UINT16, height); context->state = RFX_STATE_SEND_HEADERS; context->expectedDataBlockType = WBT_FRAME_BEGIN; context->frameIdx = 0; @@ -458,7 +460,7 @@ static INLINE BOOL rfx_process_message_sync(RFX_CONTEXT* WINPR_RESTRICT context, WINPR_ASSERT(context); WINPR_ASSERT(context->priv); - context->decodedHeaderBlocks &= ~RFX_DECODED_SYNC; + context->decodedHeaderBlocks &= (uint32_t)~RFX_DECODED_SYNC; /* RFX_SYNC */ if (!Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, 6)) @@ -491,7 +493,7 @@ static INLINE BOOL rfx_process_message_codec_versions(RFX_CONTEXT* WINPR_RESTRIC WINPR_ASSERT(context); WINPR_ASSERT(context->priv); - context->decodedHeaderBlocks &= ~RFX_DECODED_VERSIONS; + context->decodedHeaderBlocks &= (uint32_t)~RFX_DECODED_VERSIONS; if (!Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, 4)) return FALSE; @@ -536,7 +538,7 @@ static INLINE BOOL rfx_process_message_channels(RFX_CONTEXT* WINPR_RESTRICT cont WINPR_ASSERT(context); WINPR_ASSERT(context->priv); - context->decodedHeaderBlocks &= ~RFX_DECODED_CHANNELS; + context->decodedHeaderBlocks &= (uint32_t)~RFX_DECODED_CHANNELS; if (!Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, 1)) return FALSE; @@ -594,7 +596,7 @@ static INLINE BOOL rfx_process_message_context(RFX_CONTEXT* WINPR_RESTRICT conte WINPR_ASSERT(context); WINPR_ASSERT(context->priv); - context->decodedHeaderBlocks &= ~RFX_DECODED_CONTEXT; + context->decodedHeaderBlocks &= (uint32_t)~RFX_DECODED_CONTEXT; if (!Stream_CheckAndLogRequiredLengthWLog(context->priv->log, s, 5)) return FALSE; @@ -820,7 +822,7 @@ static INLINE BOOL rfx_allocate_tiles(RFX_MESSAGE* WINPR_RESTRICT message, size_ message->tiles = tmpTiles; if (!allocOnly) - message->numTiles = count; + message->numTiles = WINPR_ASSERTING_INT_CAST(UINT16, count); else { WINPR_ASSERT(message->numTiles <= count); @@ -1586,7 +1588,8 @@ static INLINE void CALLBACK rfx_compose_message_tile_work_callback(PTP_CALLBACK_ static INLINE BOOL computeRegion(const RFX_RECT* WINPR_RESTRICT rects, size_t numRects, REGION16* WINPR_RESTRICT region, size_t width, size_t height) { - const RECTANGLE_16 mainRect = { 0, 0, width, height }; + const RECTANGLE_16 mainRect = { 0, 0, WINPR_ASSERTING_INT_CAST(UINT16, width), + WINPR_ASSERTING_INT_CAST(UINT16, height) }; WINPR_ASSERT(rects); for (size_t i = 0; i < numRects; i++) @@ -1724,7 +1727,7 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* WINPR_RESTRICT context, if (!(message->rects = winpr_aligned_calloc(regionNbRects, sizeof(RFX_RECT), 32))) goto skip_encoding_loop; - message->numRects = regionNbRects; + message->numRects = WINPR_ASSERTING_INT_CAST(UINT16, regionNbRects); for (UINT32 i = 0; i < regionNbRects; i++, regionRect++) { @@ -1746,8 +1749,8 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* WINPR_RESTRICT context, if ((yIdx == endTileY) && (gridRelY + 64 > height)) tileHeight = height - gridRelY; - currentTileRect.top = gridRelY; - currentTileRect.bottom = gridRelY + tileHeight; + currentTileRect.top = WINPR_ASSERTING_INT_CAST(UINT16, gridRelY); + currentTileRect.bottom = WINPR_ASSERTING_INT_CAST(UINT16, gridRelY + tileHeight); for (UINT32 xIdx = startTileX, gridRelX = startTileX * 64; xIdx <= endTileX; xIdx++, gridRelX += 64) @@ -1757,13 +1760,15 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* WINPR_RESTRICT context, const BYTE* cpv; BYTE* pv; } cnv; - int tileWidth = 64; + UINT32 tileWidth = 64; if ((xIdx == endTileX) && (gridRelX + 64 > width)) - tileWidth = width - gridRelX; + { + tileWidth = (width - gridRelX); + } - currentTileRect.left = gridRelX; - currentTileRect.right = gridRelX + tileWidth; + currentTileRect.left = WINPR_ASSERTING_INT_CAST(UINT16, gridRelX); + currentTileRect.right = WINPR_ASSERTING_INT_CAST(UINT16, gridRelX + tileWidth); /* checks if this tile is already treated */ if (region16_intersects_rect(&tilesRegion, ¤tTileRect)) @@ -1773,11 +1778,12 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* WINPR_RESTRICT context, if (!tile) goto skip_encoding_loop; - tile->xIdx = xIdx; - tile->yIdx = yIdx; - tile->x = gridRelX; - tile->y = gridRelY; + tile->xIdx = WINPR_ASSERTING_INT_CAST(UINT16, xIdx); + tile->yIdx = WINPR_ASSERTING_INT_CAST(UINT16, yIdx); + tile->x = WINPR_ASSERTING_INT_CAST(UINT16, gridRelX); + tile->y = WINPR_ASSERTING_INT_CAST(UINT16, gridRelY); tile->scanline = scanline; + tile->width = tileWidth; tile->height = tileHeight; const UINT32 ax = gridRelX; @@ -2050,7 +2056,8 @@ static INLINE BOOL rfx_write_message_tileset(RFX_CONTEXT* WINPR_RESTRICT context Stream_Write_UINT16(s, CBT_TILESET); /* subtype (2 bytes) */ Stream_Write_UINT16(s, 0); /* idx (2 bytes) */ Stream_Write_UINT16(s, context->properties); /* properties (2 bytes) */ - Stream_Write_UINT8(s, message->numQuant); /* numQuant (1 byte) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, message->numQuant)); /* numQuant (1 byte) */ Stream_Write_UINT8(s, 0x40); /* tileSize (1 byte) */ Stream_Write_UINT16(s, message->numTiles); /* numTiles (2 bytes) */ Stream_Write_UINT32(s, message->tilesDataSize); /* tilesDataSize (4 bytes) */ @@ -2059,7 +2066,8 @@ static INLINE BOOL rfx_write_message_tileset(RFX_CONTEXT* WINPR_RESTRICT context for (size_t i = 0; i < message->numQuant * 5ul; i++) { WINPR_ASSERT(quantVals); - Stream_Write_UINT8(s, quantVals[0] + (quantVals[1] << 4)); + Stream_Write_UINT8(s, + WINPR_ASSERTING_INT_CAST(uint8_t, quantVals[0] + (quantVals[1] << 4))); quantVals += 2; } diff --git a/libfreerdp/codec/rfx_bitstream.h b/libfreerdp/codec/rfx_bitstream.h index 3e8424211..f3cb7701e 100644 --- a/libfreerdp/codec/rfx_bitstream.h +++ b/libfreerdp/codec/rfx_bitstream.h @@ -20,88 +20,115 @@ #ifndef FREERDP_LIB_CODEC_RFX_BITSTREAM_H #define FREERDP_LIB_CODEC_RFX_BITSTREAM_H +#include +#include + #include -typedef struct +#ifdef __cplusplus +extern "C" { - BYTE* buffer; - int nbytes; - int byte_pos; - int bits_left; -} RFX_BITSTREAM; +#endif -#define rfx_bitstream_attach(bs, _buffer, _nbytes) \ - do \ - { \ - bs->buffer = (BYTE*)(_buffer); \ - bs->nbytes = (_nbytes); \ - bs->byte_pos = 0; \ - bs->bits_left = 8; \ - } while (0) + typedef struct + { + BYTE* buffer; + uint32_t nbytes; + uint32_t byte_pos; + uint32_t bits_left; + } RFX_BITSTREAM; -#define rfx_bitstream_get_bits(bs, _nbits, _r) \ - do \ - { \ - int nbits = _nbits; \ - int b; \ - UINT16 n = 0; \ - while (bs->byte_pos < bs->nbytes && nbits > 0) \ - { \ - b = nbits; \ - if (b > bs->bits_left) \ - b = bs->bits_left; \ - if (n) \ - n <<= b; \ - n |= (bs->buffer[bs->byte_pos] >> (bs->bits_left - b)) & ((1 << b) - 1); \ - bs->bits_left -= b; \ - nbits -= b; \ - if (bs->bits_left == 0) \ - { \ - bs->bits_left = 8; \ - bs->byte_pos++; \ - } \ - } \ - _r = n; \ - } while (0) + static inline void rfx_bitstream_attach(RFX_BITSTREAM* bs, BYTE* WINPR_RESTRICT buffer, + size_t nbytes) + { + WINPR_ASSERT(bs); + bs->buffer = (buffer); -#define rfx_bitstream_put_bits(bs, _bits, _nbits) \ - do \ - { \ - UINT16 bits = (_bits); \ - int nbits = (_nbits); \ - int b; \ - while (bs->byte_pos < bs->nbytes && nbits > 0) \ - { \ - b = nbits; \ - if (b > bs->bits_left) \ - b = bs->bits_left; \ - bs->buffer[bs->byte_pos] |= ((bits >> (nbits - b)) & ((1 << b) - 1)) \ - << (bs->bits_left - b); \ - bs->bits_left -= b; \ - nbits -= b; \ - if (bs->bits_left == 0) \ - { \ - bs->bits_left = 8; \ - bs->byte_pos++; \ - } \ - } \ - } while (0) -#define rfx_bitstream_flush(bs) \ - do \ - { \ - if (bs->bits_left != 8) \ - { \ - int _nbits = 8 - bs->bits_left; \ - rfx_bitstream_put_bits(bs, 0, _nbits); \ - } \ - } while (0) + WINPR_ASSERT(nbytes <= UINT32_MAX); + bs->nbytes = WINPR_ASSERTING_INT_CAST(uint32_t, nbytes); + bs->byte_pos = 0; + bs->bits_left = 8; + } -#define rfx_bitstream_eos(_bs) ((_bs)->byte_pos >= (_bs)->nbytes) -#define rfx_bitstream_left(_bs) \ - ((_bs)->byte_pos >= (_bs)->nbytes \ - ? 0 \ - : ((_bs)->nbytes - (_bs)->byte_pos - 1) * 8 + (_bs)->bits_left) -#define rfx_bitstream_get_processed_bytes(_bs) \ - ((_bs)->bits_left < 8 ? (_bs)->byte_pos + 1 : (_bs)->byte_pos) + static inline uint32_t rfx_bitstream_get_bits(RFX_BITSTREAM* bs, uint32_t nbits) + { + UINT16 n = 0; + while (bs->byte_pos < bs->nbytes && nbits > 0) + { + uint32_t b = nbits; + if (b > bs->bits_left) + b = bs->bits_left; + if (n) + n <<= b; + n |= (bs->buffer[bs->byte_pos] >> (bs->bits_left - b)) & ((1 << b) - 1); + bs->bits_left -= b; + nbits -= b; + if (bs->bits_left == 0) + { + bs->bits_left = 8; + bs->byte_pos++; + } + } + return n; + } + static inline void rfx_bitstream_put_bits(RFX_BITSTREAM* bs, uint32_t _bits, uint32_t _nbits) + { + UINT16 bits = WINPR_ASSERTING_INT_CAST(UINT16, _bits); + + uint32_t nbits = (_nbits); + while (bs->byte_pos < bs->nbytes && nbits > 0) + { + uint32_t b = nbits; + if (b > bs->bits_left) + b = bs->bits_left; + bs->buffer[bs->byte_pos] |= ((bits >> (nbits - b)) & ((1 << b) - 1)) + << (bs->bits_left - b); + bs->bits_left -= b; + nbits -= b; + if (bs->bits_left == 0) + { + bs->bits_left = 8; + bs->byte_pos++; + } + } + } + + static inline void rfx_bitstream_flush(RFX_BITSTREAM* bs) + { + WINPR_ASSERT(bs); + if (bs->bits_left != 8) + { + uint32_t _nbits = 8 - bs->bits_left; + rfx_bitstream_put_bits(bs, 0, _nbits); + } + } + + static inline BOOL rfx_bitstream_eos(RFX_BITSTREAM* bs) + { + WINPR_ASSERT(bs); + return ((bs)->byte_pos >= (bs)->nbytes); + } + + static inline uint32_t rfx_bitstream_left(RFX_BITSTREAM* bs) + { + WINPR_ASSERT(bs); + + if ((bs)->byte_pos >= (bs)->nbytes) + return 0; + + return ((bs)->nbytes - (bs)->byte_pos - 1) * 8 + (bs)->bits_left; + } + + static inline uint32_t rfx_bitstream_get_processed_bytes(RFX_BITSTREAM* bs) + { + WINPR_ASSERT(bs); + if ((bs)->bits_left < 8) + return (bs)->byte_pos + 1; + return (bs)->byte_pos; + } + +#ifdef __cplusplus +} +#endif #endif /* FREERDP_LIB_CODEC_RFX_BITSTREAM_H */ diff --git a/libfreerdp/codec/rfx_dwt.c b/libfreerdp/codec/rfx_dwt.c index b06fc8cf1..198251928 100644 --- a/libfreerdp/codec/rfx_dwt.c +++ b/libfreerdp/codec/rfx_dwt.c @@ -17,6 +17,9 @@ * limitations under the License. */ +#include +#include + #include #include @@ -47,13 +50,13 @@ static INLINE void rfx_dwt_2d_decode_block(INT16* WINPR_RESTRICT buffer, INT16* for (size_t y = 0; y < subband_width; y++) { /* Even coefficients */ - l_dst[0] = ll[0] - ((hl[0] + hl[0] + 1) >> 1); - h_dst[0] = lh[0] - ((hh[0] + hh[0] + 1) >> 1); + l_dst[0] = WINPR_ASSERTING_INT_CAST(int16_t, ll[0] - ((hl[0] + hl[0] + 1) >> 1)); + h_dst[0] = WINPR_ASSERTING_INT_CAST(int16_t, lh[0] - ((hh[0] + hh[0] + 1) >> 1)); for (size_t n = 1; n < subband_width; n++) { const size_t x = n << 1; - l_dst[x] = ll[n] - ((hl[n - 1] + hl[n] + 1) >> 1); - h_dst[x] = lh[n] - ((hh[n - 1] + hh[n] + 1) >> 1); + l_dst[x] = WINPR_ASSERTING_INT_CAST(int16_t, ll[n] - ((hl[n - 1] + hl[n] + 1) >> 1)); + h_dst[x] = WINPR_ASSERTING_INT_CAST(int16_t, lh[n] - ((hh[n - 1] + hh[n] + 1) >> 1)); } /* Odd coefficients */ @@ -61,13 +64,20 @@ static INLINE void rfx_dwt_2d_decode_block(INT16* WINPR_RESTRICT buffer, INT16* for (; n < subband_width - 1; n++) { const size_t x = n << 1; - l_dst[x + 1] = (hl[n] << 1) + ((l_dst[x] + l_dst[x + 2]) >> 1); - h_dst[x + 1] = (hh[n] << 1) + ((h_dst[x] + h_dst[x + 2]) >> 1); + + const int ld = (hl[n] << 1) + ((l_dst[x] + l_dst[x + 2]) >> 1); + const int hd = (hh[n] << 1) + ((h_dst[x] + h_dst[x + 2]) >> 1); + + l_dst[x + 1] = WINPR_ASSERTING_INT_CAST(INT16, ld); + h_dst[x + 1] = WINPR_ASSERTING_INT_CAST(INT16, hd); } const size_t x = n << 1; - l_dst[x + 1] = (hl[n] << 1) + (l_dst[x]); - h_dst[x + 1] = (hh[n] << 1) + (h_dst[x]); + + const int ld = (hl[n] << 1) + (l_dst[x]); + const int hd = (hh[n] << 1) + (h_dst[x]); + l_dst[x + 1] = WINPR_ASSERTING_INT_CAST(INT16, ld); + h_dst[x + 1] = WINPR_ASSERTING_INT_CAST(INT16, hd); ll += subband_width; hl += subband_width; @@ -85,7 +95,8 @@ static INLINE void rfx_dwt_2d_decode_block(INT16* WINPR_RESTRICT buffer, INT16* const INT16* h = idwt + x + subband_width * total_width; INT16* dst = buffer + x; - *dst = *l - ((*h * 2 + 1) >> 1); + const int dd = *l - ((*h * 2 + 1) >> 1); + *dst = WINPR_ASSERTING_INT_CAST(INT16, dd); for (size_t n = 1; n < subband_width; n++) { @@ -93,15 +104,18 @@ static INLINE void rfx_dwt_2d_decode_block(INT16* WINPR_RESTRICT buffer, INT16* h += total_width; /* Even coefficients */ - dst[2 * total_width] = *l - ((*(h - total_width) + *h + 1) >> 1); + const int d2 = *l - ((*(h - total_width) + *h + 1) >> 1); + dst[2 * total_width] = WINPR_ASSERTING_INT_CAST(INT16, d2); /* Odd coefficients */ - dst[total_width] = (*(h - total_width) << 1) + ((*dst + dst[2 * total_width]) >> 1); + const int d = (*(h - total_width) << 1) + ((*dst + dst[2 * total_width]) >> 1); + dst[total_width] = WINPR_ASSERTING_INT_CAST(INT16, d); dst += 2 * total_width; } - dst[total_width] = (*h << 1) + ((*dst * 2) >> 1); + const int d = (*h << 1) + ((*dst * 2) >> 1); + dst[total_width] = WINPR_ASSERTING_INT_CAST(INT16, d); } } @@ -141,12 +155,14 @@ static void rfx_dwt_2d_encode_block(INT16* WINPR_RESTRICT buffer, INT16* WINPR_R src = buffer + 1ULL * y * total_width + x; /* H */ - *h = (src[total_width] - - ((src[0] + src[n < subband_width - 1 ? 2 * total_width : 0]) >> 1)) >> - 1; + *h = WINPR_ASSERTING_INT_CAST( + int16_t, (src[total_width] - + ((src[0] + src[n < subband_width - 1 ? 2 * total_width : 0]) >> 1)) >> + 1); /* L */ - *l = src[0] + (n == 0 ? *h : (*(h - total_width) + *h) >> 1); + *l = WINPR_ASSERTING_INT_CAST(int16_t, + src[0] + (n == 0 ? *h : (*(h - total_width) + *h) >> 1)); } } @@ -171,10 +187,12 @@ static void rfx_dwt_2d_encode_block(INT16* WINPR_RESTRICT buffer, INT16* WINPR_R UINT32 x = n << 1; /* HL */ - hl[n] = - (l_src[x + 1] - ((l_src[x] + l_src[n < subband_width - 1 ? x + 2 : x]) >> 1)) >> 1; + hl[n] = WINPR_ASSERTING_INT_CAST( + int16_t, + (l_src[x + 1] - ((l_src[x] + l_src[n < subband_width - 1 ? x + 2 : x]) >> 1)) >> 1); /* LL */ - ll[n] = l_src[x] + (n == 0 ? hl[n] : (hl[n - 1] + hl[n]) >> 1); + ll[n] = WINPR_ASSERTING_INT_CAST( + int16_t, l_src[x] + (n == 0 ? hl[n] : (hl[n - 1] + hl[n]) >> 1)); } /* H */ @@ -183,10 +201,12 @@ static void rfx_dwt_2d_encode_block(INT16* WINPR_RESTRICT buffer, INT16* WINPR_R UINT32 x = n << 1; /* HH */ - hh[n] = - (h_src[x + 1] - ((h_src[x] + h_src[n < subband_width - 1 ? x + 2 : x]) >> 1)) >> 1; + hh[n] = WINPR_ASSERTING_INT_CAST( + int16_t, + (h_src[x + 1] - ((h_src[x] + h_src[n < subband_width - 1 ? x + 2 : x]) >> 1)) >> 1); /* LH */ - lh[n] = h_src[x] + (n == 0 ? hh[n] : (hh[n - 1] + hh[n]) >> 1); + lh[n] = WINPR_ASSERTING_INT_CAST( + int16_t, h_src[x] + (n == 0 ? hh[n] : (hh[n - 1] + hh[n]) >> 1)); } ll += subband_width; diff --git a/libfreerdp/codec/rfx_encode.c b/libfreerdp/codec/rfx_encode.c index 850462b8b..c56fab416 100644 --- a/libfreerdp/codec/rfx_encode.c +++ b/libfreerdp/codec/rfx_encode.c @@ -37,13 +37,11 @@ #include "rfx_encode.h" -static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width, int height, - int rowstride, UINT32 pixel_format, +static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, uint32_t width, + uint32_t height, uint32_t rowstride, UINT32 pixel_format, const BYTE* WINPR_RESTRICT palette, INT16* WINPR_RESTRICT r_buf, INT16* WINPR_RESTRICT g_buf, INT16* WINPR_RESTRICT b_buf) { - int x_exceed = 0; - int y_exceed = 0; const BYTE* src = NULL; INT16 r = 0; INT16 g = 0; @@ -51,10 +49,10 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width INT16* r_last = NULL; INT16* g_last = NULL; INT16* b_last = NULL; - x_exceed = 64 - width; - y_exceed = 64 - height; + uint32_t x_exceed = 64 - width; + uint32_t y_exceed = 64 - height; - for (int y = 0; y < height; y++) + for (uint32_t y = 0; y < height; y++) { src = rgb_data + 1ULL * y * rowstride; @@ -62,7 +60,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width { case PIXEL_FORMAT_BGRX32: case PIXEL_FORMAT_BGRA32: - for (int x = 0; x < width; x++) + for (uint32_t x = 0; x < width; x++) { *b_buf++ = (INT16)(*src++); *g_buf++ = (INT16)(*src++); @@ -74,7 +72,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width case PIXEL_FORMAT_XBGR32: case PIXEL_FORMAT_ABGR32: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { src++; *b_buf++ = (INT16)(*src++); @@ -86,7 +84,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width case PIXEL_FORMAT_RGBX32: case PIXEL_FORMAT_RGBA32: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { *r_buf++ = (INT16)(*src++); *g_buf++ = (INT16)(*src++); @@ -98,7 +96,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width case PIXEL_FORMAT_XRGB32: case PIXEL_FORMAT_ARGB32: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { src++; *r_buf++ = (INT16)(*src++); @@ -109,7 +107,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width break; case PIXEL_FORMAT_BGR24: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { *b_buf++ = (INT16)(*src++); *g_buf++ = (INT16)(*src++); @@ -119,7 +117,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width break; case PIXEL_FORMAT_RGB24: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { *r_buf++ = (INT16)(*src++); *g_buf++ = (INT16)(*src++); @@ -129,7 +127,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width break; case PIXEL_FORMAT_BGR16: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { *b_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); *g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); @@ -140,7 +138,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width break; case PIXEL_FORMAT_RGB16: - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { *r_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); *g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); @@ -154,11 +152,10 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width if (!palette) break; - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { - int shift = 0; BYTE idx = 0; - shift = (7 - (x % 8)); + const size_t shift = (7 - (x % 8)); idx = ((*src) >> shift) & 1; idx |= (((*(src + 1)) >> shift) & 1) << 1; idx |= (((*(src + 2)) >> shift) & 1) << 2; @@ -178,7 +175,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width if (!palette) break; - for (int x = 0; x < width; x++) + for (size_t x = 0; x < width; x++) { int idx = (*src) * 3; *r_buf++ = (INT16)palette[idx]; @@ -201,7 +198,7 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width g = *(g_buf - 1); b = *(b_buf - 1); - for (int x = 0; x < x_exceed; x++) + for (size_t x = 0; x < x_exceed; x++) { *r_buf++ = r; *g_buf++ = g; @@ -235,10 +232,9 @@ static void rfx_encode_format_rgb(const BYTE* WINPR_RESTRICT rgb_data, int width static void rfx_encode_component(RFX_CONTEXT* WINPR_RESTRICT context, const UINT32* WINPR_RESTRICT quantization_values, INT16* WINPR_RESTRICT data, BYTE* WINPR_RESTRICT buffer, - int buffer_size, int* WINPR_RESTRICT size) + uint32_t buffer_size, uint32_t* WINPR_RESTRICT size) { - INT16* dwt_buffer = NULL; - dwt_buffer = BufferPool_Take(context->priv->BufferPool, -1); /* dwt_buffer */ + INT16* dwt_buffer = BufferPool_Take(context->priv->BufferPool, -1); /* dwt_buffer */ PROFILER_ENTER(context->priv->prof_rfx_encode_component) PROFILER_ENTER(context->priv->prof_rfx_dwt_2d_encode) context->dwt_2d_encode(data, dwt_buffer); @@ -250,10 +246,12 @@ static void rfx_encode_component(RFX_CONTEXT* WINPR_RESTRICT context, rfx_differential_encode(data + 4032, 64); PROFILER_EXIT(context->priv->prof_rfx_differential_encode) PROFILER_ENTER(context->priv->prof_rfx_rlgr_encode) - *size = context->rlgr_encode(context->mode, data, 4096, buffer, buffer_size); + const int rc = context->rlgr_encode(context->mode, data, 4096, buffer, buffer_size); PROFILER_EXIT(context->priv->prof_rfx_rlgr_encode) PROFILER_EXIT(context->priv->prof_rfx_encode_component) BufferPool_Return(context->priv->BufferPool, dwt_buffer); + + *size = WINPR_ASSERTING_INT_CAST(uint32_t, rc); } void rfx_encode_rgb(RFX_CONTEXT* WINPR_RESTRICT context, RFX_TILE* WINPR_RESTRICT tile) @@ -265,9 +263,9 @@ void rfx_encode_rgb(RFX_CONTEXT* WINPR_RESTRICT context, RFX_TILE* WINPR_RESTRIC } cnv; BYTE* pBuffer = NULL; INT16* pSrcDst[3]; - int YLen = 0; - int CbLen = 0; - int CrLen = 0; + uint32_t YLen = 0; + uint32_t CbLen = 0; + uint32_t CrLen = 0; UINT32* YQuant = NULL; UINT32* CbQuant = NULL; UINT32* CrQuant = NULL; @@ -306,9 +304,9 @@ void rfx_encode_rgb(RFX_CONTEXT* WINPR_RESTRICT context, RFX_TILE* WINPR_RESTRIC rfx_encode_component(context, YQuant, pSrcDst[0], tile->YData, 4096, &YLen); rfx_encode_component(context, CbQuant, pSrcDst[1], tile->CbData, 4096, &CbLen); rfx_encode_component(context, CrQuant, pSrcDst[2], tile->CrData, 4096, &CrLen); - tile->YLen = (UINT16)YLen; - tile->CbLen = (UINT16)CbLen; - tile->CrLen = (UINT16)CrLen; + tile->YLen = WINPR_ASSERTING_INT_CAST(UINT16, YLen); + tile->CbLen = WINPR_ASSERTING_INT_CAST(UINT16, CbLen); + tile->CrLen = WINPR_ASSERTING_INT_CAST(UINT16, CrLen); PROFILER_EXIT(context->priv->prof_rfx_encode_rgb) BufferPool_Return(context->priv->BufferPool, pBuffer); } diff --git a/libfreerdp/codec/rfx_quantization.c b/libfreerdp/codec/rfx_quantization.c index db9977547..f46a10f8d 100644 --- a/libfreerdp/codec/rfx_quantization.c +++ b/libfreerdp/codec/rfx_quantization.c @@ -17,6 +17,9 @@ * limitations under the License. */ +#include +#include + #include #include @@ -72,16 +75,14 @@ void rfx_quantization_decode(INT16* WINPR_RESTRICT buffer, const UINT32* WINPR_R static INLINE void rfx_quantization_encode_block(INT16* WINPR_RESTRICT buffer, size_t buffer_size, UINT32 factor) { - INT16 half = 0; - if (factor == 0) return; - half = (1 << (factor - 1)); + const INT16 half = WINPR_ASSERTING_INT_CAST(INT16, 1 << (factor - 1)); /* Could probably use prims->rShiftC_16s(dst+half, factor, dst, buffer_size); */ for (INT16* dst = buffer; buffer_size > 0; dst++, buffer_size--) { - *dst = (*dst + half) >> factor; + *dst = WINPR_ASSERTING_INT_CAST(INT16, (*dst + half) >> factor); } } diff --git a/libfreerdp/codec/rfx_rlgr.c b/libfreerdp/codec/rfx_rlgr.c index 1317dbda2..489b09f8e 100644 --- a/libfreerdp/codec/rfx_rlgr.c +++ b/libfreerdp/codec/rfx_rlgr.c @@ -28,6 +28,8 @@ #include #include +#include +#include #include #include #include @@ -63,16 +65,24 @@ * Update the passed parameter and clamp it to the range [0, KPMAX] * Return the value of parameter right-shifted by LSGR */ -#define UpdateParam(_param, _deltaP, _k) \ - do \ - { \ - (_param) += (_deltaP); \ - if ((_param) > KPMAX) \ - (_param) = KPMAX; \ - if ((_param) < 0) \ - (_param) = 0; \ - (_k) = ((_param) >> LSGR); \ - } while (0) +static inline uint32_t UpdateParam(uint32_t* param, int32_t deltaP) +{ + WINPR_ASSERT(param); + if (deltaP < 0) + { + const uint32_t udeltaP = WINPR_ASSERTING_INT_CAST(uint32_t, -deltaP); + if (udeltaP > *param) + *param = 0; + else + *param -= udeltaP; + } + else + *param += WINPR_ASSERTING_INT_CAST(uint32_t, deltaP); + + if ((*param) > KPMAX) + (*param) = KPMAX; + return (*param) >> LSGR; +} static BOOL g_LZCNT = FALSE; @@ -80,6 +90,10 @@ static INIT_ONCE rfx_rlgr_init_once = INIT_ONCE_STATIC_INIT; static BOOL CALLBACK rfx_rlgr_init(PINIT_ONCE once, PVOID param, PVOID* context) { + WINPR_UNUSED(once); + WINPR_UNUSED(param); + WINPR_UNUSED(context); + g_LZCNT = IsProcessorFeaturePresentEx(PF_EX_LZCNT); return TRUE; } @@ -138,16 +152,16 @@ static INLINE UINT32 lzcnt_s(UINT32 x) int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 SrcSize, INT16* WINPR_RESTRICT pDstData, UINT32 rDstSize) { - int vk = 0; + uint32_t vk = 0; size_t run = 0; - int cnt = 0; + size_t cnt = 0; size_t size = 0; size_t offset = 0; INT16 mag = 0; UINT32 k = 0; - INT32 kp = 0; + UINT32 kp = 0; UINT32 kr = 0; - INT32 krp = 0; + UINT32 krp = 0; UINT16 code = 0; UINT32 sign = 0; UINT32 nIdx = 0; @@ -196,10 +210,10 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 size_t nbits = BitStream_GetRemainingLength(bs); - if ((size_t)cnt > nbits) - cnt = (int)nbits; + if (cnt > nbits) + cnt = WINPR_ASSERTING_INT_CAST(uint32_t, nbits); - vk = cnt; + vk = WINPR_ASSERTING_INT_CAST(uint32_t, cnt); while ((cnt == 32) && (BitStream_GetRemainingLength(bs) > 0)) { @@ -209,8 +223,8 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 nbits = BitStream_GetRemainingLength(bs); - if ((size_t)cnt > nbits) - cnt = (int)nbits; + if (cnt > nbits) + cnt = nbits; vk += cnt; } @@ -260,10 +274,10 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 nbits = BitStream_GetRemainingLength(bs); - if ((size_t)cnt > nbits) - cnt = (int)nbits; + if (cnt > nbits) + cnt = nbits; - vk = cnt; + vk = WINPR_ASSERTING_INT_CAST(uint32_t, cnt); while ((cnt == 32) && (BitStream_GetRemainingLength(bs) > 0)) { @@ -273,8 +287,8 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 nbits = BitStream_GetRemainingLength(bs); - if ((size_t)cnt > nbits) - cnt = (int)nbits; + if (cnt > nbits) + cnt = nbits; vk += cnt; } @@ -306,9 +320,9 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 { /* update kr, krp params */ - krp -= 2; - - if (krp < 0) + if (krp > 2) + krp -= 2; + else krp = 0; kr = krp >> LSGR; @@ -327,9 +341,9 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 /* update k, kp params */ - kp -= DN_GR; - - if (kp < 0) + if (kp > DN_GR) + kp -= DN_GR; + else kp = 0; k = kp >> LSGR; @@ -337,17 +351,17 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 /* compute magnitude from code */ if (sign) - mag = ((INT16)(code + 1)) * -1; + mag = WINPR_ASSERTING_INT_CAST(int16_t, (code + 1)) * -1; else - mag = (INT16)(code + 1); + mag = WINPR_ASSERTING_INT_CAST(int16_t, code + 1); /* write to output stream */ - offset = (pOutput - pDstData); + offset = WINPR_ASSERTING_INT_CAST(size_t, (pOutput)-pDstData); size = run; if ((offset + size) > rDstSize) - size = DstSize - offset; + size = WINPR_ASSERTING_INT_CAST(size_t, DstSize) - offset; if (size) { @@ -371,10 +385,10 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 size_t nbits = BitStream_GetRemainingLength(bs); - if ((size_t)cnt > nbits) - cnt = (int)nbits; + if (cnt > nbits) + cnt = nbits; - vk = cnt; + vk = WINPR_ASSERTING_INT_CAST(uint32_t, cnt); while ((cnt == 32) && (BitStream_GetRemainingLength(bs) > 0)) { @@ -384,8 +398,8 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 nbits = BitStream_GetRemainingLength(bs); - if ((size_t)cnt > nbits) - cnt = (int)nbits; + if (cnt > nbits) + cnt = nbits; vk += cnt; } @@ -417,12 +431,12 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 { /* update kr, krp params */ - krp -= 2; - - if (krp < 0) + if (krp > 2) + krp -= 2; + else krp = 0; - kr = krp >> LSGR; + kr = (krp >> LSGR) & UINT32_MAX; } else if (vk != 1) { @@ -455,9 +469,9 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 { /* update k, kp params */ - kp -= DQ_GR; - - if (kp < 0) + if (kp > DQ_GR) + kp -= DQ_GR; + else kp = 0; k = kp >> LSGR; @@ -468,9 +482,9 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 */ if (code & 1) - mag = ((INT16)((code + 1) >> 1)) * -1; + mag = WINPR_ASSERTING_INT_CAST(INT16, (code + 1) >> 1) * -1; else - mag = (INT16)(code >> 1); + mag = WINPR_ASSERTING_INT_CAST(INT16, code >> 1); } if ((pOutput - pDstData) < DstSize) @@ -485,8 +499,8 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 if (code) { - mag = (UINT32)code; - nIdx = 32 - lzcnt_s(mag); + mag = WINPR_ASSERTING_INT_CAST(int16_t, code); + nIdx = 32 - lzcnt_s(WINPR_ASSERTING_INT_CAST(uint32_t, mag)); } if (BitStream_GetRemainingLength(bs) < nIdx) @@ -505,9 +519,9 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 { /* update k, kp params */ - kp -= (2 * DQ_GR); - - if (kp < 0) + if (kp > 2 * DQ_GR) + kp -= (2 * DQ_GR); + else kp = 0; k = kp >> LSGR; @@ -525,9 +539,9 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 } if (val1 & 1) - mag = ((INT16)((val1 + 1) >> 1)) * -1; + mag = WINPR_ASSERTING_INT_CAST(int16_t, (val1 + 1) >> 1) * -1; else - mag = (INT16)(val1 >> 1); + mag = WINPR_ASSERTING_INT_CAST(int16_t, val1 >> 1); if ((pOutput - pDstData) < DstSize) { @@ -536,29 +550,29 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 } if (val2 & 1) - mag = ((INT16)((val2 + 1) >> 1)) * -1; + mag = WINPR_ASSERTING_INT_CAST(int16_t, (val2 + 1) >> 1) * -1; else - mag = (INT16)(val2 >> 1); + mag = WINPR_ASSERTING_INT_CAST(int16_t, val2 >> 1); if ((pOutput - pDstData) < DstSize) { - *pOutput = mag; + *pOutput = WINPR_ASSERTING_INT_CAST(int16_t, mag); pOutput++; } } } } - offset = (pOutput - pDstData); + offset = WINPR_ASSERTING_INT_CAST(size_t, (pOutput - pDstData)); if (offset < rDstSize) { - size = DstSize - offset; + size = WINPR_ASSERTING_INT_CAST(size_t, DstSize) - offset; ZeroMemory(pOutput, size * 2); pOutput += size; } - offset = (pOutput - pDstData); + offset = WINPR_ASSERTING_INT_CAST(size_t, (pOutput - pDstData)); if ((DstSize < 0) || (offset != (size_t)DstSize)) return -1; @@ -585,31 +599,38 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* WINPR_RESTRICT pSrcData, UINT32 #define OutputBits(numBits, bitPattern) rfx_bitstream_put_bits(bs, bitPattern, numBits) /* Emit a bit (0 or 1), count number of times, to the output bitstream */ -#define OutputBit(count, bit) \ - do \ - { \ - UINT16 _b = ((bit) ? 0xFFFF : 0); \ - int _c = (count); \ - for (; _c > 0; _c -= 16) \ - rfx_bitstream_put_bits(bs, _b, (_c > 16 ? 16 : _c)); \ - } while (0) +static inline void OutputBit(RFX_BITSTREAM* bs, uint32_t count, UINT8 bit) +{ + UINT16 _b = ((bit) ? 0xFFFF : 0); + const uint32_t rem = count % 16; + for (uint32_t x = 0; x < count - rem; x += 16) + rfx_bitstream_put_bits(bs, _b, 16); + + if (rem > 0) + rfx_bitstream_put_bits(bs, _b, rem); +} /* Converts the input value to (2 * abs(input) - sign(input)), where sign(input) = (input < 0 ? 1 : * 0) and returns it */ -#define Get2MagSign(input) ((input) >= 0 ? 2 * (input) : -2 * (input)-1) +static inline UINT32 Get2MagSign(INT32 input) +{ + if (input >= 0) + return WINPR_ASSERTING_INT_CAST(UINT32, 2 * input); + return WINPR_ASSERTING_INT_CAST(UINT32, -2 * input - 1); +} /* Outputs the Golomb/Rice encoding of a non-negative integer */ #define CodeGR(krp, val) rfx_rlgr_code_gr(bs, krp, val) -static void rfx_rlgr_code_gr(RFX_BITSTREAM* bs, int* krp, UINT32 val) +static void rfx_rlgr_code_gr(RFX_BITSTREAM* bs, uint32_t* krp, UINT32 val) { - int kr = *krp >> LSGR; + uint32_t kr = *krp >> LSGR; /* unary part of GR code */ - UINT32 vk = (val) >> kr; - OutputBit(vk, 1); - OutputBit(1, 0); + const uint32_t vk = val >> kr; + OutputBit(bs, vk, 1); + OutputBit(bs, 1, 0); /* remainder part of GR code, if needed */ if (kr) @@ -620,22 +641,21 @@ static void rfx_rlgr_code_gr(RFX_BITSTREAM* bs, int* krp, UINT32 val) /* update krp, only if it is not equal to 1 */ if (vk == 0) { - UpdateParam(*krp, -2, kr); + (void)UpdateParam(krp, -2); } else if (vk > 1) { - UpdateParam(*krp, vk, kr); + (void)UpdateParam(krp, WINPR_CXX_COMPAT_CAST(int32_t, vk)); } } int rfx_rlgr_encode(RLGR_MODE mode, const INT16* WINPR_RESTRICT data, UINT32 data_size, BYTE* WINPR_RESTRICT buffer, UINT32 buffer_size) { - int k = 0; - int kp = 0; - int krp = 0; + uint32_t k = 0; + uint32_t kp = 0; + uint32_t krp = 0; RFX_BITSTREAM* bs = NULL; - int processed_size = 0; if (!(bs = (RFX_BITSTREAM*)winpr_aligned_calloc(1, sizeof(RFX_BITSTREAM), 32))) return 0; @@ -654,9 +674,9 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* WINPR_RESTRICT data, UINT32 dat if (k) { - int numZeros = 0; - int runmax = 0; - int sign = 0; + uint32_t numZeros = 0; + uint32_t runmax = 0; + BYTE sign = 0; /* RUN-LENGTH MODE */ @@ -673,14 +693,14 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* WINPR_RESTRICT data, UINT32 dat runmax = 1 << k; while (numZeros >= runmax) { - OutputBit(1, 0); /* output a zero bit */ + OutputBit(bs, 1, 0); /* output a zero bit */ numZeros -= runmax; - UpdateParam(kp, UP_GR, k); /* update kp, k */ + k = UpdateParam(&kp, UP_GR); /* update kp, k */ runmax = 1 << k; } /* output a 1 to terminate runs */ - OutputBit(1, 1); + OutputBit(bs, 1, 1); /* output the remaining run length using k bits */ OutputBits(k, numZeros); @@ -693,10 +713,10 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* WINPR_RESTRICT data, UINT32 dat (UINT32)(input < 0 ? -input : input); /* absolute value of input coefficient */ sign = (input < 0 ? 1 : 0); /* sign of input coefficient */ - OutputBit(1, sign); /* output the sign bit */ + OutputBit(bs, 1, sign); /* output the sign bit */ CodeGR(&krp, mag ? mag - 1 : 0); /* output GR code for (mag - 1) */ - UpdateParam(kp, -DN_GR, k); + k = UpdateParam(&kp, -DN_GR); } else { @@ -718,11 +738,11 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* WINPR_RESTRICT data, UINT32 dat and the update direction is reversed */ if (twoMs) { - UpdateParam(kp, -DQ_GR, k); + k = UpdateParam(&kp, -DQ_GR); } else { - UpdateParam(kp, UQ_GR, k); + k = UpdateParam(&kp, UQ_GR); } } else /* mode == RLGR3 */ @@ -753,19 +773,19 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* WINPR_RESTRICT data, UINT32 dat if (twoMs1 && twoMs2) { - UpdateParam(kp, -2 * DQ_GR, k); + k = UpdateParam(&kp, -2 * DQ_GR); } else if (!twoMs1 && !twoMs2) { - UpdateParam(kp, 2 * UQ_GR, k); + k = UpdateParam(&kp, 2 * UQ_GR); } } } } rfx_bitstream_flush(bs); - processed_size = rfx_bitstream_get_processed_bytes(bs); + uint32_t processed_size = rfx_bitstream_get_processed_bytes(bs); winpr_aligned_free(bs); - return processed_size; + return WINPR_ASSERTING_INT_CAST(int, processed_size); } diff --git a/libfreerdp/codec/sse/nsc_sse2.c b/libfreerdp/codec/sse/nsc_sse2.c index 0c5495d63..1d5291050 100644 --- a/libfreerdp/codec/sse/nsc_sse2.c +++ b/libfreerdp/codec/sse/nsc_sse2.c @@ -17,6 +17,8 @@ * limitations under the License. */ +#include +#include #include #include @@ -40,39 +42,30 @@ static BOOL nsc_encode_argb_to_aycocg_sse2(NSC_CONTEXT* context, const BYTE* data, UINT32 scanline) { size_t y = 0; - UINT16 rw = 0; - BYTE ccl = 0; - const BYTE* src = NULL; - BYTE* yplane = NULL; - BYTE* coplane = NULL; - BYTE* cgplane = NULL; - BYTE* aplane = NULL; - __m128i r_val; - __m128i g_val; - __m128i b_val; - __m128i a_val; - __m128i y_val; - __m128i co_val; - __m128i cg_val; - UINT32 tempWidth = 0; if (!context || !data || (scanline == 0)) return FALSE; - tempWidth = ROUND_UP_TO(context->width, 8); - rw = (context->ChromaSubsamplingLevel > 0 ? tempWidth : context->width); - ccl = context->ColorLossLevel; + const UINT16 tempWidth = ROUND_UP_TO(context->width, 8); + const UINT16 rw = (context->ChromaSubsamplingLevel > 0 ? tempWidth : context->width); + + const BYTE ccl = WINPR_ASSERTING_INT_CAST(BYTE, context->ColorLossLevel); for (; y < context->height; y++) { - src = data + (context->height - 1 - y) * scanline; - yplane = context->priv->PlaneBuffers[0] + y * rw; - coplane = context->priv->PlaneBuffers[1] + y * rw; - cgplane = context->priv->PlaneBuffers[2] + y * rw; - aplane = context->priv->PlaneBuffers[3] + y * context->width; + const BYTE* src = data + (context->height - 1 - y) * scanline; + BYTE* yplane = context->priv->PlaneBuffers[0] + y * rw; + BYTE* coplane = context->priv->PlaneBuffers[1] + y * rw; + BYTE* cgplane = context->priv->PlaneBuffers[2] + y * rw; + BYTE* aplane = context->priv->PlaneBuffers[3] + y * context->width; for (UINT16 x = 0; x < context->width; x += 8) { + __m128i r_val = { 0 }; + __m128i g_val = { 0 }; + __m128i b_val = { 0 }; + __m128i a_val = { 0 }; + switch (context->format) { case PIXEL_FORMAT_BGRX32: @@ -142,65 +135,105 @@ static BOOL nsc_encode_argb_to_aycocg_sse2(NSC_CONTEXT* context, const BYTE* dat a_val = _mm_set1_epi16(0xFF); src += 24; break; - case PIXEL_FORMAT_BGR16: - b_val = _mm_set_epi16((((*(src + 15)) & 0xF8) | ((*(src + 15)) >> 5)), - (((*(src + 13)) & 0xF8) | ((*(src + 13)) >> 5)), - (((*(src + 11)) & 0xF8) | ((*(src + 11)) >> 5)), - (((*(src + 9)) & 0xF8) | ((*(src + 9)) >> 5)), - (((*(src + 7)) & 0xF8) | ((*(src + 7)) >> 5)), - (((*(src + 5)) & 0xF8) | ((*(src + 5)) >> 5)), - (((*(src + 3)) & 0xF8) | ((*(src + 3)) >> 5)), - (((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5))); + b_val = _mm_set_epi16(WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 15)) & 0xF8) | + ((*(src + 15)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 13)) & 0xF8) | + ((*(src + 13)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 11)) & 0xF8) | + ((*(src + 11)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 9)) & 0xF8) | + ((*(src + 9)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 7)) & 0xF8) | + ((*(src + 7)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 5)) & 0xF8) | + ((*(src + 5)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 3)) & 0xF8) | + ((*(src + 3)) >> 5)), + WINPR_ASSERTING_INT_CAST(INT16, ((*(src + 1)) & 0xF8) | + ((*(src + 1)) >> 5))); g_val = _mm_set_epi16( - ((((*(src + 15)) & 0x07) << 5) | (((*(src + 14)) & 0xE0) >> 3)), - ((((*(src + 13)) & 0x07) << 5) | (((*(src + 12)) & 0xE0) >> 3)), - ((((*(src + 11)) & 0x07) << 5) | (((*(src + 10)) & 0xE0) >> 3)), - ((((*(src + 9)) & 0x07) << 5) | (((*(src + 8)) & 0xE0) >> 3)), - ((((*(src + 7)) & 0x07) << 5) | (((*(src + 6)) & 0xE0) >> 3)), - ((((*(src + 5)) & 0x07) << 5) | (((*(src + 4)) & 0xE0) >> 3)), - ((((*(src + 3)) & 0x07) << 5) | (((*(src + 2)) & 0xE0) >> 3)), - ((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3))); + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 15)) & 0x07) << 5) | + (((*(src + 14)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 13)) & 0x07) << 5) | + (((*(src + 12)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 11)) & 0x07) << 5) | + (((*(src + 10)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 9)) & 0x07) << 5) | + (((*(src + 8)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 7)) & 0x07) << 5) | + (((*(src + 6)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 5)) & 0x07) << 5) | + (((*(src + 4)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 3)) & 0x07) << 5) | + (((*(src + 2)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 1)) & 0x07) << 5) | + (((*src) & 0xE0) >> 3))); r_val = _mm_set_epi16( - ((((*(src + 14)) & 0x1F) << 3) | (((*(src + 14)) >> 2) & 0x07)), - ((((*(src + 12)) & 0x1F) << 3) | (((*(src + 12)) >> 2) & 0x07)), - ((((*(src + 10)) & 0x1F) << 3) | (((*(src + 10)) >> 2) & 0x07)), - ((((*(src + 8)) & 0x1F) << 3) | (((*(src + 8)) >> 2) & 0x07)), - ((((*(src + 6)) & 0x1F) << 3) | (((*(src + 6)) >> 2) & 0x07)), - ((((*(src + 4)) & 0x1F) << 3) | (((*(src + 4)) >> 2) & 0x07)), - ((((*(src + 2)) & 0x1F) << 3) | (((*(src + 2)) >> 2) & 0x07)), - ((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07))); + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 14)) & 0x1F) << 3) | + (((*(src + 14)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 12)) & 0x1F) << 3) | + (((*(src + 12)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 10)) & 0x1F) << 3) | + (((*(src + 10)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 8)) & 0x1F) << 3) | + (((*(src + 8)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 6)) & 0x1F) << 3) | + (((*(src + 6)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 4)) & 0x1F) << 3) | + (((*(src + 4)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 2)) & 0x1F) << 3) | + (((*(src + 2)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, + (((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07))); a_val = _mm_set1_epi16(0xFF); src += 16; break; case PIXEL_FORMAT_RGB16: - r_val = _mm_set_epi16((((*(src + 15)) & 0xF8) | ((*(src + 15)) >> 5)), - (((*(src + 13)) & 0xF8) | ((*(src + 13)) >> 5)), - (((*(src + 11)) & 0xF8) | ((*(src + 11)) >> 5)), - (((*(src + 9)) & 0xF8) | ((*(src + 9)) >> 5)), - (((*(src + 7)) & 0xF8) | ((*(src + 7)) >> 5)), - (((*(src + 5)) & 0xF8) | ((*(src + 5)) >> 5)), - (((*(src + 3)) & 0xF8) | ((*(src + 3)) >> 5)), - (((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5))); + r_val = _mm_set_epi16( + WINPR_ASSERTING_INT_CAST(INT16, ((src[15] & 0xF8) | (src[15] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[13] & 0xF8) | (src[13] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[11] & 0xF8) | (src[11] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[9] & 0xF8) | (src[9] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[7] & 0xF8) | (src[7] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[5] & 0xF8) | (src[5] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[3] & 0xF8) | (src[3] >> 5))), + WINPR_ASSERTING_INT_CAST(INT16, ((src[1] & 0xF8) | (src[1] >> 5)))); g_val = _mm_set_epi16( - ((((*(src + 15)) & 0x07) << 5) | (((*(src + 14)) & 0xE0) >> 3)), - ((((*(src + 13)) & 0x07) << 5) | (((*(src + 12)) & 0xE0) >> 3)), - ((((*(src + 11)) & 0x07) << 5) | (((*(src + 10)) & 0xE0) >> 3)), - ((((*(src + 9)) & 0x07) << 5) | (((*(src + 8)) & 0xE0) >> 3)), - ((((*(src + 7)) & 0x07) << 5) | (((*(src + 6)) & 0xE0) >> 3)), - ((((*(src + 5)) & 0x07) << 5) | (((*(src + 4)) & 0xE0) >> 3)), - ((((*(src + 3)) & 0x07) << 5) | (((*(src + 2)) & 0xE0) >> 3)), - ((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3))); + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 15)) & 0x07) << 5) | + (((*(src + 14)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 13)) & 0x07) << 5) | + (((*(src + 12)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 11)) & 0x07) << 5) | + (((*(src + 10)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 9)) & 0x07) << 5) | + (((*(src + 8)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 7)) & 0x07) << 5) | + (((*(src + 6)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 5)) & 0x07) << 5) | + (((*(src + 4)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 3)) & 0x07) << 5) | + (((*(src + 2)) & 0xE0) >> 3)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 1)) & 0x07) << 5) | + (((*src) & 0xE0) >> 3))); b_val = _mm_set_epi16( - ((((*(src + 14)) & 0x1F) << 3) | (((*(src + 14)) >> 2) & 0x07)), - ((((*(src + 12)) & 0x1F) << 3) | (((*(src + 12)) >> 2) & 0x07)), - ((((*(src + 10)) & 0x1F) << 3) | (((*(src + 10)) >> 2) & 0x07)), - ((((*(src + 8)) & 0x1F) << 3) | (((*(src + 8)) >> 2) & 0x07)), - ((((*(src + 6)) & 0x1F) << 3) | (((*(src + 6)) >> 2) & 0x07)), - ((((*(src + 4)) & 0x1F) << 3) | (((*(src + 4)) >> 2) & 0x07)), - ((((*(src + 2)) & 0x1F) << 3) | (((*(src + 2)) >> 2) & 0x07)), - ((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07))); + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 14)) & 0x1F) << 3) | + (((*(src + 14)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 12)) & 0x1F) << 3) | + (((*(src + 12)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 10)) & 0x1F) << 3) | + (((*(src + 10)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 8)) & 0x1F) << 3) | + (((*(src + 8)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 6)) & 0x1F) << 3) | + (((*(src + 6)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 4)) & 0x1F) << 3) | + (((*(src + 4)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, (((*(src + 2)) & 0x1F) << 3) | + (((*(src + 2)) >> 2) & 0x07)), + WINPR_ASSERTING_INT_CAST(INT16, + (((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07))); a_val = _mm_set1_epi16(0xFF); src += 16; break; @@ -275,12 +308,12 @@ static BOOL nsc_encode_argb_to_aycocg_sse2(NSC_CONTEXT* context, const BYTE* dat break; } - y_val = _mm_srai_epi16(r_val, 2); + __m128i y_val = _mm_srai_epi16(r_val, 2); y_val = _mm_add_epi16(y_val, _mm_srai_epi16(g_val, 1)); y_val = _mm_add_epi16(y_val, _mm_srai_epi16(b_val, 2)); - co_val = _mm_sub_epi16(r_val, b_val); + __m128i co_val = _mm_sub_epi16(r_val, b_val); co_val = _mm_srai_epi16(co_val, ccl); - cg_val = _mm_sub_epi16(g_val, _mm_srai_epi16(r_val, 1)); + __m128i cg_val = _mm_sub_epi16(g_val, _mm_srai_epi16(r_val, 1)); cg_val = _mm_sub_epi16(cg_val, _mm_srai_epi16(b_val, 1)); cg_val = _mm_srai_epi16(cg_val, ccl); y_val = _mm_packus_epi16(y_val, y_val); @@ -310,9 +343,9 @@ static BOOL nsc_encode_argb_to_aycocg_sse2(NSC_CONTEXT* context, const BYTE* dat if (context->ChromaSubsamplingLevel > 0 && (y % 2) == 1) { - yplane = context->priv->PlaneBuffers[0] + y * rw; - coplane = context->priv->PlaneBuffers[1] + y * rw; - cgplane = context->priv->PlaneBuffers[2] + y * rw; + BYTE* yplane = context->priv->PlaneBuffers[0] + y * rw; + BYTE* coplane = context->priv->PlaneBuffers[1] + y * rw; + BYTE* cgplane = context->priv->PlaneBuffers[2] + y * rw; CopyMemory(yplane, yplane - rw, rw); CopyMemory(coplane, coplane - rw, rw); CopyMemory(cgplane, cgplane - rw, rw); diff --git a/libfreerdp/codec/sse/rfx_sse2.c b/libfreerdp/codec/sse/rfx_sse2.c index 189cea293..a0d3701e5 100644 --- a/libfreerdp/codec/sse/rfx_sse2.c +++ b/libfreerdp/codec/sse/rfx_sse2.c @@ -18,6 +18,8 @@ * limitations under the License. */ +#include +#include #include #include @@ -47,12 +49,6 @@ #define ATTRIBUTES __gnu_inline__, __always_inline__ #endif -#define mm_between_epi16(_val, _min, _max) \ - do \ - { \ - (_val) = _mm_min_epi16(_max, _mm_max_epi16(_val, _min)); \ - } while (0) - static __inline void __attribute__((ATTRIBUTES)) mm_prefetch_buffer(char* WINPR_RESTRICT buffer, size_t num_bytes) { @@ -71,17 +67,17 @@ static __inline void __attribute__((ATTRIBUTES)) rfx_quantization_decode_block_sse2(INT16* WINPR_RESTRICT buffer, const size_t buffer_size, const UINT32 factor) { - __m128i a; __m128i* ptr = (__m128i*)buffer; - __m128i* buf_end = (__m128i*)(buffer + buffer_size); + const __m128i* buf_end = (__m128i*)(buffer + buffer_size); if (factor == 0) return; do { - a = _mm_load_si128(ptr); - a = _mm_slli_epi16(a, factor); + const __m128i la = _mm_load_si128(ptr); + const __m128i a = _mm_slli_epi16(la, WINPR_ASSERTING_INT_CAST(int, factor)); + _mm_store_si128(ptr, a); ptr++; } while (ptr < buf_end); @@ -107,23 +103,21 @@ static void rfx_quantization_decode_sse2(INT16* WINPR_RESTRICT buffer, } static __inline void __attribute__((ATTRIBUTES)) -rfx_quantization_encode_block_sse2(INT16* WINPR_RESTRICT buffer, const int buffer_size, - const UINT32 factor) +rfx_quantization_encode_block_sse2(INT16* WINPR_RESTRICT buffer, const unsigned buffer_size, + const INT16 factor) { - __m128i a; __m128i* ptr = (__m128i*)buffer; - __m128i* buf_end = (__m128i*)(buffer + buffer_size); - __m128i half; + const __m128i* buf_end = (const __m128i*)(buffer + buffer_size); if (factor == 0) return; - half = _mm_set1_epi16(1 << (factor - 1)); + const __m128i half = _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(INT16, 1 << (factor - 1))); do { - a = _mm_load_si128(ptr); - a = _mm_add_epi16(a, half); + const __m128i la = _mm_load_si128(ptr); + __m128i a = _mm_add_epi16(la, half); a = _mm_srai_epi16(a, factor); _mm_store_si128(ptr, a); ptr++; @@ -135,48 +129,57 @@ static void rfx_quantization_encode_sse2(INT16* WINPR_RESTRICT buffer, { WINPR_ASSERT(buffer); WINPR_ASSERT(quantization_values); + for (size_t x = 0; x < 10; x++) + { + WINPR_ASSERT(quantization_values[x] >= 6); + WINPR_ASSERT(quantization_values[x] <= INT16_MAX + 6); + } mm_prefetch_buffer((char*)buffer, 4096 * sizeof(INT16)); - rfx_quantization_encode_block_sse2(buffer, 1024, quantization_values[8] - 6); /* HL1 */ - rfx_quantization_encode_block_sse2(buffer + 1024, 1024, quantization_values[7] - 6); /* LH1 */ - rfx_quantization_encode_block_sse2(buffer + 2048, 1024, quantization_values[9] - 6); /* HH1 */ - rfx_quantization_encode_block_sse2(buffer + 3072, 256, quantization_values[5] - 6); /* HL2 */ - rfx_quantization_encode_block_sse2(buffer + 3328, 256, quantization_values[4] - 6); /* LH2 */ - rfx_quantization_encode_block_sse2(buffer + 3584, 256, quantization_values[6] - 6); /* HH2 */ - rfx_quantization_encode_block_sse2(buffer + 3840, 64, quantization_values[2] - 6); /* HL3 */ - rfx_quantization_encode_block_sse2(buffer + 3904, 64, quantization_values[1] - 6); /* LH3 */ - rfx_quantization_encode_block_sse2(buffer + 3968, 64, quantization_values[3] - 6); /* HH3 */ - rfx_quantization_encode_block_sse2(buffer + 4032, 64, quantization_values[0] - 6); /* LL3 */ + rfx_quantization_encode_block_sse2( + buffer, 1024, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[8] - 6)); /* HL1 */ + rfx_quantization_encode_block_sse2( + buffer + 1024, 1024, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[7] - 6)); /* LH1 */ + rfx_quantization_encode_block_sse2( + buffer + 2048, 1024, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[9] - 6)); /* HH1 */ + rfx_quantization_encode_block_sse2( + buffer + 3072, 256, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[5] - 6)); /* HL2 */ + rfx_quantization_encode_block_sse2( + buffer + 3328, 256, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[4] - 6)); /* LH2 */ + rfx_quantization_encode_block_sse2( + buffer + 3584, 256, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[6] - 6)); /* HH2 */ + rfx_quantization_encode_block_sse2( + buffer + 3840, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[2] - 6)); /* HL3 */ + rfx_quantization_encode_block_sse2( + buffer + 3904, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[1] - 6)); /* LH3 */ + rfx_quantization_encode_block_sse2( + buffer + 3968, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[3] - 6)); /* HH3 */ + rfx_quantization_encode_block_sse2( + buffer + 4032, 64, WINPR_ASSERTING_INT_CAST(INT16, quantization_values[0] - 6)); /* LL3 */ rfx_quantization_encode_block_sse2(buffer, 4096, 5); } static __inline void __attribute__((ATTRIBUTES)) rfx_dwt_2d_decode_block_horiz_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRICT h, - INT16* WINPR_RESTRICT dst, int subband_width) + INT16* WINPR_RESTRICT dst, size_t subband_width) { INT16* l_ptr = l; INT16* h_ptr = h; INT16* dst_ptr = dst; int first = 0; int last = 0; - __m128i l_n; - __m128i h_n; - __m128i h_n_m; - __m128i tmp_n; - __m128i dst_n; - __m128i dst_n_p; __m128i dst1; __m128i dst2; - for (int y = 0; y < subband_width; y++) + for (size_t y = 0; y < subband_width; y++) { /* Even coefficients */ - for (int n = 0; n < subband_width; n += 8) + for (size_t n = 0; n < subband_width; n += 8) { /* dst[2n] = l[n] - ((h[n-1] + h[n] + 1) >> 1); */ - l_n = _mm_load_si128((__m128i*)l_ptr); - h_n = _mm_load_si128((__m128i*)h_ptr); - h_n_m = _mm_loadu_si128((__m128i*)(h_ptr - 1)); + __m128i l_n = _mm_load_si128((__m128i*)l_ptr); + __m128i h_n = _mm_load_si128((__m128i*)h_ptr); + __m128i h_n_m = _mm_loadu_si128((__m128i*)(h_ptr - 1)); if (n == 0) { @@ -184,10 +187,10 @@ rfx_dwt_2d_decode_block_horiz_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRIC h_n_m = _mm_insert_epi16(h_n_m, first, 0); } - tmp_n = _mm_add_epi16(h_n, h_n_m); + __m128i tmp_n = _mm_add_epi16(h_n, h_n_m); tmp_n = _mm_add_epi16(tmp_n, _mm_set1_epi16(1)); tmp_n = _mm_srai_epi16(tmp_n, 1); - dst_n = _mm_sub_epi16(l_n, tmp_n); + const __m128i dst_n = _mm_sub_epi16(l_n, tmp_n); _mm_store_si128((__m128i*)l_ptr, dst_n); l_ptr += 8; h_ptr += 8; @@ -197,13 +200,13 @@ rfx_dwt_2d_decode_block_horiz_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRIC h_ptr -= subband_width; /* Odd coefficients */ - for (int n = 0; n < subband_width; n += 8) + for (size_t n = 0; n < subband_width; n += 8) { /* dst[2n + 1] = (h[n] << 1) + ((dst[2n] + dst[2n + 2]) >> 1); */ - h_n = _mm_load_si128((__m128i*)h_ptr); + __m128i h_n = _mm_load_si128((__m128i*)h_ptr); h_n = _mm_slli_epi16(h_n, 1); - dst_n = _mm_load_si128((__m128i*)(l_ptr)); - dst_n_p = _mm_loadu_si128((__m128i*)(l_ptr + 1)); + __m128i dst_n = _mm_load_si128((__m128i*)(l_ptr)); + __m128i dst_n_p = _mm_loadu_si128((__m128i*)(l_ptr + 1)); if (n == subband_width - 8) { @@ -211,7 +214,7 @@ rfx_dwt_2d_decode_block_horiz_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRIC dst_n_p = _mm_insert_epi16(dst_n_p, last, 7); } - tmp_n = _mm_add_epi16(dst_n_p, dst_n); + __m128i tmp_n = _mm_add_epi16(dst_n_p, dst_n); tmp_n = _mm_srai_epi16(tmp_n, 1); tmp_n = _mm_add_epi16(tmp_n, h_n); dst1 = _mm_unpacklo_epi16(dst_n, tmp_n); @@ -227,40 +230,33 @@ rfx_dwt_2d_decode_block_horiz_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRIC static __inline void __attribute__((ATTRIBUTES)) rfx_dwt_2d_decode_block_vert_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRICT h, - INT16* WINPR_RESTRICT dst, int subband_width) + INT16* WINPR_RESTRICT dst, size_t subband_width) { INT16* l_ptr = l; INT16* h_ptr = h; INT16* dst_ptr = dst; - __m128i l_n; - __m128i h_n; - __m128i tmp_n; - __m128i h_n_m; - __m128i dst_n; - __m128i dst_n_m; - __m128i dst_n_p; - int total_width = subband_width + subband_width; + const size_t total_width = subband_width + subband_width; /* Even coefficients */ - for (int n = 0; n < subband_width; n++) + for (size_t n = 0; n < subband_width; n++) { - for (int x = 0; x < total_width; x += 8) + for (size_t x = 0; x < total_width; x += 8) { /* dst[2n] = l[n] - ((h[n-1] + h[n] + 1) >> 1); */ - l_n = _mm_load_si128((__m128i*)l_ptr); - h_n = _mm_load_si128((__m128i*)h_ptr); - tmp_n = _mm_add_epi16(h_n, _mm_set1_epi16(1)); + const __m128i l_n = _mm_load_si128((__m128i*)l_ptr); + const __m128i h_n = _mm_load_si128((__m128i*)h_ptr); + __m128i tmp_n = _mm_add_epi16(h_n, _mm_set1_epi16(1)); if (n == 0) tmp_n = _mm_add_epi16(tmp_n, h_n); else { - h_n_m = _mm_loadu_si128((__m128i*)(h_ptr - total_width)); + const __m128i h_n_m = _mm_loadu_si128((__m128i*)(h_ptr - total_width)); tmp_n = _mm_add_epi16(tmp_n, h_n_m); } tmp_n = _mm_srai_epi16(tmp_n, 1); - dst_n = _mm_sub_epi16(l_n, tmp_n); + const __m128i dst_n = _mm_sub_epi16(l_n, tmp_n); _mm_store_si128((__m128i*)dst_ptr, dst_n); l_ptr += 8; h_ptr += 8; @@ -274,26 +270,26 @@ rfx_dwt_2d_decode_block_vert_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRICT dst_ptr = dst + total_width; /* Odd coefficients */ - for (int n = 0; n < subband_width; n++) + for (size_t n = 0; n < subband_width; n++) { - for (int x = 0; x < total_width; x += 8) + for (size_t x = 0; x < total_width; x += 8) { /* dst[2n + 1] = (h[n] << 1) + ((dst[2n] + dst[2n + 2]) >> 1); */ - h_n = _mm_load_si128((__m128i*)h_ptr); - dst_n_m = _mm_load_si128((__m128i*)(dst_ptr - total_width)); + __m128i h_n = _mm_load_si128((__m128i*)h_ptr); + __m128i dst_n_m = _mm_load_si128((__m128i*)(dst_ptr - total_width)); h_n = _mm_slli_epi16(h_n, 1); - tmp_n = dst_n_m; + __m128i tmp_n = dst_n_m; if (n == subband_width - 1) tmp_n = _mm_add_epi16(tmp_n, dst_n_m); else { - dst_n_p = _mm_loadu_si128((__m128i*)(dst_ptr + total_width)); + const __m128i dst_n_p = _mm_loadu_si128((__m128i*)(dst_ptr + total_width)); tmp_n = _mm_add_epi16(tmp_n, dst_n_p); } tmp_n = _mm_srai_epi16(tmp_n, 1); - dst_n = _mm_add_epi16(tmp_n, h_n); + const __m128i dst_n = _mm_add_epi16(tmp_n, h_n); _mm_store_si128((__m128i*)dst_ptr, dst_n); h_ptr += 8; dst_ptr += 8; @@ -305,27 +301,21 @@ rfx_dwt_2d_decode_block_vert_sse2(INT16* WINPR_RESTRICT l, INT16* WINPR_RESTRICT static __inline void __attribute__((ATTRIBUTES)) rfx_dwt_2d_decode_block_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RESTRICT idwt, - int subband_width) + size_t subband_width) { - INT16* hl = NULL; - INT16* lh = NULL; - INT16* hh = NULL; - INT16* ll = NULL; - INT16* l_dst = NULL; - INT16* h_dst = NULL; mm_prefetch_buffer((char*)idwt, 4ULL * subband_width * sizeof(INT16)); /* Inverse DWT in horizontal direction, results in 2 sub-bands in L, H order in tmp buffer idwt. */ /* The 4 sub-bands are stored in HL(0), LH(1), HH(2), LL(3) order. */ /* The lower part L uses LL(3) and HL(0). */ /* The higher part H uses LH(1) and HH(2). */ - ll = buffer + 3ULL * subband_width * subband_width; - hl = buffer; - l_dst = idwt; + INT16* ll = buffer + 3ULL * subband_width * subband_width; + INT16* hl = buffer; + INT16* l_dst = idwt; rfx_dwt_2d_decode_block_horiz_sse2(ll, hl, l_dst, subband_width); - lh = buffer + 1ULL * subband_width * subband_width; - hh = buffer + 2ULL * subband_width * subband_width; - h_dst = idwt + 2ULL * subband_width * subband_width; + INT16* lh = buffer + 1ULL * subband_width * subband_width; + INT16* hh = buffer + 2ULL * subband_width * subband_width; + INT16* h_dst = idwt + 2ULL * subband_width * subband_width; rfx_dwt_2d_decode_block_horiz_sse2(lh, hh, h_dst, subband_width); /* Inverse DWT in vertical direction, results are stored in original buffer. */ rfx_dwt_2d_decode_block_vert_sse2(l_dst, h_dst, buffer, subband_width); @@ -344,43 +334,34 @@ static void rfx_dwt_2d_decode_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RE static __inline void __attribute__((ATTRIBUTES)) rfx_dwt_2d_encode_block_vert_sse2(INT16* WINPR_RESTRICT src, INT16* WINPR_RESTRICT l, - INT16* WINPR_RESTRICT h, int subband_width) + INT16* WINPR_RESTRICT h, size_t subband_width) { - int total_width = 0; - __m128i src_2n; - __m128i src_2n_1; - __m128i src_2n_2; - __m128i h_n; - __m128i h_n_m; - __m128i l_n; - total_width = subband_width << 1; + const size_t total_width = subband_width << 1; - for (int n = 0; n < subband_width; n++) + for (size_t n = 0; n < subband_width; n++) { - for (int x = 0; x < total_width; x += 8) + for (size_t x = 0; x < total_width; x += 8) { - src_2n = _mm_load_si128((__m128i*)src); - src_2n_1 = _mm_load_si128((__m128i*)(src + total_width)); + __m128i src_2n = _mm_load_si128((__m128i*)src); + __m128i src_2n_1 = _mm_load_si128((__m128i*)(src + total_width)); + __m128i src_2n_2 = src_2n; if (n < subband_width - 1) src_2n_2 = _mm_load_si128((__m128i*)(src + 2ULL * total_width)); - else - src_2n_2 = src_2n; /* h[n] = (src[2n + 1] - ((src[2n] + src[2n + 2]) >> 1)) >> 1 */ - h_n = _mm_add_epi16(src_2n, src_2n_2); + __m128i h_n = _mm_add_epi16(src_2n, src_2n_2); h_n = _mm_srai_epi16(h_n, 1); h_n = _mm_sub_epi16(src_2n_1, h_n); h_n = _mm_srai_epi16(h_n, 1); _mm_store_si128((__m128i*)h, h_n); - if (n == 0) - h_n_m = h_n; - else + __m128i h_n_m = h_n; + if (n != 0) h_n_m = _mm_load_si128((__m128i*)(h - total_width)); /* l[n] = src[2n] + ((h[n - 1] + h[n]) >> 1) */ - l_n = _mm_add_epi16(h_n_m, h_n); + __m128i l_n = _mm_add_epi16(h_n_m, h_n); l_n = _mm_srai_epi16(l_n, 1); l_n = _mm_add_epi16(l_n, src_2n); _mm_store_si128((__m128i*)l, l_n); @@ -395,44 +376,37 @@ rfx_dwt_2d_encode_block_vert_sse2(INT16* WINPR_RESTRICT src, INT16* WINPR_RESTRI static __inline void __attribute__((ATTRIBUTES)) rfx_dwt_2d_encode_block_horiz_sse2(INT16* WINPR_RESTRICT src, INT16* WINPR_RESTRICT l, - INT16* WINPR_RESTRICT h, int subband_width) + INT16* WINPR_RESTRICT h, size_t subband_width) { - int first = 0; - __m128i src_2n; - __m128i src_2n_1; - __m128i src_2n_2; - __m128i h_n; - __m128i h_n_m; - __m128i l_n; - - for (int y = 0; y < subband_width; y++) + for (size_t y = 0; y < subband_width; y++) { - for (int n = 0; n < subband_width; n += 8) + for (size_t n = 0; n < subband_width; n += 8) { /* The following 3 Set operations consumes more than half of the total DWT processing * time! */ - src_2n = + __m128i src_2n = _mm_set_epi16(src[14], src[12], src[10], src[8], src[6], src[4], src[2], src[0]); - src_2n_1 = + __m128i src_2n_1 = _mm_set_epi16(src[15], src[13], src[11], src[9], src[7], src[5], src[3], src[1]); - src_2n_2 = _mm_set_epi16(n == subband_width - 8 ? src[14] : src[16], src[14], src[12], - src[10], src[8], src[6], src[4], src[2]); + __m128i src_2n_2 = + _mm_set_epi16(((n + 8) == subband_width) ? src[14] : src[16], src[14], src[12], + src[10], src[8], src[6], src[4], src[2]); /* h[n] = (src[2n + 1] - ((src[2n] + src[2n + 2]) >> 1)) >> 1 */ - h_n = _mm_add_epi16(src_2n, src_2n_2); + __m128i h_n = _mm_add_epi16(src_2n, src_2n_2); h_n = _mm_srai_epi16(h_n, 1); h_n = _mm_sub_epi16(src_2n_1, h_n); h_n = _mm_srai_epi16(h_n, 1); _mm_store_si128((__m128i*)h, h_n); - h_n_m = _mm_loadu_si128((__m128i*)(h - 1)); + __m128i h_n_m = _mm_loadu_si128((__m128i*)(h - 1)); if (n == 0) { - first = _mm_extract_epi16(h_n_m, 1); + int first = _mm_extract_epi16(h_n_m, 1); h_n_m = _mm_insert_epi16(h_n_m, first, 0); } /* l[n] = src[2n] + ((h[n - 1] + h[n]) >> 1) */ - l_n = _mm_add_epi16(h_n_m, h_n); + __m128i l_n = _mm_add_epi16(h_n_m, h_n); l_n = _mm_srai_epi16(l_n, 1); l_n = _mm_add_epi16(l_n, src_2n); _mm_store_si128((__m128i*)l, l_n); @@ -445,27 +419,21 @@ rfx_dwt_2d_encode_block_horiz_sse2(INT16* WINPR_RESTRICT src, INT16* WINPR_RESTR static __inline void __attribute__((ATTRIBUTES)) rfx_dwt_2d_encode_block_sse2(INT16* WINPR_RESTRICT buffer, INT16* WINPR_RESTRICT dwt, - int subband_width) + size_t subband_width) { - INT16* hl = NULL; - INT16* lh = NULL; - INT16* hh = NULL; - INT16* ll = NULL; - INT16* l_src = NULL; - INT16* h_src = NULL; mm_prefetch_buffer((char*)dwt, 4ULL * subband_width * sizeof(INT16)); /* DWT in vertical direction, results in 2 sub-bands in L, H order in tmp buffer dwt. */ - l_src = dwt; - h_src = dwt + 2ULL * subband_width * subband_width; + INT16* l_src = dwt; + INT16* h_src = dwt + 2ULL * subband_width * subband_width; rfx_dwt_2d_encode_block_vert_sse2(buffer, l_src, h_src, subband_width); /* DWT in horizontal direction, results in 4 sub-bands in HL(0), LH(1), HH(2), LL(3) order, * stored in original buffer. */ /* The lower part L generates LL(3) and HL(0). */ /* The higher part H generates LH(1) and HH(2). */ - ll = buffer + 3ULL * subband_width * subband_width; - hl = buffer; - lh = buffer + 1ULL * subband_width * subband_width; - hh = buffer + 2ULL * subband_width * subband_width; + INT16* ll = buffer + 3ULL * subband_width * subband_width; + INT16* hl = buffer; + INT16* lh = buffer + 1ULL * subband_width * subband_width; + INT16* hh = buffer + 2ULL * subband_width * subband_width; rfx_dwt_2d_encode_block_horiz_sse2(l_src, ll, hl, subband_width); rfx_dwt_2d_encode_block_horiz_sse2(h_src, lh, hh, subband_width); } diff --git a/libfreerdp/codec/test/TestFreeRDPCodecCopy.c b/libfreerdp/codec/test/TestFreeRDPCodecCopy.c index 01725b6b9..bd990ac07 100644 --- a/libfreerdp/codec/test/TestFreeRDPCodecCopy.c +++ b/libfreerdp/codec/test/TestFreeRDPCodecCopy.c @@ -32,7 +32,7 @@ static BOOL TestFreeRDPImageCopy(UINT32 w, UINT32 h, UINT32 srcFormat, UINT32 ds NULL, 0); const UINT64 end = winpr_GetUnixTimeNS(); - double ms = end - start; + double ms = (double)(end - start); ms /= 1000000.0; (void)fprintf(stdout, @@ -70,7 +70,7 @@ static BOOL TestFreeRDPImageCopy_no_overlap(UINT32 w, UINT32 h, UINT32 srcFormat srcStep, 0, 0, NULL, 0); const UINT64 end = winpr_GetUnixTimeNS(); - double ms = end - start; + double ms = (double)(end - start); ms /= 1000000.0; (void)fprintf(stdout, diff --git a/libfreerdp/codec/test/TestFreeRDPCodecPlanar.c b/libfreerdp/codec/test/TestFreeRDPCodecPlanar.c index 52f32a2bb..04a910f4b 100644 --- a/libfreerdp/codec/test/TestFreeRDPCodecPlanar.c +++ b/libfreerdp/codec/test/TestFreeRDPCodecPlanar.c @@ -5426,7 +5426,7 @@ static BOOL CompareBitmap(const BYTE* srcA, UINT32 srcAFormat, const BYTE* srcB, double maxDiff = NAN; const UINT32 srcABits = FreeRDPGetBitsPerPixel(srcAFormat); const UINT32 srcBBits = FreeRDPGetBitsPerPixel(srcBFormat); - UINT32 diff = fabs((double)srcABits - srcBBits); + UINT32 diff = WINPR_ASSERTING_INT_CAST(uint32_t, fabs((double)srcABits - srcBBits)); /* No support for 8bpp */ if ((srcABits < 15) || (srcBBits < 15)) @@ -5757,9 +5757,10 @@ static BOOL FuzzPlanar(void) ", nDstHeight=%" PRIu32 ", nDstStep=%" PRIu32 ", total size=%" PRIuz "\n", FreeRDPGetColorFormatName(DstFormat), nXDst, nYDst, nDstWidth, nDstHeight, nDstStep, sizeof(dstData)); - freerdp_planar_switch_bgr(planar, prand(2) % 2); + freerdp_planar_switch_bgr(planar, ((prand(2) % 2) != 0) ? TRUE : FALSE); planar_decompress(planar, data, dataSize, prand(4096), prand(4096), dstData, DstFormat, - nDstStep, nXDst, nYDst, nDstWidth, nDstHeight, prand(2)); + nDstStep, nXDst, nYDst, nDstWidth, nDstHeight, + ((prand(2) % 2) != 0) ? TRUE : FALSE); } rc = TRUE; diff --git a/libfreerdp/codec/test/TestFreeRDPCodecProgressive.c b/libfreerdp/codec/test/TestFreeRDPCodecProgressive.c index 3b0cda61b..43c49e7d9 100644 --- a/libfreerdp/codec/test/TestFreeRDPCodecProgressive.c +++ b/libfreerdp/codec/test/TestFreeRDPCodecProgressive.c @@ -156,13 +156,13 @@ static void sample_file_free(EGFX_SAMPLE_FILE* file) file->size = 0; } -static void test_fill_image_alpha_channel(BYTE* data, int width, int height, BYTE value) +static void test_fill_image_alpha_channel(BYTE* data, UINT32 width, UINT32 height, BYTE value) { UINT32* pixel = NULL; - for (int i = 0; i < height; i++) + for (UINT32 i = 0; i < height; i++) { - for (int j = 0; j < width; j++) + for (UINT32 j = 0; j < width; j++) { pixel = (UINT32*)&data[((1ULL * i * width) + j) * 4ULL]; *pixel = ((*pixel & 0x00FFFFFF) | (value << 24)); @@ -180,15 +180,15 @@ static void* test_image_memset32(UINT32* ptr, UINT32 fill, size_t length) return (void*)ptr; } -static int test_image_fill(BYTE* pDstData, int nDstStep, int nXDst, int nYDst, int nWidth, - int nHeight, UINT32 color) +static int test_image_fill(BYTE* pDstData, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, + UINT32 nWidth, UINT32 nHeight, UINT32 color) { UINT32* pDstPixel = NULL; if (nDstStep < 0) nDstStep = 4 * nWidth; - for (int y = 0; y < nHeight; y++) + for (UINT32 y = 0; y < nHeight; y++) { pDstPixel = (UINT32*)&pDstData[((nYDst + y) * nDstStep) + (nXDst * 4)]; test_image_memset32(pDstPixel, color, nWidth); @@ -197,13 +197,13 @@ static int test_image_fill(BYTE* pDstData, int nDstStep, int nXDst, int nYDst, i return 1; } -static int test_image_fill_quarter(BYTE* pDstData, int nDstStep, int nWidth, int nHeight, - UINT32 color, int quarter) +static int test_image_fill_quarter(BYTE* pDstData, UINT32 nDstStep, UINT32 nWidth, UINT32 nHeight, + UINT32 color, UINT32 quarter) { - int x = 0; - int y = 0; - int width = 0; - int height = 0; + UINT32 x = 0; + UINT32 y = 0; + UINT32 width = 0; + UINT32 height = 0; switch (quarter) { @@ -242,8 +242,8 @@ static int test_image_fill_quarter(BYTE* pDstData, int nDstStep, int nWidth, int return 1; } -static int test_image_fill_unused_quarters(BYTE* pDstData, int nDstStep, int nWidth, int nHeight, - UINT32 color, int quarter) +static int test_image_fill_unused_quarters(BYTE* pDstData, UINT32 nDstStep, UINT32 nWidth, + UINT32 nHeight, UINT32 color, UINT32 quarter) { return 1; @@ -888,8 +888,8 @@ static int test_progressive_decode(PROGRESSIVE_CONTEXT* progressive, EGFX_SAMPLE if ((nWidth <= 0) || (nHeight <= 0)) continue; - nXSrc = nXDst - tile->x; - nYSrc = nYDst - tile->y; + nXSrc = nXDst - WINPR_ASSERTING_INT_CAST(int, tile->x); + nYSrc = nYDst - WINPR_ASSERTING_INT_CAST(int, tile->y); freerdp_image_copy(g_DstData, PIXEL_FORMAT_XRGB32, g_DstStep, nXDst, nYDst, nWidth, nHeight, tile->data, PIXEL_FORMAT_XRGB32, 64 * 4, nXSrc, nYSrc, NULL, FREERDP_FLIP_NONE); diff --git a/libfreerdp/codec/test/TestFreeRDPRegion.c b/libfreerdp/codec/test/TestFreeRDPRegion.c index e192d2d99..e8a18573e 100644 --- a/libfreerdp/codec/test/TestFreeRDPRegion.c +++ b/libfreerdp/codec/test/TestFreeRDPRegion.c @@ -67,7 +67,8 @@ static int test_basic(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 2 || !compareRectangles(rects, r1_r2, nbRects)) + if (!rects || nbRects != 2 || + !compareRectangles(rects, r1_r2, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; /* clear region */ @@ -114,7 +115,8 @@ static int test_r1_r3(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 3 || !compareRectangles(rects, r1_r3, nbRects)) + if (!rects || nbRects != 3 || + !compareRectangles(rects, r1_r3, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; /* R3 + R1 */ @@ -128,7 +130,8 @@ static int test_r1_r3(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 3 || !compareRectangles(rects, r1_r3, nbRects)) + if (!rects || nbRects != 3 || + !compareRectangles(rects, r1_r3, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -173,7 +176,8 @@ static int test_r9_r10(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 3 || !compareRectangles(rects, r9_r10, nbRects)) + if (!rects || nbRects != 3 || + !compareRectangles(rects, r9_r10, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -214,7 +218,8 @@ static int test_r1_r5(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 3 || !compareRectangles(rects, r1_r5, nbRects)) + if (!rects || nbRects != 3 || + !compareRectangles(rects, r1_r5, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -254,7 +259,8 @@ static int test_r1_r6(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 1 || !compareRectangles(rects, &r1, nbRects)) + if (!rects || nbRects != 1 || + !compareRectangles(rects, &r1, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -303,7 +309,8 @@ static int test_r1_r2_r4(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 2 || !compareRectangles(rects, r1_r2_r4, nbRects)) + if (!rects || nbRects != 2 || + !compareRectangles(rects, r1_r2_r4, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -350,7 +357,8 @@ static int test_r1_r7_r8(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 5 || !compareRectangles(rects, r1_r7_r8, nbRects)) + if (!rects || nbRects != 5 || + !compareRectangles(rects, r1_r7_r8, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; region16_clear(®ion); @@ -366,7 +374,8 @@ static int test_r1_r7_r8(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 5 || !compareRectangles(rects, r1_r7_r8, nbRects)) + if (!rects || nbRects != 5 || + !compareRectangles(rects, r1_r7_r8, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; region16_clear(®ion); @@ -382,7 +391,8 @@ static int test_r1_r7_r8(void) rects = region16_rects(®ion, &nbRects); - if (!rects || nbRects != 5 || !compareRectangles(rects, r1_r7_r8, nbRects)) + if (!rects || nbRects != 5 || + !compareRectangles(rects, r1_r7_r8, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -557,7 +567,8 @@ static int test_r1_inter_r3(void) rects = region16_rects(&intersection, &nbRects); - if (!rects || nbRects != 1 || !compareRectangles(rects, r1_inter_r3, nbRects)) + if (!rects || nbRects != 1 || + !compareRectangles(rects, r1_inter_r3, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -617,7 +628,8 @@ static int test_r1_r3_inter_r11(void) rects = region16_rects(&intersection, &nbRects); - if (!rects || nbRects != 1 || !compareRectangles(rects, r1_r3_inter_r11, nbRects)) + if (!rects || nbRects != 1 || + !compareRectangles(rects, r1_r3_inter_r11, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; retCode = 0; @@ -684,7 +696,8 @@ static int test_norbert_case(void) rects = region16_rects(&intersection, &nbRects); - if (!rects || nbRects != 5 || !compareRectangles(rects, inRectangles, nbRects)) + if (!rects || nbRects != 5 || + !compareRectangles(rects, inRectangles, WINPR_ASSERTING_INT_CAST(int, nbRects))) goto out; if (!compareRectangles(region16_extents(&intersection), &expected_inter_extents, 1)) diff --git a/libfreerdp/codec/xcrush.c b/libfreerdp/codec/xcrush.c index 81670b20f..3638b43c1 100644 --- a/libfreerdp/codec/xcrush.c +++ b/libfreerdp/codec/xcrush.c @@ -20,6 +20,8 @@ */ #include +#include + #include #include @@ -159,10 +161,10 @@ static const char* xcrush_get_level_1_compression_flags_string(UINT32 flags) } #endif -static UINT32 xcrush_update_hash(const BYTE* WINPR_RESTRICT data, UINT32 size) +static UINT16 xcrush_update_hash(const BYTE* WINPR_RESTRICT data, UINT32 size) { const BYTE* end = NULL; - UINT32 seed = 5381; /* same value as in djb2 */ + UINT16 seed = 5381; /* same value as in djb2 */ WINPR_ASSERT(data); WINPR_ASSERT(size >= 4); @@ -181,7 +183,7 @@ static UINT32 xcrush_update_hash(const BYTE* WINPR_RESTRICT data, UINT32 size) data += 4; } - return (UINT16)seed; + return seed; } static int xcrush_append_chunk(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, @@ -204,8 +206,8 @@ static int xcrush_append_chunk(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, if (size >= 15) { - UINT32 seed = xcrush_update_hash(&data[*beg], (UINT16)size); - xcrush->Signatures[xcrush->SignatureIndex].size = size; + const UINT16 seed = xcrush_update_hash(&data[*beg], WINPR_ASSERTING_INT_CAST(UINT16, size)); + xcrush->Signatures[xcrush->SignatureIndex].size = WINPR_ASSERTING_INT_CAST(UINT16, size); xcrush->Signatures[xcrush->SignatureIndex].seed = seed; xcrush->SignatureIndex++; *beg = end; @@ -368,9 +370,6 @@ static int xcrush_insert_chunk(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, XCRUSH_SIGNATURE* WINPR_RESTRICT signature, UINT32 offset, XCRUSH_CHUNK** WINPR_RESTRICT pPrevChunk) { - UINT32 seed = 0; - UINT32 index = 0; - WINPR_ASSERT(xcrush); if (xcrush->ChunkHead >= 65530) @@ -385,16 +384,13 @@ static int xcrush_insert_chunk(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, xcrush->ChunkTail += 10000; } - index = xcrush->ChunkHead++; + const UINT32 index = xcrush->ChunkHead++; if (xcrush->ChunkHead >= 65534) return -3001; /* error */ xcrush->Chunks[index].offset = offset; - seed = signature->seed; - - if (seed >= 65536) - return -3002; /* error */ + const UINT16 seed = signature->seed; if (xcrush->NextChunks[seed]) { @@ -406,7 +402,7 @@ static int xcrush_insert_chunk(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, } xcrush->Chunks[index].next = xcrush->NextChunks[seed] & 0xFFFF; - xcrush->NextChunks[seed] = index; + xcrush->NextChunks[seed] = WINPR_ASSERTING_INT_CAST(UINT16, index); return 1; } @@ -687,7 +683,6 @@ static int xcrush_generate_output(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, UINT32 MatchIndex = 0; UINT32 MatchOffset = 0; UINT16 MatchLength = 0; - UINT32 MatchCount = 0; UINT32 CurrentOffset = 0; UINT32 MatchOffsetDiff = 0; UINT32 HistoryOffsetDiff = 0; @@ -698,13 +693,13 @@ static int xcrush_generate_output(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, WINPR_ASSERT(OutputSize >= 2); WINPR_ASSERT(pDstSize); - MatchCount = xcrush->OptimizedMatchCount; + const UINT32 MatchCount = xcrush->OptimizedMatchCount; OutputEnd = &OutputBuffer[OutputSize]; if (&OutputBuffer[2] >= &OutputBuffer[OutputSize]) return -6001; /* error */ - winpr_Data_Write_UINT16(OutputBuffer, MatchCount); + winpr_Data_Write_UINT16(OutputBuffer, WINPR_ASSERTING_INT_CAST(UINT16, MatchCount)); MatchDetails = (RDP61_MATCH_DETAILS*)&OutputBuffer[2]; Literals = (BYTE*)&MatchDetails[MatchCount]; @@ -713,10 +708,16 @@ static int xcrush_generate_output(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, for (MatchIndex = 0; MatchIndex < MatchCount; MatchIndex++) { + const UINT32 len = xcrush->OptimizedMatches[MatchIndex].MatchLength; winpr_Data_Write_UINT16(&MatchDetails[MatchIndex].MatchLength, - xcrush->OptimizedMatches[MatchIndex].MatchLength); + WINPR_ASSERTING_INT_CAST(UINT16, len)); + + const UINT32 moff = xcrush->OptimizedMatches[MatchIndex].MatchOffset; + WINPR_ASSERT(moff >= HistoryOffset); + + const UINT32 off = moff - HistoryOffset; winpr_Data_Write_UINT16(&MatchDetails[MatchIndex].MatchOutputOffset, - xcrush->OptimizedMatches[MatchIndex].MatchOffset - HistoryOffset); + WINPR_ASSERTING_INT_CAST(UINT16, off)); winpr_Data_Write_UINT32(&MatchDetails[MatchIndex].MatchHistoryOffset, xcrush->OptimizedMatches[MatchIndex].ChunkOffset); } @@ -770,7 +771,7 @@ static INLINE size_t xcrush_copy_bytes_no_overlap(BYTE* WINPR_RESTRICT dst, { // src and dst overlaps // we should copy the area that doesn't overlap repeatedly - const size_t diff = (dst > src) ? dst - src : src - dst; + const size_t diff = WINPR_ASSERTING_INT_CAST(size_t, (dst > src) ? dst - src : src - dst); const size_t rest = num % diff; const size_t end = num - rest; @@ -905,7 +906,7 @@ static int xcrush_decompress_l1(XCRUSH_CONTEXT* WINPR_RESTRICT xcrush, if (Literals < pSrcEnd) { - OutputLength = pSrcEnd - Literals; + OutputLength = WINPR_ASSERTING_INT_CAST(size_t, pSrcEnd - Literals); if ((&HistoryPtr[OutputLength] >= HistoryBufferEnd) || (&Literals[OutputLength] > pSrcEnd)) return -1012; diff --git a/libfreerdp/codec/yuv.c b/libfreerdp/codec/yuv.c index 5aeb7ed0e..bad8b4e8e 100644 --- a/libfreerdp/codec/yuv.c +++ b/libfreerdp/codec/yuv.c @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -90,8 +91,8 @@ static INLINE BOOL avc420_yuv_to_rgb(const BYTE* WINPR_RESTRICT pYUVData[3], pYUVPoint[1] = pYUVData[1] + 1ULL * rect->top / 2 * iStride[1] + rect->left / 2; pYUVPoint[2] = pYUVData[2] + 1ULL * rect->top / 2 * iStride[2] + rect->left / 2; - roi.width = width; - roi.height = height; + roi.width = WINPR_ASSERTING_INT_CAST(uint32_t, width); + roi.height = WINPR_ASSERTING_INT_CAST(uint32_t, height); if (prims->YUV420ToRGB_8u_P3AC4R(pYUVPoint, iStride, pDstPoint, nDstStep, DstFormat, &roi) != PRIMITIVES_SUCCESS) @@ -123,8 +124,8 @@ static INLINE BOOL avc444_yuv_to_rgb(const BYTE* WINPR_RESTRICT pYUVData[3], pYUVPoint[1] = pYUVData[1] + 1ULL * rect->top * iStride[1] + rect->left; pYUVPoint[2] = pYUVData[2] + 1ULL * rect->top * iStride[2] + rect->left; - roi.width = width; - roi.height = height; + roi.width = WINPR_ASSERTING_INT_CAST(uint32_t, width); + roi.height = WINPR_ASSERTING_INT_CAST(uint32_t, height); if (prims->YUV444ToRGB_8u_P3AC4R(pYUVPoint, iStride, pDstPoint, nDstStep, DstFormat, &roi) != PRIMITIVES_SUCCESS) @@ -379,9 +380,9 @@ static RECTANGLE_16 clamp(YUV_CONTEXT* WINPR_RESTRICT context, RECTANGLE_16 c = *rect; const UINT32 height = MIN(context->height, srcHeight); if (c.top > height) - c.top = height; + c.top = WINPR_ASSERTING_INT_CAST(UINT16, height); if (c.bottom > height) - c.bottom = height; + c.bottom = WINPR_ASSERTING_INT_CAST(UINT16, height); return c; } diff --git a/libfreerdp/codec/zgfx.c b/libfreerdp/codec/zgfx.c index 96d8acc3f..0310218b5 100644 --- a/libfreerdp/codec/zgfx.c +++ b/libfreerdp/codec/zgfx.c @@ -21,6 +21,8 @@ #include +#include +#include #include #include #include @@ -181,34 +183,37 @@ static INLINE void zgfx_history_buffer_ring_read(ZGFX_CONTEXT* WINPR_RESTRICT zg return; bytesLeft = (INT32)count; - index = (zgfx->HistoryIndex + zgfx->HistoryBufferSize - offset) % zgfx->HistoryBufferSize; + index = (zgfx->HistoryIndex + zgfx->HistoryBufferSize - + WINPR_ASSERTING_INT_CAST(uint32_t, offset)) % + zgfx->HistoryBufferSize; bytes = MIN(bytesLeft, offset); - if ((index + bytes) <= zgfx->HistoryBufferSize) + if ((index + WINPR_ASSERTING_INT_CAST(uint32_t, bytes)) <= zgfx->HistoryBufferSize) { - CopyMemory(dptr, &(zgfx->HistoryBuffer[index]), bytes); + CopyMemory(dptr, &(zgfx->HistoryBuffer[index]), WINPR_ASSERTING_INT_CAST(size_t, bytes)); } else { front = zgfx->HistoryBufferSize - index; CopyMemory(dptr, &(zgfx->HistoryBuffer[index]), front); - CopyMemory(&dptr[front], zgfx->HistoryBuffer, bytes - front); + CopyMemory(&dptr[front], zgfx->HistoryBuffer, + WINPR_ASSERTING_INT_CAST(uint32_t, bytes) - front); } if ((bytesLeft -= bytes) == 0) return; dptr += bytes; - valid = bytes; + valid = WINPR_ASSERTING_INT_CAST(uint32_t, bytes); do { - bytes = valid; + bytes = WINPR_ASSERTING_INT_CAST(int32_t, valid); if (bytes > bytesLeft) bytes = bytesLeft; - CopyMemory(dptr, origDst, bytes); + CopyMemory(dptr, origDst, WINPR_ASSERTING_INT_CAST(size_t, bytes)); dptr += bytes; valid <<= 1; } while ((bytesLeft -= bytes) > 0); @@ -334,8 +339,9 @@ static INLINE BOOL zgfx_decompress_segment(ZGFX_CONTEXT* WINPR_RESTRICT zgfx, if (count > sizeof(zgfx->OutputBuffer) - zgfx->OutputCount) return FALSE; - zgfx_history_buffer_ring_read( - zgfx, distance, &(zgfx->OutputBuffer[zgfx->OutputCount]), count); + zgfx_history_buffer_ring_read(zgfx, WINPR_ASSERTING_INT_CAST(int, distance), + &(zgfx->OutputBuffer[zgfx->OutputCount]), + count); zgfx_history_buffer_ring_write( zgfx, &(zgfx->OutputBuffer[zgfx->OutputCount]), count); zgfx->OutputCount += count; @@ -504,7 +510,7 @@ static BOOL zgfx_compress_segment(ZGFX_CONTEXT* WINPR_RESTRICT zgfx, wStream* WI } (*pFlags) |= ZGFX_PACKET_COMPR_TYPE_RDP8; /* RDP 8.0 compression format */ - Stream_Write_UINT8(s, (*pFlags)); /* header (1 byte) */ + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, *pFlags)); /* header (1 byte) */ Stream_Write(s, pSrcData, SrcSize); return TRUE; } @@ -586,7 +592,7 @@ int zgfx_compress_to_stream(ZGFX_CONTEXT* WINPR_RESTRICT zgfx, wStream* WINPR_RE if (posSegmentCount) { Stream_SetPosition(sDst, posSegmentCount); - Stream_Write_UINT16(sDst, fragment); + Stream_Write_UINT16(sDst, WINPR_ASSERTING_INT_CAST(uint16_t, fragment)); Stream_SetPosition(sDst, Stream_Length(sDst)); } diff --git a/libfreerdp/common/assistance.c b/libfreerdp/common/assistance.c index ab0e6436e..02d56b7fd 100644 --- a/libfreerdp/common/assistance.c +++ b/libfreerdp/common/assistance.c @@ -453,7 +453,7 @@ static BOOL freerdp_assistance_parse_attr(const char** opt, size_t* plength, con key); return FALSE; } - const size_t length = q - p; + const size_t length = WINPR_ASSERTING_INT_CAST(size_t, q - p); *opt = p; *plength = length; @@ -498,13 +498,15 @@ static BOOL freerdp_assistance_parse_attr_uint32(UINT32* opt, const char* key, c return FALSE; char buffer[64] = { 0 }; - if (size >= sizeof(buffer)) + if ((!copt && (size > 0)) || (size >= sizeof(buffer))) { WLog_WARN(TAG, "Invalid UINT32 string '%s' [%" PRIuz "]", copt, size); return FALSE; } - strncpy(buffer, copt, size); + if (size > 0) + strncpy(buffer, copt, size); + errno = 0; unsigned long val = strtoul(buffer, NULL, 0); @@ -573,7 +575,7 @@ static char* freerdp_assistance_contains_element(char* input, size_t ilen, const WINPR_ASSERT((size_t)erc < sizeof(ekey)); if ((erc <= 0) || ((size_t)erc >= sizeof(ekey))) return NULL; - const size_t offset = start - tag; + const size_t offset = WINPR_ASSERTING_INT_CAST(size_t, start - tag); dend = end = strrstr(start, ilen - offset, ekey); if (end) end += strnlen(ekey, sizeof(ekey)); @@ -587,12 +589,12 @@ static char* freerdp_assistance_contains_element(char* input, size_t ilen, const return NULL; } if (plen) - *plen = end - tag; + *plen = WINPR_ASSERTING_INT_CAST(size_t, end - tag); if (pdata) *pdata = data; if (pdlen) - *pdlen = dend - data; + *pdlen = WINPR_ASSERTING_INT_CAST(size_t, dend - data); return tag; } @@ -644,7 +646,7 @@ static BOOL freerdp_assistance_get_element(char* input, size_t ilen, const char* char* end = tag + len; *element = data; - *elen = end - data + 1; + *elen = WINPR_ASSERTING_INT_CAST(size_t, end - data + 1); return TRUE; } diff --git a/libfreerdp/common/settings.c b/libfreerdp/common/settings.c index 02f502db6..13478b057 100644 --- a/libfreerdp/common/settings.c +++ b/libfreerdp/common/settings.c @@ -29,6 +29,7 @@ #include #include +#include #include "../core/settings.h" #include "../core/capabilities.h" @@ -51,7 +52,8 @@ BOOL freerdp_addin_argv_add_argument_ex(ADDIN_ARGV* args, const char* argument, if (len == 0) len = strlen(argument); - new_argv = (char**)realloc((void*)args->argv, sizeof(char*) * (args->argc + 1)); + new_argv = (char**)realloc( + (void*)args->argv, sizeof(char*) * (WINPR_ASSERTING_INT_CAST(uint32_t, args->argc) + 1)); if (!new_argv) return FALSE; @@ -81,8 +83,10 @@ BOOL freerdp_addin_argv_del_argument(ADDIN_ARGV* args, const char* argument) if (strcmp(argument, arg) == 0) { free(arg); - memmove_s((void*)&args->argv[x], (args->argc - x) * sizeof(char*), - (void*)&args->argv[x + 1], (args->argc - x - 1) * sizeof(char*)); + memmove_s((void*)&args->argv[x], + (WINPR_ASSERTING_INT_CAST(uint32_t, args->argc - x)) * sizeof(char*), + (void*)&args->argv[x + 1], + (WINPR_ASSERTING_INT_CAST(uint32_t, args->argc - x - 1)) * sizeof(char*)); args->argv[args->argc - 1] = NULL; args->argc--; return TRUE; @@ -154,7 +158,8 @@ int freerdp_addin_set_argument_value(ADDIN_ARGV* args, const char* option, const if (p) { - if (strncmp(args->argv[i], option, p - args->argv[i]) == 0) + if (strncmp(args->argv[i], option, + WINPR_ASSERTING_INT_CAST(size_t, p - args->argv[i])) == 0) { free(args->argv[i]); args->argv[i] = str; @@ -829,7 +834,7 @@ ADDIN_ARGV* freerdp_addin_argv_clone(const ADDIN_ARGV* args) if (!args) return NULL; cnv.c = args->argv; - return freerdp_addin_argv_new(args->argc, cnv.cc); + return freerdp_addin_argv_new(WINPR_ASSERTING_INT_CAST(uint32_t, args->argc), cnv.cc); } void freerdp_dynamic_channel_collection_free(rdpSettings* settings) @@ -1780,7 +1785,7 @@ BOOL freerdp_settings_set_pointer_array(rdpSettings* settings, FreeRDP_Settings_ goto fail; if ((offset >= maxOffset) || !data) goto fail; - settings->OrderSupport[offset] = *(const BOOL*)data; + settings->OrderSupport[offset] = *(const BOOL*)data ? 1 : 0; return TRUE; case FreeRDP_GlyphCache: maxOffset = 10; @@ -1846,11 +1851,11 @@ UINT32 freerdp_settings_get_codecs_flags(const rdpSettings* settings) UINT32 flags = FREERDP_CODEC_ALL; if (settings->RemoteFxCodec == FALSE) { - flags &= ~FREERDP_CODEC_REMOTEFX; + flags &= (uint32_t)~FREERDP_CODEC_REMOTEFX; } if (settings->NSCodec == FALSE) { - flags &= ~FREERDP_CODEC_NSCODEC; + flags &= (uint32_t)~FREERDP_CODEC_NSCODEC; } /*TODO: check other codecs flags */ return flags; diff --git a/libfreerdp/core/aad.c b/libfreerdp/core/aad.c index 3f67da68e..9dd234ac9 100644 --- a/libfreerdp/core/aad.c +++ b/libfreerdp/core/aad.c @@ -72,7 +72,7 @@ static SSIZE_T stream_sprintf(wStream* s, WINPR_FORMAT_ARG const char* fmt, ...) char* ptr = Stream_PointerAs(s, char); va_start(ap, fmt); - const int rc2 = vsnprintf(ptr, rc + 1, fmt, ap); + const int rc2 = vsnprintf(ptr, WINPR_ASSERTING_INT_CAST(size_t, rc) + 1, fmt, ap); va_end(ap); if (rc != rc2) return -23; @@ -258,17 +258,17 @@ int aad_client_begin(rdpAad* aad) /* Get the host part of the hostname */ const char* hostname = freerdp_settings_get_string(settings, FreeRDP_AadServerHostname); if (!hostname) - hostname = freerdp_settings_get_server_name(settings); + hostname = freerdp_settings_get_string(settings, FreeRDP_ServerHostname); if (!hostname) { - WLog_Print(aad->log, WLOG_ERROR, "hostname == NULL"); + WLog_Print(aad->log, WLOG_ERROR, "FreeRDP_ServerHostname == NULL"); return -1; } aad->hostname = _strdup(hostname); if (!aad->hostname) { - WLog_Print(aad->log, WLOG_ERROR, "_strdup(hostname) == NULL"); + WLog_Print(aad->log, WLOG_ERROR, "_strdup(FreeRDP_ServerHostname) == NULL"); return -1; } diff --git a/libfreerdp/core/activation.c b/libfreerdp/core/activation.c index a9a68ae9e..96492eb9f 100644 --- a/libfreerdp/core/activation.c +++ b/libfreerdp/core/activation.c @@ -22,6 +22,7 @@ #include "settings.h" #include +#include #include "activation.h" #include "display.h" @@ -39,7 +40,8 @@ static BOOL rdp_write_synchronize_pdu(wStream* s, const rdpSettings* settings) if (!Stream_CheckAndLogRequiredCapacity(TAG, (s), 4)) return FALSE; Stream_Write_UINT16(s, SYNCMSGTYPE_SYNC); /* messageType (2 bytes) */ - Stream_Write_UINT16(s, PduSource); /* targetUser (2 bytes) */ + Stream_Write_UINT16(s, + WINPR_ASSERTING_INT_CAST(uint16_t, PduSource)); /* targetUser (2 bytes) */ return TRUE; } @@ -331,13 +333,14 @@ error: BOOL rdp_send_client_persistent_key_list_pdu(rdpRdp* rdp) { - UINT32 keyMaxFrag = 2042; + UINT16 keyMaxFrag = 2042; UINT64* keyList = NULL; RDP_BITMAP_PERSISTENT_INFO info = { 0 }; + WINPR_ASSERT(rdp); rdpSettings* settings = rdp->settings; - UINT32 keyCount = rdp_load_persistent_key_list(rdp, &keyList); + UINT16 keyCount = rdp_load_persistent_key_list(rdp, &keyList); - WLog_DBG(TAG, "Persistent Key List: TotalKeyCount: %" PRIu32 " MaxKeyFrag: %" PRIu32, keyCount, + WLog_DBG(TAG, "Persistent Key List: TotalKeyCount: %" PRIu16 " MaxKeyFrag: %" PRIu16, keyCount, keyMaxFrag); // MS-RDPBCGR recommends sending no more than 169 entries at once. diff --git a/libfreerdp/core/capabilities.c b/libfreerdp/core/capabilities.c index 6dc1603eb..a33a16a55 100644 --- a/libfreerdp/core/capabilities.c +++ b/libfreerdp/core/capabilities.c @@ -18,6 +18,8 @@ */ #include +#include +#include #include #include "settings.h" @@ -1838,9 +1840,13 @@ static BOOL rdp_write_offscreen_bitmap_cache_capability_set(wStream* s, const rd { offscreenSupportLevel = 0x01; Stream_Write_UINT32(s, offscreenSupportLevel); /* offscreenSupportLevel (4 bytes) */ - Stream_Write_UINT16(s, settings->OffscreenCacheSize); /* offscreenCacheSize (2 bytes) */ - Stream_Write_UINT16(s, - settings->OffscreenCacheEntries); /* offscreenCacheEntries (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST( + uint16_t, settings->OffscreenCacheSize)); /* offscreenCacheSize (2 bytes) */ + Stream_Write_UINT16( + s, + WINPR_ASSERTING_INT_CAST( + uint16_t, settings->OffscreenCacheEntries)); /* offscreenCacheEntries (2 bytes) */ } else Stream_Zero(s, 8); @@ -1969,7 +1975,7 @@ static void rdp_write_bitmap_cache_cell_info(wStream* s, BITMAP_CACHE_V2_CELL_IN * is used to indicate a persistent bitmap cache. */ WINPR_ASSERT(cellInfo); - info = (cellInfo->numEntries | (cellInfo->persistent << 31)); + info = (cellInfo->numEntries | (((UINT32)cellInfo->persistent << 31) & 0xFF000000)); Stream_Write_UINT32(s, info); } @@ -2081,7 +2087,9 @@ static BOOL rdp_write_bitmap_cache_v2_capability_set(wStream* s, const rdpSettin Stream_Write_UINT16(s, cacheFlags); /* cacheFlags (2 bytes) */ Stream_Write_UINT8(s, 0); /* pad2 (1 byte) */ - Stream_Write_UINT8(s, settings->BitmapCacheV2NumCells); /* numCellCaches (1 byte) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, + settings->BitmapCacheV2NumCells)); /* numCellCaches (1 byte) */ rdp_write_bitmap_cache_cell_info( s, &settings->BitmapCacheV2CellInfo[0]); /* bitmapCache0CellInfo (4 bytes) */ rdp_write_bitmap_cache_cell_info( @@ -2143,13 +2151,13 @@ static BOOL rdp_apply_virtual_channel_capability_set(rdpSettings* settings, cons (src->VCFlags & VCCAPS_COMPR_SC)) settings->VCFlags |= VCCAPS_COMPR_SC; else - settings->VCFlags &= ~VCCAPS_COMPR_SC; + settings->VCFlags &= (uint32_t)~VCCAPS_COMPR_SC; if (!settings->ServerMode && (settings->VCFlags & VCCAPS_COMPR_CS_8K) && (src->VCFlags & VCCAPS_COMPR_CS_8K)) settings->VCFlags |= VCCAPS_COMPR_CS_8K; else - settings->VCFlags &= ~VCCAPS_COMPR_CS_8K; + settings->VCFlags &= (uint32_t)~VCCAPS_COMPR_CS_8K; /* * When one peer does not write the VCChunkSize, the VCChunkSize must not be @@ -2289,9 +2297,13 @@ static BOOL rdp_write_draw_nine_grid_cache_capability_set(wStream* s, const rdpS const UINT32 drawNineGridSupportLevel = (settings->DrawNineGridEnabled) ? DRAW_NINEGRID_SUPPORTED_V2 : DRAW_NINEGRID_NO_SUPPORT; Stream_Write_UINT32(s, drawNineGridSupportLevel); /* drawNineGridSupportLevel (4 bytes) */ - Stream_Write_UINT16(s, settings->DrawNineGridCacheSize); /* drawNineGridCacheSize (2 bytes) */ Stream_Write_UINT16( - s, settings->DrawNineGridCacheEntries); /* drawNineGridCacheEntries (2 bytes) */ + s, WINPR_ASSERTING_INT_CAST( + uint16_t, settings->DrawNineGridCacheSize)); /* drawNineGridCacheSize (2 bytes) */ + Stream_Write_UINT16( + s, + WINPR_ASSERTING_INT_CAST( + uint16_t, settings->DrawNineGridCacheEntries)); /* drawNineGridCacheEntries (2 bytes) */ return rdp_capability_set_finish(s, header, CAPSET_TYPE_DRAW_NINE_GRID_CACHE); } @@ -2564,9 +2576,13 @@ static BOOL rdp_write_window_list_capability_set(wStream* s, const rdpSettings* const size_t header = rdp_capability_set_start(s); Stream_Write_UINT32(s, settings->RemoteWndSupportLevel); /* wndSupportLevel (4 bytes) */ - Stream_Write_UINT8(s, settings->RemoteAppNumIconCaches); /* numIconCaches (1 byte) */ - Stream_Write_UINT16(s, - settings->RemoteAppNumIconCacheEntries); /* numIconCacheEntries (2 bytes) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, + settings->RemoteAppNumIconCaches)); /* numIconCaches (1 byte) */ + Stream_Write_UINT16( + s, + WINPR_ASSERTING_INT_CAST( + uint16_t, settings->RemoteAppNumIconCacheEntries)); /* numIconCacheEntries (2 bytes) */ return rdp_capability_set_finish(s, header, CAPSET_TYPE_WINDOW); } @@ -2983,9 +2999,9 @@ static BOOL rdp_read_bitmap_codec_guid(wStream* s, GUID* guid) if (!Stream_CheckAndLogRequiredLength(TAG, s, 16)) return FALSE; Stream_Read(s, g, 16); - guid->Data1 = ((UINT32)g[3] << 24U) | ((UINT32)g[2] << 16U) | (g[1] << 8U) | g[0]; - guid->Data2 = (g[5] << 8U) | g[4]; - guid->Data3 = (g[7] << 8U) | g[6]; + guid->Data1 = ((UINT32)g[3] << 24U) | ((UINT32)g[2] << 16U) | (UINT32)(g[1] << 8U) | g[0]; + guid->Data2 = ((g[5] << 8U) | g[4]) & 0xFFFF; + guid->Data3 = ((g[7] << 8U) | g[6]) & 0xFFFF; guid->Data4[0] = g[8]; guid->Data4[1] = g[9]; guid->Data4[2] = g[10]; @@ -3447,15 +3463,14 @@ static BOOL rdp_read_bitmap_codecs_capability_set(wStream* s, rdpSettings* setti */ static BOOL rdp_write_rfx_client_capability_container(wStream* s, const rdpSettings* settings) { - UINT32 captureFlags = 0; - BYTE codecMode = 0; - WINPR_ASSERT(settings); if (!Stream_EnsureRemainingCapacity(s, 64)) return FALSE; - captureFlags = settings->RemoteFxOnly ? 0 : CARDP_CAPS_CAPTURE_NON_CAC; - codecMode = settings->RemoteFxCodecMode; + const UINT32 captureFlags = settings->RemoteFxOnly ? 0 : CARDP_CAPS_CAPTURE_NON_CAC; + + WINPR_ASSERT(settings->RemoteFxCodecMode <= UINT8_MAX); + const UINT8 codecMode = (UINT8)settings->RemoteFxCodecMode; Stream_Write_UINT16(s, 49); /* codecPropertiesLength */ /* TS_RFX_CLNT_CAPS_CONTAINER */ Stream_Write_UINT32(s, 49); /* length */ diff --git a/libfreerdp/core/childsession.c b/libfreerdp/core/childsession.c index bdd9f3a1a..8e9ae5538 100644 --- a/libfreerdp/core/childsession.c +++ b/libfreerdp/core/childsession.c @@ -59,7 +59,7 @@ static int transport_bio_named_write(BIO* bio, const char* buf, int size) DWORD written = 0; UINT64 start = GetTickCount64(); - BOOL ret = WriteFile(ptr->hFile, buf, size, &written, NULL); + BOOL ret = WriteFile(ptr->hFile, buf, WINPR_ASSERTING_INT_CAST(uint32_t, size), &written, NULL); // winpr_HexDump(TAG, WLOG_DEBUG, buf, size); if (!ret) @@ -222,14 +222,15 @@ static int transport_bio_named_read(BIO* bio, char* buf, int size) if ((size >= 0) && ret) { DataChunk chunks[2] = { 0 }; - const int nchunks = ringbuffer_peek(&ptr->readBuffer, chunks, ret); + const int nchunks = + ringbuffer_peek(&ptr->readBuffer, chunks, WINPR_ASSERTING_INT_CAST(size_t, ret)); for (int i = 0; i < nchunks; i++) { memcpy(buf, chunks[i].data, chunks[i].size); buf += chunks[i].size; } - ringbuffer_commit_read_bytes(&ptr->readBuffer, ret); + ringbuffer_commit_read_bytes(&ptr->readBuffer, WINPR_ASSERTING_INT_CAST(size_t, ret)); WLog_VRB(TAG, "(%d)=%" PRIdz " nchunks=%d", size, ret, nchunks); } diff --git a/libfreerdp/core/client.c b/libfreerdp/core/client.c index 34a5ff1fe..b83f7e9d2 100644 --- a/libfreerdp/core/client.c +++ b/libfreerdp/core/client.c @@ -1006,7 +1006,8 @@ static UINT VCAPITYPE FreeRDP_VirtualChannelInitEx( WINPR_ASSERT(pChannelOpenData); - pChannelOpenData->OpenHandle = InterlockedIncrement(&g_OpenHandleSeq); + const LONG hdl = InterlockedIncrement(&g_OpenHandleSeq); + pChannelOpenData->OpenHandle = WINPR_ASSERTING_INT_CAST(uint32_t, hdl); pChannelOpenData->channels = channels; pChannelOpenData->lpUserParam = lpUserParam; if (!HashTable_Insert(g_ChannelHandles, (void*)(UINT_PTR)pChannelOpenData->OpenHandle, @@ -1106,8 +1107,11 @@ static UINT VCAPITYPE FreeRDP_VirtualChannelInit(LPVOID* ppInitHandle, PCHANNEL_ UINT32 ChannelCount = freerdp_settings_get_uint32(settings, FreeRDP_ChannelCount); pChannelDef = &pChannel[index]; + pChannelOpenData = &channels->openDataList[channels->openDataCount]; - pChannelOpenData->OpenHandle = InterlockedIncrement(&g_OpenHandleSeq); + + const LONG hdl = InterlockedIncrement(&g_OpenHandleSeq); + pChannelOpenData->OpenHandle = WINPR_ASSERTING_INT_CAST(uint32_t, hdl); pChannelOpenData->channels = channels; if (!HashTable_Insert(g_ChannelHandles, (void*)(UINT_PTR)pChannelOpenData->OpenHandle, (void*)pChannelOpenData)) diff --git a/libfreerdp/core/client.h b/libfreerdp/core/client.h index 17e619201..be69e5692 100644 --- a/libfreerdp/core/client.h +++ b/libfreerdp/core/client.h @@ -53,8 +53,8 @@ typedef struct typedef struct { char name[CHANNEL_NAME_LEN + 1]; - int OpenHandle; - int options; + uint32_t OpenHandle; + ULONG options; int flags; void* pInterface; rdpChannels* channels; diff --git a/libfreerdp/core/connection.c b/libfreerdp/core/connection.c index 001b56944..71c1060fb 100644 --- a/libfreerdp/core/connection.c +++ b/libfreerdp/core/connection.c @@ -1419,7 +1419,7 @@ BOOL rdp_client_transition_to_state(rdpRdp* rdp, CONNECTION_STATE state) ConnectionStateChangeEventArgs stateEvent = { 0 }; rdpContext* context = rdp->context; EventArgsInit(&stateEvent, "libfreerdp"); - stateEvent.state = rdp_get_state(rdp); + stateEvent.state = WINPR_ASSERTING_INT_CAST(int32_t, rdp_get_state(rdp)); stateEvent.active = rdp_is_active_state(rdp); PubSub_OnConnectionStateChange(rdp->pubSub, context, &stateEvent); } @@ -1995,7 +1995,7 @@ fail: return status; } -const char* rdp_client_connection_state_string(int state) +const char* rdp_client_connection_state_string(UINT state) { switch (state) { diff --git a/libfreerdp/core/connection.h b/libfreerdp/core/connection.h index 224eb72eb..a8276260b 100644 --- a/libfreerdp/core/connection.h +++ b/libfreerdp/core/connection.h @@ -69,7 +69,7 @@ FREERDP_LOCAL BOOL rdp_server_reactivate(rdpRdp* rdp); FREERDP_LOCAL BOOL rdp_server_transition_to_state(rdpRdp* rdp, CONNECTION_STATE state); FREERDP_LOCAL const char* rdp_get_state_string(const rdpRdp* rdp); -FREERDP_LOCAL const char* rdp_client_connection_state_string(int state); +FREERDP_LOCAL const char* rdp_client_connection_state_string(UINT state); FREERDP_LOCAL BOOL rdp_channels_from_mcs(rdpSettings* settings, const rdpRdp* rdp); diff --git a/libfreerdp/core/credssp_auth.c b/libfreerdp/core/credssp_auth.c index 9b34a8fd9..adda839c5 100644 --- a/libfreerdp/core/credssp_auth.c +++ b/libfreerdp/core/credssp_auth.c @@ -695,10 +695,10 @@ const char* credssp_auth_pkg_name(rdpCredsspAuth* auth) return auth->pkgNameA; } -UINT32 credssp_auth_sspi_error(rdpCredsspAuth* auth) +INT32 credssp_auth_sspi_error(rdpCredsspAuth* auth) { WINPR_ASSERT(auth); - return (UINT32)auth->sspi_error; + return auth->sspi_error; } void credssp_auth_tableAndContext(rdpCredsspAuth* auth, SecurityFunctionTable** ptable, diff --git a/libfreerdp/core/credssp_auth.h b/libfreerdp/core/credssp_auth.h index e53231754..1e328867a 100644 --- a/libfreerdp/core/credssp_auth.h +++ b/libfreerdp/core/credssp_auth.h @@ -61,7 +61,7 @@ FREERDP_LOCAL BOOL credssp_auth_have_output_token(rdpCredsspAuth* auth); FREERDP_LOCAL BOOL credssp_auth_is_complete(rdpCredsspAuth* auth); FREERDP_LOCAL const char* credssp_auth_pkg_name(rdpCredsspAuth* auth); FREERDP_LOCAL size_t credssp_auth_trailer_size(rdpCredsspAuth* auth); -FREERDP_LOCAL UINT32 credssp_auth_sspi_error(rdpCredsspAuth* auth); +FREERDP_LOCAL INT32 credssp_auth_sspi_error(rdpCredsspAuth* auth); FREERDP_LOCAL void credssp_auth_tableAndContext(rdpCredsspAuth* auth, SecurityFunctionTable** ptable, diff --git a/libfreerdp/core/freerdp.c b/libfreerdp/core/freerdp.c index 715da31d9..e2a556cc4 100644 --- a/libfreerdp/core/freerdp.c +++ b/libfreerdp/core/freerdp.c @@ -409,7 +409,8 @@ static BOOL freerdp_prevent_session_lock(rdpContext* context) if (FakeMouseMotionInterval && in->lastInputTimestamp) { const time_t now = time(NULL); - if (now - in->lastInputTimestamp > FakeMouseMotionInterval) + if (WINPR_ASSERTING_INT_CAST(size_t, now) - in->lastInputTimestamp > + FakeMouseMotionInterval) { WLog_Print(context->log, WLOG_DEBUG, "fake mouse move: x=%d y=%d lastInputTimestamp=%" PRIu64 " " @@ -1220,15 +1221,13 @@ BOOL freerdp_nla_revert_to_self(rdpContext* context) UINT32 freerdp_get_nla_sspi_error(rdpContext* context) { - rdpNla* nla = NULL; - WINPR_ASSERT(context); WINPR_ASSERT(context->rdp); WINPR_ASSERT(context->rdp->transport); - nla = transport_get_nla(context->rdp->transport); + rdpNla* nla = transport_get_nla(context->rdp->transport); - return nla_get_sspi_error(nla); + return (UINT32)nla_get_sspi_error(nla); } BOOL freerdp_nla_encrypt(rdpContext* context, const SecBuffer* inBuffer, SecBuffer* outBuffer) diff --git a/libfreerdp/core/gateway/arm.c b/libfreerdp/core/gateway/arm.c index 4433f6ac9..5eabbb01f 100644 --- a/libfreerdp/core/gateway/arm.c +++ b/libfreerdp/core/gateway/arm.c @@ -142,7 +142,7 @@ static BOOL arm_tls_connect(rdpArm* arm, rdpTls* tls, UINT32 timeout) } tls->hostname = freerdp_settings_get_string(settings, FreeRDP_GatewayHostname); - tls->port = MIN(UINT16_MAX, settings->GatewayPort); + tls->port = MIN(UINT16_MAX, WINPR_ASSERTING_INT_CAST(int32_t, settings->GatewayPort)); tls->isGatewayTransport = TRUE; status = freerdp_tls_connect(tls, bufferedBio); if (status < 1) @@ -383,7 +383,7 @@ static WINPR_CIPHER_CTX* treatAuthBlob(wLog* log, const BYTE* pbInput, size_t cb return NULL; } - cbInput -= (algoSz + 1) * sizeof(WCHAR); + cbInput -= WINPR_ASSERTING_INT_CAST(size_t, (algoSz + 1)) * sizeof(WCHAR); if (cbInput < 12) { @@ -393,8 +393,9 @@ static WINPR_CIPHER_CTX* treatAuthBlob(wLog* log, const BYTE* pbInput, size_t cb /* BCRYPT_KEY_DATA_BLOB_HEADER */ wStream staticStream = { 0 }; - wStream* s = - Stream_StaticConstInit(&staticStream, pbInput + (algoSz + 1) * sizeof(WCHAR), cbInput); + wStream* s = Stream_StaticConstInit( + &staticStream, pbInput + WINPR_ASSERTING_INT_CAST(size_t, (algoSz + 1)) * sizeof(WCHAR), + cbInput); UINT32 dwMagic = 0; Stream_Read_UINT32(s, dwMagic); @@ -424,7 +425,7 @@ static WINPR_CIPHER_CTX* treatAuthBlob(wLog* log, const BYTE* pbInput, size_t cb return NULL; } - int cipherType = 0; + WINPR_CIPHER_TYPE cipherType = 0; switch (cbKeyData) { case 16: @@ -827,8 +828,9 @@ static BOOL arm_fill_rdstls(rdpArm* arm, rdpSettings* settings, const WINPR_JSON goto endOfFunction; } - BOOL status = freerdp_settings_set_pointer_len(settings, FreeRDP_RedirectionGuid, wGUID, - (wGUID_len + 1) * sizeof(WCHAR)); + BOOL status = freerdp_settings_set_pointer_len( + settings, FreeRDP_RedirectionGuid, wGUID, + WINPR_ASSERTING_INT_CAST(size_t, (wGUID_len + 1)) * sizeof(WCHAR)); if (!status) { WLog_Print(arm->log, WLOG_ERROR, "unable to set RedirectionGuid"); diff --git a/libfreerdp/core/gateway/http.c b/libfreerdp/core/gateway/http.c index 729ee6b71..3914fa2f6 100644 --- a/libfreerdp/core/gateway/http.c +++ b/libfreerdp/core/gateway/http.c @@ -1142,7 +1142,7 @@ int http_chuncked_read(BIO* bio, BYTE* pBuffer, size_t size, if (status <= 0) return (effectiveDataLen > 0 ? effectiveDataLen : status); - encodingContext->nextOffset -= status; + encodingContext->nextOffset -= WINPR_ASSERTING_INT_CAST(uint32_t, status); if (encodingContext->nextOffset == 0) { encodingContext->state = ChunkStateFooter; @@ -1313,7 +1313,7 @@ static SSIZE_T http_response_recv_line(rdpTls* tls, HttpResponse* response) end = (char*)Stream_Pointer(response->data) - s; if (string_strnstr(end, "\r\n\r\n", s) != NULL) - payloadOffset = Stream_GetPosition(response->data); + payloadOffset = WINPR_ASSERTING_INT_CAST(SSIZE_T, Stream_GetPosition(response->data)); } out_error: @@ -1357,7 +1357,7 @@ static BOOL http_response_recv_body(rdpTls* tls, HttpResponse* response, BOOL re full_len += status; } } while (ctx.state != ChunkStateEnd); - response->BodyLength = full_len; + response->BodyLength = WINPR_ASSERTING_INT_CAST(uint32_t, full_len); if (response->BodyLength > 0) response->BodyContent = &(Stream_Buffer(response->data))[payloadOffset]; } @@ -1438,7 +1438,9 @@ HttpResponse* http_response_recv(rdpTls* tls, BOOL readContentLength) char* line = Stream_BufferAs(response->data, char); char* context = NULL; - while ((line = string_strnstr(line, "\r\n", payloadOffset - (line - buffer) - 2UL))) + while ((line = string_strnstr(line, "\r\n", + WINPR_ASSERTING_INT_CAST(size_t, payloadOffset) - + WINPR_ASSERTING_INT_CAST(size_t, (line - buffer)) - 2UL))) { line += 2; count++; @@ -1469,7 +1471,8 @@ HttpResponse* http_response_recv(rdpTls* tls, BOOL readContentLength) if (!http_response_parse_header(response)) goto out_error; - response->BodyLength = Stream_GetPosition(response->data) - payloadOffset; + response->BodyLength = + Stream_GetPosition(response->data) - WINPR_ASSERTING_INT_CAST(size_t, payloadOffset); WINPR_ASSERT(response->BodyLength == 0); bodyLength = response->BodyLength; /* expected body length */ @@ -1502,7 +1505,8 @@ HttpResponse* http_response_recv(rdpTls* tls, BOOL readContentLength) } /* Fetch remaining body! */ - if (!http_response_recv_body(tls, response, readContentLength, payloadOffset, bodyLength)) + if (!http_response_recv_body(tls, response, readContentLength, + WINPR_ASSERTING_INT_CAST(size_t, payloadOffset), bodyLength)) goto out_error; } Stream_SealLength(response->data); @@ -1624,7 +1628,7 @@ size_t http_response_get_body_length(const HttpResponse* response) { WINPR_ASSERT(response); - return (SSIZE_T)response->BodyLength; + return response->BodyLength; } const char* http_response_get_auth_token(const HttpResponse* response, const char* method) diff --git a/libfreerdp/core/gateway/ncacn_http.c b/libfreerdp/core/gateway/ncacn_http.c index 46e944975..05975984f 100644 --- a/libfreerdp/core/gateway/ncacn_http.c +++ b/libfreerdp/core/gateway/ncacn_http.c @@ -33,7 +33,7 @@ #define AUTH_PKG NTLM_SSP_NAME -static wStream* rpc_auth_http_request(HttpContext* http, const char* method, int contentLength, +static wStream* rpc_auth_http_request(HttpContext* http, const char* method, size_t contentLength, const SecBuffer* authToken, const char* auth_scheme) { wStream* s = NULL; @@ -77,7 +77,6 @@ BOOL rpc_ncacn_http_send_in_channel_request(RpcChannel* inChannel) { wStream* s = NULL; SSIZE_T status = 0; - int contentLength = 0; rdpCredsspAuth* auth = NULL; HttpContext* http = NULL; const SecBuffer* buffer = NULL; @@ -93,7 +92,7 @@ BOOL rpc_ncacn_http_send_in_channel_request(RpcChannel* inChannel) if (rc < 0) return FALSE; - contentLength = (rc == 0) ? 0 : 0x40000000; + const size_t contentLength = (rc == 0) ? 0 : 0x40000000; buffer = credssp_auth_have_output_token(auth) ? credssp_auth_get_output_buffer(auth) : NULL; s = rpc_auth_http_request(http, "RPC_IN_DATA", contentLength, buffer, credssp_auth_pkg_name(auth)); @@ -206,7 +205,7 @@ BOOL rpc_ncacn_http_send_out_channel_request(RpcChannel* outChannel, BOOL replac { BOOL status = TRUE; wStream* s = NULL; - int contentLength = 0; + size_t contentLength = 0; rdpCredsspAuth* auth = NULL; HttpContext* http = NULL; const SecBuffer* buffer = NULL; diff --git a/libfreerdp/core/gateway/rdg.c b/libfreerdp/core/gateway/rdg.c index 9595c1c73..6a9a149ed 100644 --- a/libfreerdp/core/gateway/rdg.c +++ b/libfreerdp/core/gateway/rdg.c @@ -24,6 +24,7 @@ #include "../settings.h" #include +#include #include #include @@ -382,8 +383,8 @@ static BOOL rdg_read_all(rdpContext* context, rdpTls* tls, BYTE* buffer, size_t continue; } - readCount += status; - pBuffer += status; + readCount += WINPR_ASSERTING_INT_CAST(uint32_t, status); + pBuffer += WINPR_ASSERTING_INT_CAST(uint32_t, status); } return TRUE; @@ -414,8 +415,8 @@ static wStream* rdg_receive_packet(rdpRdg* rdg) return NULL; } - if (!rdg_read_all(rdg->context, rdg->tlsOut, Stream_Buffer(s) + header, - (int)packetLength - (int)header, &rdg->transferEncoding)) + if (!rdg_read_all(rdg->context, rdg->tlsOut, Stream_Buffer(s) + header, packetLength - header, + &rdg->transferEncoding)) { Stream_Free(s, TRUE); return NULL; @@ -644,7 +645,7 @@ static BOOL rdg_set_auth_header(rdpCredsspAuth* auth, HttpRequest* request) if (authToken->cbBuffer > INT_MAX) return FALSE; - base64AuthToken = crypto_base64_encode(authToken->pvBuffer, (int)authToken->cbBuffer); + base64AuthToken = crypto_base64_encode(authToken->pvBuffer, authToken->cbBuffer); } if (base64AuthToken) @@ -986,14 +987,14 @@ static BOOL rdg_process_tunnel_authorization_response(rdpRdg* rdg, wStream* s) static BOOL rdg_process_extauth_sspi(rdpRdg* rdg, wStream* s) { - UINT32 errorCode = 0; + INT32 errorCode = 0; UINT16 authBlobLen = 0; SecBuffer authToken = { 0 }; BYTE* authTokenData = NULL; WINPR_ASSERT(rdg); - Stream_Read_UINT32(s, errorCode); + Stream_Read_INT32(s, errorCode); Stream_Read_UINT16(s, authBlobLen); if (errorCode != ERROR_SUCCESS) @@ -1251,7 +1252,7 @@ static BOOL rdg_send_http_request(rdpRdg* rdg, rdpTls* tls, const char* method, return (status >= 0); } -static BOOL rdg_tls_connect(rdpRdg* rdg, rdpTls* tls, const char* peerAddress, int timeout) +static BOOL rdg_tls_connect(rdpRdg* rdg, rdpTls* tls, const char* peerAddress, UINT32 timeout) { long status = 0; BIO* layerBio = NULL; @@ -1312,7 +1313,7 @@ static BOOL rdg_tls_connect(rdpRdg* rdg, rdpTls* tls, const char* peerAddress, i } tls->hostname = settings->GatewayHostname; - tls->port = MIN(UINT16_MAX, settings->GatewayPort); + tls->port = WINPR_ASSERTING_INT_CAST(int32_t, MIN(UINT16_MAX, settings->GatewayPort)); tls->isGatewayTransport = TRUE; status = freerdp_tls_connect(tls, bufferedBio); if (status < 1) @@ -1333,7 +1334,8 @@ static BOOL rdg_tls_connect(rdpRdg* rdg, rdpTls* tls, const char* peerAddress, i } static BOOL rdg_establish_data_connection(rdpRdg* rdg, rdpTls* tls, const char* method, - const char* peerAddress, int timeout, BOOL* rpcFallback) + const char* peerAddress, UINT32 timeout, + BOOL* rpcFallback) { char buffer[64] = { 0 }; HttpResponse* response = NULL; @@ -1620,11 +1622,17 @@ static int rdg_write_websocket_data_packet(rdpRdg* rdg, const BYTE* buf, int isi if (!sWS) return FALSE; - Stream_Write_UINT16(sWS, PKT_TYPE_DATA ^ (maskingKey.u8[0] | maskingKey.u8[1] << 8)); /* Type */ - Stream_Write_UINT16(sWS, 0 ^ (maskingKey.u8[2] | maskingKey.u8[3] << 8)); /* Reserved */ - Stream_Write_UINT32(sWS, (UINT32)payloadSize ^ maskingKey.u32); /* Packet length */ - Stream_Write_UINT16(sWS, - (UINT16)isize ^ (maskingKey.u8[0] | maskingKey.u8[1] << 8)); /* Data size */ + Stream_Write_UINT16( + sWS, WINPR_ASSERTING_INT_CAST( + uint16_t, PKT_TYPE_DATA ^ (maskingKey.u8[0] | maskingKey.u8[1] << 8))); /* Type */ + Stream_Write_UINT16( + sWS, WINPR_ASSERTING_INT_CAST( + uint16_t, 0 ^ (maskingKey.u8[2] | maskingKey.u8[3] << 8))); /* Reserved */ + Stream_Write_UINT32( + sWS, WINPR_ASSERTING_INT_CAST(uint32_t, payloadSize ^ maskingKey.u32)); /* Packet length */ + Stream_Write_UINT16( + sWS, WINPR_ASSERTING_INT_CAST( + uint16_t, isize ^ (maskingKey.u8[0] | maskingKey.u8[1] << 8))); /* Data size */ /* masking key is now off by 2 bytes. fix that */ maskingKey.u32 = (maskingKey.u32 & 0xffff) << 16 | (maskingKey.u32 >> 16); @@ -1871,8 +1879,7 @@ static int rdg_read_data_packet(rdpRdg* rdg, BYTE* buffer, size_t size) return -1; status = rdg_socket_read(rdg->tlsOut->bio, (BYTE*)(&header) + readCount, - (int)sizeof(RdgPacketHeader) - (int)readCount, - &rdg->transferEncoding); + sizeof(RdgPacketHeader) - readCount, &rdg->transferEncoding); if (status <= 0) { @@ -1910,7 +1917,7 @@ static int rdg_read_data_packet(rdpRdg* rdg, BYTE* buffer, size_t size) return -1; status = rdg_socket_read(rdg->tlsOut->bio, (BYTE*)(&rdg->packetRemainingCount) + readCount, - 2 - (int)readCount, &rdg->transferEncoding); + 2 - readCount, &rdg->transferEncoding); if (status < 0) { diff --git a/libfreerdp/core/gateway/rpc.c b/libfreerdp/core/gateway/rpc.c index d90d80ad7..f08f18e15 100644 --- a/libfreerdp/core/gateway/rpc.c +++ b/libfreerdp/core/gateway/rpc.c @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -752,8 +753,9 @@ static BOOL rpc_channel_tls_connect(RpcChannel* channel, UINT32 timeout) if (channel->client->isProxy) { + WINPR_ASSERT(settings->GatewayPort <= UINT16_MAX); if (!proxy_connect(context, bufferedBio, proxyUsername, proxyPassword, - settings->GatewayHostname, settings->GatewayPort)) + settings->GatewayHostname, (UINT16)settings->GatewayPort)) { BIO_free_all(bufferedBio); return FALSE; @@ -767,7 +769,7 @@ static BOOL rpc_channel_tls_connect(RpcChannel* channel, UINT32 timeout) return FALSE; tls->hostname = settings->GatewayHostname; - tls->port = MIN(UINT16_MAX, settings->GatewayPort); + tls->port = WINPR_ASSERTING_INT_CAST(int32_t, MIN(UINT16_MAX, settings->GatewayPort)); tls->isGatewayTransport = TRUE; int tlsStatus = freerdp_tls_connect(tls, bufferedBio); @@ -822,7 +824,7 @@ static int rpc_in_channel_connect(RpcInChannel* inChannel, UINT32 timeout) return 1; } -static int rpc_out_channel_connect(RpcOutChannel* outChannel, int timeout) +static int rpc_out_channel_connect(RpcOutChannel* outChannel, UINT32 timeout) { rdpContext* context = NULL; @@ -854,7 +856,7 @@ static int rpc_out_channel_connect(RpcOutChannel* outChannel, int timeout) return 1; } -int rpc_out_channel_replacement_connect(RpcOutChannel* outChannel, int timeout) +int rpc_out_channel_replacement_connect(RpcOutChannel* outChannel, uint32_t timeout) { rdpContext* context = NULL; diff --git a/libfreerdp/core/gateway/rpc.h b/libfreerdp/core/gateway/rpc.h index 522f72256..236688617 100644 --- a/libfreerdp/core/gateway/rpc.h +++ b/libfreerdp/core/gateway/rpc.h @@ -775,7 +775,7 @@ FREERDP_LOCAL void rpc_channel_free(RpcChannel* channel); WINPR_ATTR_MALLOC(rpc_channel_free, 1) FREERDP_LOCAL RpcOutChannel* rpc_out_channel_new(rdpRpc* rpc, const GUID* guid); -FREERDP_LOCAL int rpc_out_channel_replacement_connect(RpcOutChannel* outChannel, int timeout); +FREERDP_LOCAL int rpc_out_channel_replacement_connect(RpcOutChannel* outChannel, uint32_t timeout); FREERDP_LOCAL BOOL rpc_in_channel_transition_to_state(RpcInChannel* inChannel, CLIENT_IN_CHANNEL_STATE state); diff --git a/libfreerdp/core/gateway/rpc_bind.c b/libfreerdp/core/gateway/rpc_bind.c index f52c53c32..ce53a4d27 100644 --- a/libfreerdp/core/gateway/rpc_bind.c +++ b/libfreerdp/core/gateway/rpc_bind.c @@ -23,6 +23,7 @@ #include #include +#include #include @@ -110,21 +111,17 @@ const p_uuid_t BTFN_UUID = { static int rpc_bind_setup(rdpRpc* rpc) { - int rc = -1; - rdpContext* context = NULL; - rdpSettings* settings = NULL; - freerdp* instance = NULL; SEC_WINNT_AUTH_IDENTITY identity = { 0 }; WINPR_ASSERT(rpc); - context = transport_get_context(rpc->transport); + rdpContext* context = transport_get_context(rpc->transport); WINPR_ASSERT(context); - settings = context->settings; + rdpSettings* settings = context->settings; WINPR_ASSERT(settings); - instance = context->instance; + freerdp* instance = context->instance; WINPR_ASSERT(instance); credssp_auth_free(rpc->auth); @@ -132,7 +129,7 @@ static int rpc_bind_setup(rdpRpc* rpc) if (!rpc->auth) return -1; - rc = utils_authenticate_gateway(instance, GW_AUTH_RPC); + auth_status rc = utils_authenticate_gateway(instance, GW_AUTH_RPC); switch (rc) { case AUTH_SUCCESS: @@ -256,7 +253,9 @@ int rpc_send_bind_pdu(rdpRpc* rpc, BOOL initial) bind_pdu.auth_verifier.auth_reserved = 0x00; bind_pdu.auth_verifier.auth_context_id = 0x00000000; offset += (8 + bind_pdu.header.auth_length); - bind_pdu.header.frag_length = offset; + + WINPR_ASSERT(offset <= UINT16_MAX); + bind_pdu.header.frag_length = (UINT16)offset; buffer = Stream_New(NULL, bind_pdu.header.frag_length); @@ -408,14 +407,19 @@ int rpc_send_rpc_auth_3_pdu(rdpRpc* rpc) auth_3_pdu.max_xmit_frag = rpc->max_xmit_frag; auth_3_pdu.max_recv_frag = rpc->max_recv_frag; offset = 20; - auth_3_pdu.auth_verifier.auth_pad_length = rpc_offset_align(&offset, 4); + + const size_t align = rpc_offset_align(&offset, 4); + WINPR_ASSERT(align <= UINT8_MAX); + auth_3_pdu.auth_verifier.auth_pad_length = (BYTE)align; auth_3_pdu.auth_verifier.auth_type = rpc_auth_pkg_to_security_provider(credssp_auth_pkg_name(rpc->auth)); auth_3_pdu.auth_verifier.auth_level = RPC_C_AUTHN_LEVEL_PKT_INTEGRITY; auth_3_pdu.auth_verifier.auth_reserved = 0x00; auth_3_pdu.auth_verifier.auth_context_id = 0x00000000; offset += (8 + auth_3_pdu.header.auth_length); - auth_3_pdu.header.frag_length = offset; + + WINPR_ASSERT(offset <= UINT16_MAX); + auth_3_pdu.header.frag_length = (UINT16)offset; buffer = Stream_New(NULL, auth_3_pdu.header.frag_length); diff --git a/libfreerdp/core/gateway/rpc_client.c b/libfreerdp/core/gateway/rpc_client.c index eede64677..b83d0cadc 100644 --- a/libfreerdp/core/gateway/rpc_client.c +++ b/libfreerdp/core/gateway/rpc_client.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -567,7 +568,6 @@ fail: static SSIZE_T rpc_client_default_out_channel_recv(rdpRpc* rpc) { SSIZE_T status = -1; - UINT32 statusCode = 0; HttpResponse* response = NULL; RpcInChannel* inChannel = NULL; RpcOutChannel* outChannel = NULL; @@ -646,7 +646,7 @@ static SSIZE_T rpc_client_default_out_channel_recv(rdpRpc* rpc) if (!response) return -1; - statusCode = http_response_get_status_code(response); + const INT16 statusCode = http_response_get_status_code(response); if (statusCode != HTTP_STATUS_OK) { @@ -1088,7 +1088,10 @@ BOOL rpc_client_write_call(rdpRpc* rpc, wStream* s, UINT16 opnum) offset = 24; stub_data_pad = rpc_offset_align(&offset, 8); offset += length; - request_pdu.auth_verifier.auth_pad_length = rpc_offset_align(&offset, 4); + + const size_t alg = rpc_offset_align(&offset, 4); + WINPR_ASSERT(alg <= UINT8_MAX); + request_pdu.auth_verifier.auth_pad_length = (UINT8)alg; request_pdu.auth_verifier.auth_type = rpc_auth_pkg_to_security_provider(credssp_auth_pkg_name(rpc->auth)); request_pdu.auth_verifier.auth_level = RPC_C_AUTHN_LEVEL_PKT_INTEGRITY; diff --git a/libfreerdp/core/gateway/rts.c b/libfreerdp/core/gateway/rts.c index 705fb47d2..3d7904089 100644 --- a/libfreerdp/core/gateway/rts.c +++ b/libfreerdp/core/gateway/rts.c @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -963,7 +964,7 @@ static BOOL rts_read_pdu_fault(wStream* s, rpcconn_fault_hdr_t* ctx, BOOL silent Stream_Read_UINT8(s, ctx->reserved); Stream_Read_UINT32(s, ctx->status); - WLog_WARN(TAG, "status=%s", Win32ErrorCode2Tag(ctx->status)); + WLog_WARN(TAG, "status=%s", Win32ErrorCode2Tag(ctx->status & 0xFFFF)); return rts_read_auth_verifier_with_stub(s, &ctx->auth_verifier, &ctx->header, silent); } diff --git a/libfreerdp/core/gateway/tsg.c b/libfreerdp/core/gateway/tsg.c index be4f09d8c..45b5e3d68 100644 --- a/libfreerdp/core/gateway/tsg.c +++ b/libfreerdp/core/gateway/tsg.c @@ -1513,8 +1513,8 @@ static BOOL tsg_ndr_read_tunnel_context(wLog* log, wStream* s, CONTEXT_HANDLE* t WINPR_ASSERT(tunnelId); Stream_Read_UINT32(s, *tunnelId); /* TunnelId (4 bytes) */ - UINT32 ReturnValue = 0; - Stream_Read_UINT32(s, ReturnValue); /* ReturnValue (4 bytes) */ + INT32 ReturnValue = 0; + Stream_Read_INT32(s, ReturnValue); /* ReturnValue (4 bytes) */ if (ReturnValue != NO_ERROR) WLog_WARN(TAG, "ReturnValue=%s", NtStatus2Tag(ReturnValue)); return TRUE; diff --git a/libfreerdp/core/gateway/websocket.c b/libfreerdp/core/gateway/websocket.c index ccc36c3b4..0cddf6a9c 100644 --- a/libfreerdp/core/gateway/websocket.c +++ b/libfreerdp/core/gateway/websocket.c @@ -443,7 +443,7 @@ int websocket_context_read(websocket_context* encodingContext, BIO* bio, BYTE* p if ((size_t)status == size) return effectiveDataLen; pBuffer += status; - size -= status; + size -= WINPR_ASSERTING_INT_CAST(size_t, status); } break; default: diff --git a/libfreerdp/core/gateway/wst.c b/libfreerdp/core/gateway/wst.c index 079c005f7..3ccc04ab9 100644 --- a/libfreerdp/core/gateway/wst.c +++ b/libfreerdp/core/gateway/wst.c @@ -744,15 +744,13 @@ static BOOL wst_parse_url(rdpWst* wst, const char* url) wst->gwhostname = NULL; if (pos - hostStart == 0) return FALSE; - wst->gwhostname = malloc(sizeof(char) * (pos - hostStart + 1)); + wst->gwhostname = strndup(hostStart, WINPR_ASSERTING_INT_CAST(size_t, (pos - hostStart))); if (!wst->gwhostname) return FALSE; - strncpy(wst->gwhostname, hostStart, (pos - hostStart)); - wst->gwhostname[pos - hostStart] = '\0'; if (*pos == ':') { - char port[6]; + char port[6] = { 0 }; char* portNumberEnd = NULL; pos++; const char* portStart = pos; @@ -760,7 +758,7 @@ static BOOL wst_parse_url(rdpWst* wst, const char* url) pos++; if (pos - portStart > 5 || pos - portStart == 0) return FALSE; - strncpy(port, portStart, (pos - portStart)); + strncpy(port, portStart, WINPR_ASSERTING_INT_CAST(size_t, (pos - portStart))); port[pos - portStart] = '\0'; long _p = strtol(port, &portNumberEnd, 10); if (portNumberEnd && (*portNumberEnd == '\0') && (_p > 0) && (_p <= UINT16_MAX)) diff --git a/libfreerdp/core/gcc.c b/libfreerdp/core/gcc.c index 0b9c835d6..319e2e5fa 100644 --- a/libfreerdp/core/gcc.c +++ b/libfreerdp/core/gcc.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -436,7 +437,9 @@ BOOL gcc_write_conference_create_request(wStream* s, wStream* userData) if (!per_write_object_identifier(s, t124_02_98_oid)) /* ITU-T T.124 (02/98) OBJECT_IDENTIFIER */ return FALSE; /* ConnectData::connectPDU (OCTET_STRING) */ - if (!per_write_length(s, Stream_GetPosition(userData) + 14)) /* connectPDU length */ + const size_t pos = Stream_GetPosition(userData); + WINPR_ASSERT(pos <= UINT16_MAX - 14); + if (!per_write_length(s, (UINT16)pos + 14)) /* connectPDU length */ return FALSE; /* ConnectGCCPDU */ if (!per_write_choice(s, 0)) /* From ConnectGCCPDU select conferenceCreateRequest (0) of type @@ -459,7 +462,9 @@ BOOL gcc_write_conference_create_request(wStream* s, wStream* userData) 4)) /* h221NonStandard, client-to-server H.221 key, "Duca" */ return FALSE; /* userData::value (OCTET_STRING) */ - return per_write_octet_string(s, Stream_Buffer(userData), Stream_GetPosition(userData), + const size_t upos = Stream_GetPosition(userData); + WINPR_ASSERT(upos <= UINT16_MAX); + return per_write_octet_string(s, Stream_Buffer(userData), (UINT16)upos, 0); /* array of client data blocks */ } @@ -559,7 +564,9 @@ BOOL gcc_write_conference_create_response(wStream* s, wStream* userData) 4)) /* h221NonStandard, server-to-client H.221 key, "McDn" */ return FALSE; /* userData (OCTET_STRING) */ - return per_write_octet_string(s, Stream_Buffer(userData), Stream_GetPosition(userData), + const size_t pos = Stream_GetPosition(userData); + WINPR_ASSERT(pos <= UINT16_MAX); + return per_write_octet_string(s, Stream_Buffer(userData), (UINT16)pos, 0); /* array of server data blocks */ } @@ -1001,7 +1008,9 @@ static UINT16 filterAndLogEarlyClientCapabilityFlags(UINT32 flags) flags, ~mask, unknown, rdp_early_client_caps_string(unknown, buffer, sizeof(buffer))); } - return filtered; + + WINPR_ASSERT(filtered <= UINT16_MAX); + return (UINT16)filtered; } static UINT16 earlyClientCapsFromSettings(const rdpSettings* settings) @@ -1407,8 +1416,10 @@ BOOL gcc_write_client_core_data(wStream* s, const rdpMcs* mcs) return FALSE; Stream_Write_UINT32(s, settings->RdpVersion); /* Version */ - Stream_Write_UINT16(s, settings->DesktopWidth); /* DesktopWidth */ - Stream_Write_UINT16(s, settings->DesktopHeight); /* DesktopHeight */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, settings->DesktopWidth)); /* DesktopWidth */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, settings->DesktopHeight)); /* DesktopHeight */ Stream_Write_UINT16(s, RNS_UD_COLOR_8BPP); /* ColorDepth, ignored because of postBeta2ColorDepth */ Stream_Write_UINT16(s, RNS_UD_SAS_DEL); /* SASSequence (Secure Access Sequence) */ @@ -1440,7 +1451,8 @@ BOOL gcc_write_client_core_data(wStream* s, const rdpMcs* mcs) highColorDepth = ColorDepthToHighColor(ColorDepth); earlyCapabilityFlags = earlyClientCapsFromSettings(settings); - connectionType = settings->ConnectionType; + WINPR_ASSERT(settings->ConnectionType <= UINT8_MAX); + connectionType = (UINT8)settings->ConnectionType; if (!Stream_EnsureRemainingCapacity(s, 6)) return FALSE; @@ -1449,7 +1461,7 @@ BOOL gcc_write_client_core_data(wStream* s, const rdpMcs* mcs) HighColorToString(highColorDepth), freerdp_supported_color_depths_string(SupportedColorDepths, dbuffer, sizeof(dbuffer)), rdp_early_client_caps_string(earlyCapabilityFlags, buffer, sizeof(buffer))); - Stream_Write_UINT16(s, highColorDepth); /* highColorDepth */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, highColorDepth)); /* highColorDepth */ Stream_Write_UINT16(s, SupportedColorDepths); /* supportedColorDepths */ Stream_Write_UINT16(s, earlyCapabilityFlags); /* earlyCapabilityFlags */ @@ -1809,8 +1821,12 @@ BOOL gcc_write_server_security_data(wStream* s, rdpMcs* mcs) if (len < 0) return FALSE; const size_t end = Stream_GetPosition(s); + + WINPR_ASSERT(end >= posHeader); + const size_t diff = end - posHeader; + WINPR_ASSERT(diff <= UINT16_MAX); Stream_SetPosition(s, posHeader); - if (!gcc_write_user_data_header(s, SC_SECURITY, end - posHeader)) + if (!gcc_write_user_data_header(s, SC_SECURITY, (UINT16)diff)) return FALSE; Stream_SetPosition(s, posCertLen); WINPR_ASSERT(len <= UINT32_MAX); @@ -1884,13 +1900,13 @@ BOOL gcc_read_client_network_data(wStream* s, rdpMcs* mcs) BOOL gcc_write_client_network_data(wStream* s, const rdpMcs* mcs) { - UINT16 length = 0; WINPR_ASSERT(s); WINPR_ASSERT(mcs); if (mcs->channelCount > 0) { - length = mcs->channelCount * 12 + 8; - if (!gcc_write_user_data_header(s, CS_NET, length)) + const size_t length = mcs->channelCount * 12 + 8; + WINPR_ASSERT(length <= UINT16_MAX); + if (!gcc_write_user_data_header(s, CS_NET, (UINT16)length)) return FALSE; Stream_Write_UINT32(s, mcs->channelCount); /* channelCount */ @@ -1960,11 +1976,13 @@ BOOL gcc_write_server_network_data(wStream* s, const rdpMcs* mcs) WINPR_ASSERT(mcs); const size_t payloadLen = 8 + mcs->channelCount * 2 + (mcs->channelCount % 2 == 1 ? 2 : 0); - if (!gcc_write_user_data_header(s, SC_NET, payloadLen)) + WINPR_ASSERT(payloadLen <= UINT16_MAX); + if (!gcc_write_user_data_header(s, SC_NET, (UINT16)payloadLen)) return FALSE; Stream_Write_UINT16(s, MCS_GLOBAL_CHANNEL_ID); /* MCSChannelId */ - Stream_Write_UINT16(s, mcs->channelCount); /* channelCount */ + Stream_Write_UINT16(s, + WINPR_ASSERTING_INT_CAST(uint16_t, mcs->channelCount)); /* channelCount */ for (UINT32 i = 0; i < mcs->channelCount; i++) { @@ -2153,7 +2171,6 @@ BOOL gcc_read_client_monitor_data(wStream* s, rdpMcs* mcs) BOOL gcc_write_client_monitor_data(wStream* s, const rdpMcs* mcs) { - UINT16 length = 0; INT32 baseX = 0; INT32 baseY = 0; const rdpSettings* settings = mcs_get_const_settings(mcs); @@ -2164,7 +2181,9 @@ BOOL gcc_write_client_monitor_data(wStream* s, const rdpMcs* mcs) WLog_DBG(TAG, "MonitorCount=%" PRIu32, settings->MonitorCount); if (settings->MonitorCount > 1) { - length = (20 * settings->MonitorCount) + 12; + const size_t len = (20 * settings->MonitorCount) + 12; + WINPR_ASSERT(len <= UINT16_MAX); + const UINT16 length = (UINT16)len; if (!gcc_write_user_data_header(s, CS_MONITOR, length)) return FALSE; Stream_Write_UINT32(s, settings->MonitorFlags); /* flags */ @@ -2186,10 +2205,10 @@ BOOL gcc_write_client_monitor_data(wStream* s, const rdpMcs* mcs) for (UINT32 i = 0; i < settings->MonitorCount; i++) { const rdpMonitor* current = &settings->MonitorDefArray[i]; - const UINT32 left = current->x - baseX; - const UINT32 top = current->y - baseY; - const UINT32 right = left + current->width - 1; - const UINT32 bottom = top + current->height - 1; + const UINT32 left = WINPR_ASSERTING_INT_CAST(uint32_t, current->x - baseX); + const UINT32 top = WINPR_ASSERTING_INT_CAST(uint32_t, current->y - baseY); + const UINT32 right = left + WINPR_ASSERTING_INT_CAST(uint32_t, current->width - 1); + const UINT32 bottom = top + WINPR_ASSERTING_INT_CAST(uint32_t, current->height - 1); const UINT32 flags = current->is_primary ? MONITOR_PRIMARY : 0; WLog_DBG(TAG, "Monitor[%" PRIu32 "]: top=%" PRIu32 ", left=%" PRIu32 ", bottom=%" PRIu32 @@ -2249,7 +2268,6 @@ BOOL gcc_read_client_monitor_extended_data(wStream* s, rdpMcs* mcs) BOOL gcc_write_client_monitor_extended_data(wStream* s, const rdpMcs* mcs) { - UINT16 length = 0; const rdpSettings* settings = mcs_get_const_settings(mcs); WINPR_ASSERT(s); @@ -2257,8 +2275,9 @@ BOOL gcc_write_client_monitor_extended_data(wStream* s, const rdpMcs* mcs) if (settings->HasMonitorAttributes) { - length = (20 * settings->MonitorCount) + 16; - if (!gcc_write_user_data_header(s, CS_MONITOR_EX, length)) + const size_t length = (20 * settings->MonitorCount) + 16; + WINPR_ASSERT(length <= UINT16_MAX); + if (!gcc_write_user_data_header(s, CS_MONITOR_EX, (UINT16)length)) return FALSE; Stream_Write_UINT32(s, settings->MonitorAttributeFlags); /* flags */ Stream_Write_UINT32(s, 20); /* monitorAttributeSize */ diff --git a/libfreerdp/core/input.c b/libfreerdp/core/input.c index 45b2ea190..86bf106fc 100644 --- a/libfreerdp/core/input.c +++ b/libfreerdp/core/input.c @@ -901,7 +901,7 @@ static BOOL input_update_last_event(rdpInput* input, BOOL mouse, UINT16 x, UINT1 if (freerdp_settings_get_uint32(input->context->settings, FreeRDP_FakeMouseMotionInterval) > 0) { const time_t now = time(NULL); - in->lastInputTimestamp = now; + in->lastInputTimestamp = WINPR_ASSERTING_INT_CAST(UINT64, now); if (mouse) { diff --git a/libfreerdp/core/listener.c b/libfreerdp/core/listener.c index 7b0e4343a..494af69bd 100644 --- a/libfreerdp/core/listener.c +++ b/libfreerdp/core/listener.c @@ -199,7 +199,7 @@ static BOOL freerdp_listener_open(freerdp_listener* instance, const char* bind_a arg = 1; ioctlsocket(sockfd, FIONBIO, &arg); #endif - status = _bind((SOCKET)sockfd, ai->ai_addr, ai->ai_addrlen); + status = _bind((SOCKET)sockfd, ai->ai_addr, WINPR_ASSERTING_INT_CAST(int, ai->ai_addrlen)); if (status != 0) { @@ -226,7 +226,7 @@ static BOOL freerdp_listener_open(freerdp_listener* instance, const char* bind_a break; } - WSAEventSelect(sockfd, listener->events[listener->num_sockfds], + WSAEventSelect((SOCKET)sockfd, listener->events[listener->num_sockfds], FD_READ | FD_ACCEPT | FD_CLOSE); listener->num_sockfds++; WLog_INFO(TAG, "Listening on [%s]:%" PRIu16, addr, port); @@ -270,7 +270,7 @@ static BOOL freerdp_listener_open_local(freerdp_listener* instance, const char* addr.sun_family = AF_UNIX; strncpy(addr.sun_path, path, sizeof(addr.sun_path) - 1); unlink(path); - status = _bind(sockfd, (struct sockaddr*)&addr, sizeof(addr)); + status = _bind((SOCKET)sockfd, (struct sockaddr*)&addr, sizeof(addr)); if (status != 0) { @@ -279,7 +279,7 @@ static BOOL freerdp_listener_open_local(freerdp_listener* instance, const char* return FALSE; } - status = _listen(sockfd, 10); + status = _listen((SOCKET)sockfd, 10); if (status != 0) { @@ -327,7 +327,8 @@ static BOOL freerdp_listener_open_from_socket(freerdp_listener* instance, int fd if (!listener->events[listener->num_sockfds]) return FALSE; - WSAEventSelect(fd, listener->events[listener->num_sockfds], FD_READ | FD_ACCEPT | FD_CLOSE); + WSAEventSelect((SOCKET)fd, listener->events[listener->num_sockfds], + FD_READ | FD_ACCEPT | FD_CLOSE); listener->num_sockfds++; WLog_INFO(TAG, "Listening on socket %d.", fd); @@ -384,7 +385,7 @@ static DWORD freerdp_listener_get_event_handles(freerdp_listener* instance, HAND events[index] = listener->events[index]; } - return listener->num_sockfds; + return WINPR_ASSERTING_INT_CAST(uint32_t, listener->num_sockfds); } BOOL freerdp_peer_set_local_and_hostname(freerdp_peer* client, @@ -481,7 +482,7 @@ static BOOL freerdp_listener_check_fds(freerdp_listener* instance) (void)WSAResetEvent(listener->events[i]); int peer_addr_size = sizeof(peer_addr); SOCKET peer_sockfd = - _accept(listener->sockfds[i], (struct sockaddr*)&peer_addr, &peer_addr_size); + _accept((SOCKET)listener->sockfds[i], (struct sockaddr*)&peer_addr, &peer_addr_size); if (peer_sockfd == (SOCKET)-1) { diff --git a/libfreerdp/core/message.c b/libfreerdp/core/message.c index 73961a026..4412148fa 100644 --- a/libfreerdp/core/message.c +++ b/libfreerdp/core/message.c @@ -195,7 +195,7 @@ static BOOL update_message_SetKeyboardImeStatus(rdpContext* context, UINT16 imeI up = update_cast(context->update); return MessageQueue_Post(up->queue, (void*)context, MakeMessageId(Update, SetKeyboardImeStatus), - (void*)(size_t)((imeId << 16UL) | imeState), + (void*)(size_t)((((UINT32)imeId << 16UL) & 0xFFFF0000) | imeState), (void*)(size_t)imeConvMode); } @@ -3104,16 +3104,14 @@ int input_message_queue_process_message(rdpInput* input, wMessage* message) int input_message_queue_process_pending_messages(rdpInput* input) { - int count = 0; int status = 1; wMessage message = { 0 }; - wMessageQueue* queue = NULL; rdp_input_internal* in = input_cast(input); if (!in->queue) return -1; - queue = in->queue; + wMessageQueue* queue = in->queue; while (MessageQueue_Peek(queue, &message, TRUE)) { @@ -3121,8 +3119,6 @@ int input_message_queue_process_pending_messages(rdpInput* input) if (!status) break; - - count++; } return status; diff --git a/libfreerdp/core/multitransport.c b/libfreerdp/core/multitransport.c index 492ec6824..b45579e5c 100644 --- a/libfreerdp/core/multitransport.c +++ b/libfreerdp/core/multitransport.c @@ -177,7 +177,7 @@ state_run_t multitransport_recv_response(rdpMultitransport* multi, wStream* s) return STATE_RUN_FAILED; UINT32 requestId = 0; - HRESULT hr = 0; + UINT32 hr = 0; Stream_Read_UINT32(s, requestId); /* requestId (4 bytes) */ Stream_Read_UINT32(s, hr); /* hrResponse (4 bytes) */ diff --git a/libfreerdp/core/nego.c b/libfreerdp/core/nego.c index 76886afd9..bff57f3ec 100644 --- a/libfreerdp/core/nego.c +++ b/libfreerdp/core/nego.c @@ -1297,9 +1297,9 @@ static const char* nego_rdp_neg_rsp_flags_str(UINT32 flags) winpr_str_append("RESTRICTED_ADMIN_MODE_SUPPORTED", buffer, sizeof(buffer), "|"); if (flags & REDIRECTED_AUTHENTICATION_MODE_SUPPORTED) winpr_str_append("REDIRECTED_AUTHENTICATION_MODE_SUPPORTED", buffer, sizeof(buffer), "|"); - if ((flags & - ~(EXTENDED_CLIENT_DATA_SUPPORTED | DYNVC_GFX_PROTOCOL_SUPPORTED | RDP_NEGRSP_RESERVED | - RESTRICTED_ADMIN_MODE_SUPPORTED | REDIRECTED_AUTHENTICATION_MODE_SUPPORTED))) + if ((flags & (uint32_t)~(EXTENDED_CLIENT_DATA_SUPPORTED | DYNVC_GFX_PROTOCOL_SUPPORTED | + RDP_NEGRSP_RESERVED | RESTRICTED_ADMIN_MODE_SUPPORTED | + REDIRECTED_AUTHENTICATION_MODE_SUPPORTED))) winpr_str_append("UNKNOWN", buffer, sizeof(buffer), "|"); return buffer; diff --git a/libfreerdp/core/nla.c b/libfreerdp/core/nla.c index 760d1a731..f08899d1f 100644 --- a/libfreerdp/core/nla.c +++ b/libfreerdp/core/nla.c @@ -117,7 +117,7 @@ struct rdp_nla rdpTransport* transport; UINT32 version; UINT32 peerVersion; - UINT32 errorCode; + INT32 errorCode; /* Lifetime of buffer nla_new -> nla_free */ SecBuffer ClientNonce; /* Depending on protocol version a random nonce or a value read from the @@ -1561,7 +1561,9 @@ static BOOL nla_encode_ts_credentials(rdpNla* nla) /* keySpec [0] INTEGER */ if (!WinPrAsn1EncContextualInteger( - enc, 0, freerdp_settings_get_uint32(settings, FreeRDP_KeySpec))) + enc, 0, + WINPR_ASSERTING_INT_CAST( + WinPrAsn1_INTEGER, freerdp_settings_get_uint32(settings, FreeRDP_KeySpec)))) goto out; for (size_t i = 0; i < ARRAYSIZE(cspData_fields); i++) @@ -1770,7 +1772,8 @@ BOOL nla_send(rdpNla* nla) /* version [0] INTEGER */ WLog_DBG(TAG, " ----->> protocol version %" PRIu32, nla->version); - if (!WinPrAsn1EncContextualInteger(enc, 0, nla->version)) + if (!WinPrAsn1EncContextualInteger(enc, 0, + WINPR_ASSERTING_INT_CAST(WinPrAsn1_INTEGER, nla->version))) goto fail; /* negoTokens [1] SEQUENCE OF SEQUENCE */ @@ -1809,7 +1812,8 @@ BOOL nla_send(rdpNla* nla) { WLog_DBG(TAG, " ----->> error code %s 0x%08" PRIx32, NtStatus2Tag(nla->errorCode), nla->errorCode); - if (!WinPrAsn1EncContextualInteger(enc, 4, nla->errorCode)) + if (!WinPrAsn1EncContextualInteger( + enc, 4, WINPR_ASSERTING_INT_CAST(WinPrAsn1_INTEGER, nla->errorCode))) goto fail; } @@ -1932,7 +1936,7 @@ static int nla_decode_ts_request(rdpNla* nla, wStream* s) /* errorCode [4] INTEGER */ if (WinPrAsn1DecReadInteger(&dec2, &val) == 0) return -1; - nla->errorCode = (UINT)val; + nla->errorCode = val; WLog_DBG(TAG, " <<----- error code %s 0x%08" PRIx32, NtStatus2Tag(nla->errorCode), nla->errorCode); break; @@ -2200,10 +2204,10 @@ DWORD nla_get_error(rdpNla* nla) { if (!nla) return ERROR_INTERNAL_ERROR; - return nla->errorCode; + return (UINT32)nla->errorCode; } -UINT32 nla_get_sspi_error(rdpNla* nla) +INT32 nla_get_sspi_error(rdpNla* nla) { WINPR_ASSERT(nla); return credssp_auth_sspi_error(nla->auth); diff --git a/libfreerdp/core/nla.h b/libfreerdp/core/nla.h index 8a1573777..ffb4ed807 100644 --- a/libfreerdp/core/nla.h +++ b/libfreerdp/core/nla.h @@ -58,7 +58,7 @@ FREERDP_LOCAL BOOL nla_set_state(rdpNla* nla, NLA_STATE state); FREERDP_LOCAL const char* nla_get_state_str(NLA_STATE state); FREERDP_LOCAL DWORD nla_get_error(rdpNla* nla); -FREERDP_LOCAL UINT32 nla_get_sspi_error(rdpNla* nla); +FREERDP_LOCAL INT32 nla_get_sspi_error(rdpNla* nla); FREERDP_LOCAL BOOL nla_set_service_principal(rdpNla* nla, const char* service, const char* hostname); diff --git a/libfreerdp/core/orders.c b/libfreerdp/core/orders.c index 6c5f98ab6..f14fb8630 100644 --- a/libfreerdp/core/orders.c +++ b/libfreerdp/core/orders.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -189,7 +190,7 @@ static BYTE get_bmf_bpp(UINT32 bmf, BOOL* pValid) if (pValid) *pValid = TRUE; /* Mask out highest bit */ - switch (bmf & (~CACHED_BRUSH)) + switch (bmf & (uint32_t)(~CACHED_BRUSH)) { case 1: return 1; @@ -676,7 +677,7 @@ static INLINE BOOL update_read_2byte_unsigned(wStream* s, UINT32* value) if (!Stream_CheckAndLogRequiredLength(TAG, s, 1)) return FALSE; - *value = (byte & 0x7F) << 8; + *value = ((byte & 0x7F) << 8) & 0xFFFF; Stream_Read_UINT8(s, byte); *value |= byte; } @@ -774,14 +775,11 @@ static INLINE BOOL update_write_2byte_signed(wStream* s, INT32 value) } static INLINE BOOL update_read_4byte_unsigned(wStream* s, UINT32* value) { - BYTE byte = 0; - BYTE count = 0; - if (!Stream_CheckAndLogRequiredLength(TAG, s, 1)) return FALSE; - Stream_Read_UINT8(s, byte); - count = (byte & 0xC0) >> 6; + const UINT32 byte = Stream_Get_UINT8(s); + const BYTE count = WINPR_ASSERTING_INT_CAST(uint8_t, (byte & 0xC0) >> 6); if (!Stream_CheckAndLogRequiredLength(TAG, s, count)) return FALSE; @@ -793,27 +791,21 @@ static INLINE BOOL update_read_4byte_unsigned(wStream* s, UINT32* value) break; case 1: - *value = (byte & 0x3F) << 8; - Stream_Read_UINT8(s, byte); - *value |= byte; + *value = ((byte & 0x3F) << 8) & 0xFFFF; + *value |= Stream_Get_UINT8(s); break; case 2: - *value = (byte & 0x3F) << 16; - Stream_Read_UINT8(s, byte); - *value |= (byte << 8); - Stream_Read_UINT8(s, byte); - *value |= byte; + *value = ((byte & 0x3F) << 16) & 0xFFFFFF; + *value |= ((Stream_Get_UINT8(s) << 8)) & 0xFFFF; + *value |= Stream_Get_UINT8(s); break; case 3: - *value = (byte & 0x3F) << 24; - Stream_Read_UINT8(s, byte); - *value |= (byte << 16); - Stream_Read_UINT8(s, byte); - *value |= (byte << 8); - Stream_Read_UINT8(s, byte); - *value |= byte; + *value = ((byte & 0x3F) << 24) & 0xFF000000; + *value |= ((Stream_Get_UINT8(s) << 16)) & 0xFF0000; + *value |= ((Stream_Get_UINT8(s) << 8)) & 0xFF00; + *value |= Stream_Get_UINT8(s); break; default: @@ -874,7 +866,7 @@ static INLINE BOOL update_read_delta(wStream* s, INT32* value) Stream_Read_UINT8(s, byte); if (byte & 0x40) - uvalue = (byte | ~0x3F); + uvalue = (byte | ((~0x3F) & 0xFF)); else uvalue = (byte & 0x3F); @@ -2552,7 +2544,7 @@ BOOL update_write_cache_bitmap_v2_order(wStream* s, CACHE_BITMAP_V2_ORDER* cache WINPR_ASSERT(cache_bitmap_v2->cacheId <= 3); WINPR_ASSERT(bitsPerPixelId <= 0x0f); WINPR_ASSERT(cache_bitmap_v2->flags <= 0x1FF); - *flags = (UINT16)((cache_bitmap_v2->cacheId & 0x0003) | (bitsPerPixelId << 3) | + *flags = (UINT16)((cache_bitmap_v2->cacheId & 0x0003) | ((bitsPerPixelId << 3) & 0xFFFF) | ((cache_bitmap_v2->flags << 7) & 0xFF80)); if (cache_bitmap_v2->flags & CBR2_PERSISTENT_KEY_PRESENT) @@ -3020,19 +3012,18 @@ static BOOL update_decompress_brush(wStream* s, BYTE* output, size_t outSize, BY if (!Stream_CheckAndLogRequiredLengthOfSize(TAG, s, 4ULL + bytesPerPixel, 4ULL)) return FALSE; - for (INT8 y = 7; y >= 0; y--) + for (size_t y = 0; y < 7; y++) { for (size_t x = 0; x < 8; x++) { - UINT32 index = 0; if ((x % 4) == 0) Stream_Read_UINT8(s, byte); - index = ((byte >> ((3 - (x % 4)) * 2)) & 0x03); + const uint32_t index = ((byte >> ((3 - (x % 4)) * 2)) & 0x03); for (size_t k = 0; k < bytesPerPixel; k++) { - const size_t dstIndex = ((8ULL * y + x) * bytesPerPixel) + k; + const size_t dstIndex = ((8ULL * (7ULL - y) + x) * bytesPerPixel) + k; const size_t srcIndex = (index * bytesPerPixel) + k; if (dstIndex >= outSize) return FALSE; @@ -3196,11 +3187,11 @@ BOOL update_write_cache_brush_order(wStream* s, const CACHE_BRUSH_ORDER* cache_b else { /* uncompressed brush */ - int scanline = (cache_brush->bpp / 8) * 8; + const size_t scanline = 8ULL * (cache_brush->bpp / 8); - for (int i = 7; i >= 0; i--) + for (size_t i = 0; i <= 7; i++) { - Stream_Write(s, &cache_brush->data[1LL * i * scanline], scanline); + Stream_Write(s, &cache_brush->data[1LL * (7 - i) * scanline], scanline); } } } @@ -3500,8 +3491,6 @@ update_read_draw_gdiplus_cache_end_order(wStream* s, } static BOOL update_read_field_flags(wStream* s, UINT32* fieldFlags, BYTE flags, BYTE fieldBytes) { - BYTE byte = 0; - if (flags & ORDER_ZERO_FIELD_BYTE_BIT0) fieldBytes--; @@ -3518,10 +3507,10 @@ static BOOL update_read_field_flags(wStream* s, UINT32* fieldFlags, BYTE flags, *fieldFlags = 0; - for (int i = 0; i < fieldBytes; i++) + for (size_t i = 0; i < fieldBytes; i++) { - Stream_Read_UINT8(s, byte); - *fieldFlags |= byte << (i * 8); + const UINT32 byte = Stream_Get_UINT8(s); + *fieldFlags |= (byte << (i * 8ULL)) & 0xFFFFFFFF; } return TRUE; @@ -3613,8 +3602,10 @@ static BOOL update_read_bounds(wStream* s, rdpBounds* bounds) return TRUE; } -BOOL update_write_bounds(wStream* s, ORDER_INFO* orderInfo) +BOOL update_write_bounds(wStream* s, const ORDER_INFO* orderInfo) { + WINPR_ASSERT(orderInfo); + if (!(orderInfo->controlFlags & ORDER_BOUNDS)) return TRUE; @@ -4073,7 +4064,6 @@ static BOOL update_recv_secondary_order(rdpUpdate* update, wStream* s, BYTE flag BYTE orderType = 0; UINT16 extraFlags = 0; INT16 orderLength = 0; - INT32 orderLengthFull = 0; rdp_update_internal* up = update_cast(update); rdpContext* context = update->context; rdpSettings* settings = context->settings; @@ -4106,14 +4096,14 @@ static BOOL update_recv_secondary_order(rdpUpdate* update, wStream* s, BYTE flag /* orderLength might be negative without the adjusted header data. * Account for that here so all further checks operate on the correct value. */ - orderLengthFull = orderLength + 7; - if (orderLengthFull < 0) + if (orderLength < 0) { WLog_Print(up->log, WLOG_ERROR, "orderLength %" PRIu16 " must be >= 7", orderLength); return FALSE; } - if (!Stream_CheckAndLogRequiredLength(TAG, s, (size_t)orderLengthFull)) + const size_t orderLengthFull = WINPR_ASSERTING_INT_CAST(size_t, orderLength) + 7ULL; + if (!Stream_CheckAndLogRequiredLength(TAG, s, orderLengthFull)) return FALSE; if (!check_secondary_order_supported(up->log, settings, orderType, name)) @@ -4236,7 +4226,7 @@ static BOOL update_recv_secondary_order(rdpUpdate* update, wStream* s, BYTE flag WLog_Print(up->log, WLOG_ERROR, "%s %s failed", secondary_order_str, name); } - end = start + orderLengthFull; + end = start + WINPR_ASSERTING_INT_CAST(size_t, orderLengthFull); pos = Stream_GetPosition(s); if (pos > end) { diff --git a/libfreerdp/core/orders.h b/libfreerdp/core/orders.h index 121a52292..5500e5181 100644 --- a/libfreerdp/core/orders.h +++ b/libfreerdp/core/orders.h @@ -196,7 +196,7 @@ FREERDP_LOCAL BOOL update_recv_order(rdpUpdate* update, wStream* s); FREERDP_LOCAL BOOL update_write_field_flags(wStream* s, UINT32 fieldFlags, BYTE flags, BYTE fieldBytes); -FREERDP_LOCAL BOOL update_write_bounds(wStream* s, ORDER_INFO* orderInfo); +FREERDP_LOCAL BOOL update_write_bounds(wStream* s, const ORDER_INFO* orderInfo); FREERDP_LOCAL size_t update_approximate_dstblt_order(ORDER_INFO* orderInfo, const DSTBLT_ORDER* dstblt); diff --git a/libfreerdp/core/peer.c b/libfreerdp/core/peer.c index f25720951..67807d92c 100644 --- a/libfreerdp/core/peer.c +++ b/libfreerdp/core/peer.c @@ -25,6 +25,7 @@ #include "settings.h" #include +#include #include #include @@ -990,8 +991,8 @@ static state_run_t peer_recv_callback_internal(rdpTransport* transport, wStream* const UINT32 h = freerdp_settings_get_uint32(settings, FreeRDP_DesktopHeight); const rdpMonitor primary = { .x = 0, .y = 0, - .width = w, - .height = h, + .width = WINPR_ASSERTING_INT_CAST(int32_t, w), + .height = WINPR_ASSERTING_INT_CAST(int32_t, h), .is_primary = TRUE, .orig_screen = 0, .attributes = { .physicalWidth = w, @@ -1375,18 +1376,15 @@ static const char* os_major_type_to_string(UINT16 osMajorType) const char* freerdp_peer_os_major_type_string(freerdp_peer* client) { - rdpContext* context = NULL; - UINT16 osMajorType = 0; - WINPR_ASSERT(client); - context = client->context; + rdpContext* context = client->context; WINPR_ASSERT(context); WINPR_ASSERT(context->settings); - osMajorType = freerdp_settings_get_uint32(context->settings, FreeRDP_OsMajorType); - - return os_major_type_to_string(osMajorType); + const UINT32 osMajorType = freerdp_settings_get_uint32(context->settings, FreeRDP_OsMajorType); + WINPR_ASSERT(osMajorType <= UINT16_MAX); + return os_major_type_to_string((UINT16)osMajorType); } static const char* os_minor_type_to_string(UINT16 osMinorType) @@ -1422,18 +1420,15 @@ static const char* os_minor_type_to_string(UINT16 osMinorType) const char* freerdp_peer_os_minor_type_string(freerdp_peer* client) { - rdpContext* context = NULL; - UINT16 osMinorType = 0; - WINPR_ASSERT(client); - context = client->context; + rdpContext* context = client->context; WINPR_ASSERT(context); WINPR_ASSERT(context->settings); - osMinorType = freerdp_settings_get_uint32(context->settings, FreeRDP_OsMinorType); - - return os_minor_type_to_string(osMinorType); + const UINT32 osMinorType = freerdp_settings_get_uint32(context->settings, FreeRDP_OsMinorType); + WINPR_ASSERT(osMinorType <= UINT16_MAX); + return os_minor_type_to_string((UINT16)osMinorType); } freerdp_peer* freerdp_peer_new(int sockfd) diff --git a/libfreerdp/core/proxy.c b/libfreerdp/core/proxy.c index 0677c1d3e..05b6f1214 100644 --- a/libfreerdp/core/proxy.c +++ b/libfreerdp/core/proxy.c @@ -665,7 +665,7 @@ static BOOL http_proxy_connect(rdpContext* context, BIO* bufferedBio, const char goto fail; } - resultsize += status; + resultsize += WINPR_ASSERTING_INT_CAST(size_t, status); } /* Extract HTTP status line */ diff --git a/libfreerdp/core/rdp.c b/libfreerdp/core/rdp.c index b4ff18ced..03f4232c1 100644 --- a/libfreerdp/core/rdp.c +++ b/libfreerdp/core/rdp.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "rdp.h" @@ -306,7 +307,10 @@ BOOL rdp_read_share_control_header(rdpRdp* rdp, wStream* s, UINT16* tpktLength, WLog_Print(rdp->log, WLOG_DEBUG, "type=%s, tpktLength=%" PRIuz ", remainingLength=%" PRIuz, pdu_type_to_str(*type, buffer, sizeof(buffer)), len, remLen); if (remainingLength) - *remainingLength = remLen; + { + WINPR_ASSERT(remLen <= UINT16_MAX); + *remainingLength = (UINT16)remLen; + } return Stream_CheckAndLogRequiredLengthWLog(rdp->log, s, remLen); } @@ -324,9 +328,9 @@ BOOL rdp_write_share_control_header(rdpRdp* rdp, wStream* s, size_t length, UINT return FALSE; length -= RDP_PACKET_HEADER_MAX_LENGTH; /* Share Control Header */ - Stream_Write_UINT16(s, length); /* totalLength */ - Stream_Write_UINT16(s, type | 0x10); /* pduType */ - Stream_Write_UINT16(s, channel_id); /* pduSource */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, length)); /* totalLength */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, type | 0x10)); /* pduType */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, channel_id)); /* pduSource */ return TRUE; } @@ -370,7 +374,8 @@ BOOL rdp_write_share_data_header(rdpRdp* rdp, wStream* s, size_t length, BYTE ty Stream_Write_UINT32(s, share_id); /* shareId (4 bytes) */ Stream_Write_UINT8(s, 0); /* pad1 (1 byte) */ Stream_Write_UINT8(s, STREAM_LOW); /* streamId (1 byte) */ - Stream_Write_UINT16(s, length); /* uncompressedLength (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, length)); /* uncompressedLength (2 bytes) */ Stream_Write_UINT8(s, type); /* pduType2, Data PDU Type (1 byte) */ Stream_Write_UINT8(s, 0); /* compressedType (1 byte) */ Stream_Write_UINT16(s, 0); /* compressedLength (2 bytes) */ @@ -535,7 +540,6 @@ fail: BOOL rdp_read_header(rdpRdp* rdp, wStream* s, UINT16* length, UINT16* channelId) { BYTE li = 0; - BYTE byte = 0; BYTE code = 0; BYTE choice = 0; UINT16 initiator = 0; @@ -640,7 +644,7 @@ BOOL rdp_read_header(rdpRdp* rdp, wStream* s, UINT16* length, UINT16* channelId) if (!per_read_integer16(s, channelId, 0)) /* channelId */ return FALSE; - Stream_Read_UINT8(s, byte); /* dataPriority + Segmentation (0x70) */ + const uint8_t byte = Stream_Get_UINT8(s); /* dataPriority + Segmentation (0x70) */ if (!per_read_length(s, length)) /* userData (OCTET_STRING) */ return FALSE; @@ -676,14 +680,14 @@ BOOL rdp_write_header(rdpRdp* rdp, wStream* s, size_t length, UINT16 channelId) if ((rdp->sec_flags & SEC_ENCRYPT) && (rdp->settings->EncryptionMethods == ENCRYPTION_METHOD_FIPS)) { - const UINT16 body_length = length - RDP_PACKET_HEADER_MAX_LENGTH; + const UINT16 body_length = (UINT16)length - RDP_PACKET_HEADER_MAX_LENGTH; const UINT16 pad = 8 - (body_length % 8); if (pad != 8) length += pad; } - if (!mcs_write_domain_mcspdu_header(s, MCSPDU, length, 0)) + if (!mcs_write_domain_mcspdu_header(s, MCSPDU, (UINT16)length, 0)) return FALSE; if (!per_write_integer16(s, rdp->mcs->userId, MCS_BASE_CHANNEL_ID)) /* initiator */ return FALSE; @@ -699,7 +703,8 @@ BOOL rdp_write_header(rdpRdp* rdp, wStream* s, size_t length, UINT16 channelId) * the data first and then store the header. */ length = (length - RDP_PACKET_HEADER_MAX_LENGTH) | 0x8000; - Stream_Write_UINT16_BE(s, length); /* userData (OCTET_STRING) */ + Stream_Write_UINT16_BE( + s, WINPR_ASSERTING_INT_CAST(uint16_t, length)); /* userData (OCTET_STRING) */ return TRUE; } @@ -716,7 +721,8 @@ static BOOL rdp_security_stream_out(rdpRdp* rdp, wStream* s, size_t length, UINT if (sec_flags != 0) { - if (!rdp_write_security_header(rdp, s, sec_flags)) + WINPR_ASSERT(sec_flags <= UINT16_MAX); + if (!rdp_write_security_header(rdp, s, (UINT16)sec_flags)) return FALSE; if (sec_flags & SEC_ENCRYPT) @@ -728,7 +734,7 @@ static BOOL rdp_security_stream_out(rdpRdp* rdp, wStream* s, size_t length, UINT if (rdp->settings->EncryptionMethods == ENCRYPTION_METHOD_FIPS) { BYTE* data = Stream_PointerAs(s, BYTE) + 12; - const size_t size = (data - Stream_Buffer(s)); + const size_t size = WINPR_ASSERTING_INT_CAST(size_t, (data - Stream_Buffer(s))); if (size > length) goto unlock; @@ -745,7 +751,7 @@ static BOOL rdp_security_stream_out(rdpRdp* rdp, wStream* s, size_t length, UINT if (*pad) memset(data + length, 0, *pad); - Stream_Write_UINT8(s, *pad); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, *pad)); if (!Stream_CheckAndLogRequiredCapacityWLog(rdp->log, s, 8)) goto unlock; @@ -974,7 +980,6 @@ static BOOL rdp_recv_server_shutdown_denied_pdu(rdpRdp* rdp, wStream* s) static BOOL rdp_recv_server_set_keyboard_indicators_pdu(rdpRdp* rdp, wStream* s) { - UINT16 unitId = 0; UINT16 ledFlags = 0; WINPR_ASSERT(rdp); @@ -987,7 +992,7 @@ static BOOL rdp_recv_server_set_keyboard_indicators_pdu(rdpRdp* rdp, wStream* s) if (!Stream_CheckAndLogRequiredLengthWLog(rdp->log, s, 4)) return FALSE; - Stream_Read_UINT16(s, unitId); /* unitId (2 bytes) */ + const uint16_t unitId = Stream_Get_UINT16(s); /* unitId (2 bytes) */ Stream_Read_UINT16(s, ledFlags); /* ledFlags (2 bytes) */ return IFCALLRESULT(TRUE, context->update->SetKeyboardIndicators, context, ledFlags); } @@ -1450,7 +1455,6 @@ BOOL rdp_decrypt(rdpRdp* rdp, wStream* s, UINT16* pLength, UINT16 securityFlags) BYTE cmac[8] = { 0 }; BYTE wmac[8] = { 0 }; BOOL status = FALSE; - INT32 length = 0; WINPR_ASSERT(rdp); WINPR_ASSERT(rdp->settings); @@ -1460,7 +1464,7 @@ BOOL rdp_decrypt(rdpRdp* rdp, wStream* s, UINT16* pLength, UINT16 securityFlags) if (!security_lock(rdp)) return FALSE; - length = *pLength; + INT32 length = *pLength; if (rdp->settings->EncryptionMethods == ENCRYPTION_METHOD_NONE) return TRUE; @@ -1494,10 +1498,10 @@ BOOL rdp_decrypt(rdpRdp* rdp, wStream* s, UINT16* pLength, UINT16 securityFlags) goto unlock; } - if (!security_fips_decrypt(Stream_Pointer(s), length, rdp)) + if (!security_fips_decrypt(Stream_Pointer(s), (size_t)length, rdp)) goto unlock; - if (!security_fips_check_signature(Stream_ConstPointer(s), length - pad, sig, 8, rdp)) + if (!security_fips_check_signature(Stream_ConstPointer(s), (size_t)padLength, sig, 8, rdp)) goto unlock; Stream_SetLength(s, Stream_Length(s) - pad); @@ -1517,15 +1521,15 @@ BOOL rdp_decrypt(rdpRdp* rdp, wStream* s, UINT16* pLength, UINT16 securityFlags) goto unlock; } - if (!security_decrypt(Stream_PointerAs(s, BYTE), length, rdp)) + if (!security_decrypt(Stream_PointerAs(s, BYTE), (size_t)length, rdp)) goto unlock; if (securityFlags & SEC_SECURE_CHECKSUM) - status = security_salted_mac_signature(rdp, Stream_ConstPointer(s), length, FALSE, cmac, - sizeof(cmac)); + status = security_salted_mac_signature(rdp, Stream_ConstPointer(s), (UINT32)length, + FALSE, cmac, sizeof(cmac)); else - status = - security_mac_signature(rdp, Stream_ConstPointer(s), length, cmac, sizeof(cmac)); + status = security_mac_signature(rdp, Stream_ConstPointer(s), (UINT32)length, cmac, + sizeof(cmac)); if (!status) goto unlock; @@ -1543,7 +1547,7 @@ BOOL rdp_decrypt(rdpRdp* rdp, wStream* s, UINT16* pLength, UINT16 securityFlags) // return FALSE; } - *pLength = length; + *pLength = (UINT16)length; } res = TRUE; unlock: @@ -2577,11 +2581,12 @@ void* rdp_get_io_callback_context(rdpRdp* rdp) const char* rdp_finalize_flags_to_str(UINT32 flags, char* buffer, size_t size) { char number[32] = { 0 }; - const UINT32 mask = ~(FINALIZE_SC_SYNCHRONIZE_PDU | FINALIZE_SC_CONTROL_COOPERATE_PDU | - FINALIZE_SC_CONTROL_GRANTED_PDU | FINALIZE_SC_FONT_MAP_PDU | - FINALIZE_CS_SYNCHRONIZE_PDU | FINALIZE_CS_CONTROL_COOPERATE_PDU | - FINALIZE_CS_CONTROL_REQUEST_PDU | FINALIZE_CS_PERSISTENT_KEY_LIST_PDU | - FINALIZE_CS_FONT_LIST_PDU | FINALIZE_DEACTIVATE_REACTIVATE); + const UINT32 mask = + (uint32_t)~(FINALIZE_SC_SYNCHRONIZE_PDU | FINALIZE_SC_CONTROL_COOPERATE_PDU | + FINALIZE_SC_CONTROL_GRANTED_PDU | FINALIZE_SC_FONT_MAP_PDU | + FINALIZE_CS_SYNCHRONIZE_PDU | FINALIZE_CS_CONTROL_COOPERATE_PDU | + FINALIZE_CS_CONTROL_REQUEST_PDU | FINALIZE_CS_PERSISTENT_KEY_LIST_PDU | + FINALIZE_CS_FONT_LIST_PDU | FINALIZE_DEACTIVATE_REACTIVATE); if (flags & FINALIZE_SC_SYNCHRONIZE_PDU) winpr_str_append("FINALIZE_SC_SYNCHRONIZE_PDU", buffer, size, "|"); diff --git a/libfreerdp/core/server.c b/libfreerdp/core/server.c index 8c5a21343..fe307b98d 100644 --- a/libfreerdp/core/server.c +++ b/libfreerdp/core/server.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include #include @@ -81,7 +83,8 @@ static BOOL wts_queue_receive_data(rdpPeerChannel* channel, const BYTE* Buffer, if (!messageCtx) return FALSE; - messageCtx->channelId = channel->channelId; + WINPR_ASSERT(channel->channelId <= UINT16_MAX); + messageCtx->channelId = (UINT16)channel->channelId; messageCtx->length = Length; messageCtx->offset = 0; buffer = (BYTE*)(messageCtx + 1); @@ -93,12 +96,14 @@ static BOOL wts_queue_send_item(rdpPeerChannel* channel, BYTE* Buffer, UINT32 Le { BYTE* buffer = NULL; UINT32 length = 0; - UINT16 channelId = 0; + WINPR_ASSERT(channel); WINPR_ASSERT(channel->vcm); buffer = Buffer; length = Length; - channelId = channel->channelId; + + WINPR_ASSERT(channel->channelId <= UINT16_MAX); + const UINT16 channelId = (UINT16)channel->channelId; return MessageQueue_Post(channel->vcm->queue, (void*)(UINT_PTR)channelId, 0, (void*)buffer, (void*)(UINT_PTR)length); } @@ -200,15 +205,16 @@ static BOOL wts_read_drdynvc_create_response(rdpPeerChannel* channel, wStream* s static BOOL wts_read_drdynvc_data_first(rdpPeerChannel* channel, wStream* s, int cbLen, UINT32 length) { - int value = 0; WINPR_ASSERT(channel); WINPR_ASSERT(s); - value = wts_read_variable_uint(s, cbLen, &channel->dvc_total_length); + const UINT32 value = wts_read_variable_uint(s, cbLen, &channel->dvc_total_length); if (value == 0) return FALSE; - - length -= value; + if (value > length) + length = 0; + else + length -= value; if (length > channel->dvc_total_length) return FALSE; @@ -390,12 +396,12 @@ static int wts_write_variable_uint(wStream* s, UINT32 val) if (val <= 0xFF) { cb = 0; - Stream_Write_UINT8(s, val); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, val)); } else if (val <= 0xFFFF) { cb = 1; - Stream_Write_UINT16(s, val); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, val)); } else { @@ -416,7 +422,7 @@ static void wts_write_drdynvc_header(wStream* s, BYTE Cmd, UINT32 ChannelId) Stream_GetPointer(s, bm); Stream_Seek_UINT8(s); cbChId = wts_write_variable_uint(s, ChannelId); - *bm = ((Cmd & 0x0F) << 4) | cbChId; + *bm = (((Cmd & 0x0F) << 4) | cbChId) & 0xFF; } static BOOL wts_write_drdynvc_create_request(wStream* s, UINT32 ChannelId, const char* ChannelName) @@ -1319,7 +1325,9 @@ HANDLE WINAPI FreeRDP_WTSVirtualChannelOpen(HANDLE hServer, DWORD SessionId, LPS { const UINT32 VCChunkSize = freerdp_settings_get_uint32(context->settings, FreeRDP_VCChunkSize); - channel = channel_new(vcm, client, joined_channel->ChannelId, index, + + WINPR_ASSERT(index <= UINT16_MAX); + channel = channel_new(vcm, client, joined_channel->ChannelId, (UINT16)index, RDP_PEER_CHANNEL_TYPE_SVC, VCChunkSize, pVirtualName); if (!channel) @@ -1400,7 +1408,8 @@ HANDLE WINAPI FreeRDP_WTSVirtualChannelOpenEx(DWORD SessionId, LPSTR pVirtualNam return NULL; } - channel->channelId = InterlockedIncrement(&vcm->dvc_channel_id_seq); + const LONG hdl = InterlockedIncrement(&vcm->dvc_channel_id_seq); + channel->channelId = WINPR_ASSERTING_INT_CAST(uint32_t, hdl); if (!HashTable_Insert(vcm->dynamicVirtualChannels, &channel->channelId, channel)) { @@ -1600,11 +1609,11 @@ BOOL WINAPI FreeRDP_WTSVirtualChannelWrite(HANDLE hChannelHandle, PCHAR Buffer, if (first && (Length > Stream_GetRemainingLength(s))) { cbLen = wts_write_variable_uint(s, Length); - buffer[0] = (DATA_FIRST_PDU << 4) | (cbLen << 2) | cbChId; + buffer[0] = ((DATA_FIRST_PDU << 4) | (cbLen << 2) | cbChId) & 0xFF; } else { - buffer[0] = (DATA_PDU << 4) | cbChId; + buffer[0] = ((DATA_PDU << 4) | cbChId) & 0xFF; } first = FALSE; diff --git a/libfreerdp/core/streamdump.c b/libfreerdp/core/streamdump.c index e94cc5674..4ae571642 100644 --- a/libfreerdp/core/streamdump.c +++ b/libfreerdp/core/streamdump.c @@ -80,7 +80,7 @@ static return FALSE; if (pOffset) - (void)_fseeki64(fp, *pOffset, SEEK_SET); + (void)_fseeki64(fp, WINPR_ASSERTING_INT_CAST(int64_t, *pOffset), SEEK_SET); r = fread(&ts, 1, sizeof(ts), fp); if (r != sizeof(ts)) @@ -211,7 +211,7 @@ SSIZE_T stream_dump_append(const rdpContext* context, UINT32 flags, wStream* s, if (!fp) return -1; - r = _fseeki64(fp, *offset, SEEK_SET); + r = _fseeki64(fp, WINPR_ASSERTING_INT_CAST(int64_t, *offset), SEEK_SET); if (r < 0) goto fail; @@ -239,7 +239,7 @@ SSIZE_T stream_dump_get(const rdpContext* context, UINT32* flags, wStream* s, si fp = stream_dump_get_file(context->settings, "rb"); if (!fp) return -1; - r = _fseeki64(fp, *offset, SEEK_SET); + r = _fseeki64(fp, WINPR_ASSERTING_INT_CAST(int64_t, *offset), SEEK_SET); if (r < 0) goto fail; diff --git a/libfreerdp/core/surface.c b/libfreerdp/core/surface.c index 43256f87a..3a2830c8b 100644 --- a/libfreerdp/core/surface.c +++ b/libfreerdp/core/surface.c @@ -22,6 +22,7 @@ #include "settings.h" #include +#include #include #include @@ -293,11 +294,11 @@ static BOOL update_write_surfcmd_bitmap_ex(wStream* s, const TS_BITMAP_DATA_EX* BOOL update_write_surfcmd_surface_bits(wStream* s, const SURFACE_BITS_COMMAND* cmd) { - UINT16 cmdType = 0; if (!Stream_EnsureRemainingCapacity(s, SURFCMD_SURFACE_BITS_HEADER_LENGTH)) return FALSE; - cmdType = cmd->cmdType; + WINPR_ASSERT(cmd->cmdType <= UINT16_MAX); + UINT16 cmdType = (UINT16)cmd->cmdType; switch (cmdType) { case CMDTYPE_SET_SURFACE_BITS: @@ -312,11 +313,11 @@ BOOL update_write_surfcmd_surface_bits(wStream* s, const SURFACE_BITS_COMMAND* c break; } - Stream_Write_UINT16(s, cmdType); - Stream_Write_UINT16(s, cmd->destLeft); - Stream_Write_UINT16(s, cmd->destTop); - Stream_Write_UINT16(s, cmd->destRight); - Stream_Write_UINT16(s, cmd->destBottom); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, cmdType)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, cmd->destLeft)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, cmd->destTop)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, cmd->destRight)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, cmd->destBottom)); return update_write_surfcmd_bitmap_ex(s, &cmd->bmp); } diff --git a/libfreerdp/core/tcp.c b/libfreerdp/core/tcp.c index 53e3a412f..3c3c08ab2 100644 --- a/libfreerdp/core/tcp.c +++ b/libfreerdp/core/tcp.c @@ -593,7 +593,7 @@ static long transport_bio_buffered_ctrl(BIO* bio, int cmd, long arg1, void* arg2 break; case BIO_CTRL_WPENDING: - status = ringbuffer_used(&ptr->xmitBuffer); + status = WINPR_ASSERTING_INT_CAST(long, ringbuffer_used(&ptr->xmitBuffer)); break; case BIO_CTRL_PENDING: @@ -815,9 +815,8 @@ static BOOL freerdp_tcp_connect_timeout(rdpContext* context, int sockfd, struct socklen_t addrlen, UINT32 timeout) { BOOL rc = FALSE; - HANDLE handles[2]; - int status = 0; - int count = 0; + HANDLE handles[2] = { 0 }; + DWORD count = 0; u_long arg = 0; DWORD tout = (timeout > 0) ? timeout : INFINITE; @@ -826,22 +825,23 @@ static BOOL freerdp_tcp_connect_timeout(rdpContext* context, int sockfd, struct if (!handles[count]) return FALSE; - status = WSAEventSelect(sockfd, handles[count++], FD_READ | FD_WRITE | FD_CONNECT | FD_CLOSE); + const int wsastatus = WSAEventSelect((SOCKET)sockfd, handles[count++], + FD_READ | FD_WRITE | FD_CONNECT | FD_CLOSE); - if (status < 0) + if (wsastatus < 0) { WLog_ERR(TAG, "WSAEventSelect failed with %d", WSAGetLastError()); goto fail; } handles[count++] = utils_get_abort_event(context->rdp); - status = _connect(sockfd, addr, addrlen); + const int constatus = _connect((SOCKET)sockfd, addr, WINPR_ASSERTING_INT_CAST(int, addrlen)); - if (status < 0) + if (constatus < 0) { - status = WSAGetLastError(); + const int estatus = WSAGetLastError(); - switch (status) + switch (estatus) { case WSAEINPROGRESS: case WSAEWOULDBLOCK: @@ -852,9 +852,9 @@ static BOOL freerdp_tcp_connect_timeout(rdpContext* context, int sockfd, struct } } - status = WaitForMultipleObjects(count, handles, FALSE, tout); + const DWORD wstatus = WaitForMultipleObjects(count, handles, FALSE, tout); - if (WAIT_OBJECT_0 != status) + if (WAIT_OBJECT_0 != wstatus) goto fail; const SSIZE_T res = recv(sockfd, NULL, 0, 0); @@ -865,7 +865,7 @@ static BOOL freerdp_tcp_connect_timeout(rdpContext* context, int sockfd, struct goto fail; } - status = WSAEventSelect(sockfd, handles[0], 0); + const int status = WSAEventSelect((SOCKET)sockfd, handles[0], 0); if (status < 0) { @@ -873,7 +873,7 @@ static BOOL freerdp_tcp_connect_timeout(rdpContext* context, int sockfd, struct goto fail; } - if (_ioctlsocket(sockfd, FIONBIO, &arg) != 0) + if (_ioctlsocket((SOCKET)sockfd, FIONBIO, &arg) != 0) goto fail; rc = TRUE; @@ -903,7 +903,6 @@ static int freerdp_tcp_connect_multi(rdpContext* context, char** hostnames, cons UINT32 count, UINT16 port, UINT32 timeout) { UINT32 sindex = count; - int status = -1; SOCKET sockfd = INVALID_SOCKET; struct addrinfo* addr = NULL; struct addrinfo* result = NULL; @@ -923,7 +922,7 @@ static int freerdp_tcp_connect_multi(rdpContext* context, char** hostnames, cons int curPort = port; if (ports) - curPort = ports[index]; + curPort = WINPR_ASSERTING_INT_CAST(int, ports[index]); result = freerdp_tcp_resolve_host(hostnames[index], curPort, 0); @@ -965,9 +964,10 @@ static int freerdp_tcp_connect_multi(rdpContext* context, char** hostnames, cons continue; /* blocking tcp connect */ - status = _connect(sockfd, addr->ai_addr, addr->ai_addrlen); + const int rc = + _connect(sockfd, addr->ai_addr, WINPR_ASSERTING_INT_CAST(int, addr->ai_addrlen)); - if (status >= 0) + if (rc >= 0) { /* connection success */ sindex = index; @@ -1361,7 +1361,7 @@ static int freerdp_tcp_layer_read(void* userContext, void* data, int bytes) int status = 0; (void)WSAResetEvent(tcpLayer->hEvent); - status = _recv(tcpLayer->sockfd, data, bytes, 0); + status = _recv((SOCKET)tcpLayer->sockfd, data, bytes, 0); if (status > 0) return status; @@ -1396,7 +1396,7 @@ static int freerdp_tcp_layer_write(void* userContext, const void* data, int byte int error = 0; int status = 0; - status = _send(tcpLayer->sockfd, data, bytes, 0); + status = _send((SOCKET)tcpLayer->sockfd, data, bytes, 0); if (status <= 0) { @@ -1424,7 +1424,7 @@ static BOOL freerdp_tcp_layer_close(void* userContext) rdpTcpLayer* tcpLayer = (rdpTcpLayer*)userContext; if (tcpLayer->sockfd >= 0) - closesocket(tcpLayer->sockfd); + closesocket((SOCKET)tcpLayer->sockfd); if (tcpLayer->hEvent) (void)CloseHandle(tcpLayer->hEvent); @@ -1521,7 +1521,7 @@ rdpTransportLayer* freerdp_tcp_connect_layer(rdpContext* context, const char* ho goto fail; /* WSAEventSelect automatically sets the socket in non-blocking mode */ - if (WSAEventSelect(sockfd, tcpLayer->hEvent, FD_READ | FD_ACCEPT | FD_CLOSE)) + if (WSAEventSelect((SOCKET)sockfd, tcpLayer->hEvent, FD_READ | FD_ACCEPT | FD_CLOSE)) { WLog_ERR(TAG, "WSAEventSelect returned 0x%08X", WSAGetLastError()); goto fail; @@ -1533,7 +1533,7 @@ rdpTransportLayer* freerdp_tcp_connect_layer(rdpContext* context, const char* ho fail: if (sockfd >= 0) - closesocket(sockfd); + closesocket((SOCKET)sockfd); transport_layer_free(layer); return NULL; } diff --git a/libfreerdp/core/test/TestSettings.c b/libfreerdp/core/test/TestSettings.c index 755618e7b..c19d5df8a 100644 --- a/libfreerdp/core/test/TestSettings.c +++ b/libfreerdp/core/test/TestSettings.c @@ -480,7 +480,7 @@ static BOOL check_key_helpers(size_t key, const char* stype) FreeRDP_ReceivedCapabilities, FreeRDP_OrderSupport, FreeRDP_MonitorIds }; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); if (!name) { printf("[%s] missing name for key %" PRIuz "\n", stype, key); @@ -504,7 +504,7 @@ static BOOL check_key_helpers(size_t key, const char* stype) printf("[%s] missing reverse type for key %s [%" PRIuz "]\n", stype, name, key); return FALSE; } - rc = freerdp_settings_get_type_for_key(key); + rc = freerdp_settings_get_type_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); if (rc < 0) { printf("[%s] missing reverse name for key %s [%" PRIuz "]\n", stype, name, key); @@ -527,7 +527,7 @@ static BOOL check_key_helpers(size_t key, const char* stype) for (size_t x = 0; x < ARRAYSIZE(clear_keys); x++) { const size_t id = clear_keys[x]; - const char* foo = freerdp_settings_get_name_for_key(id); + const char* foo = freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, id)); if (!freerdp_settings_set_pointer_len(settings, id, NULL, 0)) { printf("[%s] freerdp_settings_set_pointer_len(%s, NULL, 0) failed\n", stype, foo); @@ -1527,7 +1527,8 @@ int TestSettings(int argc, char* argv[]) for (size_t x = 0; x < ARRAYSIZE(bool_list_indices); x++) { const size_t key = bool_list_indices[x]; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = + freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); const BOOL val = freerdp_settings_get_bool(settings, key); const BOOL cval = freerdp_settings_get_bool(cloned, key); if (val != cval) @@ -1567,7 +1568,8 @@ int TestSettings(int argc, char* argv[]) for (size_t x = 0; x < ARRAYSIZE(uint16_list_indices); x++) { const size_t key = uint16_list_indices[x]; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = + freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); const UINT16 val = freerdp_settings_get_uint16(settings, key); const UINT16 cval = freerdp_settings_get_uint16(cloned, key); if (val != cval) @@ -1587,7 +1589,8 @@ int TestSettings(int argc, char* argv[]) for (size_t x = 0; x < ARRAYSIZE(uint32_list_indices); x++) { const size_t key = uint32_list_indices[x]; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = + freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); const UINT32 val = freerdp_settings_get_uint32(settings, key); const UINT32 cval = freerdp_settings_get_uint32(cloned, key); if (val != cval) @@ -1607,7 +1610,8 @@ int TestSettings(int argc, char* argv[]) for (size_t x = 0; x < ARRAYSIZE(int32_list_indices); x++) { const size_t key = int32_list_indices[x]; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = + freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); const INT32 val = freerdp_settings_get_int32(settings, key); const INT32 cval = freerdp_settings_get_int32(cloned, key); if (val != cval) @@ -1627,7 +1631,8 @@ int TestSettings(int argc, char* argv[]) for (size_t x = 0; x < ARRAYSIZE(uint64_list_indices); x++) { const size_t key = uint64_list_indices[x]; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = + freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); const UINT64 val = freerdp_settings_get_uint64(settings, key); const UINT64 cval = freerdp_settings_get_uint64(cloned, key); if (val != cval) @@ -1669,7 +1674,8 @@ int TestSettings(int argc, char* argv[]) const size_t key = string_list_indices[x]; const char val[] = "test-string"; const char* res = NULL; - const char* name = freerdp_settings_get_name_for_key(key); + const char* name = + freerdp_settings_get_name_for_key(WINPR_ASSERTING_INT_CAST(SSIZE_T, key)); const char* oval = freerdp_settings_get_string(settings, key); const char* cval = freerdp_settings_get_string(cloned, key); if ((oval != cval) && (strcmp(oval, cval) != 0)) diff --git a/libfreerdp/core/timezone.c b/libfreerdp/core/timezone.c index 5369a50bb..5b07a7482 100644 --- a/libfreerdp/core/timezone.c +++ b/libfreerdp/core/timezone.c @@ -200,17 +200,17 @@ BOOL rdp_read_client_time_zone(wStream* s, rdpSettings* settings) if (!tz) return FALSE; - Stream_Read_UINT32(s, tz->Bias); /* Bias */ + Stream_Read_INT32(s, tz->Bias); /* Bias */ /* standardName (64 bytes) */ Stream_Read(s, tz->StandardName, sizeof(tz->StandardName)); if (!rdp_read_system_time(s, &tz->StandardDate)) /* StandardDate */ return FALSE; - Stream_Read_UINT32(s, tz->StandardBias); /* StandardBias */ + Stream_Read_INT32(s, tz->StandardBias); /* StandardBias */ /* daylightName (64 bytes) */ Stream_Read(s, tz->DaylightName, sizeof(tz->DaylightName)); if (!rdp_read_system_time(s, &tz->DaylightDate)) /* DaylightDate */ return FALSE; - Stream_Read_UINT32(s, tz->DaylightBias); /* DaylightBias */ + Stream_Read_INT32(s, tz->DaylightBias); /* DaylightBias */ log_timezone(tz, 0); return TRUE; } diff --git a/libfreerdp/core/transport.c b/libfreerdp/core/transport.c index 53635a5cd..1513b5d3e 100644 --- a/libfreerdp/core/transport.c +++ b/libfreerdp/core/transport.c @@ -228,7 +228,7 @@ fail: if (socketBio) BIO_free_all(socketBio); else - closesocket(sockfd); + closesocket((SOCKET)sockfd); return FALSE; } @@ -308,7 +308,7 @@ static BOOL transport_default_connect_tls(rdpTransport* transport) tls->hostname = settings->ServerHostname; tls->serverName = settings->UserSpecifiedServerName; - tls->port = MIN(UINT16_MAX, settings->ServerPort); + tls->port = WINPR_ASSERTING_INT_CAST(int32_t, MIN(UINT16_MAX, settings->ServerPort)); if (tls->port == 0) tls->port = 3389; @@ -844,7 +844,7 @@ static SSIZE_T transport_read_layer(rdpTransport* transport, BYTE* data, size_t VALGRIND_MAKE_MEM_DEFINED(data + read, bytes - read); #endif read += status; - rdp->inBytes += status; + rdp->inBytes += WINPR_ASSERTING_INT_CAST(uint64_t, status); } return read; @@ -1023,7 +1023,7 @@ static SSIZE_T parse_default_mode_pdu(rdpTransport* transport, wStream* stream) SSIZE_T transport_parse_pdu(rdpTransport* transport, wStream* s, BOOL* incomplete) { - size_t pduLength = 0; + SSIZE_T pduLength = 0; if (!transport) return -1; @@ -1042,20 +1042,20 @@ SSIZE_T transport_parse_pdu(rdpTransport* transport, wStream* s, BOOL* incomplet else pduLength = parse_default_mode_pdu(transport, s); - if (pduLength == 0) - return 0; + if (pduLength <= 0) + return pduLength; if (pduLength > SSIZE_MAX) return -1; const size_t len = Stream_Length(s); - if (len > pduLength) + if (len > WINPR_ASSERTING_INT_CAST(size_t, pduLength)) return -1; if (incomplete) - *incomplete = len < pduLength; + *incomplete = len < WINPR_ASSERTING_INT_CAST(size_t, pduLength); - return (int)pduLength; + return pduLength; } static int transport_default_read_pdu(rdpTransport* transport, wStream* s) @@ -2019,7 +2019,8 @@ static long transport_layer_bio_ctrl(BIO* bio, int cmd, long arg1, void* arg2) case BIO_C_WAIT_READ: { int timeout = (int)arg1; - BOOL r = IFCALLRESULT(FALSE, layer->Wait, layer->userContext, FALSE, timeout); + BOOL r = IFCALLRESULT(FALSE, layer->Wait, layer->userContext, FALSE, + WINPR_ASSERTING_INT_CAST(uint32_t, timeout)); /* Convert timeout to error return */ if (!r) { @@ -2034,7 +2035,8 @@ static long transport_layer_bio_ctrl(BIO* bio, int cmd, long arg1, void* arg2) case BIO_C_WAIT_WRITE: { int timeout = (int)arg1; - BOOL r = IFCALLRESULT(FALSE, layer->Wait, layer->userContext, TRUE, timeout); + BOOL r = IFCALLRESULT(FALSE, layer->Wait, layer->userContext, TRUE, + WINPR_ASSERTING_INT_CAST(uint32_t, timeout)); /* Convert timeout to error return */ if (!r) { diff --git a/libfreerdp/core/update.c b/libfreerdp/core/update.c index 7842f51fe..0a6de5908 100644 --- a/libfreerdp/core/update.c +++ b/libfreerdp/core/update.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include "settings.h" #include "update.h" @@ -171,27 +173,35 @@ static BOOL update_write_bitmap_data(rdpUpdate* update_pub, wStream* s, BITMAP_D } } - Stream_Write_UINT16(s, bitmapData->destLeft); - Stream_Write_UINT16(s, bitmapData->destTop); - Stream_Write_UINT16(s, bitmapData->destRight); - Stream_Write_UINT16(s, bitmapData->destBottom); - Stream_Write_UINT16(s, bitmapData->width); - Stream_Write_UINT16(s, bitmapData->height); - Stream_Write_UINT16(s, bitmapData->bitsPerPixel); - Stream_Write_UINT16(s, bitmapData->flags); - Stream_Write_UINT16(s, bitmapData->bitmapLength); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->destLeft)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->destTop)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->destRight)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->destBottom)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->width)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->height)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->bitsPerPixel)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->flags)); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, bitmapData->bitmapLength)); if (bitmapData->flags & BITMAP_COMPRESSION) { if (!(bitmapData->flags & NO_BITMAP_COMPRESSION_HDR)) { - Stream_Write_UINT16(s, - bitmapData->cbCompFirstRowSize); /* cbCompFirstRowSize (2 bytes) */ - Stream_Write_UINT16(s, - bitmapData->cbCompMainBodySize); /* cbCompMainBodySize (2 bytes) */ - Stream_Write_UINT16(s, bitmapData->cbScanWidth); /* cbScanWidth (2 bytes) */ - Stream_Write_UINT16(s, - bitmapData->cbUncompressedSize); /* cbUncompressedSize (2 bytes) */ + Stream_Write_UINT16( + s, + WINPR_ASSERTING_INT_CAST( + uint16_t, bitmapData->cbCompFirstRowSize)); /* cbCompFirstRowSize (2 bytes) */ + Stream_Write_UINT16( + s, + WINPR_ASSERTING_INT_CAST( + uint16_t, bitmapData->cbCompMainBodySize)); /* cbCompMainBodySize (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, + bitmapData->cbScanWidth)); /* cbScanWidth (2 bytes) */ + Stream_Write_UINT16( + s, + WINPR_ASSERTING_INT_CAST( + uint16_t, bitmapData->cbUncompressedSize)); /* cbUncompressedSize (2 bytes) */ } Stream_Write(s, bitmapData->bitmapDataStream, bitmapData->bitmapLength); @@ -249,7 +259,8 @@ static BOOL update_write_bitmap_update(rdpUpdate* update, wStream* s, return FALSE; Stream_Write_UINT16(s, UPDATE_TYPE_BITMAP); /* updateType */ - Stream_Write_UINT16(s, bitmapUpdate->number); /* numberRectangles (2 bytes) */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST( + uint16_t, bitmapUpdate->number)); /* numberRectangles (2 bytes) */ /* rectangles */ for (UINT32 i = 0; i < bitmapUpdate->number; i++) @@ -676,8 +687,9 @@ POINTER_NEW_UPDATE* update_read_pointer_new(rdpUpdate* update, wStream* s) goto fail; } + WINPR_ASSERT(pointer_new->xorBpp <= UINT8_MAX); if (!s_update_read_pointer_color( - s, &pointer_new->colorPtrAttr, pointer_new->xorBpp, + s, &pointer_new->colorPtrAttr, (UINT8)pointer_new->xorBpp, update->context->settings->LargePointerFlag)) /* colorPtrAttr */ goto fail; @@ -1121,9 +1133,9 @@ static BOOL update_bounds_equals(rdpBounds* bounds1, rdpBounds* bounds2) return FALSE; } -static int update_prepare_bounds(rdpContext* context, ORDER_INFO* orderInfo) +static size_t update_prepare_bounds(rdpContext* context, ORDER_INFO* orderInfo) { - int length = 0; + size_t length = 0; rdp_update_internal* update = NULL; WINPR_ASSERT(context); @@ -1195,23 +1207,25 @@ static size_t update_prepare_order_info(rdpContext* context, ORDER_INFO* orderIn return length; } -static int update_write_order_info(rdpContext* context, wStream* s, ORDER_INFO* orderInfo, +static int update_write_order_info(rdpContext* context, wStream* s, const ORDER_INFO* orderInfo, size_t offset) { - size_t position = 0; - WINPR_UNUSED(context); WINPR_ASSERT(orderInfo); + WINPR_ASSERT(orderInfo->controlFlags <= UINT8_MAX); + + const size_t position = Stream_GetPosition(s); + const UINT8 controlFlags = (UINT8)orderInfo->controlFlags; - position = Stream_GetPosition(s); Stream_SetPosition(s, offset); - Stream_Write_UINT8(s, orderInfo->controlFlags); /* controlFlags (1 byte) */ + Stream_Write_UINT8(s, controlFlags); /* controlFlags (1 byte) */ if (orderInfo->controlFlags & ORDER_TYPE_CHANGE) - Stream_Write_UINT8(s, orderInfo->orderType); /* orderType (1 byte) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, orderInfo->orderType)); /* orderType (1 byte) */ if (!update_write_field_flags( - s, orderInfo->fieldFlags, orderInfo->controlFlags, + s, orderInfo->fieldFlags, controlFlags, get_primary_drawing_order_field_bytes(orderInfo->orderType, NULL))) return -1; if (!update_write_bounds(s, orderInfo)) @@ -1369,7 +1383,8 @@ static BOOL update_send_surface_frame_marker(rdpContext* context, if (!s) return FALSE; - if (!update_write_surfcmd_frame_marker(s, surfaceFrameMarker->frameAction, + WINPR_ASSERT(surfaceFrameMarker->frameAction <= UINT16_MAX); + if (!update_write_surfcmd_frame_marker(s, (UINT16)surfaceFrameMarker->frameAction, surfaceFrameMarker->frameId) || !fastpath_send_update_pdu(rdp->fastpath, FASTPATH_UPDATETYPE_SURFCMDS, s, FALSE)) goto out_fail; @@ -2180,8 +2195,10 @@ static BOOL update_send_pointer_position(rdpContext* context, if (!Stream_EnsureRemainingCapacity(s, 16)) goto out_fail; - Stream_Write_UINT16(s, pointerPosition->xPos); /* xPos (2 bytes) */ - Stream_Write_UINT16(s, pointerPosition->yPos); /* yPos (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, pointerPosition->xPos)); /* xPos (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, pointerPosition->yPos)); /* yPos (2 bytes) */ ret = fastpath_send_update_pdu(rdp->fastpath, FASTPATH_UPDATETYPE_PTR_POSITION, s, FALSE); out_fail: Stream_Release(s); @@ -2300,7 +2317,8 @@ static BOOL update_send_pointer_new(rdpContext* context, const POINTER_NEW_UPDAT if (!Stream_EnsureRemainingCapacity(s, 16)) goto out_fail; - Stream_Write_UINT16(s, pointer_new->xorBpp); /* xorBpp (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, pointer_new->xorBpp)); /* xorBpp (2 bytes) */ update_write_pointer_color(s, &pointer_new->colorPtrAttr); ret = fastpath_send_update_pdu(rdp->fastpath, FASTPATH_UPDATETYPE_POINTER, s, FALSE); out_fail: @@ -2324,7 +2342,8 @@ static BOOL update_send_pointer_cached(rdpContext* context, if (!s) return FALSE; - Stream_Write_UINT16(s, pointer_cached->cacheIndex); /* cacheIndex (2 bytes) */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST( + uint16_t, pointer_cached->cacheIndex)); /* cacheIndex (2 bytes) */ ret = fastpath_send_update_pdu(rdp->fastpath, FASTPATH_UPDATETYPE_CACHED, s, FALSE); Stream_Release(s); return ret; @@ -2557,7 +2576,7 @@ static BOOL update_send_new_or_existing_window(rdpContext* context, if ((orderInfo->fieldFlags & WINDOW_ORDER_FIELD_SHOW) != 0) { - Stream_Write_UINT8(s, stateOrder->showState); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, stateOrder->showState)); } if ((orderInfo->fieldFlags & WINDOW_ORDER_FIELD_TITLE) != 0) @@ -2592,7 +2611,7 @@ static BOOL update_send_new_or_existing_window(rdpContext* context, if ((orderInfo->fieldFlags & WINDOW_ORDER_FIELD_RP_CONTENT) != 0) { - Stream_Write_UINT8(s, stateOrder->RPContent); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, stateOrder->RPContent)); } if ((orderInfo->fieldFlags & WINDOW_ORDER_FIELD_ROOT_PARENT) != 0) @@ -2620,7 +2639,7 @@ static BOOL update_send_new_or_existing_window(rdpContext* context, if ((orderInfo->fieldFlags & WINDOW_ORDER_FIELD_WND_RECTS) != 0) { - Stream_Write_UINT16(s, stateOrder->numWindowRects); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, stateOrder->numWindowRects)); Stream_Write(s, stateOrder->windowRects, stateOrder->numWindowRects * sizeof(RECTANGLE_16)); } @@ -2632,7 +2651,7 @@ static BOOL update_send_new_or_existing_window(rdpContext* context, if ((orderInfo->fieldFlags & WINDOW_ORDER_FIELD_VISIBILITY) != 0) { - Stream_Write_UINT16(s, stateOrder->numVisibilityRects); + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST(uint16_t, stateOrder->numVisibilityRects)); Stream_Write(s, stateOrder->visibilityRects, stateOrder->numVisibilityRects * sizeof(RECTANGLE_16)); } @@ -2729,19 +2748,26 @@ static BOOL update_send_window_icon(rdpContext* context, const WINDOW_ORDER_INFO Stream_Write_UINT32(s, orderInfo->fieldFlags); /* FieldsPresentFlags (4 bytes) */ Stream_Write_UINT32(s, orderInfo->windowId); /* WindowID (4 bytes) */ /* Write body */ - Stream_Write_UINT16(s, iconInfo->cacheEntry); /* CacheEntry (2 bytes) */ - Stream_Write_UINT8(s, iconInfo->cacheId); /* CacheId (1 byte) */ - Stream_Write_UINT8(s, iconInfo->bpp); /* Bpp (1 byte) */ - Stream_Write_UINT16(s, iconInfo->width); /* Width (2 bytes) */ - Stream_Write_UINT16(s, iconInfo->height); /* Height (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo->cacheEntry)); /* CacheEntry (2 bytes) */ + Stream_Write_UINT8(s, + WINPR_ASSERTING_INT_CAST(uint8_t, iconInfo->cacheId)); /* CacheId (1 byte) */ + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, iconInfo->bpp)); /* Bpp (1 byte) */ + Stream_Write_UINT16(s, + WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo->width)); /* Width (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo->height)); /* Height (2 bytes) */ if (iconInfo->bpp <= 8) { - Stream_Write_UINT16(s, iconInfo->cbColorTable); /* CbColorTable (2 bytes) */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST( + uint16_t, iconInfo->cbColorTable)); /* CbColorTable (2 bytes) */ } - Stream_Write_UINT16(s, iconInfo->cbBitsMask); /* CbBitsMask (2 bytes) */ - Stream_Write_UINT16(s, iconInfo->cbBitsColor); /* CbBitsColor (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo->cbBitsMask)); /* CbBitsMask (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo->cbBitsColor)); /* CbBitsColor (2 bytes) */ Stream_Write(s, iconInfo->bitsMask, iconInfo->cbBitsMask); /* BitsMask (variable) */ if (iconInfo->bpp <= 8) @@ -2786,8 +2812,10 @@ static BOOL update_send_window_cached_icon(rdpContext* context, const WINDOW_ORD Stream_Write_UINT32(s, orderInfo->fieldFlags); /* FieldsPresentFlags (4 bytes) */ Stream_Write_UINT32(s, orderInfo->windowId); /* WindowID (4 bytes) */ /* Write body */ - Stream_Write_UINT16(s, cachedIcon->cacheEntry); /* CacheEntry (2 bytes) */ - Stream_Write_UINT8(s, cachedIcon->cacheId); /* CacheId (1 byte) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, cachedIcon->cacheEntry)); /* CacheEntry (2 bytes) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, cachedIcon->cacheId)); /* CacheId (1 byte) */ update->numberOrders++; return TRUE; } @@ -2938,20 +2966,29 @@ update_send_new_or_existing_notification_icons(rdpContext* context, if ((orderInfo->fieldFlags & WINDOW_ORDER_ICON) != 0) { - ICON_INFO iconInfo = iconStateOrder->icon; - Stream_Write_UINT16(s, iconInfo.cacheEntry); /* CacheEntry (2 bytes) */ - Stream_Write_UINT8(s, iconInfo.cacheId); /* CacheId (1 byte) */ - Stream_Write_UINT8(s, iconInfo.bpp); /* Bpp (1 byte) */ - Stream_Write_UINT16(s, iconInfo.width); /* Width (2 bytes) */ - Stream_Write_UINT16(s, iconInfo.height); /* Height (2 bytes) */ + const ICON_INFO iconInfo = iconStateOrder->icon; + + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo.cacheEntry)); /* CacheEntry (2 bytes) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, iconInfo.cacheId)); /* CacheId (1 byte) */ + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(uint8_t, iconInfo.bpp)); /* Bpp (1 byte) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo.width)); /* Width (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo.height)); /* Height (2 bytes) */ if (iconInfo.bpp <= 8) { - Stream_Write_UINT16(s, iconInfo.cbColorTable); /* CbColorTable (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, + iconInfo.cbColorTable)); /* CbColorTable (2 bytes) */ } - Stream_Write_UINT16(s, iconInfo.cbBitsMask); /* CbBitsMask (2 bytes) */ - Stream_Write_UINT16(s, iconInfo.cbBitsColor); /* CbBitsColor (2 bytes) */ + Stream_Write_UINT16( + s, WINPR_ASSERTING_INT_CAST(uint16_t, iconInfo.cbBitsMask)); /* CbBitsMask (2 bytes) */ + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST( + uint16_t, iconInfo.cbBitsColor)); /* CbBitsColor (2 bytes) */ Stream_Write(s, iconInfo.bitsMask, iconInfo.cbBitsMask); /* BitsMask (variable) */ if (iconInfo.bpp <= 8) @@ -2963,9 +3000,11 @@ update_send_new_or_existing_notification_icons(rdpContext* context, } else if ((orderInfo->fieldFlags & WINDOW_ORDER_CACHED_ICON) != 0) { - CACHED_ICON_INFO cachedIcon = iconStateOrder->cachedIcon; - Stream_Write_UINT16(s, cachedIcon.cacheEntry); /* CacheEntry (2 bytes) */ - Stream_Write_UINT8(s, cachedIcon.cacheId); /* CacheId (1 byte) */ + const CACHED_ICON_INFO cachedIcon = iconStateOrder->cachedIcon; + Stream_Write_UINT16(s, WINPR_ASSERTING_INT_CAST( + uint16_t, cachedIcon.cacheEntry)); /* CacheEntry (2 bytes) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST(uint8_t, cachedIcon.cacheId)); /* CacheId (1 byte) */ } update->numberOrders++; @@ -3063,12 +3102,15 @@ static BOOL update_send_monitored_desktop(rdpContext* context, const WINDOW_ORDE if (orderInfo->fieldFlags & WINDOW_ORDER_FIELD_DESKTOP_ZORDER) { - Stream_Write_UINT8(s, monitoredDesktop->numWindowIds); /* numWindowIds (1 byte) */ + Stream_Write_UINT8( + s, WINPR_ASSERTING_INT_CAST( + uint8_t, monitoredDesktop->numWindowIds)); /* numWindowIds (1 byte) */ /* windowIds */ for (UINT32 i = 0; i < monitoredDesktop->numWindowIds; i++) { - Stream_Write_UINT32(s, monitoredDesktop->windowIds[i]); + Stream_Write_UINT32(s, + WINPR_ASSERTING_INT_CAST(uint32_t, monitoredDesktop->windowIds[i])); } } diff --git a/libfreerdp/core/window.c b/libfreerdp/core/window.c index 0954a99fd..752fca352 100644 --- a/libfreerdp/core/window.c +++ b/libfreerdp/core/window.c @@ -401,8 +401,8 @@ static BOOL update_read_window_state_order(wStream* s, WINDOW_ORDER_INFO* orderI if (!Stream_CheckAndLogRequiredLength(TAG, s, 8)) return FALSE; - Stream_Read_UINT32(s, windowState->visibleOffsetX); /* visibleOffsetX (4 bytes) */ - Stream_Read_UINT32(s, windowState->visibleOffsetY); /* visibleOffsetY (4 bytes) */ + Stream_Read_INT32(s, windowState->visibleOffsetX); /* visibleOffsetX (4 bytes) */ + Stream_Read_INT32(s, windowState->visibleOffsetY); /* visibleOffsetY (4 bytes) */ } if (orderInfo->fieldFlags & WINDOW_ORDER_FIELD_VISIBILITY) diff --git a/libfreerdp/crypto/ber.c b/libfreerdp/crypto/ber.c index 6a64e2af8..7e3324b67 100644 --- a/libfreerdp/crypto/ber.c +++ b/libfreerdp/crypto/ber.c @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -605,7 +606,7 @@ BOOL ber_read_integer(wStream* s, UINT32* value) BYTE byte = 0; Stream_Read_UINT8(s, byte); Stream_Read_UINT16_BE(s, *value); - *value += (byte << 16); + *value += (byte << 16) & 0xFF0000; } else if (length == 4) { @@ -643,7 +644,7 @@ size_t ber_write_integer(wStream* s, UINT32 value) ber_write_universal_tag(s, BER_TAG_INTEGER, FALSE); ber_write_length(s, 1); - Stream_Write_UINT8(s, value); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(UINT8, value)); return 3; } else if (value < 0x8000) @@ -651,7 +652,7 @@ size_t ber_write_integer(wStream* s, UINT32 value) ber_write_universal_tag(s, BER_TAG_INTEGER, FALSE); ber_write_length(s, 2); - Stream_Write_UINT16_BE(s, value); + Stream_Write_UINT16_BE(s, WINPR_ASSERTING_INT_CAST(UINT16, value)); return 4; } else if (value < 0x800000) @@ -659,8 +660,8 @@ size_t ber_write_integer(wStream* s, UINT32 value) ber_write_universal_tag(s, BER_TAG_INTEGER, FALSE); ber_write_length(s, 3); - Stream_Write_UINT8(s, (value >> 16)); - Stream_Write_UINT16_BE(s, (value & 0xFFFF)); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(UINT8, value >> 16)); + Stream_Write_UINT16_BE(s, WINPR_ASSERTING_INT_CAST(UINT16, value)); return 5; } else if (value < 0x80000000) diff --git a/libfreerdp/crypto/certificate.c b/libfreerdp/crypto/certificate.c index 886f761b0..5b6590a43 100644 --- a/libfreerdp/crypto/certificate.c +++ b/libfreerdp/crypto/certificate.c @@ -789,7 +789,7 @@ static BOOL certificate_read_server_proprietary_certificate(rdpCertificate* cert if (!Stream_CheckAndLogRequiredLength(TAG, s, 4)) return FALSE; - sigdatalen = Stream_PointerAs(s, const BYTE) - sigdata; + sigdatalen = WINPR_ASSERTING_INT_CAST(size_t, Stream_PointerAs(s, const BYTE) - sigdata); Stream_Read_UINT16(s, wSignatureBlobType); if (wSignatureBlobType != BB_RSA_SIGNATURE_BLOB) @@ -1564,7 +1564,7 @@ BOOL freerdp_certificate_get_public_key(const rdpCertificate* cert, BYTE** Publi goto exit; } - *PublicKey = optr = ptr = (BYTE*)calloc(length, sizeof(BYTE)); + *PublicKey = optr = ptr = (BYTE*)calloc(WINPR_ASSERTING_INT_CAST(size_t, length), sizeof(BYTE)); if (!ptr) goto exit; @@ -1642,7 +1642,7 @@ BOOL freerdp_certificate_publickey_encrypt(const rdpCertificate* cert, const BYT if (!ctx) return FALSE; - size_t outputSize = EVP_PKEY_size(pkey); + size_t outputSize = WINPR_ASSERTING_INT_CAST(size_t, EVP_PKEY_size(pkey)); output = malloc(outputSize); if (output == NULL) goto out; @@ -1692,7 +1692,7 @@ BYTE* freerdp_certificate_get_der(const rdpCertificate* cert, size_t* pLength) if (rc <= 0) return NULL; - BYTE* ptr = calloc(rc + 1, sizeof(BYTE)); + BYTE* ptr = calloc(WINPR_ASSERTING_INT_CAST(size_t, rc) + 1, sizeof(BYTE)); if (!ptr) return NULL; BYTE* i2d_ptr = ptr; @@ -1771,7 +1771,7 @@ char* freerdp_certificate_get_param(const rdpCertificate* cert, enum FREERDP_CER } #endif - const size_t bnsize = BN_num_bytes(bn); + const size_t bnsize = WINPR_ASSERTING_INT_CAST(size_t, BN_num_bytes(bn)); char* rc = calloc(bnsize + 1, sizeof(char)); if (!rc) goto fail; diff --git a/libfreerdp/crypto/crypto.c b/libfreerdp/crypto/crypto.c index 806f9b986..3cf94cefb 100644 --- a/libfreerdp/crypto/crypto.c +++ b/libfreerdp/crypto/crypto.c @@ -106,13 +106,13 @@ static SSIZE_T crypto_rsa_common(const BYTE* input, size_t length, UINT32 key_le output_length = BN_bn2bin(y, output); if (output_length < 0) goto fail; - if ((size_t)output_length > out_length) + if (WINPR_ASSERTING_INT_CAST(size_t, output_length) > out_length) goto fail; - crypto_reverse(output, output_length); + crypto_reverse(output, WINPR_ASSERTING_INT_CAST(size_t, output_length)); if ((size_t)output_length < key_length) { - size_t diff = key_length - output_length; + size_t diff = key_length - WINPR_ASSERTING_INT_CAST(size_t, output_length); if ((size_t)output_length + diff > out_length) diff = out_length - (size_t)output_length; memset(output + output_length, 0, diff); @@ -209,7 +209,7 @@ char* crypto_read_pem(const char* WINPR_RESTRICT filename, size_t* WINPR_RESTRIC if (rc < 0) goto fail; - pem = calloc(size + 1, sizeof(char)); + pem = calloc(WINPR_ASSERTING_INT_CAST(size_t, size) + 1, sizeof(char)); if (!pem) goto fail; @@ -218,7 +218,7 @@ char* crypto_read_pem(const char* WINPR_RESTRICT filename, size_t* WINPR_RESTRIC goto fail; if (plength) - *plength = strnlen(pem, size); + *plength = strnlen(pem, WINPR_ASSERTING_INT_CAST(size_t, size)); (void)fclose(fp); return pem; diff --git a/libfreerdp/crypto/der.c b/libfreerdp/crypto/der.c index 5fcbf4ac9..0080bfd4e 100644 --- a/libfreerdp/crypto/der.c +++ b/libfreerdp/crypto/der.c @@ -20,6 +20,8 @@ #include #include +#include +#include #include @@ -38,18 +40,18 @@ int der_write_length(wStream* s, int length) if (length > 0x7F && length <= 0xFF) { Stream_Write_UINT8(s, 0x81); - Stream_Write_UINT8(s, length); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(UINT8, length)); return 2; } else if (length > 0xFF) { Stream_Write_UINT8(s, 0x82); - Stream_Write_UINT16_BE(s, length); + Stream_Write_UINT16_BE(s, WINPR_ASSERTING_INT_CAST(UINT16, length)); return 3; } else { - Stream_Write_UINT8(s, length); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(UINT8, length)); return 1; } } @@ -89,7 +91,7 @@ void der_write_octet_string(wStream* s, BYTE* oct_str, int length) { der_write_universal_tag(s, ER_TAG_OCTET_STRING, FALSE); der_write_length(s, length); - Stream_Write(s, oct_str, length); + Stream_Write(s, oct_str, WINPR_ASSERTING_INT_CAST(size_t, length)); } int der_skip_sequence_tag(int length) diff --git a/libfreerdp/crypto/er.c b/libfreerdp/crypto/er.c index 845d3eecc..61e3beb20 100644 --- a/libfreerdp/crypto/er.c +++ b/libfreerdp/crypto/er.c @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -279,7 +280,7 @@ BOOL er_read_bit_string(wStream* s, int* length, BYTE* padding) BOOL er_write_bit_string_tag(wStream* s, UINT32 length, BYTE padding, BOOL flag) { er_write_universal_tag(s, ER_TAG_BIT_STRING, FALSE); - er_write_length(s, length, flag); + er_write_length(s, WINPR_ASSERTING_INT_CAST(int, length), flag); Stream_Write_UINT8(s, padding); return TRUE; } @@ -304,7 +305,7 @@ void er_write_octet_string(wStream* s, BYTE* oct_str, int length, BOOL flag) { er_write_universal_tag(s, ER_TAG_OCTET_STRING, FALSE); er_write_length(s, length, flag); - Stream_Write(s, oct_str, length); + Stream_Write(s, oct_str, WINPR_ASSERTING_INT_CAST(size_t, length)); } int er_write_octet_string_tag(wStream* s, int length, BOOL flag) @@ -362,7 +363,7 @@ BOOL er_read_integer(wStream* s, UINT32* value) if (value == NULL) { - Stream_Seek(s, length); + Stream_Seek(s, WINPR_ASSERTING_INT_CAST(size_t, length)); return TRUE; } @@ -379,7 +380,7 @@ BOOL er_read_integer(wStream* s, UINT32* value) BYTE byte = 0; Stream_Read_UINT8(s, byte); Stream_Read_UINT16_BE(s, *value); - *value += (byte << 16); + *value += (byte << 16) & 0xFF0000; } else if (length == 4) { @@ -406,13 +407,13 @@ int er_write_integer(wStream* s, INT32 value) if (value <= 127 && value >= -128) { er_write_length(s, 1, FALSE); - Stream_Write_UINT8(s, value); + Stream_Write_INT8(s, WINPR_ASSERTING_INT_CAST(INT8, value)); return 2; } else if (value <= 32767 && value >= -32768) { er_write_length(s, 2, FALSE); - Stream_Write_INT16_BE(s, value); + Stream_Write_INT16_BE(s, WINPR_ASSERTING_INT_CAST(INT16, value)); return 3; } else diff --git a/libfreerdp/crypto/per.c b/libfreerdp/crypto/per.c index 642f13598..924742334 100644 --- a/libfreerdp/crypto/per.c +++ b/libfreerdp/crypto/per.c @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -51,7 +52,7 @@ BOOL per_read_length(wStream* s, UINT16* length) return FALSE; byte &= ~(0x80); - *length = (byte << 8); + *length = WINPR_ASSERTING_INT_CAST(UINT16, byte << 8); Stream_Read_UINT8(s, byte); *length += byte; } @@ -272,7 +273,7 @@ BOOL per_write_integer(wStream* s, UINT32 integer) return FALSE; if (!Stream_EnsureRemainingCapacity(s, 1)) return FALSE; - Stream_Write_UINT8(s, integer); + Stream_Write_UINT8(s, WINPR_ASSERTING_INT_CAST(UINT8, integer)); } else if (integer <= UINT16_MAX) { @@ -280,7 +281,7 @@ BOOL per_write_integer(wStream* s, UINT32 integer) return FALSE; if (!Stream_EnsureRemainingCapacity(s, 2)) return FALSE; - Stream_Write_UINT16_BE(s, integer); + Stream_Write_UINT16_BE(s, WINPR_ASSERTING_INT_CAST(UINT16, integer)); } else if (integer <= UINT32_MAX) { @@ -596,7 +597,7 @@ BOOL per_write_numeric_string(wStream* s, const BYTE* num_str, UINT16 length, UI c1 = (c1 - 0x30) % 10; c2 = (c2 - 0x30) % 10; - const BYTE num = (c1 << 4) | c2; + const BYTE num = WINPR_ASSERTING_INT_CAST(BYTE, (c1 << 4) | c2); Stream_Write_UINT8(s, num); /* string */ } diff --git a/libfreerdp/crypto/privatekey.c b/libfreerdp/crypto/privatekey.c index 09d1593ac..75799826f 100644 --- a/libfreerdp/crypto/privatekey.c +++ b/libfreerdp/crypto/privatekey.c @@ -375,7 +375,7 @@ size_t freerdp_key_get_bits(const rdpPrivateKey* key) rc = EVP_PKEY_get_bits(key->evp); #endif - return rc; + return WINPR_ASSERTING_INT_CAST(size_t, rc); } BOOL freerdp_key_generate(rdpPrivateKey* key, size_t key_length) @@ -530,7 +530,7 @@ BYTE* freerdp_key_get_param(const rdpPrivateKey* key, enum FREERDP_KEY_PARAM par buf = NULL; } else - *plength = length; + *plength = WINPR_ASSERTING_INT_CAST(size_t, length); fail: BN_free(bn); diff --git a/libfreerdp/crypto/tls.c b/libfreerdp/crypto/tls.c index 6fa85f880..86901eb0a 100644 --- a/libfreerdp/crypto/tls.c +++ b/libfreerdp/crypto/tls.c @@ -783,7 +783,7 @@ static BOOL tls_prepare(rdpTls* tls, BIO* underlying, SSL_METHOD* method, int op } SSL_CTX_set_mode(tls->ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_ENABLE_PARTIAL_WRITE); - SSL_CTX_set_options(tls->ctx, options); + SSL_CTX_set_options(tls->ctx, WINPR_ASSERTING_INT_CAST(uint64_t, options)); SSL_CTX_set_read_ahead(tls->ctx, 1); #if OPENSSL_VERSION_NUMBER >= 0x10100000L || defined(LIBRESSL_VERSION_NUMBER) UINT16 version = freerdp_settings_get_uint16(settings, FreeRDP_TLSMinVersion); @@ -800,7 +800,7 @@ static BOOL tls_prepare(rdpTls* tls, BIO* underlying, SSL_METHOD* method, int op } #endif #if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) - SSL_CTX_set_security_level(tls->ctx, settings->TlsSecLevel); + SSL_CTX_set_security_level(tls->ctx, WINPR_ASSERTING_INT_CAST(int, settings->TlsSecLevel)); #endif if (settings->AllowedTlsCiphers) diff --git a/libfreerdp/crypto/x509_utils.c b/libfreerdp/crypto/x509_utils.c index af056e515..569517312 100644 --- a/libfreerdp/crypto/x509_utils.c +++ b/libfreerdp/crypto/x509_utils.c @@ -237,9 +237,9 @@ that must be freed with OPENSSL_free. typedef struct string_list { char** strings; - int allocated; - int count; - int maximum; + size_t allocated; + size_t count; + size_t maximum; } string_list; static void string_list_initialize(string_list* list) @@ -250,12 +250,12 @@ static void string_list_initialize(string_list* list) list->maximum = INT_MAX; } -static void string_list_allocate(string_list* list, int allocate_count) +static void string_list_allocate(string_list* list, size_t allocate_count) { if (!list->strings && list->allocated == 0) { - list->strings = (char**)calloc((size_t)allocate_count, sizeof(char*)); - list->allocated = list->strings ? allocate_count : -1; + list->strings = (char**)calloc(allocate_count, sizeof(char*)); + list->allocated = list->strings ? allocate_count : 0; list->count = 0; } } @@ -274,6 +274,8 @@ static int extract_string(GENERAL_NAME* name, void* data, int index, int count) unsigned char* cstring = 0; ASN1_STRING* str = NULL; + WINPR_UNUSED(index); + switch (name->type) { case GEN_URI: @@ -299,7 +301,7 @@ static int extract_string(GENERAL_NAME* name, void* data, int index, int count) return 1; } - string_list_allocate(list, count); + string_list_allocate(list, WINPR_ASSERTING_INT_CAST(WINPR_CIPHER_TYPE, count)); if (list->allocated <= 0) { @@ -340,9 +342,9 @@ typedef struct object_list { ASN1_OBJECT* type_id; char** strings; - int allocated; - int count; - int maximum; + size_t allocated; + size_t count; + size_t maximum; } object_list; static void object_list_initialize(object_list* list) @@ -354,12 +356,12 @@ static void object_list_initialize(object_list* list) list->maximum = INT_MAX; } -static void object_list_allocate(object_list* list, int allocate_count) +static void object_list_allocate(object_list* list, size_t allocate_count) { - if (!list->strings && list->allocated == 0) + if (!list->strings && (list->allocated == 0) && (allocate_count > 0)) { list->strings = (char**)calloc(allocate_count, sizeof(list->strings[0])); - list->allocated = list->strings ? allocate_count : -1; + list->allocated = list->strings ? allocate_count : 0; list->count = 0; } } @@ -368,16 +370,16 @@ static char* object_string(ASN1_TYPE* object) { char* result = NULL; unsigned char* utf8String = NULL; - int length = 0; + /* TODO: check that object.type is a string type. */ - length = ASN1_STRING_to_UTF8(&utf8String, object->value.asn1_string); + const int length = ASN1_STRING_to_UTF8(&utf8String, object->value.asn1_string); if (length < 0) { return 0; } - result = _strdup((char*)utf8String); + result = strndup((char*)utf8String, WINPR_ASSERTING_INT_CAST(size_t, length)); OPENSSL_free(utf8String); return result; } @@ -391,6 +393,10 @@ static void object_list_free(object_list* list) static int extract_othername_object_as_string(GENERAL_NAME* name, void* data, int index, int count) { object_list* list = data; + WINPR_UNUSED(index); + + if (count < 0) + return -1; if (name->type != GEN_OTHERNAME) { @@ -402,7 +408,7 @@ static int extract_othername_object_as_string(GENERAL_NAME* name, void* data, in return 1; } - object_list_allocate(list, count); + object_list_allocate(list, WINPR_ASSERTING_INT_CAST(size_t, count)); if (list->allocated <= 0) { @@ -447,7 +453,7 @@ char* x509_utils_get_email(const X509* x509) char* x509_utils_get_upn(const X509* x509) { char* result = 0; - object_list list; + object_list list = { 0 }; object_list_initialize(&list); list.type_id = OBJ_nid2obj(NID_ms_upn); list.maximum = 1; @@ -515,7 +521,7 @@ char** x509_utils_get_dns_names(const X509* x509, size_t* count, size_t** length string_list list = { 0 }; string_list_initialize(&list); map_subject_alt_name(x509, GEN_DNS, extract_string, &list); - (*count) = (size_t)list.count; + (*count) = list.count; if (list.count <= 0) { @@ -525,8 +531,8 @@ char** x509_utils_get_dns_names(const X509* x509, size_t* count, size_t** length /* lengths are not useful, since we converted the strings to utf-8, there cannot be nul-bytes in them. */ - result = (char**)calloc((size_t)list.count, sizeof(*result)); - (*lengths) = calloc((size_t)list.count, sizeof(**lengths)); + result = (char**)calloc(list.count, sizeof(*result)); + (*lengths) = calloc(list.count, sizeof(**lengths)); if (!result || !(*lengths)) { @@ -538,7 +544,7 @@ char** x509_utils_get_dns_names(const X509* x509, size_t* count, size_t** length return NULL; } - for (int i = 0; i < list.count; i++) + for (size_t i = 0; i < list.count; i++) { result[i] = list.strings[i]; (*lengths)[i] = strlen(result[i]); diff --git a/libfreerdp/gdi/bitmap.c b/libfreerdp/gdi/bitmap.c index 8fa3a2515..372f242d4 100644 --- a/libfreerdp/gdi/bitmap.c +++ b/libfreerdp/gdi/bitmap.c @@ -60,7 +60,7 @@ UINT32 gdi_GetPixel(HGDI_DC hdc, UINT32 nXPos, UINT32 nYPos) BYTE* gdi_GetPointer(HGDI_BITMAP hBmp, UINT32 X, UINT32 Y) { UINT32 bpp = FreeRDPGetBytesPerPixel(hBmp->format); - return &hBmp->data[(Y * hBmp->width * bpp) + X * bpp]; + return &hBmp->data[(Y * WINPR_ASSERTING_INT_CAST(uint32_t, hBmp->width) * bpp) + X * bpp]; } /** @@ -128,8 +128,8 @@ HGDI_BITMAP gdi_CreateBitmapEx(UINT32 nWidth, UINT32 nHeight, UINT32 format, UIN else hBitmap->scanline = nWidth * FreeRDPGetBytesPerPixel(hBitmap->format); - hBitmap->width = nWidth; - hBitmap->height = nHeight; + hBitmap->width = WINPR_ASSERTING_INT_CAST(int, nWidth); + hBitmap->height = WINPR_ASSERTING_INT_CAST(int, nHeight); hBitmap->data = data; hBitmap->free = fkt_free; return hBitmap; @@ -329,7 +329,9 @@ static INLINE BOOL BitBlt_write(HGDI_DC hdcDest, HGDI_DC hdcSrc, INT32 nXDest, I case GDI_BS_HATCHED: case GDI_BS_PATTERN: { - const BYTE* patp = gdi_get_brush_pointer(hdcDest, nXDest + x, nYDest + y); + const BYTE* patp = + gdi_get_brush_pointer(hdcDest, WINPR_ASSERTING_INT_CAST(uint32_t, nXDest + x), + WINPR_ASSERTING_INT_CAST(uint32_t, nYDest + y)); if (!patp) { @@ -623,9 +625,14 @@ BOOL gdi_BitBlt(HGDI_DC hdcDest, INT32 nXDest, INT32 nYDest, INT32 nWidth, INT32 if (!hSrcBmp || !hDstBmp) return FALSE; - if (!freerdp_image_copy(hDstBmp->data, hDstBmp->format, hDstBmp->scanline, nXDest, - nYDest, nWidth, nHeight, hSrcBmp->data, hSrcBmp->format, - hSrcBmp->scanline, nXSrc, nYSrc, palette, FREERDP_FLIP_NONE)) + if (!freerdp_image_copy( + hDstBmp->data, hDstBmp->format, hDstBmp->scanline, + WINPR_ASSERTING_INT_CAST(UINT32, nXDest), + WINPR_ASSERTING_INT_CAST(UINT32, nYDest), + WINPR_ASSERTING_INT_CAST(UINT32, nWidth), + WINPR_ASSERTING_INT_CAST(UINT32, nHeight), hSrcBmp->data, hSrcBmp->format, + hSrcBmp->scanline, WINPR_ASSERTING_INT_CAST(UINT32, nXSrc), + WINPR_ASSERTING_INT_CAST(UINT32, nYSrc), palette, FREERDP_FLIP_NONE)) return FALSE; break; @@ -644,9 +651,14 @@ BOOL gdi_BitBlt(HGDI_DC hdcDest, INT32 nXDest, INT32 nYDest, INT32 nWidth, INT32 if (!hSrcBmp || !hDstBmp) return FALSE; - if (!freerdp_image_copy(hDstBmp->data, hDstBmp->format, hDstBmp->scanline, nXDest, - nYDest, nWidth, nHeight, hSrcBmp->data, hSrcBmp->format, - hSrcBmp->scanline, nXSrc, nYSrc, palette, FREERDP_FLIP_NONE)) + if (!freerdp_image_copy( + hDstBmp->data, hDstBmp->format, hDstBmp->scanline, + WINPR_ASSERTING_INT_CAST(UINT32, nXDest), + WINPR_ASSERTING_INT_CAST(UINT32, nYDest), + WINPR_ASSERTING_INT_CAST(UINT32, nWidth), + WINPR_ASSERTING_INT_CAST(UINT32, nHeight), hSrcBmp->data, hSrcBmp->format, + hSrcBmp->scanline, WINPR_ASSERTING_INT_CAST(UINT32, nXSrc), + WINPR_ASSERTING_INT_CAST(UINT32, nYSrc), palette, FREERDP_FLIP_NONE)) return FALSE; break; diff --git a/libfreerdp/gdi/brush.h b/libfreerdp/gdi/brush.h index b7e86c2fc..6845c5d73 100644 --- a/libfreerdp/gdi/brush.h +++ b/libfreerdp/gdi/brush.h @@ -22,6 +22,8 @@ #ifndef FREERDP_LIB_GDI_BRUSH_H #define FREERDP_LIB_GDI_BRUSH_H +#include + #include #include @@ -41,7 +43,7 @@ extern "C" if (!hdc || !hdc->brush) return GDI_BS_NULL; - return hdc->brush->style; + return WINPR_ASSERTING_INT_CAST(UINT32, hdc->brush->style); } #ifdef __cplusplus diff --git a/libfreerdp/gdi/drawing.c b/libfreerdp/gdi/drawing.c index 750c878a6..8dec55361 100644 --- a/libfreerdp/gdi/drawing.c +++ b/libfreerdp/gdi/drawing.c @@ -107,7 +107,7 @@ UINT32 gdi_SetBkColor(HGDI_DC hdc, UINT32 crColor) * @return background mode */ -UINT32 gdi_GetBkMode(HGDI_DC hdc) +INT32 gdi_GetBkMode(HGDI_DC hdc) { return hdc->bkMode; } diff --git a/libfreerdp/gdi/drawing.h b/libfreerdp/gdi/drawing.h index 96bdf65a4..f786e6461 100644 --- a/libfreerdp/gdi/drawing.h +++ b/libfreerdp/gdi/drawing.h @@ -34,7 +34,7 @@ extern "C" FREERDP_LOCAL INT32 gdi_SetROP2(HGDI_DC hdc, INT32 fnDrawMode); FREERDP_LOCAL UINT32 gdi_GetBkColor(HGDI_DC hdc); FREERDP_LOCAL UINT32 gdi_SetBkColor(HGDI_DC hdc, UINT32 crColor); - FREERDP_LOCAL UINT32 gdi_GetBkMode(HGDI_DC hdc); + FREERDP_LOCAL INT32 gdi_GetBkMode(HGDI_DC hdc); FREERDP_LOCAL INT32 gdi_SetBkMode(HGDI_DC hdc, INT32 iBkMode); FREERDP_LOCAL UINT32 gdi_SetTextColor(HGDI_DC hdc, UINT32 crColor); diff --git a/libfreerdp/gdi/gdi.c b/libfreerdp/gdi/gdi.c index f80ada8ae..b88e4cd68 100644 --- a/libfreerdp/gdi/gdi.c +++ b/libfreerdp/gdi/gdi.c @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -445,9 +446,13 @@ gdiBitmap* gdi_bitmap_new_ex(rdpGdi* gdi, int width, int height, int bpp, BYTE* bpp); if (!data) - bitmap->bitmap = gdi_CreateCompatibleBitmap(gdi->hdc, width, height); + bitmap->bitmap = + gdi_CreateCompatibleBitmap(gdi->hdc, WINPR_ASSERTING_INT_CAST(uint32_t, width), + WINPR_ASSERTING_INT_CAST(uint32_t, height)); else - bitmap->bitmap = gdi_create_bitmap(gdi, width, height, bpp, data); + bitmap->bitmap = gdi_create_bitmap(gdi, WINPR_ASSERTING_INT_CAST(uint32_t, width), + WINPR_ASSERTING_INT_CAST(uint32_t, height), + WINPR_ASSERTING_INT_CAST(uint32_t, bpp), data); if (!bitmap->bitmap) goto fail_bitmap_bitmap; @@ -488,43 +493,39 @@ BOOL gdi_bitmap_update(rdpContext* context, const BITMAP_UPDATE* bitmapUpdate) for (UINT32 index = 0; index < bitmapUpdate->number; index++) { + BOOL rc = FALSE; const BITMAP_DATA* bitmap = &(bitmapUpdate->rectangles[index]); rdpBitmap* bmp = Bitmap_Alloc(context); if (!bmp) - { - WLog_ERR(TAG, "Bitmap_Alloc failed"); - return FALSE; - } + goto fail; - Bitmap_SetDimensions(bmp, bitmap->width, bitmap->height); - Bitmap_SetRectangle(bmp, bitmap->destLeft, bitmap->destTop, bitmap->destRight, - bitmap->destBottom); + if (!Bitmap_SetDimensions(bmp, WINPR_ASSERTING_INT_CAST(UINT16, bitmap->width), + WINPR_ASSERTING_INT_CAST(UINT16, bitmap->height))) + goto fail; + + if (!Bitmap_SetRectangle(bmp, WINPR_ASSERTING_INT_CAST(UINT16, bitmap->destLeft), + WINPR_ASSERTING_INT_CAST(UINT16, bitmap->destTop), + WINPR_ASSERTING_INT_CAST(UINT16, bitmap->destRight), + WINPR_ASSERTING_INT_CAST(UINT16, bitmap->destBottom))) + goto fail; if (!bmp->Decompress(context, bmp, bitmap->bitmapDataStream, bitmap->width, bitmap->height, bitmap->bitsPerPixel, bitmap->bitmapLength, bitmap->compressed, RDP_CODEC_ID_NONE)) - { - WLog_ERR(TAG, "bmp->Decompress failed"); - Bitmap_Free(context, bmp); - return FALSE; - } + goto fail; if (!bmp->New(context, bmp)) - { - WLog_ERR(TAG, "bmp->New failed"); - Bitmap_Free(context, bmp); - return FALSE; - } + goto fail; if (!bmp->Paint(context, bmp)) - { - WLog_ERR(TAG, "bmp->Paint failed"); - Bitmap_Free(context, bmp); - return FALSE; - } + goto fail; + rc = TRUE; + fail: Bitmap_Free(context, bmp); + if (!rc) + return FALSE; } return TRUE; @@ -675,8 +676,8 @@ static BOOL gdi_patblt(rdpContext* context, PATBLT_ORDER* patblt) if (hbrush) { - hbrush->nXOrg = brush->x; - hbrush->nYOrg = brush->y; + hbrush->nXOrg = WINPR_ASSERTING_INT_CAST(int32_t, brush->x); + hbrush->nYOrg = WINPR_ASSERTING_INT_CAST(int32_t, brush->y); gdi->drawing->hdc->brush = hbrush; ret = gdi_BitBlt(gdi->drawing->hdc, patblt->nLeftRect, patblt->nTopRect, patblt->nWidth, patblt->nHeight, gdi->primary->hdc, nXSrc, nYSrc, rop, &gdi->palette); @@ -786,7 +787,7 @@ static BOOL gdi_line_to(rdpContext* context, const LINE_TO_ORDER* lineTo) return FALSE; gdi_SelectObject(gdi->drawing->hdc, (HGDIOBJECT)hPen); - gdi_SetROP2(gdi->drawing->hdc, lineTo->bRop2); + gdi_SetROP2(gdi->drawing->hdc, WINPR_ASSERTING_INT_CAST(int32_t, lineTo->bRop2)); gdi_MoveToEx(gdi->drawing->hdc, lineTo->nXStart, lineTo->nYStart, NULL); gdi_LineTo(gdi->drawing->hdc, lineTo->nXEnd, lineTo->nYEnd); gdi_DeleteObject((HGDIOBJECT)hPen); @@ -811,7 +812,7 @@ static BOOL gdi_polyline(rdpContext* context, const POLYLINE_ORDER* polyline) return FALSE; gdi_SelectObject(gdi->drawing->hdc, (HGDIOBJECT)hPen); - gdi_SetROP2(gdi->drawing->hdc, polyline->bRop2); + gdi_SetROP2(gdi->drawing->hdc, WINPR_ASSERTING_INT_CAST(int32_t, polyline->bRop2)); x = polyline->xStart; y = polyline->yStart; gdi_ClipCoords(gdi->drawing->hdc, &x, &y, &w, &h, NULL, NULL); @@ -947,8 +948,8 @@ static BOOL gdi_mem3blt(rdpContext* context, MEM3BLT_ORDER* mem3blt) goto out_fail; } - gdi->drawing->hdc->brush->nXOrg = brush->x; - gdi->drawing->hdc->brush->nYOrg = brush->y; + gdi->drawing->hdc->brush->nXOrg = WINPR_ASSERTING_INT_CAST(int32_t, brush->x); + gdi->drawing->hdc->brush->nYOrg = WINPR_ASSERTING_INT_CAST(int32_t, brush->y); ret = gdi_BitBlt(gdi->drawing->hdc, mem3blt->nLeftRect, mem3blt->nTopRect, mem3blt->nWidth, mem3blt->nHeight, bitmap->hdc, mem3blt->nXSrc, mem3blt->nYSrc, gdi_rop3_code_checked(mem3blt->bRop), &gdi->palette); @@ -1086,8 +1087,8 @@ static BOOL gdi_surface_bits(rdpContext* context, const SURFACE_BITS_COMMAND* cm case RDP_CODEC_ID_IMAGE_REMOTEFX: if (!rfx_process_message(context->codecs->rfx, cmd->bmp.bitmapData, cmd->bmp.bitmapDataLength, cmdRect.left, cmdRect.top, - gdi->primary_buffer, gdi->dstFormat, gdi->stride, gdi->height, - ®ion)) + gdi->primary_buffer, gdi->dstFormat, gdi->stride, + WINPR_ASSERTING_INT_CAST(uint32_t, gdi->height), ®ion)) { WLog_ERR(TAG, "Failed to process RemoteFX message"); goto out; @@ -1148,7 +1149,10 @@ static BOOL gdi_surface_bits(rdpContext* context, const SURFACE_BITS_COMMAND* cm UINT32 width = rects[i].right - rects[i].left; UINT32 height = rects[i].bottom - rects[i].top; - if (!gdi_InvalidateRegion(gdi->primary->hdc, left, top, width, height)) + if (!gdi_InvalidateRegion(gdi->primary->hdc, WINPR_ASSERTING_INT_CAST(int32_t, left), + WINPR_ASSERTING_INT_CAST(int32_t, top), + WINPR_ASSERTING_INT_CAST(int32_t, width), + WINPR_ASSERTING_INT_CAST(int32_t, height))) { WLog_ERR(TAG, "Failed to update invalid region"); goto out; @@ -1228,7 +1232,8 @@ static BOOL gdi_init_primary(rdpGdi* gdi, UINT32 stride, UINT32 format, BYTE* bu if (stride > 0) gdi->stride = stride; else - gdi->stride = gdi->width * FreeRDPGetBytesPerPixel(gdi->dstFormat); + gdi->stride = WINPR_ASSERTING_INT_CAST(uint32_t, gdi->width) * + FreeRDPGetBytesPerPixel(gdi->dstFormat); if (!gdi->primary) goto fail_primary; @@ -1238,12 +1243,15 @@ static BOOL gdi_init_primary(rdpGdi* gdi, UINT32 stride, UINT32 format, BYTE* bu if (!buffer) { - gdi->primary->bitmap = gdi_CreateCompatibleBitmap(gdi->hdc, gdi->width, gdi->height); + gdi->primary->bitmap = + gdi_CreateCompatibleBitmap(gdi->hdc, WINPR_ASSERTING_INT_CAST(uint32_t, gdi->width), + WINPR_ASSERTING_INT_CAST(uint32_t, gdi->height)); } else { - gdi->primary->bitmap = - gdi_CreateBitmapEx(gdi->width, gdi->height, gdi->dstFormat, gdi->stride, buffer, pfree); + gdi->primary->bitmap = gdi_CreateBitmapEx(WINPR_ASSERTING_INT_CAST(uint32_t, gdi->width), + WINPR_ASSERTING_INT_CAST(uint32_t, gdi->height), + gdi->dstFormat, gdi->stride, buffer, pfree); } if (!gdi->primary->bitmap) @@ -1372,8 +1380,10 @@ BOOL gdi_init_ex(freerdp* instance, UINT32 format, UINT32 stride, BYTE* buffer, goto fail; gdi->context = context; - gdi->width = freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth); - gdi->height = freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopHeight); + gdi->width = WINPR_ASSERTING_INT_CAST( + int32_t, freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopWidth)); + gdi->height = WINPR_ASSERTING_INT_CAST( + int32_t, freerdp_settings_get_uint32(context->settings, FreeRDP_DesktopHeight)); gdi->dstFormat = format; /* default internal buffer format */ WLog_Print(gdi->log, WLOG_INFO, "Local framebuffer format %s", @@ -1449,7 +1459,10 @@ BOOL gdi_send_suppress_output(rdpGdi* gdi, BOOL suppress) update = gdi->context->update; rect.left = 0; rect.top = 0; - rect.right = freerdp_settings_get_uint32(settings, FreeRDP_DesktopWidth); - rect.bottom = freerdp_settings_get_uint32(settings, FreeRDP_DesktopHeight); + + const UINT32 w = freerdp_settings_get_uint32(settings, FreeRDP_DesktopWidth); + const UINT32 h = freerdp_settings_get_uint32(settings, FreeRDP_DesktopHeight); + rect.right = WINPR_ASSERTING_INT_CAST(UINT16, w); + rect.bottom = WINPR_ASSERTING_INT_CAST(UINT16, h); return update->SuppressOutput(gdi->context, !suppress, &rect); } diff --git a/libfreerdp/gdi/gdi.h b/libfreerdp/gdi/gdi.h index d39b18a44..dc3924bfe 100644 --- a/libfreerdp/gdi/gdi.h +++ b/libfreerdp/gdi/gdi.h @@ -20,6 +20,8 @@ #ifndef FREERDP_LIB_GDI_CORE_H #define FREERDP_LIB_GDI_CORE_H +#include + #include "graphics.h" #include "brush.h" @@ -32,12 +34,12 @@ FREERDP_LOCAL void gdi_bitmap_free_ex(gdiBitmap* gdi_bmp); static INLINE BYTE* gdi_get_bitmap_pointer(HGDI_DC hdcBmp, INT32 x, INT32 y) { - BYTE* p = NULL; HGDI_BITMAP hBmp = (HGDI_BITMAP)hdcBmp->selectedObject; if ((x >= 0) && (y >= 0) && (x < hBmp->width) && (y < hBmp->height)) { - p = hBmp->data + (y * hBmp->scanline) + (x * FreeRDPGetBytesPerPixel(hdcBmp->format)); + BYTE* p = hBmp->data + (WINPR_ASSERTING_INT_CAST(size_t, y) * hBmp->scanline) + + (WINPR_ASSERTING_INT_CAST(size_t, x) * FreeRDPGetBytesPerPixel(hdcBmp->format)); return p; } else @@ -72,15 +74,14 @@ static INLINE BYTE* gdi_get_brush_pointer(HGDI_DC hdcBrush, UINT32 x, UINT32 y) * at the brush origin and copy across the client area. * Calculate the offset of the mapped pixel in the brush bitmap according to * brush origin and dest coordinates */ - x = (x + hBmpBrush->width - (hdcBrush->brush->nXOrg % hBmpBrush->width)) % - hBmpBrush->width; - y = (y + hBmpBrush->height - (hdcBrush->brush->nYOrg % hBmpBrush->height)) % - hBmpBrush->height; + const UINT32 w = WINPR_ASSERTING_INT_CAST(UINT32, hBmpBrush->width); + const UINT32 h = WINPR_ASSERTING_INT_CAST(UINT32, hBmpBrush->height); + x = (x + w - (WINPR_ASSERTING_INT_CAST(UINT32, hdcBrush->brush->nXOrg) % w)) % w; + y = (y + h - (WINPR_ASSERTING_INT_CAST(UINT32, hdcBrush->brush->nYOrg) % h)) % h; p = hBmpBrush->data + (y * hBmpBrush->scanline) + (x * FreeRDPGetBytesPerPixel(hBmpBrush->format)); return p; } - break; default: break; diff --git a/libfreerdp/gdi/gfx.c b/libfreerdp/gdi/gfx.c index 531a75d65..ad8ee2fdb 100644 --- a/libfreerdp/gdi/gfx.c +++ b/libfreerdp/gdi/gfx.c @@ -23,6 +23,9 @@ #include "../core/update.h" +#include +#include + #include #include #include @@ -1035,20 +1038,20 @@ static UINT gdi_SurfaceCommand(RdpgfxClientContext* context, const RDPGFX_SURFAC gdi = (rdpGdi*)context->custom; EnterCriticalSection(&context->mux); + const UINT16 codecId = WINPR_ASSERTING_INT_CAST(UINT16, cmd->codecId); WLog_Print(gdi->log, WLOG_TRACE, "surfaceId=%" PRIu32 ", codec=%s [%" PRIu32 "], contextId=%" PRIu32 ", format=%s, " "left=%" PRIu32 ", top=%" PRIu32 ", right=%" PRIu32 ", bottom=%" PRIu32 ", width=%" PRIu32 ", height=%" PRIu32 " " "length=%" PRIu32 ", data=%p, extra=%p", - cmd->surfaceId, rdpgfx_get_codec_id_string(cmd->codecId), cmd->codecId, - cmd->contextId, FreeRDPGetColorFormatName(cmd->format), cmd->left, cmd->top, - cmd->right, cmd->bottom, cmd->width, cmd->height, cmd->length, (void*)cmd->data, - (void*)cmd->extra); + cmd->surfaceId, rdpgfx_get_codec_id_string(codecId), cmd->codecId, cmd->contextId, + FreeRDPGetColorFormatName(cmd->format), cmd->left, cmd->top, cmd->right, cmd->bottom, + cmd->width, cmd->height, cmd->length, (void*)cmd->data, (void*)cmd->extra); #if defined(WITH_GFX_FRAME_DUMP) dump_cmd(cmd, gdi->frameId); #endif - switch (cmd->codecId) + switch (codecId) { case RDPGFX_CODECID_UNCOMPRESSED: status = gdi_SurfaceCommand_Uncompressed(gdi, context, cmd); @@ -1084,13 +1087,13 @@ static UINT gdi_SurfaceCommand(RdpgfxClientContext* context, const RDPGFX_SURFAC break; case RDPGFX_CODECID_CAPROGRESSIVE_V2: - WLog_WARN(TAG, "SurfaceCommand %s [0x%08" PRIX32 "] not implemented", - rdpgfx_get_codec_id_string(cmd->codecId), cmd->codecId); + WLog_WARN(TAG, "SurfaceCommand %s [0x%08" PRIX16 "] not implemented", + rdpgfx_get_codec_id_string(codecId), codecId); break; default: - WLog_WARN(TAG, "Invalid SurfaceCommand %s [0x%08" PRIX32 "]", - rdpgfx_get_codec_id_string(cmd->codecId), cmd->codecId); + WLog_WARN(TAG, "Invalid SurfaceCommand %s [0x%08" PRIX16 "]", + rdpgfx_get_codec_id_string(codecId), codecId); break; } @@ -1251,8 +1254,9 @@ static BOOL intersect_rect(const RECTANGLE_16* rect, const gdiGfxSurface* surfac return FALSE; prect->left = rect->left; prect->top = rect->top; - prect->right = MIN(rect->right, surface->width); - prect->bottom = MIN(rect->bottom, surface->height); + + prect->right = MIN(rect->right, WINPR_ASSERTING_INT_CAST(UINT16, surface->width)); + prect->bottom = MIN(rect->bottom, WINPR_ASSERTING_INT_CAST(UINT16, surface->height)); return TRUE; } diff --git a/libfreerdp/gdi/graphics.c b/libfreerdp/gdi/graphics.c index bcb9f9776..cc668b35a 100644 --- a/libfreerdp/gdi/graphics.c +++ b/libfreerdp/gdi/graphics.c @@ -125,7 +125,9 @@ static BOOL gdi_Bitmap_Paint(rdpContext* context, rdpBitmap* bitmap) gdiBitmap* gdi_bitmap = (gdiBitmap*)bitmap; UINT32 width = bitmap->right - bitmap->left + 1; UINT32 height = bitmap->bottom - bitmap->top + 1; - return gdi_BitBlt(context->gdi->primary->hdc, bitmap->left, bitmap->top, width, height, + return gdi_BitBlt(context->gdi->primary->hdc, WINPR_ASSERTING_INT_CAST(int, bitmap->left), + WINPR_ASSERTING_INT_CAST(int, bitmap->top), + WINPR_ASSERTING_INT_CAST(int, width), WINPR_ASSERTING_INT_CAST(int, height), gdi_bitmap->hdc, 0, 0, GDI_SRCCOPY, &context->gdi->palette); } @@ -163,7 +165,7 @@ static BOOL gdi_Bitmap_Decompress(rdpContext* context, rdpBitmap* bitmap, const if (!rfx_process_message(context->codecs->rfx, pSrcData, SrcSize, bitmap->left, bitmap->top, bitmap->data, bitmap->format, gdi->stride, - gdi->height, &invalidRegion)) + WINPR_ASSERTING_INT_CAST(UINT32, gdi->height), &invalidRegion)) { WLog_ERR(TAG, "rfx_process_message failed"); return FALSE; diff --git a/libfreerdp/gdi/line.c b/libfreerdp/gdi/line.c index 1c4a6ab29..b5b7b8ba7 100644 --- a/libfreerdp/gdi/line.c +++ b/libfreerdp/gdi/line.c @@ -37,7 +37,7 @@ #include "clipping.h" #include "line.h" -static BOOL gdi_rop_color(UINT32 rop, BYTE* pixelPtr, UINT32 pen, UINT32 format) +static BOOL gdi_rop_color(INT32 rop, BYTE* pixelPtr, UINT32 pen, UINT32 format) { WINPR_ASSERT(pixelPtr); const UINT32 srcPixel = FreeRDPReadColor(pixelPtr, format); @@ -116,7 +116,7 @@ static BOOL gdi_rop_color(UINT32 rop, BYTE* pixelPtr, UINT32 pen, UINT32 format) return FreeRDPWriteColor(pixelPtr, format, dstPixel); } -BOOL gdi_LineTo(HGDI_DC hdc, UINT32 nXEnd, UINT32 nYEnd) +BOOL gdi_LineTo(HGDI_DC hdc, INT32 nXEnd, INT32 nYEnd) { INT32 e2 = 0; UINT32 pen = 0; @@ -126,8 +126,8 @@ BOOL gdi_LineTo(HGDI_DC hdc, UINT32 nXEnd, UINT32 nYEnd) const INT32 x1 = hdc->pen->posX; const INT32 y1 = hdc->pen->posY; - const INT32 x2 = nXEnd; - const INT32 y2 = nYEnd; + const INT32 x2 = WINPR_ASSERTING_INT_CAST(int32_t, nXEnd); + const INT32 y2 = WINPR_ASSERTING_INT_CAST(int32_t, nYEnd); const INT32 dx = (x1 > x2) ? x1 - x2 : x2 - x1; const INT32 dy = (y1 > y2) ? y1 - y2 : y2 - y1; const INT32 sx = (x1 < x2) ? 1 : -1; @@ -299,7 +299,7 @@ BOOL gdi_PolyPolyline(HGDI_DC hdc, GDI_POINT* lppt, const UINT32* lpdwPolyPoints * @return nonzero on success, 0 otherwise */ -BOOL gdi_MoveToEx(HGDI_DC hdc, UINT32 X, UINT32 Y, HGDI_POINT lpPoint) +BOOL gdi_MoveToEx(HGDI_DC hdc, INT32 X, INT32 Y, HGDI_POINT lpPoint) { WINPR_ASSERT(hdc); diff --git a/libfreerdp/gdi/line.h b/libfreerdp/gdi/line.h index 9549deec8..fa90a3136 100644 --- a/libfreerdp/gdi/line.h +++ b/libfreerdp/gdi/line.h @@ -30,12 +30,12 @@ extern "C" { #endif - FREERDP_LOCAL BOOL gdi_LineTo(HGDI_DC hdc, UINT32 nXEnd, UINT32 nYEnd); + FREERDP_LOCAL BOOL gdi_LineTo(HGDI_DC hdc, INT32 nXEnd, INT32 nYEnd); FREERDP_LOCAL BOOL gdi_PolylineTo(HGDI_DC hdc, GDI_POINT* lppt, DWORD cCount); FREERDP_LOCAL BOOL gdi_Polyline(HGDI_DC hdc, GDI_POINT* lppt, UINT32 cPoints); FREERDP_LOCAL BOOL gdi_PolyPolyline(HGDI_DC hdc, GDI_POINT* lppt, const UINT32* lpdwPolyPoints, DWORD cCount); - FREERDP_LOCAL BOOL gdi_MoveToEx(HGDI_DC hdc, UINT32 X, UINT32 Y, HGDI_POINT lpPoint); + FREERDP_LOCAL BOOL gdi_MoveToEx(HGDI_DC hdc, INT32 X, INT32 Y, HGDI_POINT lpPoint); #ifdef __cplusplus } diff --git a/libfreerdp/gdi/shape.c b/libfreerdp/gdi/shape.c index 90207e522..2a906b77b 100644 --- a/libfreerdp/gdi/shape.c +++ b/libfreerdp/gdi/shape.c @@ -71,10 +71,14 @@ static void Ellipse_Bresenham(HGDI_DC hdc, int x1, int y1, int x2, int y2) do { - gdi_SetPixel(hdc, x2, y1, 0); - gdi_SetPixel(hdc, x1, y1, 0); - gdi_SetPixel(hdc, x1, y2, 0); - gdi_SetPixel(hdc, x2, y2, 0); + gdi_SetPixel(hdc, WINPR_ASSERTING_INT_CAST(UINT32, x2), + WINPR_ASSERTING_INT_CAST(UINT32, y1), 0); + gdi_SetPixel(hdc, WINPR_ASSERTING_INT_CAST(UINT32, x1), + WINPR_ASSERTING_INT_CAST(UINT32, y1), 0); + gdi_SetPixel(hdc, WINPR_ASSERTING_INT_CAST(UINT32, x1), + WINPR_ASSERTING_INT_CAST(UINT32, y2), 0); + gdi_SetPixel(hdc, WINPR_ASSERTING_INT_CAST(UINT32, x2), + WINPR_ASSERTING_INT_CAST(UINT32, y2), 0); e2 = 2 * e; if (e2 >= dx) @@ -94,8 +98,13 @@ static void Ellipse_Bresenham(HGDI_DC hdc, int x1, int y1, int x2, int y2) while (y1 - y2 < b) { - gdi_SetPixel(hdc, x1 - 1, ++y1, 0); - gdi_SetPixel(hdc, x1 - 1, --y2, 0); + y1++; + y2--; + + gdi_SetPixel(hdc, WINPR_ASSERTING_INT_CAST(uint32_t, x1 - 1), + WINPR_ASSERTING_INT_CAST(uint32_t, y1), 0); + gdi_SetPixel(hdc, WINPR_ASSERTING_INT_CAST(uint32_t, x1 - 1), + WINPR_ASSERTING_INT_CAST(uint32_t, y2), 0); } } @@ -166,7 +175,7 @@ BOOL gdi_FillRect(HGDI_DC hdc, const HGDI_RECT rect, HGDI_BRUSH hbr) for (INT32 y = 1; y < nHeight; y++) { BYTE* dstp = gdi_get_bitmap_pointer(hdc, nXDest, nYDest + y); - memcpy(dstp, srcp, 1ull * nWidth * formatSize); + memcpy(dstp, srcp, 1ull * WINPR_ASSERTING_INT_CAST(size_t, nWidth) * formatSize); } break; @@ -181,9 +190,15 @@ BOOL gdi_FillRect(HGDI_DC hdc, const HGDI_RECT rect, HGDI_BRUSH hbr) for (INT32 x = 0; x < nWidth; x++) { const size_t yOffset = - ((1ULL * nYDest + y) * hbr->pattern->width % hbr->pattern->height) * + ((1ULL * WINPR_ASSERTING_INT_CAST(size_t, nYDest) + + WINPR_ASSERTING_INT_CAST(size_t, y)) * + WINPR_ASSERTING_INT_CAST(size_t, hbr->pattern->width) % + WINPR_ASSERTING_INT_CAST(size_t, hbr->pattern->height)) * formatSize; - const size_t xOffset = ((1ULL * nXDest + x) % hbr->pattern->width) * formatSize; + const size_t xOffset = ((1ULL * WINPR_ASSERTING_INT_CAST(size_t, nXDest) + + WINPR_ASSERTING_INT_CAST(size_t, x)) % + WINPR_ASSERTING_INT_CAST(size_t, hbr->pattern->width)) * + formatSize; const BYTE* patp = &hbr->pattern->data[yOffset + xOffset]; if (monochrome) diff --git a/libfreerdp/gdi/test/TestGdiRect.c b/libfreerdp/gdi/test/TestGdiRect.c index 7e86b0246..444aed1e4 100644 --- a/libfreerdp/gdi/test/TestGdiRect.c +++ b/libfreerdp/gdi/test/TestGdiRect.c @@ -23,7 +23,9 @@ static int test_gdi_PtInRect(void) UINT32 right = 60; UINT32 bottom = 80; - if (!(hRect = gdi_CreateRect(left, top, right, bottom))) + if (!(hRect = gdi_CreateRect( + WINPR_ASSERTING_INT_CAST(int, left), WINPR_ASSERTING_INT_CAST(int, top), + WINPR_ASSERTING_INT_CAST(int, right), WINPR_ASSERTING_INT_CAST(int, bottom)))) { printf("gdi_CreateRect failed\n"); return rc; @@ -44,16 +46,18 @@ static int test_gdi_PtInRect(void) if (!gdi_PtInRect(hRect, 30, 50)) goto fail; - if (!gdi_PtInRect(hRect, left, top)) + if (!gdi_PtInRect(hRect, WINPR_ASSERTING_INT_CAST(int, left), + WINPR_ASSERTING_INT_CAST(int, top))) goto fail; - if (!gdi_PtInRect(hRect, right, bottom)) + if (!gdi_PtInRect(hRect, WINPR_ASSERTING_INT_CAST(int, right), + WINPR_ASSERTING_INT_CAST(int, bottom))) goto fail; - if (!gdi_PtInRect(hRect, right, 60)) + if (!gdi_PtInRect(hRect, WINPR_ASSERTING_INT_CAST(int, right), 60)) goto fail; - if (!gdi_PtInRect(hRect, 40, bottom)) + if (!gdi_PtInRect(hRect, 40, WINPR_ASSERTING_INT_CAST(int, bottom))) goto fail; rc = 0; @@ -89,7 +93,9 @@ static int test_gdi_FillRect(void) hdc->format = PIXEL_FORMAT_XRGB32; - if (!(hRect = gdi_CreateRect(left, top, right, bottom))) + if (!(hRect = gdi_CreateRect( + WINPR_ASSERTING_INT_CAST(int, left), WINPR_ASSERTING_INT_CAST(int, top), + WINPR_ASSERTING_INT_CAST(int, right), WINPR_ASSERTING_INT_CAST(int, bottom)))) { printf("gdi_CreateRect failed\n"); goto fail; @@ -111,7 +117,8 @@ static int test_gdi_FillRect(void) rawPixel = gdi_GetPixel(hdc, x, y); pixel = FreeRDPConvertColor(rawPixel, hdc->format, PIXEL_FORMAT_ARGB32, NULL); - if (gdi_PtInRect(hRect, x, y)) + if (gdi_PtInRect(hRect, WINPR_ASSERTING_INT_CAST(int, x), + WINPR_ASSERTING_INT_CAST(int, y))) { if (pixel == color) { diff --git a/libfreerdp/gdi/test/helpers.c b/libfreerdp/gdi/test/helpers.c index 45d00b4bb..752acb0c1 100644 --- a/libfreerdp/gdi/test/helpers.c +++ b/libfreerdp/gdi/test/helpers.c @@ -53,18 +53,18 @@ HGDI_BITMAP test_convert_to_bitmap(const BYTE* src, UINT32 SrcFormat, UINT32 Src return bmp; } -static void test_dump_data(unsigned char* p, int len, int width, const char* name) +static void test_dump_data(unsigned char* p, size_t len, size_t width, const char* name) { unsigned char* line = p; - int thisline = 0; - int offset = 0; - printf("\n%s[%d][%d]:\n", name, len / width, width); + const size_t stride = (width > 0) ? len / width : 1; + size_t offset = 0; + printf("\n%s[%" PRIuz "][%" PRIuz "]:\n", name, stride, width); while (offset < len) { - int i = 0; - printf("%04x ", offset); - thisline = len - offset; + size_t i = 0; + printf("%04" PRIxz " ", offset); + size_t thisline = len - offset; if (thisline > width) thisline = width; @@ -86,16 +86,20 @@ static void test_dump_data(unsigned char* p, int len, int width, const char* nam void test_dump_bitmap(HGDI_BITMAP hBmp, const char* name) { - UINT32 stride = hBmp->width * FreeRDPGetBytesPerPixel(hBmp->format); - test_dump_data(hBmp->data, hBmp->height * stride, stride, name); + const size_t stride = + WINPR_ASSERTING_INT_CAST(size_t, hBmp->width) * FreeRDPGetBytesPerPixel(hBmp->format); + test_dump_data(hBmp->data, stride * WINPR_ASSERTING_INT_CAST(uint32_t, hBmp->height), stride, + name); } static BOOL CompareBitmaps(HGDI_BITMAP hBmp1, HGDI_BITMAP hBmp2, const gdiPalette* palette) { const BYTE* p1 = hBmp1->data; const BYTE* p2 = hBmp2->data; - const UINT32 minw = (hBmp1->width < hBmp2->width) ? hBmp1->width : hBmp2->width; - const UINT32 minh = (hBmp1->height < hBmp2->height) ? hBmp1->height : hBmp2->height; + const UINT32 minw = WINPR_ASSERTING_INT_CAST( + uint32_t, (hBmp1->width < hBmp2->width) ? hBmp1->width : hBmp2->width); + const UINT32 minh = WINPR_ASSERTING_INT_CAST( + uint32_t, (hBmp1->height < hBmp2->height) ? hBmp1->height : hBmp2->height); for (UINT32 y = 0; y < minh; y++) { diff --git a/libfreerdp/primitives/prim_YCoCg.c b/libfreerdp/primitives/prim_YCoCg.c index a19075431..5fabb7d0d 100644 --- a/libfreerdp/primitives/prim_YCoCg.c +++ b/libfreerdp/primitives/prim_YCoCg.c @@ -43,8 +43,8 @@ static pstatus_t general_YCoCgToRGB_8u_AC4R(const BYTE* pSrc, INT32 srcStep, BYT for (size_t y = 0; y < height; y++) { - const BYTE* sptr = &pSrc[y * srcStep]; - BYTE* dptr = &pDst[y * dstStep]; + const BYTE* sptr = &pSrc[y * WINPR_ASSERTING_INT_CAST(uint32_t, srcStep)]; + BYTE* dptr = &pDst[y * WINPR_ASSERTING_INT_CAST(uint32_t, dstStep)]; for (size_t x = 0; x < width; x++) { /* Note: shifts must be done before sign-conversion. */ diff --git a/libfreerdp/primitives/prim_YUV.c b/libfreerdp/primitives/prim_YUV.c index 6024d609b..e5f0e517b 100644 --- a/libfreerdp/primitives/prim_YUV.c +++ b/libfreerdp/primitives/prim_YUV.c @@ -22,6 +22,8 @@ */ #include +#include +#include #include @@ -509,8 +511,8 @@ static pstatus_t general_YUV420ToRGB_8u_P3AC4R(const BYTE* WINPR_RESTRICT pSrc[3 pV = pSrc[2]; lastCol = roi->width & 0x01; lastRow = roi->height & 0x01; - nWidth = (roi->width + 1) & ~0x0001; - nHeight = (roi->height + 1) & ~0x0001; + nWidth = (roi->width + 1) & (uint32_t)~0x0001; + nHeight = (roi->height + 1) & (uint32_t)~0x0001; halfWidth = nWidth / 2; halfHeight = nHeight / 2; srcPad[0] = (srcStep[0] - nWidth); @@ -615,19 +617,22 @@ static pstatus_t general_YUV420ToRGB_8u_P3AC4R(const BYTE* WINPR_RESTRICT pSrc[3 * | U | = ( | -29 -99 128 | | G | ) >> 8 + | 128 | * | V | ( | 128 -116 -12 | | B | ) | 128 | */ -static INLINE BYTE RGB2Y(BYTE R, BYTE G, BYTE B) +static INLINE BYTE RGB2Y(INT32 R, INT32 G, INT32 B) { - return (54 * R + 183 * G + 18 * B) >> 8; + const INT32 val = ((54 * R + 183 * G + 18 * B) >> 8); + return WINPR_ASSERTING_INT_CAST(BYTE, val); } -static INLINE BYTE RGB2U(BYTE R, BYTE G, BYTE B) +static INLINE BYTE RGB2U(INT32 R, INT32 G, INT32 B) { - return ((-29 * R - 99 * G + 128 * B) >> 8) + 128; + const INT32 val = (((-29 * R - 99 * G + 128 * B) >> 8) + 128); + return WINPR_ASSERTING_INT_CAST(BYTE, val); } static INLINE BYTE RGB2V(INT32 R, INT32 G, INT32 B) { - return ((128 * R - 116 * G - 12 * B) >> 8) + 128; + const INT32 val = (((128 * R - 116 * G - 12 * B) >> 8) + 128); + return WINPR_ASSERTING_INT_CAST(BYTE, val); } // NOLINTBEGIN(readability-non-const-parameter) @@ -769,41 +774,49 @@ static INLINE pstatus_t general_RGBToYUV420_RGBX(const BYTE* WINPR_RESTRICT pSrc for (UINT32 x = 0; x < roi->width; x += 2) { - BYTE R = 0; - BYTE G = 0; - BYTE B = 0; - INT32 Ra = 0; - INT32 Ga = 0; - INT32 Ba = 0; + BYTE R = *(src + x1 + 0); + BYTE G = *(src + x1 + 1); + BYTE B = *(src + x1 + 2); /* row 1, pixel 1 */ - Ra = R = *(src + x1 + 0); - Ga = G = *(src + x1 + 1); - Ba = B = *(src + x1 + 2); + INT32 Ra = R; + INT32 Ga = G; + INT32 Ba = B; ydst[y1] = RGB2Y(R, G, B); if (x < max_x) { /* row 1, pixel 2 */ - Ra += R = *(src + x2 + 0); - Ga += G = *(src + x2 + 1); - Ba += B = *(src + x2 + 2); + R = *(src + x2 + 0); + G = *(src + x2 + 1); + B = *(src + x2 + 2); + Ra += R; + Ga += G; + Ba += B; ydst[y2] = RGB2Y(R, G, B); } if (y < max_y) { /* row 2, pixel 1 */ - Ra += R = *(src + x3 + 0); - Ga += G = *(src + x3 + 1); - Ba += B = *(src + x3 + 2); + R = *(src + x3 + 0); + G = *(src + x3 + 1); + B = *(src + x3 + 2); + + Ra += R; + Ga += G; + Ba += B; ydst[y3] = RGB2Y(R, G, B); if (x < max_x) { /* row 2, pixel 2 */ - Ra += R = *(src + x4 + 0); - Ga += G = *(src + x4 + 1); - Ba += B = *(src + x4 + 2); + R = *(src + x4 + 0); + G = *(src + x4 + 1); + B = *(src + x4 + 2); + + Ra += R; + Ga += G; + Ba += B; ydst[y4] = RGB2Y(R, G, B); } } @@ -1006,8 +1019,8 @@ static INLINE void general_RGBToAVC444YUV_BGRX_DOUBLE_ROW( /* 2x 2y pixel in luma UV plane use averaging */ { - const BYTE Uavg = ((UINT16)U1e + (UINT16)U2e + (UINT16)U1o + (UINT16)U2o) / 4; - const BYTE Vavg = ((UINT16)V1e + (UINT16)V2e + (UINT16)V1o + (UINT16)V2o) / 4; + const BYTE Uavg = WINPR_ASSERTING_INT_CAST(BYTE, ((UINT16)U1e + U2e + U1o + U2o) / 4); + const BYTE Vavg = WINPR_ASSERTING_INT_CAST(BYTE, ((UINT16)V1e + V2e + V1o + V2o) / 4); *b2++ = Uavg; *b3++ = Vavg; } @@ -1053,7 +1066,7 @@ static INLINE pstatus_t general_RGBToAVC444YUV_BGRX(const BYTE* WINPR_RESTRICT p const BYTE* srcEven = y < roi->height ? pSrc + 1ULL * y * srcStep : pMaxSrc; const BYTE* srcOdd = !last ? pSrc + 1ULL * (y + 1) * srcStep : pMaxSrc; const UINT32 i = y >> 1; - const UINT32 n = (i & ~7) + i; + const UINT32 n = (i & (uint32_t)~7) + i; BYTE* b1Even = pDst1[0] + 1ULL * y * dst1Step[0]; BYTE* b1Odd = !last ? (b1Even + dst1Step[0]) : NULL; BYTE* b2 = pDst1[1] + 1ULL * (y / 2) * dst1Step[1]; @@ -1147,8 +1160,8 @@ static INLINE void general_RGBToAVC444YUV_RGBX_DOUBLE_ROW( /* 2x 2y pixel in luma UV plane use averaging */ { - const BYTE Uavg = ((UINT16)U1e + (UINT16)U2e + (UINT16)U1o + (UINT16)U2o) / 4; - const BYTE Vavg = ((UINT16)V1e + (UINT16)V2e + (UINT16)V1o + (UINT16)V2o) / 4; + const BYTE Uavg = WINPR_ASSERTING_INT_CAST(BYTE, ((UINT16)U1e + U2e + U1o + U2o) / 4); + const BYTE Vavg = WINPR_ASSERTING_INT_CAST(BYTE, ((UINT16)V1e + V2e + V1o + V2o) / 4); *b2++ = Uavg; *b3++ = Vavg; } @@ -1194,7 +1207,7 @@ static INLINE pstatus_t general_RGBToAVC444YUV_RGBX(const BYTE* WINPR_RESTRICT p const BYTE* srcEven = y < roi->height ? pSrc + 1ULL * y * srcStep : pMaxSrc; const BYTE* srcOdd = !last ? pSrc + 1ULL * (y + 1) * srcStep : pMaxSrc; const UINT32 i = y >> 1; - const UINT32 n = (i & ~7) + i; + const UINT32 n = (i & (uint32_t)~7) + i; BYTE* b1Even = pDst1[0] + 1ULL * y * dst1Step[0]; BYTE* b1Odd = !last ? (b1Even + dst1Step[0]) : NULL; BYTE* b2 = pDst1[1] + 1ULL * (y / 2) * dst1Step[1]; @@ -1297,8 +1310,10 @@ static INLINE void general_RGBToAVC444YUV_ANY_DOUBLE_ROW( /* 2x 2y pixel in luma UV plane use averaging */ { - const BYTE Uavg = ((UINT16)U1e + (UINT16)U2e + (UINT16)U1o + (UINT16)U2o) / 4; - const BYTE Vavg = ((UINT16)V1e + (UINT16)V2e + (UINT16)V1o + (UINT16)V2o) / 4; + const BYTE Uavg = WINPR_ASSERTING_INT_CAST( + BYTE, ((UINT16)U1e + (UINT16)U2e + (UINT16)U1o + (UINT16)U2o) / 4); + const BYTE Vavg = WINPR_ASSERTING_INT_CAST( + BYTE, ((UINT16)V1e + (UINT16)V2e + (UINT16)V1o + (UINT16)V2o) / 4); *b2++ = Uavg; *b3++ = Vavg; } @@ -1397,7 +1412,7 @@ static INLINE pstatus_t general_RGBToAVC444YUV_ANY( const BYTE* srcEven = y < roi->height ? pSrc + y * srcStep : pMaxSrc; const BYTE* srcOdd = !last ? pSrc + (y + 1) * srcStep : pMaxSrc; const UINT32 i = (UINT32)y >> 1; - const UINT32 n = (i & ~7) + i; + const UINT32 n = (i & (uint32_t)~7) + i; BYTE* b1Even = pDst1[0] + y * dst1Step[0]; BYTE* b1Odd = !last ? (b1Even + dst1Step[0]) : NULL; BYTE* b2 = pDst1[1] + (y / 2) * dst1Step[1]; diff --git a/libfreerdp/primitives/prim_colors.c b/libfreerdp/primitives/prim_colors.c index f488d965b..d05f89729 100644 --- a/libfreerdp/primitives/prim_colors.c +++ b/libfreerdp/primitives/prim_colors.c @@ -17,7 +17,11 @@ * permissions and limitations under the License. */ +#include + #include +#include +#include #include #include @@ -30,6 +34,47 @@ #define MINMAX(_v_, _l_, _h_) ((_v_) < (_l_) ? (_l_) : ((_v_) > (_h_) ? (_h_) : (_v_))) #endif /* !MINMAX */ /* ------------------------------------------------------------------------- */ + +/* pregenerated table for ycbcr constants: [0,27] + * + * rounded integer values derived from the following formula: + * + * { (1.402525f * 2^divisor), (0.714401f * 2^divisor), (0.343730f * 2^divisor), (1.769905f * + * 2^divisor) } + */ + +static const INT32 ycbcr_constants[][4] = { { 1, 1, 0, 2 }, + { 3, 1, 1, 4 }, + { 6, 3, 1, 7 }, + { 11, 6, 3, 14 }, + { 22, 11, 5, 28 }, + { 45, 23, 11, 57 }, + { 90, 46, 22, 113 }, + { 180, 91, 44, 227 }, + { 359, 183, 88, 453 }, + { 718, 366, 176, 906 }, + { 1436, 732, 352, 1812 }, + { 2872, 1463, 704, 3625 }, + { 5745, 2926, 1408, 7250 }, + { 11489, 5852, 2816, 14499 }, + { 22979, 11705, 5632, 28998 }, + { 45958, 23409, 11263, 57996 }, + { 91916, 46819, 22527, 115992 }, + { 183832, 93638, 45053, 231985 }, + { 367664, 187276, 90107, 463970 }, + { 735327, 374552, 180214, 927940 }, + { 1470654, 749104, 360427, 1855880 }, + { 2941308, 1498207, 720854, 3711760 }, + { 5882616, 2996415, 1441708, 7423520 }, + { 11765232, 5992830, 2883416, 14847039 }, + { 23530465, 11985660, 5766832, 29694078 }, + { 47060930, 23971320, 11533665, 59388157 }, + { 94121859, 47942640, 23067330, 118776314 }, + { 188243719, 95885279, 46134660, 237552628 }, + { 376487438, 191770558, 92269319, 475105256 }, + { 752974876, 383541116, 184538639, 950210512 }, + { 1505949752, 767082233, 369077277, 1900421023 } }; + static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 srcStep, BYTE* WINPR_RESTRICT pDst, UINT32 dstStep, UINT32 DstFormat, @@ -47,20 +92,22 @@ static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* WINPR_RESTRIC { for (UINT32 x = 0; x < roi->width; x++) { - INT16 R = 0; - INT16 G = 0; - INT16 B = 0; const INT32 divisor = 16; const INT32 Y = (INT32)((UINT32)((*pY++) + 4096) << divisor); const INT32 Cb = (*pCb++); const INT32 Cr = (*pCr++); - const INT64 CrR = Cr * (INT64)(1.402525f * (1 << divisor)) * 1LL; - const INT64 CrG = Cr * (INT64)(0.714401f * (1 << divisor)) * 1LL; - const INT64 CbG = Cb * (INT64)(0.343730f * (1 << divisor)) * 1LL; - const INT64 CbB = Cb * (INT64)(1.769905f * (1 << divisor)) * 1LL; - R = ((INT16)((CrR + Y) >> divisor) >> 5); - G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5); - B = ((INT16)((CbB + Y) >> divisor) >> 5); + + const INT32 CrR = WINPR_ASSERTING_INT_CAST( + int32_t, Cr* ycbcr_constants[divisor][0]); //(1.402525f * 2^divisor); + const INT32 CrG = WINPR_ASSERTING_INT_CAST( + int32_t, Cr* ycbcr_constants[divisor][1]); //(0.714401f * 2^divisor); + const INT32 CbG = WINPR_ASSERTING_INT_CAST( + int32_t, Cb* ycbcr_constants[divisor][2]); //(0.343730f * 2^divisor); + const INT32 CbB = WINPR_ASSERTING_INT_CAST( + int32_t, Cb* ycbcr_constants[divisor][3]); //(1.769905f * 2^divisor); + const INT16 R = WINPR_ASSERTING_INT_CAST(int16_t, ((CrR + Y) >> divisor) >> 5); + const INT16 G = WINPR_ASSERTING_INT_CAST(int16_t, ((Y - CbG - CrG) >> divisor) >> 5); + const INT16 B = WINPR_ASSERTING_INT_CAST(int16_t, ((CbB + Y) >> divisor) >> 5); pRGB = writePixelBGRX(pRGB, formatSize, DstFormat, CLIP(R), CLIP(G), CLIP(B), 0); } @@ -95,13 +142,13 @@ static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_general(const INT16* WINPR_REST const INT32 Y = (INT32)((UINT32)((*pY++) + 4096) << divisor); const INT32 Cb = (*pCb++); const INT32 Cr = (*pCr++); - const INT64 CrR = Cr * (INT64)(1.402525f * (1 << divisor)) * 1LL; - const INT64 CrG = Cr * (INT64)(0.714401f * (1 << divisor)) * 1LL; - const INT64 CbG = Cb * (INT64)(0.343730f * (1 << divisor)) * 1LL; - const INT64 CbB = Cb * (INT64)(1.769905f * (1 << divisor)) * 1LL; - const INT64 R = (CrR + Y) >> (divisor + 5); - const INT64 G = (Y - CbG - CrG) >> (divisor + 5); - const INT64 B = (CbB + Y) >> (divisor + 5); + const INT32 CrR = Cr * ycbcr_constants[divisor][0]; + const INT32 CrG = Cr * ycbcr_constants[divisor][1]; + const INT32 CbG = Cb * ycbcr_constants[divisor][2]; + const INT32 CbB = Cb * ycbcr_constants[divisor][3]; + const INT32 R = (CrR + Y) >> (divisor + 5); + const INT32 G = (Y - CbG - CrG) >> (divisor + 5); + const INT32 B = (CbB + Y) >> (divisor + 5); pRGB = writePixel(pRGB, formatSize, DstFormat, CLIP(R), CLIP(G), CLIP(B), 0); } @@ -156,8 +203,10 @@ general_yCbCrToRGB_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], INT32 srcSte INT16* rptr = pDst[0]; INT16* gptr = pDst[1]; INT16* bptr = pDst[2]; - UINT32 srcbump = (srcStep - (roi->width * sizeof(UINT16))) / sizeof(UINT16); - UINT32 dstbump = (dstStep - (roi->width * sizeof(UINT16))) / sizeof(UINT16); + UINT32 srcbump = (WINPR_ASSERTING_INT_CAST(uint32_t, srcStep) - (roi->width * sizeof(UINT16))) / + sizeof(UINT16); + UINT32 dstbump = (WINPR_ASSERTING_INT_CAST(uint32_t, dstStep) - (roi->width * sizeof(UINT16))) / + sizeof(UINT16); for (UINT32 y = 0; y < roi->height; y++) { @@ -193,9 +242,9 @@ general_yCbCrToRGB_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], INT32 srcSte * B: 1.770 << 16 = 115998 */ cy = (INT32)((UINT32)(cy + 4096) << 16); - r = cy + cr * 91947LL; - g = cy - cb * 22544LL - cr * 46792LL; - b = cy + cb * 115998LL; + r = cy + cr * ycbcr_constants[16][0]; + g = cy - cb * ycbcr_constants[16][1] - cr * ycbcr_constants[16][2]; + b = cy + cb * ycbcr_constants[16][3]; *rptr++ = CLIP(r >> 21); *gptr++ = CLIP(g >> 21); *bptr++ = CLIP(b >> 21); @@ -234,8 +283,10 @@ general_RGBToYCbCr_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], INT32 srcSte INT16* yptr = pDst[0]; INT16* cbptr = pDst[1]; INT16* crptr = pDst[2]; - UINT32 srcbump = (srcStep - (roi->width * sizeof(UINT16))) / sizeof(UINT16); - UINT32 dstbump = (dstStep - (roi->width * sizeof(UINT16))) / sizeof(UINT16); + UINT32 srcbump = (WINPR_ASSERTING_INT_CAST(uint32_t, srcStep) - (roi->width * sizeof(UINT16))) / + sizeof(UINT16); + UINT32 dstbump = (WINPR_ASSERTING_INT_CAST(uint32_t, dstStep) - (roi->width * sizeof(UINT16))) / + sizeof(UINT16); for (UINT32 y = 0; y < roi->height; y++) { @@ -284,7 +335,15 @@ static INLINE void writeScanlineGeneric(BYTE* dst, DWORD formatSize, UINT32 DstF fkt_writePixel writePixel = getPixelWriteFunction(DstFormat, FALSE); for (UINT32 x = 0; x < width; x++) - dst = writePixel(dst, formatSize, DstFormat, *r++, *g++, *b++, 0); + { + const INT16 pr = *r++; + const INT16 pg = *g++; + const INT16 pb = *b++; + + dst = + writePixel(dst, formatSize, DstFormat, WINPR_ASSERTING_INT_CAST(UINT8, pr), + WINPR_ASSERTING_INT_CAST(UINT8, pg), WINPR_ASSERTING_INT_CAST(UINT8, pb), 0); + } } static INLINE void writeScanlineRGB(BYTE* dst, DWORD formatSize, UINT32 DstFormat, const INT16* r, diff --git a/libfreerdp/primitives/prim_copy.c b/libfreerdp/primitives/prim_copy.c index 39a66a2a7..2c33805f9 100644 --- a/libfreerdp/primitives/prim_copy.c +++ b/libfreerdp/primitives/prim_copy.c @@ -60,14 +60,20 @@ static BOOL memory_regions_overlap_2d(const BYTE* p1, int p1Step, int p1Size, co if (p1m <= p2m) { - ULONG_PTR p1mEnd = p1m + 1ull * (height - 1) * p1Step + 1ull * width * p1Size; + ULONG_PTR p1mEnd = p1m + + 1ull * (WINPR_ASSERTING_INT_CAST(uint32_t, height - 1)) * + WINPR_ASSERTING_INT_CAST(uint32_t, p1Step) + + 1ull * WINPR_ASSERTING_INT_CAST(uint32_t, width* p1Size); if (p1mEnd > p2m) return TRUE; } else { - ULONG_PTR p2mEnd = p2m + 1ull * (height - 1) * p2Step + 1ull * width * p2Size; + ULONG_PTR p2mEnd = p2m + + 1ull * (WINPR_ASSERTING_INT_CAST(uintptr_t, height - 1)) * + WINPR_ASSERTING_INT_CAST(uintptr_t, p2Step) + + 1ull * WINPR_ASSERTING_INT_CAST(uintptr_t, width* p2Size); if (p2mEnd > p1m) return TRUE; @@ -102,7 +108,7 @@ static pstatus_t general_copy_8u_AC4r(const BYTE* pSrc, INT32 srcStep, BYTE* pDs { const BYTE* src = pSrc; BYTE* dst = pDst; - int rowbytes = width * sizeof(UINT32); + const size_t rowbytes = WINPR_ASSERTING_INT_CAST(size_t, width) * sizeof(UINT32); if ((width == 0) || (height == 0)) return PRIMITIVES_SUCCESS; @@ -112,7 +118,7 @@ static pstatus_t general_copy_8u_AC4r(const BYTE* pSrc, INT32 srcStep, BYTE* pDs { do { - generic->copy(src, dst, rowbytes); + generic->copy(src, dst, WINPR_ASSERTING_INT_CAST(int32_t, rowbytes)); src += srcStep; dst += dstStep; } while (--height); @@ -267,7 +273,8 @@ pstatus_t generic_image_copy_no_overlap_memcpy( &pSrcData[srcVMultiplier * (y + nYSrc) * nSrcStep + srcVOffset]; BYTE* WINPR_RESTRICT dstLine = &pDstData[dstVMultiplier * (y + nYDst) * nDstStep + dstVOffset]; - memcpy(&dstLine[xDstOffset], &srcLine[xSrcOffset], copyDstWidth); + memcpy(&dstLine[xDstOffset], &srcLine[xSrcOffset], + WINPR_ASSERTING_INT_CAST(size_t, copyDstWidth)); } return PRIMITIVES_SUCCESS; diff --git a/libfreerdp/primitives/prim_internal.h b/libfreerdp/primitives/prim_internal.h index fbc0d9dae..6cbfa2564 100644 --- a/libfreerdp/primitives/prim_internal.h +++ b/libfreerdp/primitives/prim_internal.h @@ -46,6 +46,22 @@ #endif #if defined(SSE_AVX_INTRINSICS_ENABLED) +#include +static inline __m128i mm_set_epu32(uint32_t val1, uint32_t val2, uint32_t val3, uint32_t val4) +{ + return _mm_set_epi32((int32_t)val1, (int32_t)val2, (int32_t)val3, (int32_t)val4); +} + +static inline __m128i mm_set1_epu32(uint32_t val) +{ + return _mm_set1_epi32((int32_t)val); +} + +static inline __m128i mm_set1_epu8(uint8_t val) +{ + return _mm_set1_epi8((int8_t)val); +} + /* Use lddqu for unaligned; load for 16-byte aligned. */ #define LOAD_SI128(_ptr_) \ (((const ULONG_PTR)(_ptr_)&0x0f) ? _mm_lddqu_si128((const __m128i*)(_ptr_)) \ diff --git a/libfreerdp/primitives/prim_shift.c b/libfreerdp/primitives/prim_shift.c index 1f7d34afb..07342bbed 100644 --- a/libfreerdp/primitives/prim_shift.c +++ b/libfreerdp/primitives/prim_shift.c @@ -14,6 +14,8 @@ */ #include +#include +#include #include #include @@ -24,7 +26,8 @@ /* ------------------------------------------------------------------------- */ static INLINE INT16 shift(INT16 val, UINT32 sh) { - return val << sh; + const INT16 rc = (int16_t)(((UINT32)val << sh) & 0xFFFF); + return WINPR_ASSERTING_INT_CAST(INT16, rc); } static INLINE pstatus_t general_lShiftC_16s_inplace(INT16* WINPR_RESTRICT pSrcDst, UINT32 val, @@ -62,8 +65,8 @@ static INLINE pstatus_t general_rShiftC_16s(const INT16* pSrc, UINT32 val, INT16 if (val >= 16) return -1; - while (len--) - *pDst++ = *pSrc++ >> val; + for (UINT32 x = 0; x < len; x++) + pDst[x] = WINPR_ASSERTING_INT_CAST(int16_t, pSrc[x] >> val); return PRIMITIVES_SUCCESS; } @@ -77,8 +80,8 @@ static INLINE pstatus_t general_lShiftC_16u(const UINT16* pSrc, UINT32 val, UINT if (val >= 16) return -1; - while (len--) - *pDst++ = (INT16)(*pSrc++ << val); + for (UINT32 x = 0; x < len; x++) + pDst[x] = WINPR_ASSERTING_INT_CAST(UINT16, ((pSrc[x] << val) & 0xFFFF)); return PRIMITIVES_SUCCESS; } @@ -92,8 +95,8 @@ static INLINE pstatus_t general_rShiftC_16u(const UINT16* pSrc, UINT32 val, UINT if (val >= 16) return -1; - while (len--) - *pDst++ = *pSrc++ >> val; + for (UINT32 x = 0; x < len; x++) + pDst[x] = pSrc[x] >> val; return PRIMITIVES_SUCCESS; } @@ -105,9 +108,9 @@ static INLINE pstatus_t general_shiftC_16s(const INT16* pSrc, INT32 val, INT16* return PRIMITIVES_SUCCESS; if (val < 0) - return general_rShiftC_16s(pSrc, -val, pDst, len); + return general_rShiftC_16s(pSrc, WINPR_ASSERTING_INT_CAST(UINT32, -val), pDst, len); else - return general_lShiftC_16s(pSrc, val, pDst, len); + return general_lShiftC_16s(pSrc, WINPR_ASSERTING_INT_CAST(UINT32, val), pDst, len); } /* ------------------------------------------------------------------------- */ @@ -117,9 +120,9 @@ static INLINE pstatus_t general_shiftC_16u(const UINT16* pSrc, INT32 val, UINT16 return PRIMITIVES_SUCCESS; if (val < 0) - return general_rShiftC_16u(pSrc, -val, pDst, len); + return general_rShiftC_16u(pSrc, WINPR_ASSERTING_INT_CAST(UINT32, -val), pDst, len); else - return general_lShiftC_16u(pSrc, val, pDst, len); + return general_lShiftC_16u(pSrc, WINPR_ASSERTING_INT_CAST(UINT32, val), pDst, len); } /* ------------------------------------------------------------------------- */ diff --git a/libfreerdp/primitives/prim_sign.c b/libfreerdp/primitives/prim_sign.c index 6243a46ba..ad1cbf29d 100644 --- a/libfreerdp/primitives/prim_sign.c +++ b/libfreerdp/primitives/prim_sign.c @@ -29,7 +29,7 @@ static pstatus_t general_sign_16s(const INT16* pSrc, INT16* pDst, UINT32 len) while (len--) { INT16 src = *pSrc++; - *pDst++ = (src < 0) ? (-1) : ((src > 0) ? 1 : 0); + *pDst++ = WINPR_ASSERTING_INT_CAST(int16_t, (src < 0) ? (-1) : ((src > 0) ? 1 : 0)); } return PRIMITIVES_SUCCESS; diff --git a/libfreerdp/primitives/prim_templates.h b/libfreerdp/primitives/prim_templates.h index 5ab85a8e8..19bb3f3fe 100644 --- a/libfreerdp/primitives/prim_templates.h +++ b/libfreerdp/primitives/prim_templates.h @@ -43,14 +43,13 @@ /* ---------------------------------------------------------------------------- * SCD = Source, Constant, Destination */ -#define SSE3_SCD_ROUTINE(_name_, _type_, _fallback_, _op_, _slowWay_) \ +#define SSE3_SCD_ROUTINE(_name_, _type_, _fallback_, _op_, _op_type_, _slowWay_) \ static pstatus_t _name_(const _type_* pSrc, UINT32 val, _type_* pDst, UINT32 len) \ { \ INT32 shifts = 0; \ UINT32 offBeatMask; \ const _type_* sptr = pSrc; \ _type_* dptr = pDst; \ - int count; \ if (val == 0) \ return PRIMITIVES_SUCCESS; \ if (val >= 16) \ @@ -81,7 +80,7 @@ return PRIMITIVES_SUCCESS; \ } \ /* Use 8 128-bit SSE registers. */ \ - count = len >> (8 - shifts); \ + size_t count = len >> (8 - shifts); \ len -= count << (8 - shifts); \ if ((const ULONG_PTR)sptr & 0x0f) \ { \ @@ -103,14 +102,14 @@ sptr += (16 / sizeof(_type_)); \ __m128i xmm7 = _mm_lddqu_si128((const __m128i*)sptr); \ sptr += (16 / sizeof(_type_)); \ - xmm0 = _op_(xmm0, val); \ - xmm1 = _op_(xmm1, val); \ - xmm2 = _op_(xmm2, val); \ - xmm3 = _op_(xmm3, val); \ - xmm4 = _op_(xmm4, val); \ - xmm5 = _op_(xmm5, val); \ - xmm6 = _op_(xmm6, val); \ - xmm7 = _op_(xmm7, val); \ + xmm0 = _op_(xmm0, (_op_type_)val); \ + xmm1 = _op_(xmm1, (_op_type_)val); \ + xmm2 = _op_(xmm2, (_op_type_)val); \ + xmm3 = _op_(xmm3, (_op_type_)val); \ + xmm4 = _op_(xmm4, (_op_type_)val); \ + xmm5 = _op_(xmm5, (_op_type_)val); \ + xmm6 = _op_(xmm6, (_op_type_)val); \ + xmm7 = _op_(xmm7, (_op_type_)val); \ _mm_store_si128((__m128i*)dptr, xmm0); \ dptr += (16 / sizeof(_type_)); \ _mm_store_si128((__m128i*)dptr, xmm1); \ @@ -149,14 +148,14 @@ sptr += (16 / sizeof(_type_)); \ __m128i xmm7 = _mm_load_si128((const __m128i*)sptr); \ sptr += (16 / sizeof(_type_)); \ - xmm0 = _op_(xmm0, val); \ - xmm1 = _op_(xmm1, val); \ - xmm2 = _op_(xmm2, val); \ - xmm3 = _op_(xmm3, val); \ - xmm4 = _op_(xmm4, val); \ - xmm5 = _op_(xmm5, val); \ - xmm6 = _op_(xmm6, val); \ - xmm7 = _op_(xmm7, val); \ + xmm0 = _op_(xmm0, (_op_type_)val); \ + xmm1 = _op_(xmm1, (_op_type_)val); \ + xmm2 = _op_(xmm2, (_op_type_)val); \ + xmm3 = _op_(xmm3, (_op_type_)val); \ + xmm4 = _op_(xmm4, (_op_type_)val); \ + xmm5 = _op_(xmm5, (_op_type_)val); \ + xmm6 = _op_(xmm6, (_op_type_)val); \ + xmm7 = _op_(xmm7, (_op_type_)val); \ _mm_store_si128((__m128i*)dptr, xmm0); \ dptr += (16 / sizeof(_type_)); \ _mm_store_si128((__m128i*)dptr, xmm1); \ @@ -182,7 +181,7 @@ { \ __m128i xmm0 = LOAD_SI128(sptr); \ sptr += (16 / sizeof(_type_)); \ - xmm0 = _op_(xmm0, val); \ + xmm0 = _op_(xmm0, (_op_type_)val); \ _mm_store_si128((__m128i*)dptr, xmm0); \ dptr += (16 / sizeof(_type_)); \ } \ @@ -198,113 +197,114 @@ * SCD = Source, Constant, Destination * PRE = preload xmm0 with the constant. */ -#define SSE3_SCD_PRE_ROUTINE(_name_, _type_, _fallback_, _op_, _slowWay_) \ - static pstatus_t _name_(const _type_* pSrc, _type_ val, _type_* pDst, INT32 len) \ - { \ - int shifts = 0; \ - UINT32 offBeatMask; \ - const _type_* sptr = pSrc; \ - _type_* dptr = pDst; \ - size_t count; \ - __m128i xmm0; \ - if (len < 16) /* pointless if too small */ \ - { \ - return _fallback_(pSrc, val, pDst, len); \ - } \ - if (sizeof(_type_) == 1) \ - shifts = 1; \ - else if (sizeof(_type_) == 2) \ - shifts = 2; \ - else if (sizeof(_type_) == 4) \ - shifts = 3; \ - else if (sizeof(_type_) == 8) \ - shifts = 4; \ - offBeatMask = (1 << (shifts - 1)) - 1; \ - if ((ULONG_PTR)pDst & offBeatMask) \ - { \ - /* Incrementing the pointer skips over 16-byte boundary. */ \ - return _fallback_(pSrc, val, pDst, len); \ - } \ - /* Get to the 16-byte boundary now. */ \ - while ((ULONG_PTR)dptr & 0x0f) \ - { \ - _slowWay_; \ - if (--len == 0) \ - return PRIMITIVES_SUCCESS; \ - } \ - /* Use 4 128-bit SSE registers. */ \ - count = len >> (7 - shifts); \ - len -= count << (7 - shifts); \ - xmm0 = _mm_set1_epi32(val); \ - if ((const ULONG_PTR)sptr & 0x0f) \ - { \ - while (count--) \ - { \ - __m128i xmm1 = _mm_lddqu_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - __m128i xmm2 = _mm_lddqu_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - __m128i xmm3 = _mm_lddqu_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - __m128i xmm4 = _mm_lddqu_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - xmm1 = _op_(xmm1, xmm0); \ - xmm2 = _op_(xmm2, xmm0); \ - xmm3 = _op_(xmm3, xmm0); \ - xmm4 = _op_(xmm4, xmm0); \ - _mm_store_si128((__m128i*)dptr, xmm1); \ - dptr += (16 / sizeof(_type_)); \ - _mm_store_si128((__m128i*)dptr, xmm2); \ - dptr += (16 / sizeof(_type_)); \ - _mm_store_si128((__m128i*)dptr, xmm3); \ - dptr += (16 / sizeof(_type_)); \ - _mm_store_si128((__m128i*)dptr, xmm4); \ - dptr += (16 / sizeof(_type_)); \ - } \ - } \ - else \ - { \ - while (count--) \ - { \ - __m128i xmm1 = _mm_load_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - __m128i xmm2 = _mm_load_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - __m128i xmm3 = _mm_load_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - __m128i xmm4 = _mm_load_si128((const __m128i*)sptr); \ - sptr += (16 / sizeof(_type_)); \ - xmm1 = _op_(xmm1, xmm0); \ - xmm2 = _op_(xmm2, xmm0); \ - xmm3 = _op_(xmm3, xmm0); \ - xmm4 = _op_(xmm4, xmm0); \ - _mm_store_si128((__m128i*)dptr, xmm1); \ - dptr += (16 / sizeof(_type_)); \ - _mm_store_si128((__m128i*)dptr, xmm2); \ - dptr += (16 / sizeof(_type_)); \ - _mm_store_si128((__m128i*)dptr, xmm3); \ - dptr += (16 / sizeof(_type_)); \ - _mm_store_si128((__m128i*)dptr, xmm4); \ - dptr += (16 / sizeof(_type_)); \ - } \ - } \ - /* Use a single 128-bit SSE register. */ \ - count = len >> (5 - shifts); \ - len -= count << (5 - shifts); \ - while (count--) \ - { \ - __m128i xmm1 = LOAD_SI128(sptr); \ - sptr += (16 / sizeof(_type_)); \ - xmm1 = _op_(xmm1, xmm0); \ - _mm_store_si128((__m128i*)dptr, xmm1); \ - dptr += (16 / sizeof(_type_)); \ - } \ - /* Finish off the remainder. */ \ - while (len--) \ - { \ - _slowWay_; \ - } \ - return PRIMITIVES_SUCCESS; \ +#define SSE3_SCD_PRE_ROUTINE(_name_, _type_, _fallback_, _op_, _slowWay_) \ + static pstatus_t _name_(const _type_* pSrc, _type_ val, _type_* pDst, INT32 ilen) \ + { \ + size_t len = WINPR_ASSERTING_INT_CAST(size_t, ilen); \ + int shifts = 0; \ + UINT32 offBeatMask; \ + const _type_* sptr = pSrc; \ + _type_* dptr = pDst; \ + size_t count; \ + __m128i xmm0; \ + if (len < 16) /* pointless if too small */ \ + { \ + return _fallback_(pSrc, val, pDst, WINPR_ASSERTING_INT_CAST(int32_t, len)); \ + } \ + if (sizeof(_type_) == 1) \ + shifts = 1; \ + else if (sizeof(_type_) == 2) \ + shifts = 2; \ + else if (sizeof(_type_) == 4) \ + shifts = 3; \ + else if (sizeof(_type_) == 8) \ + shifts = 4; \ + offBeatMask = (1 << (shifts - 1)) - 1; \ + if ((ULONG_PTR)pDst & offBeatMask) \ + { \ + /* Incrementing the pointer skips over 16-byte boundary. */ \ + return _fallback_(pSrc, val, pDst, WINPR_ASSERTING_INT_CAST(int32_t, len)); \ + } \ + /* Get to the 16-byte boundary now. */ \ + while ((ULONG_PTR)dptr & 0x0f) \ + { \ + _slowWay_; \ + if (--len == 0) \ + return PRIMITIVES_SUCCESS; \ + } \ + /* Use 4 128-bit SSE registers. */ \ + count = len >> (7 - shifts); \ + len -= count << (7 - shifts); \ + xmm0 = mm_set1_epu32(val); \ + if ((const ULONG_PTR)sptr & 0x0f) \ + { \ + while (count--) \ + { \ + __m128i xmm1 = _mm_lddqu_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + __m128i xmm2 = _mm_lddqu_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + __m128i xmm3 = _mm_lddqu_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + __m128i xmm4 = _mm_lddqu_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + xmm1 = _op_(xmm1, xmm0); \ + xmm2 = _op_(xmm2, xmm0); \ + xmm3 = _op_(xmm3, xmm0); \ + xmm4 = _op_(xmm4, xmm0); \ + _mm_store_si128((__m128i*)dptr, xmm1); \ + dptr += (16 / sizeof(_type_)); \ + _mm_store_si128((__m128i*)dptr, xmm2); \ + dptr += (16 / sizeof(_type_)); \ + _mm_store_si128((__m128i*)dptr, xmm3); \ + dptr += (16 / sizeof(_type_)); \ + _mm_store_si128((__m128i*)dptr, xmm4); \ + dptr += (16 / sizeof(_type_)); \ + } \ + } \ + else \ + { \ + while (count--) \ + { \ + __m128i xmm1 = _mm_load_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + __m128i xmm2 = _mm_load_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + __m128i xmm3 = _mm_load_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + __m128i xmm4 = _mm_load_si128((const __m128i*)sptr); \ + sptr += (16 / sizeof(_type_)); \ + xmm1 = _op_(xmm1, xmm0); \ + xmm2 = _op_(xmm2, xmm0); \ + xmm3 = _op_(xmm3, xmm0); \ + xmm4 = _op_(xmm4, xmm0); \ + _mm_store_si128((__m128i*)dptr, xmm1); \ + dptr += (16 / sizeof(_type_)); \ + _mm_store_si128((__m128i*)dptr, xmm2); \ + dptr += (16 / sizeof(_type_)); \ + _mm_store_si128((__m128i*)dptr, xmm3); \ + dptr += (16 / sizeof(_type_)); \ + _mm_store_si128((__m128i*)dptr, xmm4); \ + dptr += (16 / sizeof(_type_)); \ + } \ + } \ + /* Use a single 128-bit SSE register. */ \ + count = len >> (5 - shifts); \ + len -= count << (5 - shifts); \ + while (count--) \ + { \ + __m128i xmm1 = LOAD_SI128(sptr); \ + sptr += (16 / sizeof(_type_)); \ + xmm1 = _op_(xmm1, xmm0); \ + _mm_store_si128((__m128i*)dptr, xmm1); \ + dptr += (16 / sizeof(_type_)); \ + } \ + /* Finish off the remainder. */ \ + while (len--) \ + { \ + _slowWay_; \ + } \ + return PRIMITIVES_SUCCESS; \ } /* ---------------------------------------------------------------------------- diff --git a/libfreerdp/primitives/sse/prim_YCoCg_ssse3.c b/libfreerdp/primitives/sse/prim_YCoCg_ssse3.c index 98d59b7bc..c2ad2a632 100644 --- a/libfreerdp/primitives/sse/prim_YCoCg_ssse3.c +++ b/libfreerdp/primitives/sse/prim_YCoCg_ssse3.c @@ -42,8 +42,11 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_invert(const BYTE* WINPR_RESTRICT pSr { const BYTE* sptr = pSrc; BYTE* dptr = pDst; - int sRowBump = srcStep - width * sizeof(UINT32); - int dRowBump = dstStep - width * sizeof(UINT32); + + WINPR_ASSERT(srcStep / sizeof(UINT32) >= width); + WINPR_ASSERT(dstStep / sizeof(UINT32) >= width); + const size_t sRowBump = srcStep - width * sizeof(UINT32); + const size_t dRowBump = dstStep - width * sizeof(UINT32); /* Shift left by "shift" and divide by two is the same as shift * left by "shift-1". */ @@ -62,8 +65,9 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_invert(const BYTE* WINPR_RESTRICT pSr if ((width < 8) || (ULONG_PTR)dptr & 0x03) { /* Too small, or we'll never hit a 16-byte boundary. Punt. */ - return generic->YCoCgToRGB_8u_AC4R(pSrc, srcStep, pDst, DstFormat, dstStep, width, height, - shift, withAlpha); + return generic->YCoCgToRGB_8u_AC4R(pSrc, WINPR_ASSERTING_INT_CAST(INT32, srcStep), pDst, + DstFormat, WINPR_ASSERTING_INT_CAST(INT32, dstStep), + width, height, shift, withAlpha); } for (UINT32 h = 0; h < height; h++) @@ -80,8 +84,9 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_invert(const BYTE* WINPR_RESTRICT pSr if (startup > width) startup = width; - status = generic->YCoCgToRGB_8u_AC4R(sptr, srcStep, dptr, DstFormat, dstStep, startup, - 1, shift, withAlpha); + status = generic->YCoCgToRGB_8u_AC4R( + sptr, WINPR_ASSERTING_INT_CAST(INT32, srcStep), dptr, DstFormat, + WINPR_ASSERTING_INT_CAST(INT32, dstStep), startup, 1, shift, withAlpha); if (status != PRIMITIVES_SUCCESS) return status; @@ -139,11 +144,11 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_invert(const BYTE* WINPR_RESTRICT pSr if (withAlpha) R7 = _mm_unpackhi_epi64(R5, R5); else - R7 = _mm_set1_epi32(0xFFFFFFFFU); + R7 = mm_set1_epu32(0xFFFFFFFFU); /* R7 = a7a6a5a4 a3a2a1a0 a7a6a5a4 a3a2a1a0 */ /* Expand Y's from 8-bit unsigned to 16-bit signed. */ - R1 = _mm_set1_epi32(0); + R1 = mm_set1_epu32(0); R0 = _mm_unpacklo_epi8(R5, R1); /* R0 = 00y700y6 00y500y4 00y300y2 00y100y0 */ /* Shift Co's and Cg's by (shift-1). -1 covers division by two. @@ -152,7 +157,7 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_invert(const BYTE* WINPR_RESTRICT pSr * version and then mask. */ R6 = _mm_slli_epi16(R6, dataShift); - R1 = _mm_set1_epi8(mask); + R1 = mm_set1_epu8(mask); R6 = _mm_and_si128(R6, R1); /* R6 = shifted o7o6o5o4 o3o2o1o0 g7g6g5g4 g3g2g1g0 */ /* Expand Co's from 8-bit signed to 16-bit signed */ @@ -203,8 +208,9 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_invert(const BYTE* WINPR_RESTRICT pSr if (w > 0) { pstatus_t status = 0; - status = generic->YCoCgToRGB_8u_AC4R(sptr, srcStep, dptr, DstFormat, dstStep, w, 1, - shift, withAlpha); + status = generic->YCoCgToRGB_8u_AC4R( + sptr, WINPR_ASSERTING_INT_CAST(INT32, srcStep), dptr, DstFormat, + WINPR_ASSERTING_INT_CAST(INT32, dstStep), w, 1, shift, withAlpha); if (status != PRIMITIVES_SUCCESS) return status; @@ -228,8 +234,8 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_no_invert(const BYTE* WINPR_RESTRICT { const BYTE* sptr = pSrc; BYTE* dptr = pDst; - int sRowBump = srcStep - width * sizeof(UINT32); - int dRowBump = dstStep - width * sizeof(UINT32); + size_t sRowBump = srcStep - width * sizeof(UINT32); + size_t dRowBump = dstStep - width * sizeof(UINT32); /* Shift left by "shift" and divide by two is the same as shift * left by "shift-1". */ @@ -248,13 +254,14 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_no_invert(const BYTE* WINPR_RESTRICT if ((width < 8) || (ULONG_PTR)dptr & 0x03) { /* Too small, or we'll never hit a 16-byte boundary. Punt. */ - return generic->YCoCgToRGB_8u_AC4R(pSrc, srcStep, pDst, DstFormat, dstStep, width, height, - shift, withAlpha); + return generic->YCoCgToRGB_8u_AC4R(pSrc, WINPR_ASSERTING_INT_CAST(INT32, srcStep), pDst, + DstFormat, WINPR_ASSERTING_INT_CAST(INT32, dstStep), + width, height, shift, withAlpha); } for (UINT32 h = 0; h < height; h++) { - int w = width; + UINT32 w = width; BOOL onStride = 0; /* Get to a 16-byte destination boundary. */ @@ -266,8 +273,9 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_no_invert(const BYTE* WINPR_RESTRICT if (startup > width) startup = width; - status = generic->YCoCgToRGB_8u_AC4R(sptr, srcStep, dptr, DstFormat, dstStep, startup, - 1, shift, withAlpha); + status = generic->YCoCgToRGB_8u_AC4R( + sptr, WINPR_ASSERTING_INT_CAST(INT32, srcStep), dptr, DstFormat, + WINPR_ASSERTING_INT_CAST(INT32, dstStep), startup, 1, shift, withAlpha); if (status != PRIMITIVES_SUCCESS) return status; @@ -325,11 +333,11 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_no_invert(const BYTE* WINPR_RESTRICT if (withAlpha) R7 = _mm_unpackhi_epi64(R5, R5); else - R7 = _mm_set1_epi32(0xFFFFFFFFU); + R7 = mm_set1_epu32(0xFFFFFFFFU); /* R7 = a7a6a5a4 a3a2a1a0 a7a6a5a4 a3a2a1a0 */ /* Expand Y's from 8-bit unsigned to 16-bit signed. */ - R1 = _mm_set1_epi32(0); + R1 = mm_set1_epu32(0); R0 = _mm_unpacklo_epi8(R5, R1); /* R0 = 00y700y6 00y500y4 00y300y2 00y100y0 */ /* Shift Co's and Cg's by (shift-1). -1 covers division by two. @@ -338,7 +346,7 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_no_invert(const BYTE* WINPR_RESTRICT * version and then mask. */ R6 = _mm_slli_epi16(R6, dataShift); - R1 = _mm_set1_epi8(mask); + R1 = mm_set1_epu8(mask); R6 = _mm_and_si128(R6, R1); /* R6 = shifted o7o6o5o4 o3o2o1o0 g7g6g5g4 g3g2g1g0 */ /* Expand Co's from 8-bit signed to 16-bit signed */ @@ -393,14 +401,16 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R_no_invert(const BYTE* WINPR_RESTRICT if (w > 0) { pstatus_t status = 0; - status = generic->YCoCgToRGB_8u_AC4R(sptr, srcStep, dptr, DstFormat, dstStep, w, 1, - shift, withAlpha); + status = generic->YCoCgToRGB_8u_AC4R( + sptr, WINPR_ASSERTING_INT_CAST(INT32, srcStep), dptr, DstFormat, + WINPR_ASSERTING_INT_CAST(INT32, dstStep), WINPR_ASSERTING_INT_CAST(UINT32, w), 1, + shift, withAlpha); if (status != PRIMITIVES_SUCCESS) return status; - sptr += w * sizeof(UINT32); - dptr += w * sizeof(UINT32); + sptr += WINPR_ASSERTING_INT_CAST(UINT32, w) * sizeof(UINT32); + dptr += WINPR_ASSERTING_INT_CAST(UINT32, w) * sizeof(UINT32); } sptr += sRowBump; @@ -420,13 +430,15 @@ static pstatus_t ssse3_YCoCgRToRGB_8u_AC4R(const BYTE* WINPR_RESTRICT pSrc, INT3 { case PIXEL_FORMAT_BGRX32: case PIXEL_FORMAT_BGRA32: - return ssse3_YCoCgRToRGB_8u_AC4R_invert(pSrc, srcStep, pDst, DstFormat, dstStep, width, - height, shift, withAlpha); + return ssse3_YCoCgRToRGB_8u_AC4R_invert( + pSrc, WINPR_ASSERTING_INT_CAST(UINT32, srcStep), pDst, DstFormat, + WINPR_ASSERTING_INT_CAST(UINT32, dstStep), width, height, shift, withAlpha); case PIXEL_FORMAT_RGBX32: case PIXEL_FORMAT_RGBA32: - return ssse3_YCoCgRToRGB_8u_AC4R_no_invert(pSrc, srcStep, pDst, DstFormat, dstStep, - width, height, shift, withAlpha); + return ssse3_YCoCgRToRGB_8u_AC4R_no_invert( + pSrc, WINPR_ASSERTING_INT_CAST(UINT32, srcStep), pDst, DstFormat, + WINPR_ASSERTING_INT_CAST(UINT32, dstStep), width, height, shift, withAlpha); default: return generic->YCoCgToRGB_8u_AC4R(pSrc, srcStep, pDst, DstFormat, dstStep, width, diff --git a/libfreerdp/primitives/sse/prim_YUV_ssse3.c b/libfreerdp/primitives/sse/prim_YUV_ssse3.c index 96bcc4b31..e3107dcfd 100644 --- a/libfreerdp/primitives/sse/prim_YUV_ssse3.c +++ b/libfreerdp/primitives/sse/prim_YUV_ssse3.c @@ -43,50 +43,20 @@ static primitives_t* generic = NULL; static __m128i* ssse3_YUV444Pixel(__m128i* WINPR_RESTRICT dst, __m128i Yraw, __m128i Uraw, __m128i Vraw, UINT8 pos) { - /* Visual Studio 2010 doesn't like _mm_set_epi32 in array initializer list */ - /* Note: This also applies to Visual Studio 2013 before Update 4 */ -#if !defined(_MSC_VER) || (_MSC_VER > 1600) - const __m128i mapY[] = { _mm_set_epi32(0x80800380, 0x80800280, 0x80800180, 0x80800080), - _mm_set_epi32(0x80800780, 0x80800680, 0x80800580, 0x80800480), - _mm_set_epi32(0x80800B80, 0x80800A80, 0x80800980, 0x80800880), - _mm_set_epi32(0x80800F80, 0x80800E80, 0x80800D80, 0x80800C80) }; - const __m128i mapUV[] = { _mm_set_epi32(0x80038002, 0x80018000, 0x80808080, 0x80808080), - _mm_set_epi32(0x80078006, 0x80058004, 0x80808080, 0x80808080), - _mm_set_epi32(0x800B800A, 0x80098008, 0x80808080, 0x80808080), - _mm_set_epi32(0x800F800E, 0x800D800C, 0x80808080, 0x80808080) }; - const __m128i mask[] = { _mm_set_epi32(0x80038080, 0x80028080, 0x80018080, 0x80008080), - _mm_set_epi32(0x80800380, 0x80800280, 0x80800180, 0x80800080), - _mm_set_epi32(0x80808003, 0x80808002, 0x80808001, 0x80808000) }; -#else - /* Note: must be in little-endian format ! */ - const __m128i mapY[] = { { 0x80, 0x00, 0x80, 0x80, 0x80, 0x01, 0x80, 0x80, 0x80, 0x02, 0x80, - 0x80, 0x80, 0x03, 0x80, 0x80 }, - { 0x80, 0x04, 0x80, 0x80, 0x80, 0x05, 0x80, 0x80, 0x80, 0x06, 0x80, - 0x80, 0x80, 0x07, 0x80, 0x80 }, - { 0x80, 0x08, 0x80, 0x80, 0x80, 0x09, 0x80, 0x80, 0x80, 0x0a, 0x80, - 0x80, 0x80, 0x0b, 0x80, 0x80 }, - { 0x80, 0x0c, 0x80, 0x80, 0x80, 0x0d, 0x80, 0x80, 0x80, 0x0e, 0x80, - 0x80, 0x80, 0x0f, 0x80, 0x80 } - - }; - const __m128i mapUV[] = { { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, 0x80, 0x01, - 0x80, 0x02, 0x80, 0x03, 0x80 }, - { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x04, 0x80, 0x05, - 0x80, 0x06, 0x80, 0x07, 0x80 }, - { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x08, 0x80, 0x09, - 0x80, 0x0a, 0x80, 0x0b, 0x80 }, - { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0c, 0x80, 0x0d, - 0x80, 0x0e, 0x80, 0x0f, 0x80 } }; - const __m128i mask[] = { { 0x80, 0x80, 0x00, 0x80, 0x80, 0x80, 0x01, 0x80, 0x80, 0x80, 0x02, - 0x80, 0x80, 0x80, 0x03, 0x80 }, - { 0x80, 0x00, 0x80, 0x80, 0x80, 0x01, 0x80, 0x80, 0x80, 0x02, 0x80, - 0x80, 0x80, 0x03, 0x80, 0x80 }, - { 0x00, 0x80, 0x80, 0x80, 0x01, 0x80, 0x80, 0x80, 0x02, 0x80, 0x80, - 0x80, 0x03, 0x80, 0x80, 0x80 } }; -#endif + const __m128i mapY[] = { mm_set_epu32(0x80800380, 0x80800280, 0x80800180, 0x80800080), + mm_set_epu32(0x80800780, 0x80800680, 0x80800580, 0x80800480), + mm_set_epu32(0x80800B80, 0x80800A80, 0x80800980, 0x80800880), + mm_set_epu32(0x80800F80, 0x80800E80, 0x80800D80, 0x80800C80) }; + const __m128i mapUV[] = { mm_set_epu32(0x80038002, 0x80018000, 0x80808080, 0x80808080), + mm_set_epu32(0x80078006, 0x80058004, 0x80808080, 0x80808080), + mm_set_epu32(0x800B800A, 0x80098008, 0x80808080, 0x80808080), + mm_set_epu32(0x800F800E, 0x800D800C, 0x80808080, 0x80808080) }; + const __m128i mask[] = { mm_set_epu32(0x80038080, 0x80028080, 0x80018080, 0x80008080), + mm_set_epu32(0x80800380, 0x80800280, 0x80800180, 0x80800080), + mm_set_epu32(0x80808003, 0x80808002, 0x80808001, 0x80808000) }; const __m128i c128 = _mm_set1_epi16(128); __m128i BGRX = _mm_and_si128(_mm_loadu_si128(dst), - _mm_set_epi32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000)); + mm_set_epu32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000)); { __m128i C; __m128i D; @@ -736,7 +706,7 @@ static pstatus_t ssse3_RGBToAVC444YUV_BGRX(const BYTE* WINPR_RESTRICT pSrc, UINT const BYTE* srcEven = y < roi->height ? pSrc + y * srcStep : pMaxSrc; const BYTE* srcOdd = !last ? pSrc + (y + 1) * srcStep : pMaxSrc; const size_t i = y >> 1; - const size_t n = (i & ~7) + i; + const size_t n = (i & (size_t)~7) + i; BYTE* b1Even = pDst1[0] + y * dst1Step[0]; BYTE* b1Odd = !last ? (b1Even + dst1Step[0]) : NULL; BYTE* b2 = pDst1[1] + (y / 2) * dst1Step[1]; diff --git a/libfreerdp/primitives/sse/prim_alphaComp_sse3.c b/libfreerdp/primitives/sse/prim_alphaComp_sse3.c index 0a17829f0..d945b4733 100644 --- a/libfreerdp/primitives/sse/prim_alphaComp_sse3.c +++ b/libfreerdp/primitives/sse/prim_alphaComp_sse3.c @@ -44,13 +44,6 @@ static pstatus_t sse2_alphaComp_argb(const BYTE* WINPR_RESTRICT pSrc1, UINT32 sr { const UINT32* sptr1 = (const UINT32*)pSrc1; const UINT32* sptr2 = (const UINT32*)pSrc2; - UINT32* dptr = NULL; - int linebytes = 0; - int src1Jump = 0; - int src2Jump = 0; - int dstJump = 0; - __m128i xmm0; - __m128i xmm1; if ((width <= 0) || (height <= 0)) return PRIMITIVES_SUCCESS; @@ -61,20 +54,20 @@ static pstatus_t sse2_alphaComp_argb(const BYTE* WINPR_RESTRICT pSrc1, UINT32 sr height); } - dptr = (UINT32*)pDst; - linebytes = width * sizeof(UINT32); - src1Jump = (src1Step - linebytes) / sizeof(UINT32); - src2Jump = (src2Step - linebytes) / sizeof(UINT32); - dstJump = (dstStep - linebytes) / sizeof(UINT32); - xmm0 = _mm_set1_epi32(0); - xmm1 = _mm_set1_epi16(1); + UINT32* dptr = (UINT32*)pDst; + const size_t linebytes = width * sizeof(UINT32); + const size_t src1Jump = (src1Step - linebytes) / sizeof(UINT32); + const size_t src2Jump = (src2Step - linebytes) / sizeof(UINT32); + const size_t dstJump = (dstStep - linebytes) / sizeof(UINT32); + __m128i xmm0 = mm_set1_epu32(0); + __m128i xmm1 = _mm_set1_epi16(1); for (UINT32 y = 0; y < height; ++y) { - int pixels = width; - int count = 0; + uint32_t pixels = width; + uint32_t count = 0; /* Get to the 16-byte boundary now. */ - int leadIn = 0; + uint32_t leadIn = 0; switch ((ULONG_PTR)dptr & 0x0f) { diff --git a/libfreerdp/primitives/sse/prim_colors_sse2.c b/libfreerdp/primitives/sse/prim_colors_sse2.c index 3d9192745..fa06b7408 100644 --- a/libfreerdp/primitives/sse/prim_colors_sse2.c +++ b/libfreerdp/primitives/sse/prim_colors_sse2.c @@ -41,11 +41,49 @@ static primitives_t* generic = NULL; #define CACHE_LINE_BYTES 64 -#define mm_between_epi16(_val, _min, _max) \ - do \ - { \ - (_val) = _mm_min_epi16(_max, _mm_max_epi16(_val, _min)); \ - } while (0) +/* 1.403 << 14 */ +/* -0.344 << 14 */ +/* -0.714 << 14 */ +/* 1.770 << 14 */ + +static const int32_t ycbcr_table[][4] = { { 1, 0, -1, 2 }, + { 3, -1, -1, 4 }, + { 6, -1, -3, 7 }, + { 11, -3, -6, 14 }, + { 22, -6, -11, 28 }, + { 45, -11, -23, 57 }, + { 90, -22, -46, 113 }, + { 180, -44, -91, 227 }, + { 359, -88, -183, 453 }, + { 718, -176, -366, 906 }, + { 1437, -352, -731, 1812 }, + { 2873, -705, -1462, 3625 }, + { 5747, -1409, -2925, 7250 }, + { 11493, -2818, -5849, 14500 }, + { 22987, -5636, -11698, 29000 }, + { 45974, -11272, -23396, 57999 }, + { 91947, -22544, -46793, 115999 }, + { 183894, -45089, -93585, 231997 }, + { 367788, -90178, -187171, 463995 }, + { 735576, -180355, -374342, 927990 }, + { 1471152, -360710, -748683, 1855980 }, + { 2942304, -721420, -1497367, 3711959 }, + { 5884609, -1442841, -2994733, 7423918 }, + { 11769217, -2885681, -5989466, 14847836 }, + { 23538434, -5771362, -11978932, 29695672 }, + { 47076868, -11542725, -23957864, 59391345 }, + { 94153736, -23085449, -47915729, 118782689 }, + { 188307472, -46170898, -95831458, 237565379 }, + { 376614945, -92341797, -191662916, 475130757 }, + { 753229890, -184683594, -383325831, 950261514 }, + { 1506459779, -369367187, -766651662, 1900523028 } }; + +static inline __m128i mm_between_epi16_int(__m128i val, __m128i min, __m128i max) +{ + return _mm_min_epi16(max, _mm_max_epi16(val, min)); +} + +#define mm_between_epi16(_val, _min, _max) (_val) = mm_between_epi16_int((_val), (_min), (_max)) #ifdef DO_PREFETCH /*---------------------------------------------------------------------------*/ @@ -67,23 +105,6 @@ sse2_yCbCrToRGB_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, INT16* WINPR_RESTRICT pDst[3], int dstStep, const prim_size_t* WINPR_RESTRICT roi) /* region of interest */ { - __m128i zero; - __m128i max; - __m128i r_cr; - __m128i g_cb; - __m128i g_cr; - __m128i b_cb; - __m128i c4096; - const __m128i* y_buf = NULL; - const __m128i* cb_buf = NULL; - const __m128i* cr_buf = NULL; - __m128i* r_buf = NULL; - __m128i* g_buf = NULL; - __m128i* b_buf = NULL; - int srcbump = 0; - int dstbump = 0; - int imax = 0; - if (((ULONG_PTR)(pSrc[0]) & 0x0f) || ((ULONG_PTR)(pSrc[1]) & 0x0f) || ((ULONG_PTR)(pSrc[2]) & 0x0f) || ((ULONG_PTR)(pDst[0]) & 0x0f) || ((ULONG_PTR)(pDst[1]) & 0x0f) || ((ULONG_PTR)(pDst[2]) & 0x0f) || (roi->width & 0x07) || @@ -93,21 +114,25 @@ sse2_yCbCrToRGB_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, return generic->yCbCrToRGB_16s16s_P3P3(pSrc, srcStep, pDst, dstStep, roi); } - zero = _mm_setzero_si128(); - max = _mm_set1_epi16(255); - y_buf = (const __m128i*)(pSrc[0]); - cb_buf = (const __m128i*)(pSrc[1]); - cr_buf = (const __m128i*)(pSrc[2]); - r_buf = (__m128i*)(pDst[0]); - g_buf = (__m128i*)(pDst[1]); - b_buf = (__m128i*)(pDst[2]); - r_cr = _mm_set1_epi16(22986); /* 1.403 << 14 */ - g_cb = _mm_set1_epi16(-5636); /* -0.344 << 14 */ - g_cr = _mm_set1_epi16(-11698); /* -0.714 << 14 */ - b_cb = _mm_set1_epi16(28999); /* 1.770 << 14 */ - c4096 = _mm_set1_epi16(4096); - srcbump = srcStep / sizeof(__m128i); - dstbump = dstStep / sizeof(__m128i); + const __m128i zero = _mm_setzero_si128(); + const __m128i max = _mm_set1_epi16(255); + const __m128i* y_buf = (const __m128i*)(pSrc[0]); + const __m128i* cb_buf = (const __m128i*)(pSrc[1]); + const __m128i* cr_buf = (const __m128i*)(pSrc[2]); + __m128i* r_buf = (__m128i*)(pDst[0]); + __m128i* g_buf = (__m128i*)(pDst[1]); + __m128i* b_buf = (__m128i*)(pDst[2]); + __m128i r_cr = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][0])); /* 1.403 << 14 */ + __m128i g_cb = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][1])); /* -0.344 << 14 */ + __m128i g_cr = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][2])); /* -0.714 << 14 */ + __m128i b_cb = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][3])); /* 1.770 << 14 */ + __m128i c4096 = _mm_set1_epi16(4096); + const size_t srcbump = WINPR_ASSERTING_INT_CAST(size_t, srcStep) / sizeof(__m128i); + const size_t dstbump = WINPR_ASSERTING_INT_CAST(size_t, dstStep) / sizeof(__m128i); #ifdef DO_PREFETCH /* Prefetch Y's, Cb's, and Cr's. */ @@ -130,11 +155,11 @@ sse2_yCbCrToRGB_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, cb_buf = (__m128i*)(pSrc[1]); cr_buf = (__m128i*)(pSrc[2]); #endif /* DO_PREFETCH */ - imax = roi->width * sizeof(INT16) / sizeof(__m128i); + const size_t imax = roi->width * sizeof(INT16) / sizeof(__m128i); for (UINT32 yp = 0; yp < roi->height; ++yp) { - for (int i = 0; i < imax; i++) + for (size_t i = 0; i < imax; i++) { /* In order to use SSE2 signed 16-bit integer multiplication * we need to convert the floating point factors to signed int @@ -156,34 +181,28 @@ sse2_yCbCrToRGB_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, * r = ((y+4096)>>2 + HIWORD(cr*22986)) >> 3 */ /* y = (y_r_buf[i] + 4096) >> 2 */ - __m128i y; - __m128i cb; - __m128i cr; - __m128i r; - __m128i g; - __m128i b; - y = _mm_load_si128(y_buf + i); + __m128i y = _mm_load_si128(y_buf + i); y = _mm_add_epi16(y, c4096); y = _mm_srai_epi16(y, 2); /* cb = cb_g_buf[i]; */ - cb = _mm_load_si128(cb_buf + i); + __m128i cb = _mm_load_si128(cb_buf + i); /* cr = cr_b_buf[i]; */ - cr = _mm_load_si128(cr_buf + i); + __m128i cr = _mm_load_si128(cr_buf + i); /* (y + HIWORD(cr*22986)) >> 3 */ - r = _mm_add_epi16(y, _mm_mulhi_epi16(cr, r_cr)); + __m128i r = _mm_add_epi16(y, _mm_mulhi_epi16(cr, r_cr)); r = _mm_srai_epi16(r, 3); /* r_buf[i] = CLIP(r); */ mm_between_epi16(r, zero, max); _mm_store_si128(r_buf + i, r); /* (y + HIWORD(cb*-5636) + HIWORD(cr*-11698)) >> 3 */ - g = _mm_add_epi16(y, _mm_mulhi_epi16(cb, g_cb)); + __m128i g = _mm_add_epi16(y, _mm_mulhi_epi16(cb, g_cb)); g = _mm_add_epi16(g, _mm_mulhi_epi16(cr, g_cr)); g = _mm_srai_epi16(g, 3); /* g_buf[i] = CLIP(g); */ mm_between_epi16(g, zero, max); _mm_store_si128(g_buf + i, g); /* (y + HIWORD(cb*28999)) >> 3 */ - b = _mm_add_epi16(y, _mm_mulhi_epi16(cb, b_cb)); + __m128i b = _mm_add_epi16(y, _mm_mulhi_epi16(cb, b_cb)); b = _mm_srai_epi16(b, 3); /* b_buf[i] = CLIP(b); */ mm_between_epi16(b, zero, max); @@ -209,10 +228,14 @@ sse2_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 sr { const __m128i zero = _mm_setzero_si128(); const __m128i max = _mm_set1_epi16(255); - const __m128i r_cr = _mm_set1_epi16(22986); /* 1.403 << 14 */ - const __m128i g_cb = _mm_set1_epi16(-5636); /* -0.344 << 14 */ - const __m128i g_cr = _mm_set1_epi16(-11698); /* -0.714 << 14 */ - const __m128i b_cb = _mm_set1_epi16(28999); /* 1.770 << 14 */ + const __m128i r_cr = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][0])); /* 1.403 << 14 */ + const __m128i g_cb = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][1])); /* -0.344 << 14 */ + const __m128i g_cr = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][2])); /* -0.714 << 14 */ + const __m128i b_cb = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][3])); /* 1.770 << 14 */ const __m128i c4096 = _mm_set1_epi16(4096); const INT16* y_buf = pSrc[0]; const INT16* cb_buf = pSrc[1]; @@ -268,93 +291,76 @@ sse2_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 sr * r = ((y+4096)>>2 + HIWORD(cr*22986)) >> 3 */ /* y = (y_r_buf[i] + 4096) >> 2 */ - __m128i y1; - __m128i y2; - __m128i cb1; - __m128i cb2; - __m128i cr1; - __m128i cr2; - __m128i r1; - __m128i r2; - __m128i g1; - __m128i g2; - __m128i b1; - __m128i b2; - y1 = _mm_load_si128((const __m128i*)y_buf); + __m128i y1 = _mm_load_si128((const __m128i*)y_buf); y_buf += step; y1 = _mm_add_epi16(y1, c4096); y1 = _mm_srai_epi16(y1, 2); /* cb = cb_g_buf[i]; */ - cb1 = _mm_load_si128((const __m128i*)cb_buf); + __m128i cb1 = _mm_load_si128((const __m128i*)cb_buf); cb_buf += step; /* cr = cr_b_buf[i]; */ - cr1 = _mm_load_si128((const __m128i*)cr_buf); + __m128i cr1 = _mm_load_si128((const __m128i*)cr_buf); cr_buf += step; /* (y + HIWORD(cr*22986)) >> 3 */ - r1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cr1, r_cr)); + __m128i r1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cr1, r_cr)); r1 = _mm_srai_epi16(r1, 3); /* r_buf[i] = CLIP(r); */ mm_between_epi16(r1, zero, max); /* (y + HIWORD(cb*-5636) + HIWORD(cr*-11698)) >> 3 */ - g1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, g_cb)); + __m128i g1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, g_cb)); g1 = _mm_add_epi16(g1, _mm_mulhi_epi16(cr1, g_cr)); g1 = _mm_srai_epi16(g1, 3); /* g_buf[i] = CLIP(g); */ mm_between_epi16(g1, zero, max); /* (y + HIWORD(cb*28999)) >> 3 */ - b1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, b_cb)); + __m128i b1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, b_cb)); b1 = _mm_srai_epi16(b1, 3); /* b_buf[i] = CLIP(b); */ mm_between_epi16(b1, zero, max); - y2 = _mm_load_si128((const __m128i*)y_buf); + __m128i y2 = _mm_load_si128((const __m128i*)y_buf); y_buf += step; y2 = _mm_add_epi16(y2, c4096); y2 = _mm_srai_epi16(y2, 2); /* cb = cb_g_buf[i]; */ - cb2 = _mm_load_si128((const __m128i*)cb_buf); + __m128i cb2 = _mm_load_si128((const __m128i*)cb_buf); cb_buf += step; /* cr = cr_b_buf[i]; */ - cr2 = _mm_load_si128((const __m128i*)cr_buf); + __m128i cr2 = _mm_load_si128((const __m128i*)cr_buf); cr_buf += step; /* (y + HIWORD(cr*22986)) >> 3 */ - r2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cr2, r_cr)); + __m128i r2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cr2, r_cr)); r2 = _mm_srai_epi16(r2, 3); /* r_buf[i] = CLIP(r); */ mm_between_epi16(r2, zero, max); /* (y + HIWORD(cb*-5636) + HIWORD(cr*-11698)) >> 3 */ - g2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, g_cb)); + __m128i g2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, g_cb)); g2 = _mm_add_epi16(g2, _mm_mulhi_epi16(cr2, g_cr)); g2 = _mm_srai_epi16(g2, 3); /* g_buf[i] = CLIP(g); */ mm_between_epi16(g2, zero, max); /* (y + HIWORD(cb*28999)) >> 3 */ - b2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, b_cb)); + __m128i b2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, b_cb)); b2 = _mm_srai_epi16(b2, 3); /* b_buf[i] = CLIP(b); */ mm_between_epi16(b2, zero, max); { - __m128i R0; - __m128i R1; - __m128i R2; - __m128i R3; - __m128i R4; /* The comments below pretend these are 8-byte registers * rather than 16-byte, for readability. */ - R0 = b1; /* R0 = 00B300B200B100B0 */ - R1 = b2; /* R1 = 00B700B600B500B4 */ + __m128i R0 = b1; /* R0 = 00B300B200B100B0 */ + __m128i R1 = b2; /* R1 = 00B700B600B500B4 */ R0 = _mm_packus_epi16(R0, R1); /* R0 = B7B6B5B4B3B2B1B0 */ R1 = g1; /* R1 = 00G300G200G100G0 */ - R2 = g2; /* R2 = 00G700G600G500G4 */ + __m128i R2 = g2; /* R2 = 00G700G600G500G4 */ R1 = _mm_packus_epi16(R1, R2); /* R1 = G7G6G5G4G3G2G1G0 */ R2 = R1; /* R2 = G7G6G5G4G3G2G1G0 */ R2 = _mm_unpacklo_epi8(R0, R2); /* R2 = B3G3B2G2B1G1B0G0 */ R1 = _mm_unpackhi_epi8(R0, R1); /* R1 = B7G7B6G6B5G5B4G4 */ R0 = r1; /* R0 = 00R300R200R100R0 */ - R3 = r2; /* R3 = 00R700R600R500R4 */ + __m128i R3 = r2; /* R3 = 00R700R600R500R4 */ R0 = _mm_packus_epi16(R0, R3); /* R0 = R7R6R5R4R3R2R1R0 */ - R3 = _mm_set1_epi32(0xFFFFFFFFU); /* R3 = FFFFFFFFFFFFFFFF */ - R4 = R3; /* R4 = FFFFFFFFFFFFFFFF */ + R3 = mm_set1_epu32(0xFFFFFFFFU); /* R3 = FFFFFFFFFFFFFFFF */ + __m128i R4 = R3; /* R4 = FFFFFFFFFFFFFFFF */ R4 = _mm_unpacklo_epi8(R0, R4); /* R4 = R3FFR2FFR1FFR0FF */ R3 = _mm_unpackhi_epi8(R0, R3); /* R3 = R7FFR6FFR5FFR4FF */ R0 = R4; /* R0 = R4 */ @@ -380,13 +386,13 @@ sse2_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 sr const INT32 Y = ((*y_buf++) + 4096) << divisor; const INT32 Cb = (*cb_buf++); const INT32 Cr = (*cr_buf++); - const INT32 CrR = Cr * (INT32)(1.402525f * (1 << divisor)); - const INT32 CrG = Cr * (INT32)(0.714401f * (1 << divisor)); - const INT32 CbG = Cb * (INT32)(0.343730f * (1 << divisor)); - const INT32 CbB = Cb * (INT32)(1.769905f * (1 << divisor)); - const INT16 R = ((INT16)((CrR + Y) >> divisor) >> 5); - const INT16 G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5); - const INT16 B = ((INT16)((CbB + Y) >> divisor) >> 5); + const INT32 CrR = Cr * ycbcr_table[divisor][0]; + const INT32 CrG = Cr * ycbcr_table[divisor][1]; + const INT32 CbG = Cb * ycbcr_table[divisor][2]; + const INT32 CbB = Cb * ycbcr_table[divisor][3]; + const INT16 R = WINPR_ASSERTING_INT_CAST(int16_t, (((CrR + Y) >> divisor) >> 5)); + const INT16 G = WINPR_ASSERTING_INT_CAST(int16_t, (((Y - CbG - CrG) >> divisor) >> 5)); + const INT16 B = WINPR_ASSERTING_INT_CAST(int16_t, (((CbB + Y) >> divisor) >> 5)); *d_buf++ = CLIP(B); *d_buf++ = CLIP(G); *d_buf++ = CLIP(R); @@ -407,10 +413,14 @@ sse2_yCbCrToRGB_16s8u_P3AC4R_RGBX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 sr { const __m128i zero = _mm_setzero_si128(); const __m128i max = _mm_set1_epi16(255); - const __m128i r_cr = _mm_set1_epi16(22986); /* 1.403 << 14 */ - const __m128i g_cb = _mm_set1_epi16(-5636); /* -0.344 << 14 */ - const __m128i g_cr = _mm_set1_epi16(-11698); /* -0.714 << 14 */ - const __m128i b_cb = _mm_set1_epi16(28999); /* 1.770 << 14 */ + const __m128i r_cr = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][0])); /* 1.403 << 14 */ + const __m128i g_cb = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][1])); /* -0.344 << 14 */ + const __m128i g_cr = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][2])); /* -0.714 << 14 */ + const __m128i b_cb = + _mm_set1_epi16(WINPR_ASSERTING_INT_CAST(int16_t, ycbcr_table[14][3])); /* 1.770 << 14 */ const __m128i c4096 = _mm_set1_epi16(4096); const INT16* y_buf = pSrc[0]; const INT16* cb_buf = pSrc[1]; @@ -466,93 +476,76 @@ sse2_yCbCrToRGB_16s8u_P3AC4R_RGBX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 sr * r = ((y+4096)>>2 + HIWORD(cr*22986)) >> 3 */ /* y = (y_r_buf[i] + 4096) >> 2 */ - __m128i y1; - __m128i y2; - __m128i cb1; - __m128i cb2; - __m128i cr1; - __m128i cr2; - __m128i r1; - __m128i r2; - __m128i g1; - __m128i g2; - __m128i b1; - __m128i b2; - y1 = _mm_load_si128((const __m128i*)y_buf); + __m128i y1 = _mm_load_si128((const __m128i*)y_buf); y_buf += step; y1 = _mm_add_epi16(y1, c4096); y1 = _mm_srai_epi16(y1, 2); /* cb = cb_g_buf[i]; */ - cb1 = _mm_load_si128((const __m128i*)cb_buf); + __m128i cb1 = _mm_load_si128((const __m128i*)cb_buf); cb_buf += step; /* cr = cr_b_buf[i]; */ - cr1 = _mm_load_si128((const __m128i*)cr_buf); + __m128i cr1 = _mm_load_si128((const __m128i*)cr_buf); cr_buf += step; /* (y + HIWORD(cr*22986)) >> 3 */ - r1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cr1, r_cr)); + __m128i r1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cr1, r_cr)); r1 = _mm_srai_epi16(r1, 3); /* r_buf[i] = CLIP(r); */ mm_between_epi16(r1, zero, max); /* (y + HIWORD(cb*-5636) + HIWORD(cr*-11698)) >> 3 */ - g1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, g_cb)); + __m128i g1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, g_cb)); g1 = _mm_add_epi16(g1, _mm_mulhi_epi16(cr1, g_cr)); g1 = _mm_srai_epi16(g1, 3); /* g_buf[i] = CLIP(g); */ mm_between_epi16(g1, zero, max); /* (y + HIWORD(cb*28999)) >> 3 */ - b1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, b_cb)); + __m128i b1 = _mm_add_epi16(y1, _mm_mulhi_epi16(cb1, b_cb)); b1 = _mm_srai_epi16(b1, 3); /* b_buf[i] = CLIP(b); */ mm_between_epi16(b1, zero, max); - y2 = _mm_load_si128((const __m128i*)y_buf); + __m128i y2 = _mm_load_si128((const __m128i*)y_buf); y_buf += step; y2 = _mm_add_epi16(y2, c4096); y2 = _mm_srai_epi16(y2, 2); /* cb = cb_g_buf[i]; */ - cb2 = _mm_load_si128((const __m128i*)cb_buf); + __m128i cb2 = _mm_load_si128((const __m128i*)cb_buf); cb_buf += step; /* cr = cr_b_buf[i]; */ - cr2 = _mm_load_si128((const __m128i*)cr_buf); + __m128i cr2 = _mm_load_si128((const __m128i*)cr_buf); cr_buf += step; /* (y + HIWORD(cr*22986)) >> 3 */ - r2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cr2, r_cr)); + __m128i r2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cr2, r_cr)); r2 = _mm_srai_epi16(r2, 3); /* r_buf[i] = CLIP(r); */ mm_between_epi16(r2, zero, max); /* (y + HIWORD(cb*-5636) + HIWORD(cr*-11698)) >> 3 */ - g2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, g_cb)); + __m128i g2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, g_cb)); g2 = _mm_add_epi16(g2, _mm_mulhi_epi16(cr2, g_cr)); g2 = _mm_srai_epi16(g2, 3); /* g_buf[i] = CLIP(g); */ mm_between_epi16(g2, zero, max); /* (y + HIWORD(cb*28999)) >> 3 */ - b2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, b_cb)); + __m128i b2 = _mm_add_epi16(y2, _mm_mulhi_epi16(cb2, b_cb)); b2 = _mm_srai_epi16(b2, 3); /* b_buf[i] = CLIP(b); */ mm_between_epi16(b2, zero, max); { - __m128i R0; - __m128i R1; - __m128i R2; - __m128i R3; - __m128i R4; /* The comments below pretend these are 8-byte registers * rather than 16-byte, for readability. */ - R0 = r1; /* R0 = 00R300R200R100R0 */ - R1 = r2; /* R1 = 00R700R600R500R4 */ + __m128i R0 = r1; /* R0 = 00R300R200R100R0 */ + __m128i R1 = r2; /* R1 = 00R700R600R500R4 */ R0 = _mm_packus_epi16(R0, R1); /* R0 = R7R6R5R4R3R2R1R0 */ R1 = g1; /* R1 = 00G300G200G100G0 */ - R2 = g2; /* R2 = 00G700G600G500G4 */ + __m128i R2 = g2; /* R2 = 00G700G600G500G4 */ R1 = _mm_packus_epi16(R1, R2); /* R1 = G7G6G5G4G3G2G1G0 */ R2 = R1; /* R2 = G7G6G5G4G3G2G1G0 */ R2 = _mm_unpacklo_epi8(R0, R2); /* R2 = R3G3R2G2R1G1R0G0 */ R1 = _mm_unpackhi_epi8(R0, R1); /* R1 = R7G7R6G6R5G5R4G4 */ R0 = b1; /* R0 = 00B300B200B100B0 */ - R3 = b2; /* R3 = 00B700B600B500B4 */ + __m128i R3 = b2; /* R3 = 00B700B600B500B4 */ R0 = _mm_packus_epi16(R0, R3); /* R0 = B7B6B5B4B3B2B1B0 */ - R3 = _mm_set1_epi32(0xFFFFFFFFU); /* R3 = FFFFFFFFFFFFFFFF */ - R4 = R3; /* R4 = FFFFFFFFFFFFFFFF */ + R3 = mm_set1_epu32(0xFFFFFFFFU); /* R3 = FFFFFFFFFFFFFFFF */ + __m128i R4 = R3; /* R4 = FFFFFFFFFFFFFFFF */ R4 = _mm_unpacklo_epi8(R0, R4); /* R4 = B3FFB2FFB1FFB0FF */ R3 = _mm_unpackhi_epi8(R0, R3); /* R3 = B7FFB6FFB5FFB4FF */ R0 = R4; /* R0 = R4 */ @@ -578,13 +571,13 @@ sse2_yCbCrToRGB_16s8u_P3AC4R_RGBX(const INT16* WINPR_RESTRICT pSrc[3], UINT32 sr const INT32 Y = ((*y_buf++) + 4096) << divisor; const INT32 Cb = (*cb_buf++); const INT32 Cr = (*cr_buf++); - const INT32 CrR = Cr * (INT32)(1.402525f * (1 << divisor)); - const INT32 CrG = Cr * (INT32)(0.714401f * (1 << divisor)); - const INT32 CbG = Cb * (INT32)(0.343730f * (1 << divisor)); - const INT32 CbB = Cb * (INT32)(1.769905f * (1 << divisor)); - const INT16 R = ((INT16)((CrR + Y) >> divisor) >> 5); - const INT16 G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5); - const INT16 B = ((INT16)((CbB + Y) >> divisor) >> 5); + const INT32 CrR = Cr * ycbcr_table[divisor][0]; + const INT32 CrG = Cr * ycbcr_table[divisor][1]; + const INT32 CbG = Cb * ycbcr_table[divisor][2]; + const INT32 CbB = Cb * ycbcr_table[divisor][3]; + const INT16 R = WINPR_ASSERTING_INT_CAST(int16_t, (((CrR + Y) >> divisor) >> 5)); + const INT16 G = WINPR_ASSERTING_INT_CAST(int16_t, (((Y - CbG - CrG) >> divisor) >> 5)); + const INT16 B = WINPR_ASSERTING_INT_CAST(int16_t, (((CbB + Y) >> divisor) >> 5)); *d_buf++ = CLIP(R); *d_buf++ = CLIP(G); *d_buf++ = CLIP(B); @@ -632,25 +625,12 @@ sse2_RGBToYCbCr_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, INT16* WINPR_RESTRICT pDst[3], int dstStep, const prim_size_t* WINPR_RESTRICT roi) /* region of interest */ { - __m128i min; - __m128i max; - __m128i y_r; - __m128i y_g; - __m128i y_b; - __m128i cb_r; - __m128i cb_g; - __m128i cb_b; - __m128i cr_r; - __m128i cr_g; - __m128i cr_b; const __m128i* r_buf = (const __m128i*)(pSrc[0]); const __m128i* g_buf = (const __m128i*)(pSrc[1]); const __m128i* b_buf = (const __m128i*)(pSrc[2]); __m128i* y_buf = (__m128i*)(pDst[0]); __m128i* cb_buf = (__m128i*)(pDst[1]); __m128i* cr_buf = (__m128i*)(pDst[2]); - int srcbump = 0; - int dstbump = 0; int imax = 0; if (((ULONG_PTR)(pSrc[0]) & 0x0f) || ((ULONG_PTR)(pSrc[1]) & 0x0f) || @@ -662,20 +642,20 @@ sse2_RGBToYCbCr_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, return generic->RGBToYCbCr_16s16s_P3P3(pSrc, srcStep, pDst, dstStep, roi); } - min = _mm_set1_epi16(-128 * 32); - max = _mm_set1_epi16(127 * 32); + const __m128i min = _mm_set1_epi16(-128 * 32); + const __m128i max = _mm_set1_epi16(127 * 32); - y_r = _mm_set1_epi16(9798); /* 0.299000 << 15 */ - y_g = _mm_set1_epi16(19235); /* 0.587000 << 15 */ - y_b = _mm_set1_epi16(3735); /* 0.114000 << 15 */ - cb_r = _mm_set1_epi16(-5535); /* -0.168935 << 15 */ - cb_g = _mm_set1_epi16(-10868); /* -0.331665 << 15 */ - cb_b = _mm_set1_epi16(16403); /* 0.500590 << 15 */ - cr_r = _mm_set1_epi16(16377); /* 0.499813 << 15 */ - cr_g = _mm_set1_epi16(-13714); /* -0.418531 << 15 */ - cr_b = _mm_set1_epi16(-2663); /* -0.081282 << 15 */ - srcbump = srcStep / sizeof(__m128i); - dstbump = dstStep / sizeof(__m128i); + __m128i y_r = _mm_set1_epi16(9798); /* 0.299000 << 15 */ + __m128i y_g = _mm_set1_epi16(19235); /* 0.587000 << 15 */ + __m128i y_b = _mm_set1_epi16(3735); /* 0.114000 << 15 */ + __m128i cb_r = _mm_set1_epi16(-5535); /* -0.168935 << 15 */ + __m128i cb_g = _mm_set1_epi16(-10868); /* -0.331665 << 15 */ + __m128i cb_b = _mm_set1_epi16(16403); /* 0.500590 << 15 */ + __m128i cr_r = _mm_set1_epi16(16377); /* 0.499813 << 15 */ + __m128i cr_g = _mm_set1_epi16(-13714); /* -0.418531 << 15 */ + __m128i cr_b = _mm_set1_epi16(-2663); /* -0.081282 << 15 */ + const size_t srcbump = WINPR_ASSERTING_INT_CAST(size_t, srcStep) / sizeof(__m128i); + const size_t dstbump = WINPR_ASSERTING_INT_CAST(size_t, dstStep) / sizeof(__m128i); #ifdef DO_PREFETCH /* Prefetch RGB's. */ @@ -715,21 +695,15 @@ sse2_RGBToYCbCr_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, * within the upper 16 bits we will also have to scale the RGB * values used in the multiplication by << 5+(16-n). */ - __m128i r; - __m128i g; - __m128i b; - __m128i y; - __m128i cb; - __m128i cr; - r = _mm_load_si128(r_buf + i); - g = _mm_load_si128(g_buf + i); - b = _mm_load_si128(b_buf + i); + __m128i r = _mm_load_si128(r_buf + i); + __m128i g = _mm_load_si128(g_buf + i); + __m128i b = _mm_load_si128(b_buf + i); /* r<<6; g<<6; b<<6 */ r = _mm_slli_epi16(r, 6); g = _mm_slli_epi16(g, 6); b = _mm_slli_epi16(b, 6); /* y = HIWORD(r*y_r) + HIWORD(g*y_g) + HIWORD(b*y_b) + min */ - y = _mm_mulhi_epi16(r, y_r); + __m128i y = _mm_mulhi_epi16(r, y_r); y = _mm_add_epi16(y, _mm_mulhi_epi16(g, y_g)); y = _mm_add_epi16(y, _mm_mulhi_epi16(b, y_b)); y = _mm_add_epi16(y, min); @@ -737,14 +711,14 @@ sse2_RGBToYCbCr_16s16s_P3P3(const INT16* WINPR_RESTRICT pSrc[3], int srcStep, mm_between_epi16(y, min, max); _mm_store_si128(y_buf + i, y); /* cb = HIWORD(r*cb_r) + HIWORD(g*cb_g) + HIWORD(b*cb_b) */ - cb = _mm_mulhi_epi16(r, cb_r); + __m128i cb = _mm_mulhi_epi16(r, cb_r); cb = _mm_add_epi16(cb, _mm_mulhi_epi16(g, cb_g)); cb = _mm_add_epi16(cb, _mm_mulhi_epi16(b, cb_b)); /* cb_g_buf[i] = MINMAX(cb, (-128 << 5), (127 << 5)); */ mm_between_epi16(cb, min, max); _mm_store_si128(cb_buf + i, cb); /* cr = HIWORD(r*cr_r) + HIWORD(g*cr_g) + HIWORD(b*cr_b) */ - cr = _mm_mulhi_epi16(r, cr_r); + __m128i cr = _mm_mulhi_epi16(r, cr_r); cr = _mm_add_epi16(cr, _mm_mulhi_epi16(g, cr_g)); cr = _mm_add_epi16(cr, _mm_mulhi_epi16(b, cr_b)); /* cr_b_buf[i] = MINMAX(cr, (-128 << 5), (127 << 5)); */ @@ -775,7 +749,7 @@ static pstatus_t sse2_RGBToRGB_16s8u_P3AC4R_BGRX( const UINT16* pg = (const UINT16*)(pSrc[1]); const UINT16* pb = (const UINT16*)(pSrc[2]); const UINT32 pad = roi->width % 16; - const __m128i a = _mm_set1_epi32(0xFFFFFFFFU); + const __m128i a = mm_set1_epu32(0xFFFFFFFFU); BYTE* out = NULL; UINT32 srcbump = 0; UINT32 dstbump = 0; @@ -821,16 +795,11 @@ static pstatus_t sse2_RGBToRGB_16s8u_P3AC4R_BGRX( r = _mm_packus_epi16(R0, R1); /* r = R7R6R5R4R3R2R1R0 */ } { - __m128i gbHi; - __m128i gbLo; - __m128i arHi; - __m128i arLo; - { - gbLo = _mm_unpacklo_epi8(b, g); /* R0 = G7G6G5G4G3G2G1G0 */ - gbHi = _mm_unpackhi_epi8(b, g); /* R1 = G7B7G6B7G5B5G4B4 */ - arLo = _mm_unpacklo_epi8(r, a); /* R4 = FFR3FFR2FFR1FFR0 */ - arHi = _mm_unpackhi_epi8(r, a); /* R3 = FFR7FFR6FFR5FFR4 */ - } + const __m128i gbLo = _mm_unpacklo_epi8(b, g); /* R0 = G7G6G5G4G3G2G1G0 */ + const __m128i gbHi = _mm_unpackhi_epi8(b, g); /* R1 = G7B7G6B7G5B5G4B4 */ + const __m128i arLo = _mm_unpacklo_epi8(r, a); /* R4 = FFR3FFR2FFR1FFR0 */ + const __m128i arHi = _mm_unpackhi_epi8(r, a); /* R3 = FFR7FFR6FFR5FFR4 */ + { const __m128i bgrx = _mm_unpacklo_epi16(gbLo, arLo); _mm_store_si128((__m128i*)out, bgrx); @@ -886,7 +855,7 @@ static pstatus_t sse2_RGBToRGB_16s8u_P3AC4R_RGBX( const UINT16* pg = (const UINT16*)(pSrc[1]); const UINT16* pb = (const UINT16*)(pSrc[2]); const UINT32 pad = roi->width % 16; - const __m128i a = _mm_set1_epi32(0xFFFFFFFFU); + const __m128i a = mm_set1_epu32(0xFFFFFFFFU); BYTE* out = NULL; UINT32 srcbump = 0; UINT32 dstbump = 0; @@ -997,7 +966,7 @@ static pstatus_t sse2_RGBToRGB_16s8u_P3AC4R_XBGR( const UINT16* pg = (const UINT16*)(pSrc[1]); const UINT16* pb = (const UINT16*)(pSrc[2]); const UINT32 pad = roi->width % 16; - const __m128i a = _mm_set1_epi32(0xFFFFFFFFU); + const __m128i a = mm_set1_epu32(0xFFFFFFFFU); BYTE* out = NULL; UINT32 srcbump = 0; UINT32 dstbump = 0; @@ -1107,7 +1076,7 @@ static pstatus_t sse2_RGBToRGB_16s8u_P3AC4R_XRGB( const UINT16* pr = (const UINT16*)(pSrc[0]); const UINT16* pg = (const UINT16*)(pSrc[1]); const UINT16* pb = (const UINT16*)(pSrc[2]); - const __m128i a = _mm_set1_epi32(0xFFFFFFFFU); + const __m128i a = mm_set1_epu32(0xFFFFFFFFU); const UINT32 pad = roi->width % 16; BYTE* out = NULL; UINT32 srcbump = 0; diff --git a/libfreerdp/primitives/sse/prim_copy_avx2.c b/libfreerdp/primitives/sse/prim_copy_avx2.c index 17a00f91c..d962b5eba 100644 --- a/libfreerdp/primitives/sse/prim_copy_avx2.c +++ b/libfreerdp/primitives/sse/prim_copy_avx2.c @@ -34,6 +34,13 @@ #include #include +static inline __m256i mm256_set_epu32(uint32_t i0, uint32_t i1, uint32_t i2, uint32_t i3, + uint32_t i4, uint32_t i5, uint32_t i6, uint32_t i7) +{ + return _mm256_set_epi32((int32_t)i0, (int32_t)i1, (int32_t)i2, (int32_t)i3, (int32_t)i4, + (int32_t)i5, (int32_t)i6, (int32_t)i7); +} + static INLINE pstatus_t avx2_image_copy_no_overlap_convert( BYTE* WINPR_RESTRICT pDstData, DWORD DstFormat, UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, const BYTE* WINPR_RESTRICT pSrcData, DWORD SrcFormat, @@ -52,12 +59,12 @@ static INLINE pstatus_t avx2_image_copy_bgr24_bgrx32(BYTE* WINPR_RESTRICT pDstDa const SSIZE_T srcByte = 3; const SSIZE_T dstByte = 4; - const __m256i mask = _mm256_set_epi32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000, - 0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000); - const __m256i smask = _mm256_set_epi32(0xff171615, 0xff141312, 0xff1110ff, 0xffffffff, - 0xff0b0a09, 0xff080706, 0xff050403, 0xff020100); - const __m256i shelpmask = _mm256_set_epi32(0xffffffff, 0xffffffff, 0xffffff1f, 0xff1e1d1c, - 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff); + const __m256i mask = mm256_set_epu32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000, + 0xFF000000, 0xFF000000, 0xFF000000); + const __m256i smask = mm256_set_epu32(0xff171615, 0xff141312, 0xff1110ff, 0xffffffff, + 0xff0b0a09, 0xff080706, 0xff050403, 0xff020100); + const __m256i shelpmask = mm256_set_epu32(0xffffffff, 0xffffffff, 0xffffff1f, 0xff1e1d1c, + 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff); const UINT32 rem = nWidth % 8; const SSIZE_T width = nWidth - rem; diff --git a/libfreerdp/primitives/sse/prim_copy_sse4_1.c b/libfreerdp/primitives/sse/prim_copy_sse4_1.c index a99cf5069..f83056f06 100644 --- a/libfreerdp/primitives/sse/prim_copy_sse4_1.c +++ b/libfreerdp/primitives/sse/prim_copy_sse4_1.c @@ -52,8 +52,8 @@ static INLINE pstatus_t sse_image_copy_bgr24_bgrx32(BYTE* WINPR_RESTRICT pDstDat const SSIZE_T srcByte = 3; const SSIZE_T dstByte = 4; - const __m128i mask = _mm_set_epi32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000); - const __m128i smask = _mm_set_epi32(0xff0b0a09, 0xff080706, 0xff050403, 0xff020100); + const __m128i mask = mm_set_epu32(0xFF000000, 0xFF000000, 0xFF000000, 0xFF000000); + const __m128i smask = mm_set_epu32(0xff0b0a09, 0xff080706, 0xff050403, 0xff020100); const UINT32 rem = nWidth % 4; const size_t align = nSrcStep % 64; diff --git a/libfreerdp/primitives/sse/prim_set_sse2.c b/libfreerdp/primitives/sse/prim_set_sse2.c index 6265fab21..47993adc9 100644 --- a/libfreerdp/primitives/sse/prim_set_sse2.c +++ b/libfreerdp/primitives/sse/prim_set_sse2.c @@ -52,7 +52,7 @@ static pstatus_t sse2_set_8u(BYTE val, BYTE* WINPR_RESTRICT pDst, UINT32 len) return PRIMITIVES_SUCCESS; } - xmm0 = _mm_set1_epi8(byte); + xmm0 = mm_set1_epu8(byte); /* Cover 256-byte chunks via SSE register stores. */ count = len >> 8; len -= count << 8; @@ -144,7 +144,7 @@ static pstatus_t sse2_set_32u(UINT32 val, UINT32* WINPR_RESTRICT pDst, UINT32 le return PRIMITIVES_SUCCESS; } - xmm0 = _mm_set1_epi32(val); + xmm0 = mm_set1_epu32(val); /* Cover 256-byte chunks via SSE register stores. */ count = len >> 6; len -= count << 6; diff --git a/libfreerdp/primitives/sse/prim_shift_sse3.c b/libfreerdp/primitives/sse/prim_shift_sse3.c index f5b50e51c..b6e708b48 100644 --- a/libfreerdp/primitives/sse/prim_shift_sse3.c +++ b/libfreerdp/primitives/sse/prim_shift_sse3.c @@ -31,16 +31,16 @@ static primitives_t* generic = NULL; /* ------------------------------------------------------------------------- */ -SSE3_SCD_ROUTINE(sse2_lShiftC_16s, INT16, generic->lShiftC_16s, _mm_slli_epi16, - *dptr++ = (INT16)((UINT16)*sptr++ << val)) +SSE3_SCD_ROUTINE(sse2_lShiftC_16s, INT16, generic->lShiftC_16s, _mm_slli_epi16, int16_t, + *dptr++ = (INT16)(((UINT16)*sptr++ << val) & 0xFFFF)) /* ------------------------------------------------------------------------- */ -SSE3_SCD_ROUTINE(sse2_rShiftC_16s, INT16, generic->rShiftC_16s, _mm_srai_epi16, +SSE3_SCD_ROUTINE(sse2_rShiftC_16s, INT16, generic->rShiftC_16s, _mm_srai_epi16, int16_t, *dptr++ = *sptr++ >> val) /* ------------------------------------------------------------------------- */ -SSE3_SCD_ROUTINE(sse2_lShiftC_16u, UINT16, generic->lShiftC_16u, _mm_slli_epi16, - *dptr++ = (INT16)((UINT16)*sptr++ << val)) +SSE3_SCD_ROUTINE(sse2_lShiftC_16u, UINT16, generic->lShiftC_16u, _mm_slli_epi16, int16_t, + *dptr++ = (((UINT16)*sptr++ << val) & 0xFFFF)) /* ------------------------------------------------------------------------- */ -SSE3_SCD_ROUTINE(sse2_rShiftC_16u, UINT16, generic->rShiftC_16u, _mm_srli_epi16, +SSE3_SCD_ROUTINE(sse2_rShiftC_16u, UINT16, generic->rShiftC_16u, _mm_srli_epi16, int16_t, *dptr++ = *sptr++ >> val) static pstatus_t sse2_lShiftC_16s_inplace(INT16* WINPR_RESTRICT pSrcDst, UINT32 val, UINT32 len) @@ -72,7 +72,7 @@ static pstatus_t sse2_lShiftC_16s_inplace(INT16* WINPR_RESTRICT pSrcDst, UINT32 } /* Use 8 128-bit SSE registers. */ - int count = len >> (8 - shifts); + uint32_t count = len >> (8 - shifts); len -= count << (8 - shifts); while (count--) @@ -88,14 +88,14 @@ static pstatus_t sse2_lShiftC_16s_inplace(INT16* WINPR_RESTRICT pSrcDst, UINT32 __m128i xmm6 = _mm_load_si128(src++); __m128i xmm7 = _mm_load_si128(src); - xmm0 = _mm_slli_epi16(xmm0, val); - xmm1 = _mm_slli_epi16(xmm1, val); - xmm2 = _mm_slli_epi16(xmm2, val); - xmm3 = _mm_slli_epi16(xmm3, val); - xmm4 = _mm_slli_epi16(xmm4, val); - xmm5 = _mm_slli_epi16(xmm5, val); - xmm6 = _mm_slli_epi16(xmm6, val); - xmm7 = _mm_slli_epi16(xmm7, val); + xmm0 = _mm_slli_epi16(xmm0, (int16_t)val); + xmm1 = _mm_slli_epi16(xmm1, (int16_t)val); + xmm2 = _mm_slli_epi16(xmm2, (int16_t)val); + xmm3 = _mm_slli_epi16(xmm3, (int16_t)val); + xmm4 = _mm_slli_epi16(xmm4, (int16_t)val); + xmm5 = _mm_slli_epi16(xmm5, (int16_t)val); + xmm6 = _mm_slli_epi16(xmm6, (int16_t)val); + xmm7 = _mm_slli_epi16(xmm7, (int16_t)val); __m128i* dst = (__m128i*)pSrcDst; @@ -119,7 +119,7 @@ static pstatus_t sse2_lShiftC_16s_inplace(INT16* WINPR_RESTRICT pSrcDst, UINT32 const __m128i* src = (const __m128i*)pSrcDst; __m128i xmm0 = LOAD_SI128(src); - xmm0 = _mm_slli_epi16(xmm0, val); + xmm0 = _mm_slli_epi16(xmm0, (int16_t)val); __m128i* dst = (__m128i*)pSrcDst; _mm_store_si128(dst++, xmm0); diff --git a/libfreerdp/primitives/sse/prim_sign_ssse3.c b/libfreerdp/primitives/sse/prim_sign_ssse3.c index 82b9e97c6..0cac444c0 100644 --- a/libfreerdp/primitives/sse/prim_sign_ssse3.c +++ b/libfreerdp/primitives/sse/prim_sign_ssse3.c @@ -52,7 +52,7 @@ static pstatus_t ssse3_sign_16s(const INT16* WINPR_RESTRICT pSrc, INT16* WINPR_R while ((ULONG_PTR)dptr & 0x0f) { INT16 src = *sptr++; - *dptr++ = (src < 0) ? (-1) : ((src > 0) ? 1 : 0); + *dptr++ = WINPR_ASSERTING_INT_CAST(int16_t, (src < 0) ? (-1) : ((src > 0) ? 1 : 0)); if (--len == 0) return PRIMITIVES_SUCCESS; @@ -159,7 +159,7 @@ static pstatus_t ssse3_sign_16s(const INT16* WINPR_RESTRICT pSrc, INT16* WINPR_R while (len--) { INT16 src = *sptr++; - *dptr++ = (src < 0) ? -1 : ((src > 0) ? 1 : 0); + *dptr++ = WINPR_ASSERTING_INT_CAST(int16_t, (src < 0) ? -1 : ((src > 0) ? 1 : 0)); } return PRIMITIVES_SUCCESS; diff --git a/libfreerdp/primitives/test/TestPrimitivesAndOr.c b/libfreerdp/primitives/test/TestPrimitivesAndOr.c index b3e52f6d5..8f3b141de 100644 --- a/libfreerdp/primitives/test/TestPrimitivesAndOr.c +++ b/libfreerdp/primitives/test/TestPrimitivesAndOr.c @@ -26,7 +26,7 @@ static BOOL test_and_32u_impl(const char* name, __andC_32u_t fkt, const UINT32* src, const UINT32 val, UINT32* dst, size_t size) { - pstatus_t status = fkt(src, val, dst, size); + pstatus_t status = fkt(src, val, dst, WINPR_ASSERTING_INT_CAST(int32_t, size)); if (status != PRIMITIVES_SUCCESS) return FALSE; diff --git a/libfreerdp/primitives/test/TestPrimitivesShift.c b/libfreerdp/primitives/test/TestPrimitivesShift.c index 884583838..b0d58f2c3 100644 --- a/libfreerdp/primitives/test/TestPrimitivesShift.c +++ b/libfreerdp/primitives/test/TestPrimitivesShift.c @@ -204,13 +204,12 @@ static BOOL test_rShift_16u_func(void) static BOOL test_ShiftWrapper_16s_func(void) { pstatus_t status = 0; - INT16 ALIGN(src[FUNC_TEST_SIZE + 3]); - INT16 ALIGN(d1[FUNC_TEST_SIZE + 3]); + INT16 ALIGN(src[FUNC_TEST_SIZE + 3]) = { 0 }; + INT16 ALIGN(d1[FUNC_TEST_SIZE + 3]) = { 0 }; UINT32 tmp = 0; - INT32 val = 0; winpr_RAND(&tmp, sizeof(tmp)); winpr_RAND(src, sizeof(src)); - val = tmp % 16; + INT32 val = WINPR_ASSERTING_INT_CAST(int32_t, tmp % 16); /* Negative tests */ status = generic->shiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE); @@ -271,13 +270,12 @@ static BOOL test_ShiftWrapper_16s_func(void) static BOOL test_ShiftWrapper_16u_func(void) { pstatus_t status = 0; - UINT16 ALIGN(src[FUNC_TEST_SIZE + 3]); - UINT16 ALIGN(d1[FUNC_TEST_SIZE + 3]); + UINT16 ALIGN(src[FUNC_TEST_SIZE + 3]) = { 0 }; + UINT16 ALIGN(d1[FUNC_TEST_SIZE + 3]) = { 0 }; UINT32 tmp = 0; - INT32 val = 0; winpr_RAND(&tmp, sizeof(tmp)); winpr_RAND(src, sizeof(src)); - val = tmp % 16; + INT32 val = WINPR_ASSERTING_INT_CAST(int32_t, tmp % 16); /* Negative */ status = generic->shiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE); diff --git a/libfreerdp/primitives/test/TestPrimitivesYCbCr.c b/libfreerdp/primitives/test/TestPrimitivesYCbCr.c index 64e7f9168..ca9e906c3 100644 --- a/libfreerdp/primitives/test/TestPrimitivesYCbCr.c +++ b/libfreerdp/primitives/test/TestPrimitivesYCbCr.c @@ -1486,7 +1486,8 @@ static const UINT32 TEST_XRGB_IMAGE[4096] = { 0xFF169ff8, 0xFF159ef7, 0xFF149df7, 0xFF139cf6, 0xFF129bf5, 0xFF129bf5, 0xFF129bf5, 0xFF129bf5 }; -static int test_bmp_cmp_count(const BYTE* mem1, const BYTE* mem2, int size, int channel, int margin) +static int test_bmp_cmp_count(const BYTE* mem1, const BYTE* mem2, size_t size, int channel, + int margin) { int error = 0; int count = 0; @@ -1494,7 +1495,7 @@ static int test_bmp_cmp_count(const BYTE* mem1, const BYTE* mem2, int size, int mem1 += channel; mem2 += channel; - for (int index = 0; index < size; index++) + for (size_t index = 0; index < size; index++) { if (*mem1 != *mem2) { @@ -1511,16 +1512,16 @@ static int test_bmp_cmp_count(const BYTE* mem1, const BYTE* mem2, int size, int return count; } -static int test_bmp_cmp_dump(const BYTE* actual, const BYTE* expected, int size, int channel, +static int test_bmp_cmp_dump(const BYTE* actual, const BYTE* expected, size_t size, int channel, int margin) { - int error[3]; + int error[3] = { 0 }; int count = 0; size /= 4; actual += channel; expected += channel; - for (int index = 0; index < size; index++) + for (size_t index = 0; index < size; index++) { if (*actual != *expected) { @@ -1529,8 +1530,8 @@ static int test_bmp_cmp_dump(const BYTE* actual, const BYTE* expected, int size, const INT16 Y = TEST_Y_COMPONENT[index]; const INT16 Cb = TEST_CB_COMPONENT[index]; const INT16 Cr = TEST_CR_COMPONENT[index]; - const int x = index % 64; - const int y = (index - x) / 64; + const size_t x = index % 64; + const size_t y = (index - x) / 64; BYTE R = 0; BYTE G = 0; BYTE B = 0; @@ -1546,9 +1547,9 @@ static int test_bmp_cmp_dump(const BYTE* actual, const BYTE* expected, int size, if ((error[0] > margin) || (error[1] > margin) || (error[2] > margin)) { - printf("(%2d,%2d) Y: %+5" PRId16 " Cb: %+5" PRId16 " Cr: %+5" PRId16 - " R: %03" PRIu8 "/%03" PRIu8 " G: %03" PRIu8 "/%03" PRIu8 " B: %03" PRIu8 - "/%03" PRIu8 " %d %d %d\n", + printf("(%2" PRIuz ",%2" PRIuz ") Y: %+5" PRId16 " Cb: %+5" PRId16 + " Cr: %+5" PRId16 " R: %03" PRIu8 "/%03" PRIu8 " G: %03" PRIu8 + "/%03" PRIu8 " B: %03" PRIu8 "/%03" PRIu8 " %d %d %d\n", x, y, Y, Cb, Cr, R, eR, G, eG, B, eB, R - eR, G - eG, B - eB); count++; } @@ -1638,7 +1639,9 @@ static int test_PrimitivesYCbCr(const primitives_t* prims, UINT32 format, prim_s CopyMemory(pSrcDst[2], pYCbCr[2], srcSize); PROFILER_ENTER(prof1) cnv.pi = pSrcDst; - status = prims->yCbCrToRGB_16s16s_P3P3(cnv.cpi, srcStride, pSrcDst, srcStride, &roi); + status = + prims->yCbCrToRGB_16s16s_P3P3(cnv.cpi, WINPR_ASSERTING_INT_CAST(int, srcStride), + pSrcDst, WINPR_ASSERTING_INT_CAST(int, srcStride), &roi); PROFILER_EXIT(prof1) if (status != PRIMITIVES_SUCCESS) diff --git a/libfreerdp/primitives/test/TestPrimitivesYCoCg.c b/libfreerdp/primitives/test/TestPrimitivesYCoCg.c index f6b9821e6..a31d5ebfc 100644 --- a/libfreerdp/primitives/test/TestPrimitivesYCoCg.c +++ b/libfreerdp/primitives/test/TestPrimitivesYCoCg.c @@ -52,16 +52,18 @@ static BOOL test_YCoCgRToRGB_8u_AC4R_func(UINT32 width, UINT32 height) PROFILER_CREATE(genericProf, "YCoCgRToRGB_8u_AC4R-GENERIC") PROFILER_CREATE(optProf, "YCoCgRToRGB_8u_AC4R-OPT") PROFILER_ENTER(genericProf) - status = generic->YCoCgToRGB_8u_AC4R(in, srcStride, out_c, format, dstStride, width, height, - 2, TRUE); + status = generic->YCoCgToRGB_8u_AC4R(in, WINPR_ASSERTING_INT_CAST(int, srcStride), out_c, + format, WINPR_ASSERTING_INT_CAST(int, dstStride), + width, height, 2, TRUE); PROFILER_EXIT(genericProf) if (status != PRIMITIVES_SUCCESS) goto loop_fail; PROFILER_ENTER(optProf) - status = optimized->YCoCgToRGB_8u_AC4R(in, srcStride, out_sse, format, dstStride, width, - height, 2, TRUE); + status = optimized->YCoCgToRGB_8u_AC4R( + in, WINPR_ASSERTING_INT_CAST(int, srcStride), out_sse, format, + WINPR_ASSERTING_INT_CAST(int, dstStride), width, height, 2, TRUE); PROFILER_EXIT(optProf) if (status != PRIMITIVES_SUCCESS) diff --git a/libfreerdp/primitives/test/TestPrimitivesYUV.c b/libfreerdp/primitives/test/TestPrimitivesYUV.c index a71790758..81400518a 100644 --- a/libfreerdp/primitives/test/TestPrimitivesYUV.c +++ b/libfreerdp/primitives/test/TestPrimitivesYUV.c @@ -244,8 +244,8 @@ static BOOL TestPrimitiveYUVCombine(primitives_t* prims, prim_size_t roi) if (!(paux[x] = set_padding(halfSize, padding))) goto fail; - memset(luma[x], 0xAB + 3 * x, halfSize); - memset(chroma[x], 0x80 + 2 * x, halfSize); + memset(luma[x], WINPR_ASSERTING_INT_CAST(int, 0xAB + 3 * x), halfSize); + memset(chroma[x], WINPR_ASSERTING_INT_CAST(int, 0x80 + 2 * x), halfSize); if (!check_padding(luma[x], halfSize, padding, "luma")) goto fail; diff --git a/libfreerdp/primitives/test/prim_test.c b/libfreerdp/primitives/test/prim_test.c index ffb7c6b5a..2d415df03 100644 --- a/libfreerdp/primitives/test/prim_test.c +++ b/libfreerdp/primitives/test/prim_test.c @@ -39,7 +39,7 @@ int test_sizes[] = { 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096 }; float measure_delta_time(UINT64 t0, UINT64 t1) { INT64 diff = (INT64)(t1 - t0); - double retval = diff / 1000000000.0; + double retval = ((double)diff / 1000000000.0); return (retval < 0.0) ? 0.0f : (float)retval; } @@ -47,13 +47,13 @@ float measure_delta_time(UINT64 t0, UINT64 t1) void measure_floatprint(float t, char* output, size_t len) { /* I don't want to link against -lm, so avoid log,exp,... */ - float f = 10.0; + float f = 10.0f; int i = 0; while (t > f) - f *= 10.0; + f *= 10.0f; - f /= 1000.0; + f /= 1000.0f; i = ((int)(t / f + 0.5f)) * (int)f; if (t < 0.0f) diff --git a/libfreerdp/utils/http.c b/libfreerdp/utils/http.c index 2633310b3..996f611e2 100644 --- a/libfreerdp/utils/http.c +++ b/libfreerdp/utils/http.c @@ -118,7 +118,7 @@ BOOL freerdp_http_request(const char* url, const char* body, long* status_code, goto out; } - const size_t len = path - (url + 8); + const size_t len = WINPR_ASSERTING_INT_CAST(size_t, path - (url + 8)); hostname = strndup(&url[8], len); if (!hostname) return FALSE; diff --git a/libfreerdp/utils/passphrase.c b/libfreerdp/utils/passphrase.c index d9081e527..5d3c61b59 100644 --- a/libfreerdp/utils/passphrase.c +++ b/libfreerdp/utils/passphrase.c @@ -175,7 +175,7 @@ static const char* freerdp_passphrase_read_tty(rdpContext* context, const char* { struct termios new_flags = { 0 }; new_flags = orig_flags; - new_flags.c_lflag &= ~ECHO; + new_flags.c_lflag &= (uint32_t)~ECHO; new_flags.c_lflag |= ECHONL; terminal_needs_reset = TRUE; if (tcsetattr(terminal_fildes, TCSAFLUSH, &new_flags) == -1) @@ -345,5 +345,5 @@ SSIZE_T freerdp_interruptible_get_line(rdpContext* context, char** plineptr, siz } *plineptr = ptr; *psize = used; - return used; + return WINPR_ASSERTING_INT_CAST(SSIZE_T, used); } diff --git a/libfreerdp/utils/smartcard_call.c b/libfreerdp/utils/smartcard_call.c index 9a3b3916d..4f5a2cf9a 100644 --- a/libfreerdp/utils/smartcard_call.c +++ b/libfreerdp/utils/smartcard_call.c @@ -1805,7 +1805,7 @@ LONG smartcard_irp_device_control_call(scard_call_context* smartcard, wStream* o if ((result & 0xC0000000L) == 0xC0000000L) { /* NTSTATUS error */ - *pIoStatus = (UINT32)result; + *pIoStatus = result; WLog_WARN(TAG, "IRP failure: %s (0x%08" PRIX32 "), ntstatus: 0x%08" PRIX32 "", scard_get_ioctl_string(ioControlCode, TRUE), ioControlCode, result); } diff --git a/libfreerdp/utils/smartcard_pack.c b/libfreerdp/utils/smartcard_pack.c index 0ae0aff2c..c64f4b207 100644 --- a/libfreerdp/utils/smartcard_pack.c +++ b/libfreerdp/utils/smartcard_pack.c @@ -1680,7 +1680,8 @@ LONG smartcard_unpack_redir_scard_context_(wStream* s, REDIR_SCARDCONTEXT* conte return STATUS_INVALID_PARAMETER; } - if (!smartcard_ndr_pointer_read_(s, index, &pbContextNdrPtr, file, function, line)) + if (!smartcard_ndr_pointer_read_(s, index, &pbContextNdrPtr, file, function, + WINPR_ASSERTING_INT_CAST(size_t, line))) return ERROR_INVALID_DATA; if (((context->cbContext == 0) && pbContextNdrPtr) || @@ -1781,7 +1782,8 @@ LONG smartcard_unpack_redir_scard_handle_(wStream* s, REDIR_SCARDHANDLE* handle, if (!Stream_CheckAndLogRequiredLength(TAG, s, handle->cbHandle)) return STATUS_BUFFER_TOO_SMALL; - if (!smartcard_ndr_pointer_read_(s, index, NULL, file, function, line)) + if (!smartcard_ndr_pointer_read_(s, index, NULL, file, function, + WINPR_ASSERTING_INT_CAST(size_t, line))) return ERROR_INVALID_DATA; return SCARD_S_SUCCESS;