From ea78e33d171772af445920c638583271bbb651b4 Mon Sep 17 00:00:00 2001 From: Armin Novak Date: Fri, 18 Jun 2021 10:00:43 +0200 Subject: [PATCH] server: Fixed warnings, added assertions --- include/freerdp/server/shadow.h | 20 ++-- include/freerdp/utils/profiler.h | 2 +- server/shadow/X11/x11_shadow.c | 44 ++++--- server/shadow/shadow_capture.c | 18 ++- server/shadow/shadow_client.c | 199 ++++++++++++++++++++++--------- server/shadow/shadow_encoder.c | 21 ++-- server/shadow/shadow_encoder.h | 4 +- server/shadow/shadow_lobby.c | 7 +- server/shadow/shadow_screen.c | 4 +- server/shadow/shadow_screen.h | 4 +- server/shadow/shadow_server.c | 15 ++- server/shadow/shadow_subsystem.c | 6 +- server/shadow/shadow_surface.c | 12 +- server/shadow/shadow_surface.h | 7 +- 14 files changed, 237 insertions(+), 126 deletions(-) diff --git a/include/freerdp/server/shadow.h b/include/freerdp/server/shadow.h index ae7069cbe..9ce5e0063 100644 --- a/include/freerdp/server/shadow.h +++ b/include/freerdp/server/shadow.h @@ -134,7 +134,7 @@ struct rdp_shadow_server BOOL mayInteract; BOOL shareSubRect; BOOL authentication; - int selectedMonitor; + UINT32 selectedMonitor; RECTANGLE_16 subRect; /* Codec settings */ @@ -156,11 +156,11 @@ struct rdp_shadow_surface { rdpShadowServer* server; - int x; - int y; - int width; - int height; - int scanline; + UINT16 x; + UINT16 y; + UINT32 width; + UINT32 height; + UINT32 scanline; DWORD format; BYTE* data; @@ -186,9 +186,9 @@ struct rdp_shadow_subsystem { RDP_SHADOW_ENTRY_POINTS ep; HANDLE event; - int numMonitors; - int captureFrameRate; - int selectedMonitor; + UINT32 numMonitors; + UINT32 captureFrameRate; + UINT32 selectedMonitor; MONITOR_DEF monitors[16]; MONITOR_DEF virtualScreen; @@ -321,7 +321,7 @@ extern "C" SHADOW_MSG_OUT* msg, void* lParam); FREERDP_API int shadow_client_boardcast_quit(rdpShadowServer* server, int nExitCode); - FREERDP_API int shadow_encoder_preferred_fps(rdpShadowEncoder* encoder); + FREERDP_API UINT32 shadow_encoder_preferred_fps(rdpShadowEncoder* encoder); FREERDP_API UINT32 shadow_encoder_inflight_frames(rdpShadowEncoder* encoder); FREERDP_API BOOL shadow_screen_resize(rdpShadowScreen* screen); diff --git a/include/freerdp/utils/profiler.h b/include/freerdp/utils/profiler.h index 4402de68c..431e3545a 100644 --- a/include/freerdp/utils/profiler.h +++ b/include/freerdp/utils/profiler.h @@ -63,7 +63,7 @@ extern "C" #define PROFILER_RENAME(prof, name) \ do \ { \ - } while (0) + } while (0); #define PROFILER_DEFINE(prof) #define PROFILER_CREATE(prof, name) \ diff --git a/server/shadow/X11/x11_shadow.c b/server/shadow/X11/x11_shadow.c index b775345a0..772f14055 100644 --- a/server/shadow/X11/x11_shadow.c +++ b/server/shadow/X11/x11_shadow.c @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -76,9 +77,11 @@ static int x11_shadow_pam_conv(int num_msg, const struct pam_message** msg, int pam_status = PAM_CONV_ERR; SHADOW_PAM_AUTH_DATA* appdata; struct pam_response* response; + WINPR_ASSERT(num_msg >= 0); appdata = (SHADOW_PAM_AUTH_DATA*)appdata_ptr; + WINPR_ASSERT(appdata); - if (!(response = (struct pam_response*)calloc(num_msg, sizeof(struct pam_response)))) + if (!(response = (struct pam_response*)calloc((size_t)num_msg, sizeof(struct pam_response)))) return PAM_BUF_ERR; for (index = 0; index < num_msg; index++) @@ -122,7 +125,7 @@ out_fail: } } - memset(response, 0, sizeof(struct pam_response) * num_msg); + memset(response, 0, sizeof(struct pam_response) * (size_t)num_msg); free(response); *resp = NULL; return pam_status; @@ -294,8 +297,8 @@ static BOOL x11_shadow_input_mouse_event(rdpShadowSubsystem* subsystem, rdpShado negative = TRUE; button = (negative) ? 5 : 4; - XTestFakeButtonEvent(x11->display, button, True, CurrentTime); - XTestFakeButtonEvent(x11->display, button, False, CurrentTime); + XTestFakeButtonEvent(x11->display, button, True, (unsigned long)CurrentTime); + XTestFakeButtonEvent(x11->display, button, False, (unsigned long)CurrentTime); } else { @@ -592,15 +595,15 @@ static void x11_shadow_validate_region(x11ShadowSubsystem* subsystem, int x, int static int x11_shadow_blend_cursor(x11ShadowSubsystem* subsystem) { - int x, y; - int nXSrc; - int nYSrc; - int nXDst; - int nYDst; - int nWidth; - int nHeight; - int nSrcStep; - int nDstStep; + UINT32 x, y; + UINT32 nXSrc; + UINT32 nYSrc; + UINT32 nXDst; + UINT32 nYDst; + UINT32 nWidth; + UINT32 nHeight; + UINT32 nSrcStep; + UINT32 nDstStep; BYTE* pSrcData; BYTE* pDstData; BYTE A, R, G, B; @@ -826,9 +829,14 @@ static int x11_shadow_screen_grab(x11ShadowSubsystem* subsystem) y = extents->top; width = extents->right - extents->left; height = extents->bottom - extents->top; + WINPR_ASSERT(image); + WINPR_ASSERT(image->bytes_per_line >= 0); + WINPR_ASSERT(width >= 0); + WINPR_ASSERT(height >= 0); success = freerdp_image_copy(surface->data, surface->format, surface->scanline, x, y, - width, height, (BYTE*)image->data, PIXEL_FORMAT_BGRX32, - image->bytes_per_line, x, y, NULL, FREERDP_FLIP_NONE); + (UINT32)width, (UINT32)height, (BYTE*)image->data, + PIXEL_FORMAT_BGRX32, (UINT32)image->bytes_per_line, x, y, + NULL, FREERDP_FLIP_NONE); LeaveCriticalSection(&surface->lock); if (!success) goto fail_capture; @@ -1339,8 +1347,10 @@ static int x11_shadow_subsystem_init(rdpShadowSubsystem* sub) virtualScreen = &(subsystem->common.virtualScreen); virtualScreen->left = 0; virtualScreen->top = 0; - virtualScreen->right = subsystem->width; - virtualScreen->bottom = subsystem->height; + WINPR_ASSERT(subsystem->width <= INT32_MAX); + WINPR_ASSERT(subsystem->height <= INT32_MAX); + virtualScreen->right = (INT32)subsystem->width; + virtualScreen->bottom = (INT32)subsystem->height; virtualScreen->flags = 1; WLog_INFO(TAG, "X11 Extensions: XFixes: %" PRId32 " Xinerama: %" PRId32 " XDamage: %" PRId32 diff --git a/server/shadow/shadow_capture.c b/server/shadow/shadow_capture.c index 8bd79be5d..f82db79bf 100644 --- a/server/shadow/shadow_capture.c +++ b/server/shadow/shadow_capture.c @@ -166,16 +166,22 @@ int shadow_capture_compare(BYTE* pData1, UINT32 nStep1, UINT32 nWidth, UINT32 nH if (allEqual) return 0; - rect->left = l * 16; - rect->top = t * 16; - rect->right = (r + 1) * 16; - rect->bottom = (b + 1) * 16; + WINPR_ASSERT(l * 16 <= UINT16_MAX); + WINPR_ASSERT(t * 16 <= UINT16_MAX); + WINPR_ASSERT((r + 1) * 16 <= UINT16_MAX); + WINPR_ASSERT((b + 1) * 16 <= UINT16_MAX); + rect->left = (UINT16)l * 16; + rect->top = (UINT16)t * 16; + rect->right = (UINT16)(r + 1) * 16; + rect->bottom = (UINT16)(b + 1) * 16; + WINPR_ASSERT(nWidth <= UINT16_MAX); if (rect->right > nWidth) - rect->right = nWidth; + rect->right = (UINT16)nWidth; + WINPR_ASSERT(nHeight <= UINT16_MAX); if (rect->bottom > nHeight) - rect->bottom = nHeight; + rect->bottom = (UINT16)nHeight; #ifdef WITH_DEBUG_SHADOW_CAPTURE size_t size = ncol + 1; diff --git a/server/shadow/shadow_client.c b/server/shadow/shadow_client.c index 0fc5eb945..f77afb30d 100644 --- a/server/shadow/shadow_client.c +++ b/server/shadow/shadow_client.c @@ -23,6 +23,7 @@ #endif #include +#include #include #include #include @@ -48,10 +49,19 @@ static INLINE BOOL shadow_client_rdpgfx_new_surface(rdpShadowClient* client) UINT error = CHANNEL_RC_OK; RDPGFX_CREATE_SURFACE_PDU createSurface; RDPGFX_MAP_SURFACE_TO_OUTPUT_PDU surfaceToOutput; - RdpgfxServerContext* context = client->rdpgfx; - rdpSettings* settings = ((rdpContext*)client)->settings; - createSurface.width = settings->DesktopWidth; - createSurface.height = settings->DesktopHeight; + RdpgfxServerContext* context; + rdpSettings* settings; + + WINPR_ASSERT(client); + context = client->rdpgfx; + WINPR_ASSERT(context); + settings = ((rdpContext*)client)->settings; + WINPR_ASSERT(settings); + + WINPR_ASSERT(settings->DesktopWidth <= UINT16_MAX); + WINPR_ASSERT(settings->DesktopHeight <= UINT16_MAX); + createSurface.width = (UINT16)settings->DesktopWidth; + createSurface.height = (UINT16)settings->DesktopHeight; createSurface.pixelFormat = GFX_PIXEL_FORMAT_XRGB_8888; createSurface.surfaceId = 0; surfaceToOutput.outputOriginX = 0; @@ -231,10 +241,10 @@ static void shadow_client_context_free(freerdp_peer* peer, rdpShadowClient* clie DeleteCriticalSection(&(client->lock)); } -static INLINE void shadow_client_mark_invalid(rdpShadowClient* client, int numRects, +static INLINE void shadow_client_mark_invalid(rdpShadowClient* client, UINT32 numRects, const RECTANGLE_16* rects) { - int index; + UINT32 index; RECTANGLE_16 screenRegion; rdpSettings* settings = ((rdpContext*)client)->settings; EnterCriticalSection(&(client->lock)); @@ -251,8 +261,10 @@ static INLINE void shadow_client_mark_invalid(rdpShadowClient* client, int numRe { screenRegion.left = 0; screenRegion.top = 0; - screenRegion.right = settings->DesktopWidth; - screenRegion.bottom = settings->DesktopHeight; + WINPR_ASSERT(settings->DesktopWidth <= UINT16_MAX); + WINPR_ASSERT(settings->DesktopHeight <= UINT16_MAX); + screenRegion.right = (UINT16)settings->DesktopWidth; + screenRegion.bottom = (UINT16)settings->DesktopHeight; region16_union_rect(&(client->invalidRegion), &(client->invalidRegion), &screenRegion); } @@ -267,10 +279,23 @@ static INLINE void shadow_client_mark_invalid(rdpShadowClient* client, int numRe */ static INLINE BOOL shadow_client_recalc_desktop_size(rdpShadowClient* client) { - int width, height; - rdpShadowServer* server = client->server; - rdpSettings* settings = client->context.settings; - RECTANGLE_16 viewport = { 0, 0, server->surface->width, server->surface->height }; + INT32 width, height; + rdpShadowServer* server; + rdpSettings* settings; + RECTANGLE_16 viewport = { 0 }; + + WINPR_ASSERT(client); + server = client->server; + settings = client->context.settings; + + WINPR_ASSERT(server); + WINPR_ASSERT(server->surface); + WINPR_ASSERT(settings); + + WINPR_ASSERT(server->surface->width <= UINT16_MAX); + WINPR_ASSERT(server->surface->height <= UINT16_MAX); + viewport.right = (UINT16)server->surface->width; + viewport.bottom = (UINT16)server->surface->height; if (server->shareSubRect) { @@ -280,10 +305,14 @@ static INLINE BOOL shadow_client_recalc_desktop_size(rdpShadowClient* client) width = viewport.right - viewport.left; height = viewport.bottom - viewport.top; + WINPR_ASSERT(width >= 0); + WINPR_ASSERT(width <= UINT16_MAX); + WINPR_ASSERT(height >= 0); + WINPR_ASSERT(height <= UINT16_MAX); if (settings->DesktopWidth != (UINT32)width || settings->DesktopHeight != (UINT32)height) { - settings->DesktopWidth = width; - settings->DesktopHeight = height; + settings->DesktopWidth = (UINT16)width; + settings->DesktopHeight = (UINT16)height; return TRUE; } @@ -500,16 +529,31 @@ static BOOL shadow_client_logon(freerdp_peer* peer, SEC_WINNT_AUTH_IDENTITY* ide if (identity->Flags & SEC_WINNT_AUTH_IDENTITY_UNICODE) { if (identity->User) - ConvertFromUnicode(CP_UTF8, 0, identity->User, identity->UserLength, &user, 0, NULL, - NULL); + { + int rc; + WINPR_ASSERT(identity->UserLength <= INT_MAX); + rc = ConvertFromUnicode(CP_UTF8, 0, identity->User, (int)identity->UserLength, &user, 0, + NULL, NULL); + WINPR_ASSERT(rc > 0); + } if (identity->Domain) - ConvertFromUnicode(CP_UTF8, 0, identity->Domain, identity->DomainLength, &domain, 0, - NULL, NULL); + { + int rc; + WINPR_ASSERT(identity->DomainLength <= INT_MAX); + rc = ConvertFromUnicode(CP_UTF8, 0, identity->Domain, (int)identity->DomainLength, + &domain, 0, NULL, NULL); + WINPR_ASSERT(rc > 0); + } if (identity->Password) - ConvertFromUnicode(CP_UTF8, 0, identity->Password, identity->PasswordLength, &user, 0, - NULL, NULL); + { + int rc; + WINPR_ASSERT(identity->PasswordLength <= INT_MAX); + rc = ConvertFromUnicode(CP_UTF8, 0, identity->Password, (int)identity->PasswordLength, + &user, 0, NULL, NULL); + WINPR_ASSERT(rc > 0); + } } else { @@ -786,8 +830,8 @@ static INLINE UINT32 rdpgfx_estimate_h264_avc420(RDPGFX_AVC420_BITMAP_STREAM* ha * @return TRUE on success */ static BOOL shadow_client_send_surface_gfx(rdpShadowClient* client, const BYTE* pSrcData, - int nSrcStep, int nXSrc, int nYSrc, int nWidth, - int nHeight) + UINT32 nSrcStep, UINT16 nXSrc, UINT16 nYSrc, + UINT16 nWidth, UINT16 nHeight) { UINT error = CHANNEL_RC_OK; rdpContext* context = (rdpContext*)client; @@ -809,8 +853,8 @@ static BOOL shadow_client_send_surface_gfx(rdpShadowClient* client, const BYTE* cmdstart.frameId = shadow_encoder_create_frame_id(encoder); GetSystemTime(&sTime); - cmdstart.timestamp = - sTime.wHour << 22 | sTime.wMinute << 16 | sTime.wSecond << 10 | sTime.wMilliseconds; + cmdstart.timestamp = (UINT32)(sTime.wHour << 22U | sTime.wMinute << 16U | sTime.wSecond << 10U | + sTime.wMilliseconds); cmdend.frameId = cmdstart.frameId; cmd.surfaceId = 0; cmd.codecId = 0; @@ -839,10 +883,14 @@ static BOOL shadow_client_send_surface_gfx(rdpShadowClient* client, const BYTE* return FALSE; } - regionRect.left = cmd.left; - regionRect.top = cmd.top; - regionRect.right = cmd.right; - regionRect.bottom = cmd.bottom; + WINPR_ASSERT(cmd.left <= UINT16_MAX); + WINPR_ASSERT(cmd.top <= UINT16_MAX); + WINPR_ASSERT(cmd.right <= UINT16_MAX); + WINPR_ASSERT(cmd.bottom <= UINT16_MAX); + regionRect.left = (UINT16)cmd.left; + regionRect.top = (UINT16)cmd.top; + regionRect.right = (UINT16)cmd.right; + regionRect.bottom = (UINT16)cmd.bottom; rc = avc444_compress(encoder->h264, pSrcData, cmd.format, nSrcStep, nWidth, nHeight, version, ®ionRect, &avc444.LC, &avc444.bitstream[0].data, &avc444.bitstream[0].length, &avc444.bitstream[1].data, @@ -884,10 +932,14 @@ static BOOL shadow_client_send_surface_gfx(rdpShadowClient* client, const BYTE* return FALSE; } - regionRect.left = cmd.left; - regionRect.top = cmd.top; - regionRect.right = cmd.right; - regionRect.bottom = cmd.bottom; + WINPR_ASSERT(cmd.left <= UINT16_MAX); + WINPR_ASSERT(cmd.top <= UINT16_MAX); + WINPR_ASSERT(cmd.right <= UINT16_MAX); + WINPR_ASSERT(cmd.bottom <= UINT16_MAX); + regionRect.left = (UINT16)cmd.left; + regionRect.top = (UINT16)cmd.top; + regionRect.right = (UINT16)cmd.right; + regionRect.bottom = (UINT16)cmd.bottom; rc = avc420_compress(encoder->h264, pSrcData, cmd.format, nSrcStep, nWidth, nHeight, ®ionRect, &avc420.data, &avc420.length, &avc420.meta); if (rc < 0) @@ -922,8 +974,9 @@ static BOOL shadow_client_send_surface_gfx(rdpShadowClient* client, const BYTE* * * @return TRUE on success */ -static BOOL shadow_client_send_surface_bits(rdpShadowClient* client, BYTE* pSrcData, int nSrcStep, - int nXSrc, int nYSrc, int nWidth, int nHeight) +static BOOL shadow_client_send_surface_bits(rdpShadowClient* client, BYTE* pSrcData, + UINT32 nSrcStep, UINT16 nXSrc, UINT16 nYSrc, + UINT16 nWidth, UINT16 nHeight) { BOOL ret = TRUE; size_t i; @@ -978,15 +1031,18 @@ static BOOL shadow_client_send_surface_bits(rdpShadowClient* client, BYTE* pSrcD } cmd.cmdType = CMDTYPE_STREAM_SURFACE_BITS; - cmd.bmp.codecID = settings->RemoteFxCodecId; + WINPR_ASSERT(settings->RemoteFxCodecId <= UINT16_MAX); + cmd.bmp.codecID = (UINT16)settings->RemoteFxCodecId; cmd.destLeft = 0; cmd.destTop = 0; cmd.destRight = settings->DesktopWidth; cmd.destBottom = settings->DesktopHeight; cmd.bmp.bpp = 32; cmd.bmp.flags = 0; - cmd.bmp.width = settings->DesktopWidth; - cmd.bmp.height = settings->DesktopHeight; + WINPR_ASSERT(settings->DesktopWidth <= UINT16_MAX); + WINPR_ASSERT(settings->DesktopHeight <= UINT16_MAX); + cmd.bmp.width = (UINT16)settings->DesktopWidth; + cmd.bmp.height = (UINT16)settings->DesktopHeight; cmd.skipCompression = TRUE; if (numMessages > 0) @@ -1045,7 +1101,8 @@ static BOOL shadow_client_send_surface_bits(rdpShadowClient* client, BYTE* pSrcD nsc_compose_message(encoder->nsc, s, pSrcData, nWidth, nHeight, nSrcStep); cmd.cmdType = CMDTYPE_SET_SURFACE_BITS; cmd.bmp.bpp = 32; - cmd.bmp.codecID = settings->NSCodecId; + WINPR_ASSERT(settings->NSCodecId <= UINT16_MAX); + cmd.bmp.codecID = (UINT16)settings->NSCodecId; cmd.destLeft = nXSrc; cmd.destTop = nYSrc; cmd.destRight = cmd.destLeft + nWidth; @@ -1077,15 +1134,16 @@ static BOOL shadow_client_send_surface_bits(rdpShadowClient* client, BYTE* pSrcD * * @return TRUE on success */ -static BOOL shadow_client_send_bitmap_update(rdpShadowClient* client, BYTE* pSrcData, int nSrcStep, - int nXSrc, int nYSrc, int nWidth, int nHeight) +static BOOL shadow_client_send_bitmap_update(rdpShadowClient* client, BYTE* pSrcData, + UINT32 nSrcStep, UINT16 nXSrc, UINT16 nYSrc, + UINT16 nWidth, UINT16 nHeight) { BOOL ret = TRUE; BYTE* data; BYTE* buffer; - size_t k; - int yIdx, xIdx; - int rows, cols; + UINT32 k; + UINT32 yIdx, xIdx; + UINT32 rows, cols; UINT32 DstSize; UINT32 SrcFormat; BITMAP_DATA* bitmap; @@ -1188,8 +1246,8 @@ static BOOL shadow_client_send_bitmap_update(rdpShadowClient* client, BYTE* pSrc if (settings->ColorDepth < 32) { - int bitsPerPixel = settings->ColorDepth; - int bytesPerPixel = (bitsPerPixel + 7) / 8; + UINT32 bitsPerPixel = settings->ColorDepth; + UINT32 bytesPerPixel = (bitsPerPixel + 7) / 8; DstSize = 64 * 64 * 4; buffer = encoder->grid[k]; interleaved_compress(encoder->interleaved, buffer, &DstSize, bitmap->width, @@ -1298,8 +1356,8 @@ out: static BOOL shadow_client_send_surface_update(rdpShadowClient* client, SHADOW_GFX_STATUS* pStatus) { BOOL ret = TRUE; - int nXSrc, nYSrc; - int nWidth, nHeight; + INT64 nXSrc, nYSrc; + INT64 nWidth, nHeight; rdpContext* context = (rdpContext*)client; rdpSettings* settings; rdpShadowServer* server; @@ -1308,7 +1366,7 @@ static BOOL shadow_client_send_surface_update(rdpShadowClient* client, SHADOW_GF RECTANGLE_16 surfaceRect; const RECTANGLE_16* extents; BYTE* pSrcData; - int nSrcStep; + UINT32 nSrcStep; UINT32 index; UINT32 numRects = 0; const RECTANGLE_16* rects; @@ -1341,8 +1399,10 @@ static BOOL shadow_client_send_surface_update(rdpShadowClient* client, SHADOW_GF surfaceRect.left = 0; surfaceRect.top = 0; - surfaceRect.right = surface->width; - surfaceRect.bottom = surface->height; + WINPR_ASSERT(surface->width <= UINT16_MAX); + WINPR_ASSERT(surface->height <= UINT16_MAX); + surfaceRect.right = (UINT16)surface->width; + surfaceRect.bottom = (UINT16)surface->height; region16_intersect_rect(&invalidRegion, &invalidRegion, &surfaceRect); if (server->shareSubRect) @@ -1367,12 +1427,16 @@ static BOOL shadow_client_send_surface_update(rdpShadowClient* client, SHADOW_GF /* Move to new pSrcData / nXSrc / nYSrc according to sub rect */ if (server->shareSubRect) { - int subX, subY; + INT32 subX, subY; subX = server->subRect.left; subY = server->subRect.top; nXSrc -= subX; nYSrc -= subY; - pSrcData = &pSrcData[(subY * nSrcStep) + (subX * 4)]; + WINPR_ASSERT(nXSrc >= 0); + WINPR_ASSERT(nXSrc <= UINT16_MAX); + WINPR_ASSERT(nYSrc >= 0); + WINPR_ASSERT(nYSrc <= UINT16_MAX); + pSrcData = &pSrcData[((UINT16)subY * nSrcStep) + ((UINT16)subX * 4U)]; } // WLog_INFO(TAG, "shadow_client_send_surface_update: x: %d y: %d width: %d height: %d right: %d @@ -1397,17 +1461,38 @@ static BOOL shadow_client_send_surface_update(rdpShadowClient* client, SHADOW_GF pStatus->gfxSurfaceCreated = TRUE; } - ret = shadow_client_send_surface_gfx(client, pSrcData, nSrcStep, 0, 0, nWidth, nHeight); + WINPR_ASSERT(nWidth >= 0); + WINPR_ASSERT(nWidth <= UINT16_MAX); + WINPR_ASSERT(nHeight >= 0); + WINPR_ASSERT(nHeight <= UINT16_MAX); + ret = shadow_client_send_surface_gfx(client, pSrcData, nSrcStep, 0, 0, (UINT16)nWidth, + (UINT16)nHeight); } else if (settings->RemoteFxCodec || settings->NSCodec) { - ret = shadow_client_send_surface_bits(client, pSrcData, nSrcStep, nXSrc, nYSrc, nWidth, - nHeight); + WINPR_ASSERT(nXSrc >= 0); + WINPR_ASSERT(nXSrc <= UINT16_MAX); + WINPR_ASSERT(nYSrc >= 0); + WINPR_ASSERT(nYSrc <= UINT16_MAX); + WINPR_ASSERT(nWidth >= 0); + WINPR_ASSERT(nWidth <= UINT16_MAX); + WINPR_ASSERT(nHeight >= 0); + WINPR_ASSERT(nHeight <= UINT16_MAX); + ret = shadow_client_send_surface_bits(client, pSrcData, nSrcStep, (UINT16)nXSrc, + (UINT16)nYSrc, (UINT16)nWidth, (UINT16)nHeight); } else { - ret = shadow_client_send_bitmap_update(client, pSrcData, nSrcStep, nXSrc, nYSrc, nWidth, - nHeight); + WINPR_ASSERT(nXSrc >= 0); + WINPR_ASSERT(nXSrc <= UINT16_MAX); + WINPR_ASSERT(nYSrc >= 0); + WINPR_ASSERT(nYSrc <= UINT16_MAX); + WINPR_ASSERT(nWidth >= 0); + WINPR_ASSERT(nWidth <= UINT16_MAX); + WINPR_ASSERT(nHeight >= 0); + WINPR_ASSERT(nHeight <= UINT16_MAX); + ret = shadow_client_send_bitmap_update(client, pSrcData, nSrcStep, (UINT16)nXSrc, + (UINT16)nYSrc, (UINT16)nWidth, (UINT16)nHeight); } out: diff --git a/server/shadow/shadow_encoder.c b/server/shadow/shadow_encoder.c index 30ef5b4de..7f3c862b4 100644 --- a/server/shadow/shadow_encoder.c +++ b/server/shadow/shadow_encoder.c @@ -26,7 +26,7 @@ #define TAG CLIENT_TAG("shadow") -int shadow_encoder_preferred_fps(rdpShadowEncoder* encoder) +UINT32 shadow_encoder_preferred_fps(rdpShadowEncoder* encoder) { /* Return preferred fps calculated according to the last * sent frame id and last client-acknowledged frame id. @@ -51,8 +51,7 @@ UINT32 shadow_encoder_inflight_frames(rdpShadowEncoder* encoder) UINT32 shadow_encoder_create_frame_id(rdpShadowEncoder* encoder) { UINT32 frameId; - int inFlightFrames; - inFlightFrames = shadow_encoder_inflight_frames(encoder); + UINT32 inFlightFrames = shadow_encoder_inflight_frames(encoder); /* * Calculate preferred fps according to how much frames are @@ -80,9 +79,9 @@ UINT32 shadow_encoder_create_frame_id(rdpShadowEncoder* encoder) static int shadow_encoder_init_grid(rdpShadowEncoder* encoder) { - int i, j, k; - int tileSize; - int tileCount; + UINT32 i, j, k; + UINT32 tileSize; + UINT32 tileCount; encoder->gridWidth = ((encoder->width + (encoder->maxTileWidth - 1)) / encoder->maxTileWidth); encoder->gridHeight = ((encoder->height + (encoder->maxTileHeight - 1)) / encoder->maxTileHeight); @@ -278,7 +277,7 @@ static int shadow_encoder_uninit_rfx(rdpShadowEncoder* encoder) encoder->rfx = NULL; } - encoder->codecs &= ~FREERDP_CODEC_REMOTEFX; + encoder->codecs &= (UINT32)~FREERDP_CODEC_REMOTEFX; return 1; } @@ -290,7 +289,7 @@ static int shadow_encoder_uninit_nsc(rdpShadowEncoder* encoder) encoder->nsc = NULL; } - encoder->codecs &= ~FREERDP_CODEC_NSCODEC; + encoder->codecs &= (UINT32)~FREERDP_CODEC_NSCODEC; return 1; } @@ -302,7 +301,7 @@ static int shadow_encoder_uninit_planar(rdpShadowEncoder* encoder) encoder->planar = NULL; } - encoder->codecs &= ~FREERDP_CODEC_PLANAR; + encoder->codecs &= (UINT32)~FREERDP_CODEC_PLANAR; return 1; } @@ -314,7 +313,7 @@ static int shadow_encoder_uninit_interleaved(rdpShadowEncoder* encoder) encoder->interleaved = NULL; } - encoder->codecs &= ~FREERDP_CODEC_INTERLEAVED; + encoder->codecs &= (UINT32)~FREERDP_CODEC_INTERLEAVED; return 1; } @@ -326,7 +325,7 @@ static int shadow_encoder_uninit_h264(rdpShadowEncoder* encoder) encoder->h264 = NULL; } - encoder->codecs &= ~(FREERDP_CODEC_AVC420 | FREERDP_CODEC_AVC444); + encoder->codecs &= (UINT32) ~(FREERDP_CODEC_AVC420 | FREERDP_CODEC_AVC444); return 1; } diff --git a/server/shadow/shadow_encoder.h b/server/shadow/shadow_encoder.h index 8af398620..631853621 100644 --- a/server/shadow/shadow_encoder.h +++ b/server/shadow/shadow_encoder.h @@ -51,8 +51,8 @@ struct rdp_shadow_encoder BITMAP_INTERLEAVED_CONTEXT* interleaved; H264_CONTEXT* h264; - int fps; - int maxFps; + UINT32 fps; + UINT32 maxFps; BOOL frameAck; UINT32 frameId; UINT32 lastAckframeId; diff --git a/server/shadow/shadow_lobby.c b/server/shadow/shadow_lobby.c index 2001bc4c2..ec2d84553 100644 --- a/server/shadow/shadow_lobby.c +++ b/server/shadow/shadow_lobby.c @@ -20,6 +20,7 @@ #include "config.h" #endif +#include #include #include "shadow.h" @@ -52,8 +53,10 @@ BOOL shadow_client_init_lobby(rdpShadowServer* server) invalidRect.left = 0; invalidRect.top = 0; - invalidRect.right = lobby->width; - invalidRect.bottom = lobby->height; + WINPR_ASSERT(lobby->width <= UINT16_MAX); + WINPR_ASSERT(lobby->height <= UINT16_MAX); + invalidRect.right = (UINT16)lobby->width; + invalidRect.bottom = (UINT16)lobby->height; if (server->shareSubRect) { /* If we have shared sub rect setting, only fill shared rect */ diff --git a/server/shadow/shadow_screen.c b/server/shadow/shadow_screen.c index aa15ebd78..d73219ea7 100644 --- a/server/shadow/shadow_screen.c +++ b/server/shadow/shadow_screen.c @@ -29,8 +29,8 @@ rdpShadowScreen* shadow_screen_new(rdpShadowServer* server) { - int x, y; - int width, height; + INT64 x, y; + INT64 width, height; rdpShadowScreen* screen; rdpShadowSubsystem* subsystem; MONITOR_DEF* primary; diff --git a/server/shadow/shadow_screen.h b/server/shadow/shadow_screen.h index ca39b2fb2..5049a50a7 100644 --- a/server/shadow/shadow_screen.h +++ b/server/shadow/shadow_screen.h @@ -28,8 +28,8 @@ struct rdp_shadow_screen { rdpShadowServer* server; - int width; - int height; + UINT32 width; + UINT32 height; CRITICAL_SECTION lock; REGION16 invalidRegion; diff --git a/server/shadow/shadow_server.c b/server/shadow/shadow_server.c index ad52903b1..9f982d81e 100644 --- a/server/shadow/shadow_server.c +++ b/server/shadow/shadow_server.c @@ -284,10 +284,13 @@ int shadow_server_parse_command_line(rdpShadowServer* server, int argc, char** a if ((x < 0) || (y < 0) || (w < 1) || (h < 1) || (errno != 0)) return -1; - server->subRect.left = x; - server->subRect.top = y; - server->subRect.right = x + w; - server->subRect.bottom = y + h; + if ((x > UINT16_MAX) || (y > UINT16_MAX) || (x + w > UINT16_MAX) || + (y + h > UINT16_MAX)) + return -1; + server->subRect.left = (UINT16)x; + server->subRect.top = (UINT16)y; + server->subRect.right = (UINT16)(x + w); + server->subRect.bottom = (UINT16)(y + h); server->shareSubRect = TRUE; } CommandLineSwitchCase(arg, "auth") @@ -372,7 +375,7 @@ int shadow_server_parse_command_line(rdpShadowServer* server, int argc, char** a if (arg && (arg->Flags & COMMAND_LINE_ARGUMENT_PRESENT)) { - int index; + UINT32 index; UINT32 numMonitors; MONITOR_DEF monitors[16]; numMonitors = shadow_enum_monitors(monitors, 16); @@ -385,7 +388,7 @@ int shadow_server_parse_command_line(rdpShadowServer* server, int argc, char** a if ((val < 0) || (errno != 0) || ((UINT32)val >= numMonitors)) status = COMMAND_LINE_STATUS_PRINT; - server->selectedMonitor = (int)val; + server->selectedMonitor = (UINT32)val; } else { diff --git a/server/shadow/shadow_subsystem.c b/server/shadow/shadow_subsystem.c index 902e52457..092f055cd 100644 --- a/server/shadow/shadow_subsystem.c +++ b/server/shadow/shadow_subsystem.c @@ -177,7 +177,7 @@ UINT32 shadow_enum_monitors(MONITOR_DEF* monitors, UINT32 maxMonitors) RDP_SHADOW_ENTRY_POINTS ep; if (shadow_subsystem_load_entry_points(&ep) < 0) - return -1; + return 0; numMonitors = ep.EnumMonitors(monitors, maxMonitors); @@ -197,8 +197,8 @@ int shadow_subsystem_pointer_convert_alpha_pointer_data( UINT32 x, y; BYTE* pSrc8; BYTE* pDst8; - int xorStep; - int andStep; + UINT32 xorStep; + UINT32 andStep; UINT32 andBit; BYTE* andBits; UINT32 andPixel; diff --git a/server/shadow/shadow_surface.c b/server/shadow/shadow_surface.c index 0e48c6f78..62c449e4a 100644 --- a/server/shadow/shadow_surface.c +++ b/server/shadow/shadow_surface.c @@ -23,8 +23,11 @@ #include "shadow.h" #include "shadow_surface.h" -#define ALIGN_SCREEN_SIZE(size, align) ((size + align - 1) & (~(align - 1))) -rdpShadowSurface* shadow_surface_new(rdpShadowServer* server, int x, int y, int width, int height) +#define ALIGN_SCREEN_SIZE(size, align) \ + ((((size) % (align)) != 0) ? ((size) + (align) - ((size) % (align))) : (size)) + +rdpShadowSurface* shadow_surface_new(rdpShadowServer* server, UINT16 x, UINT16 y, UINT32 width, + UINT32 height) { rdpShadowSurface* surface; surface = (rdpShadowSurface*)calloc(1, sizeof(rdpShadowSurface)); @@ -69,10 +72,11 @@ void shadow_surface_free(rdpShadowSurface* surface) free(surface); } -BOOL shadow_surface_resize(rdpShadowSurface* surface, int x, int y, int width, int height) +BOOL shadow_surface_resize(rdpShadowSurface* surface, UINT16 x, UINT16 y, UINT32 width, + UINT32 height) { BYTE* buffer = NULL; - int scanline = ALIGN_SCREEN_SIZE(width, 4) * 4; + UINT32 scanline = ALIGN_SCREEN_SIZE(width, 4) * 4; if (!surface) return FALSE; diff --git a/server/shadow/shadow_surface.h b/server/shadow/shadow_surface.h index 0f3d944ab..e376be9a5 100644 --- a/server/shadow/shadow_surface.h +++ b/server/shadow/shadow_surface.h @@ -29,10 +29,11 @@ extern "C" { #endif - rdpShadowSurface* shadow_surface_new(rdpShadowServer* server, int x, int y, int width, - int height); + rdpShadowSurface* shadow_surface_new(rdpShadowServer* server, UINT16 x, UINT16 y, UINT32 width, + UINT32 height); void shadow_surface_free(rdpShadowSurface* surface); - BOOL shadow_surface_resize(rdpShadowSurface* surface, int x, int y, int width, int height); + BOOL shadow_surface_resize(rdpShadowSurface* surface, UINT16 x, UINT16 y, UINT32 width, + UINT32 height); #ifdef __cplusplus }