From 939e922936e9c3ae8fc204968645e5e7563a2fff Mon Sep 17 00:00:00 2001
From: Armin Novak <anovak@thincast.com>
Date: Sat, 13 Jan 2024 21:01:55 +0100
Subject: [PATCH] [codec,planar] check resolution for overflow

If the codec resolution is too large return an error as the internal
buffers would otherwise overflow.
---
 libfreerdp/codec/planar.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c
index 37a0ee9f6..619e46031 100644
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -1655,7 +1655,13 @@ BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* context, UINT32
 	context->bgr = FALSE;
 	context->maxWidth = PLANAR_ALIGN(width, 4);
 	context->maxHeight = PLANAR_ALIGN(height, 4);
-	context->maxPlaneSize = context->maxWidth * context->maxHeight;
+	const UINT64 tmp = (UINT64)context->maxWidth * context->maxHeight;
+	if (tmp > UINT32_MAX)
+		return FALSE;
+	context->maxPlaneSize = tmp;
+
+	if (context->maxWidth > UINT32_MAX / 4)
+		return FALSE;
 	context->nTempStep = context->maxWidth * 4;
 
 	memset(context->planes, 0, sizeof(context->planes));