diff --git a/include/freerdp/codec/rfx.h b/include/freerdp/codec/rfx.h
index d8fa52c35..645a0d0b3 100644
--- a/include/freerdp/codec/rfx.h
+++ b/include/freerdp/codec/rfx.h
@@ -66,6 +66,7 @@ struct _RFX_TILE
 	BYTE* YData;
 	BYTE* CbData;
 	BYTE* CrData;
+	BYTE* YCbCrData;
 };
 typedef struct _RFX_TILE RFX_TILE;
 
diff --git a/libfreerdp/codec/rfx.c b/libfreerdp/codec/rfx.c
index d6e654741..1ed1b59e6 100644
--- a/libfreerdp/codec/rfx.c
+++ b/libfreerdp/codec/rfx.c
@@ -242,9 +242,11 @@ RFX_CONTEXT* rfx_context_new(BOOL encoder)
 	 * in order to allow optimized functions (SEE, NEON) to read from positions 
 	 * that are actually in front/beyond the buffer. Offset calculations are
 	 * performed at the BufferPool_Take function calls in rfx_encode/decode.c.
+	 *
+	 * We then multiply by 3 to use a single, partioned buffer for all 3 channels.
 	 */
 
-	context->priv->BufferPool = BufferPool_New(TRUE, 8192 + 32, 16);
+	context->priv->BufferPool = BufferPool_New(TRUE, (8192 + 32) * 3, 16);
 
 #ifdef _WIN32
 	{
@@ -934,14 +936,11 @@ void rfx_message_free(RFX_CONTEXT* context, RFX_MESSAGE* message)
 			{
 				tile = message->tiles[i];
 
-				if (tile->YData)
-					BufferPool_Return(context->priv->BufferPool, tile->YData);
-
-				if (tile->CbData)
-					BufferPool_Return(context->priv->BufferPool, tile->CbData);
-
-				if (tile->CrData)
-					BufferPool_Return(context->priv->BufferPool, tile->CrData);
+				if (tile->YCbCrData)
+				{
+					BufferPool_Return(context->priv->BufferPool, tile->YCbCrData);
+					tile->YCbCrData = NULL;
+				}
 
 				ObjectPool_Return(context->priv->TilePool, (void*) tile);
 			}
@@ -1154,9 +1153,12 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* context, const RFX_RECT* rects,
 			tile->YLen = 0;
 			tile->CbLen = 0;
 			tile->CrLen = 0;
-			tile->YData = (BYTE*) BufferPool_Take(context->priv->BufferPool, -1);
-			tile->CbData = (BYTE*) BufferPool_Take(context->priv->BufferPool, -1);
-			tile->CrData = (BYTE*) BufferPool_Take(context->priv->BufferPool, -1);
+
+			tile->YCbCrData = (BYTE*) BufferPool_Take(context->priv->BufferPool, -1);
+
+			tile->YData = (BYTE*) &(tile->YCbCrData[((8192 + 32) * 0) + 16]);
+			tile->CbData = (BYTE*) &(tile->YCbCrData[((8192 + 32) * 1) + 16]);
+			tile->CrData = (BYTE*) &(tile->YCbCrData[((8192 + 32) * 2) + 16]);
 
 			if (context->priv->UseThreads)
 			{
diff --git a/libfreerdp/codec/rfx_decode.c b/libfreerdp/codec/rfx_decode.c
index 2a642f9da..d91cfc52f 100644
--- a/libfreerdp/codec/rfx_decode.c
+++ b/libfreerdp/codec/rfx_decode.c
@@ -128,6 +128,7 @@ static void rfx_decode_component(RFX_CONTEXT* context, const UINT32* quantizatio
 /* stride is bytes between rows in the output buffer. */
 BOOL rfx_decode_rgb(RFX_CONTEXT* context, RFX_TILE* tile, BYTE* rgb_buffer, int stride)
 {
+	BYTE* pBuffer;
 	INT16* pSrcDst[3];
 	UINT32 *y_quants, *cb_quants, *cr_quants;
 	static const prim_size_t roi_64x64 = { 64, 64 };
@@ -139,9 +140,10 @@ BOOL rfx_decode_rgb(RFX_CONTEXT* context, RFX_TILE* tile, BYTE* rgb_buffer, int
 	cb_quants = context->quants + (tile->quantIdxCb * 10);
 	cr_quants = context->quants + (tile->quantIdxCr * 10);
 
-	pSrcDst[0] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* y_r_buffer */
-	pSrcDst[1] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cb_g_buffer */
-	pSrcDst[2] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cr_b_buffer */
+	pBuffer = (BYTE*) BufferPool_Take(context->priv->BufferPool, -1);
+	pSrcDst[0] = (INT16*)((BYTE*)(&pBuffer[((8192 + 32) * 0) + 16])); /* y_r_buffer */
+	pSrcDst[1] = (INT16*)((BYTE*)(&pBuffer[((8192 + 32) * 1) + 16])); /* cb_g_buffer */
+	pSrcDst[2] = (INT16*)((BYTE*)(&pBuffer[((8192 + 32) * 2) + 16])); /* cr_b_buffer */
 
 	rfx_decode_component(context, y_quants, tile->YData, tile->YLen, pSrcDst[0]); /* YData */
 
@@ -161,9 +163,7 @@ BOOL rfx_decode_rgb(RFX_CONTEXT* context, RFX_TILE* tile, BYTE* rgb_buffer, int
 	
 	PROFILER_EXIT(context->priv->prof_rfx_decode_rgb);
 
-	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[0] - 16);
-	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[1] - 16);
-	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[2] - 16);
+	BufferPool_Return(context->priv->BufferPool, pBuffer);
 
 	return TRUE;
 }
diff --git a/libfreerdp/codec/rfx_encode.c b/libfreerdp/codec/rfx_encode.c
index f484e58fd..ab106e291 100644
--- a/libfreerdp/codec/rfx_encode.c
+++ b/libfreerdp/codec/rfx_encode.c
@@ -219,6 +219,7 @@ static void rfx_encode_component(RFX_CONTEXT* context, const UINT32* quantizatio
 
 void rfx_encode_rgb(RFX_CONTEXT* context, RFX_TILE* tile)
 {
+	BYTE* pBuffer;
 	INT16* pSrcDst[3];
 	int YLen, CbLen, CrLen;
 	UINT32 *YQuant, *CbQuant, *CrQuant;
@@ -230,9 +231,10 @@ void rfx_encode_rgb(RFX_CONTEXT* context, RFX_TILE* tile)
 	CbQuant = context->quants + (tile->quantIdxCb * 10);
 	CrQuant = context->quants + (tile->quantIdxCr * 10);
 
-	pSrcDst[0] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* y_r_buffer */
-	pSrcDst[1] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cb_g_buffer */
-	pSrcDst[2] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cr_b_buffer */
+	pBuffer = (BYTE*) BufferPool_Take(context->priv->BufferPool, -1);
+	pSrcDst[0] = (INT16*)((BYTE*)(&pBuffer[((8192 + 32) * 0) + 16])); /* y_r_buffer */
+	pSrcDst[1] = (INT16*)((BYTE*)(&pBuffer[((8192 + 32) * 1) + 16])); /* cb_g_buffer */
+	pSrcDst[2] = (INT16*)((BYTE*)(&pBuffer[((8192 + 32) * 2) + 16])); /* cr_b_buffer */
 
 	PROFILER_ENTER(context->priv->prof_rfx_encode_rgb);
 
@@ -265,7 +267,5 @@ void rfx_encode_rgb(RFX_CONTEXT* context, RFX_TILE* tile)
 
 	PROFILER_EXIT(context->priv->prof_rfx_encode_rgb);
 
-	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[0] - 16);
-	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[1] - 16);
-	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[2] - 16);
+	BufferPool_Return(context->priv->BufferPool, pBuffer);
 }