From 161359f6ecac90b084e771dd58573a0d561dfadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Mon, 29 Jun 2015 12:43:18 -0400 Subject: [PATCH 01/14] libfreerdp-core: fix receiving of logon error info during capability exchange --- libfreerdp/core/capabilities.c | 15 ++++++++++++++- libfreerdp/core/info.c | 2 +- libfreerdp/core/rdp.c | 4 ---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/libfreerdp/core/capabilities.c b/libfreerdp/core/capabilities.c index 5f507e392..466184164 100644 --- a/libfreerdp/core/capabilities.c +++ b/libfreerdp/core/capabilities.c @@ -3594,10 +3594,23 @@ BOOL rdp_recv_demand_active(rdpRdp* rdp, wStream* s) return FALSE; } + if (pduType == PDU_TYPE_DATA) + { + /** + * We can receive a Save Session Info Data PDU containing a LogonErrorInfo + * structure at this point from the server to indicate a connection error. + */ + + if (rdp_recv_data_pdu(rdp, s) < 0) + return FALSE; + + return FALSE; + } + if (pduType != PDU_TYPE_DEMAND_ACTIVE) { if (pduType != PDU_TYPE_SERVER_REDIRECTION) - WLog_ERR(TAG, "expected PDU_TYPE_DEMAND_ACTIVE %04x, got %04x", PDU_TYPE_DEMAND_ACTIVE, pduType); + WLog_ERR(TAG, "expected PDU_TYPE_DEMAND_ACTIVE %04x, got %04x", PDU_TYPE_DEMAND_ACTIVE, pduType); return FALSE; } diff --git a/libfreerdp/core/info.c b/libfreerdp/core/info.c index 85c1d175b..fcebceb43 100644 --- a/libfreerdp/core/info.c +++ b/libfreerdp/core/info.c @@ -726,7 +726,7 @@ BOOL rdp_recv_logon_error_info(rdpRdp* rdp, wStream* s) UINT32 errorNotificationData; UINT32 errorNotificationType; - if (Stream_GetRemainingLength(s) < 4) + if (Stream_GetRemainingLength(s) < 8) return FALSE; Stream_Read_UINT32(s, errorNotificationData); /* errorNotificationData (4 bytes) */ diff --git a/libfreerdp/core/rdp.c b/libfreerdp/core/rdp.c index fac827aab..be44ad5b9 100644 --- a/libfreerdp/core/rdp.c +++ b/libfreerdp/core/rdp.c @@ -34,7 +34,6 @@ #define TAG FREERDP_TAG("core.rdp") -#ifdef WITH_DEBUG_RDP const char* DATA_PDU_TYPE_STRINGS[80] = { "?", "?", /* 0x00 - 0x01 */ @@ -72,7 +71,6 @@ const char* DATA_PDU_TYPE_STRINGS[80] = "FrameAcknowledge", "?", "?", /* 0x38 - 0x40 */ "?", "?", "?", "?", "?", "?" /* 0x41 - 0x46 */ }; -#endif /** * Read RDP Security Header.\n @@ -809,10 +807,8 @@ int rdp_recv_data_pdu(rdpRdp* rdp, wStream* s) Stream_Seek(s, SrcSize); } -#ifdef WITH_DEBUG_RDP WLog_DBG(TAG, "recv %s Data PDU (0x%02X), length: %d", type < ARRAYSIZE(DATA_PDU_TYPE_STRINGS) ? DATA_PDU_TYPE_STRINGS[type] : "???", type, length); -#endif switch (type) { From ff5b876260113f556baf4f9160413a83ebc26284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Mon, 29 Jun 2015 14:47:46 -0400 Subject: [PATCH 02/14] channels/drdynvc: silence harmless warnings with Windows 8 / Windows 2012 channel close --- channels/drdynvc/client/drdynvc_main.c | 29 +++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/channels/drdynvc/client/drdynvc_main.c b/channels/drdynvc/client/drdynvc_main.c index bb10d2df3..bda686976 100644 --- a/channels/drdynvc/client/drdynvc_main.c +++ b/channels/drdynvc/client/drdynvc_main.c @@ -47,8 +47,10 @@ static int dvcman_create_listener(IWTSVirtualChannelManager* pChannelMgr, { WLog_DBG(TAG, "create_listener: %d.%s.", dvcman->num_listeners, pszChannelName); - listener = (DVCMAN_LISTENER*) malloc(sizeof(DVCMAN_LISTENER)); - ZeroMemory(listener, sizeof(DVCMAN_LISTENER)); + listener = (DVCMAN_LISTENER*) calloc(1, sizeof(DVCMAN_LISTENER)); + + if (!listener) + return -1; listener->iface.GetConfiguration = dvcman_get_configuration; listener->iface.pInterface = NULL; @@ -407,8 +409,10 @@ int dvcman_open_channel(IWTSVirtualChannelManager* pChannelMgr, UINT32 ChannelId if (channel->status == 0) { pCallback = channel->channel_callback; + if (pCallback->OnOpen) pCallback->OnOpen(pCallback); + WLog_DBG(TAG, "open_channel: ChannelId %d", ChannelId); } @@ -426,7 +430,10 @@ int dvcman_close_channel(IWTSVirtualChannelManager* pChannelMgr, UINT32 ChannelI if (!channel) { - WLog_ERR(TAG, "ChannelId %d not found!", ChannelId); + /** + * Windows 8 / Windows Server 2012 send close requests for channels that failed to be created. + * Do not warn, simply return success here. + */ return 1; } @@ -465,8 +472,10 @@ int dvcman_receive_channel_data_first(IWTSVirtualChannelManager* pChannelMgr, UI Stream_Release(channel->dvc_data); channel->dvc_data = StreamPool_Take(channel->dvcman->pool, length); + if (!channel->dvc_data) return 1; + channel->dvc_data_length = length; return 0; @@ -660,7 +669,12 @@ static int drdynvc_send_capability_response(drdynvcPlugin* drdynvc) wStream* s; WLog_DBG(TAG, "capability_response"); + s = Stream_New(NULL, 4); + + if (!s) + return -1; + Stream_Write_UINT16(s, 0x0050); /* Cmd+Sp+cbChId+Pad. Note: MSTSC sends 0x005c */ Stream_Write_UINT16(s, drdynvc->version); @@ -767,8 +781,7 @@ static int drdynvc_process_create_request(drdynvcPlugin* drdynvc, int Sp, int cb else { WLog_DBG(TAG, "no listener"); - Stream_Write_UINT32(data_out, (UINT32)(-1)); - dvcman_close_channel(drdynvc->channel_mgr, ChannelId); + Stream_Write_UINT32(data_out, 0xFFFFFFFF); } status = drdynvc_send(drdynvc, data_out); @@ -784,6 +797,10 @@ static int drdynvc_process_create_request(drdynvcPlugin* drdynvc, int Sp, int cb { dvcman_open_channel(drdynvc->channel_mgr, ChannelId); } + else + { + dvcman_close_channel(drdynvc->channel_mgr, ChannelId); + } return 0; } @@ -915,6 +932,7 @@ void* drdynvc_get_init_handle_data(void* pInitHandle) void drdynvc_remove_init_handle_data(void* pInitHandle) { ListDictionary_Remove(g_InitHandles, pInitHandle); + if (ListDictionary_Count(g_InitHandles) < 1) { ListDictionary_Free(g_InitHandles); @@ -944,6 +962,7 @@ void drdynvc_remove_open_handle_data(DWORD openHandle) { void* pOpenHandle = (void*) (size_t) openHandle; ListDictionary_Remove(g_OpenHandles, pOpenHandle); + if (ListDictionary_Count(g_OpenHandles) < 1) { ListDictionary_Free(g_OpenHandles); From 3781e803ed7c681f4a3233ef3ad95d4400da2b4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Mon, 6 Jul 2015 16:28:52 -0400 Subject: [PATCH 03/14] channels/rdpgfx: improve frame ack handling --- channels/rdpgfx/client/rdpgfx_main.c | 27 ++++++++++++++------------- channels/rdpgfx/client/rdpgfx_main.h | 1 + client/X11/xf_gfx.c | 4 ++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/channels/rdpgfx/client/rdpgfx_main.c b/channels/rdpgfx/client/rdpgfx_main.c index eb1176732..bed2f61ba 100644 --- a/channels/rdpgfx/client/rdpgfx_main.c +++ b/channels/rdpgfx/client/rdpgfx_main.c @@ -119,7 +119,6 @@ int rdpgfx_recv_caps_confirm_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) RDPGFX_CAPSET capsSet; UINT32 capsDataLength; RDPGFX_CAPS_CONFIRM_PDU pdu; - RDPGFX_PLUGIN* gfx = (RDPGFX_PLUGIN*) callback->plugin; pdu.capsSet = &capsSet; @@ -129,8 +128,6 @@ int rdpgfx_recv_caps_confirm_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) Stream_Read_UINT32(s, capsSet.version); /* version (4 bytes) */ Stream_Read_UINT32(s, capsDataLength); /* capsDataLength (4 bytes) */ Stream_Read_UINT32(s, capsSet.flags); /* capsData (4 bytes) */ - - /*TODO: interpret this answer*/ WLog_DBG(TAG, "RecvCapsConfirmPdu: version: 0x%04X flags: 0x%04X", capsSet.version, capsSet.flags); @@ -143,7 +140,6 @@ int rdpgfx_send_frame_acknowledge_pdu(RDPGFX_CHANNEL_CALLBACK* callback, RDPGFX_ int status; wStream* s; RDPGFX_HEADER header; - RDPGFX_PLUGIN* gfx = (RDPGFX_PLUGIN*) callback->plugin; header.flags = 0; header.cmdId = RDPGFX_CMDID_FRAMEACKNOWLEDGE; @@ -374,16 +370,24 @@ int rdpgfx_recv_end_frame_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) context->EndFrame(context, &pdu); } - gfx->UnacknowledgedFrames--; - gfx->TotalDecodedFrames++; - ack.frameId = pdu.frameId; ack.totalFramesDecoded = gfx->TotalDecodedFrames; - //ack.queueDepth = SUSPEND_FRAME_ACKNOWLEDGEMENT; - ack.queueDepth = QUEUE_DEPTH_UNAVAILABLE; + if (gfx->suspendFrameAcks) + { + ack.queueDepth = SUSPEND_FRAME_ACKNOWLEDGEMENT; - rdpgfx_send_frame_acknowledge_pdu(callback, &ack); + if (gfx->TotalDecodedFrames == 0) + rdpgfx_send_frame_acknowledge_pdu(callback, &ack); + } + else + { + ack.queueDepth = QUEUE_DEPTH_UNAVAILABLE; + rdpgfx_send_frame_acknowledge_pdu(callback, &ack); + } + + gfx->UnacknowledgedFrames--; + gfx->TotalDecodedFrames++; return 1; } @@ -728,7 +732,6 @@ int rdpgfx_recv_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) int status; int beg, end; RDPGFX_HEADER header; - RDPGFX_PLUGIN* gfx = (RDPGFX_PLUGIN*) callback->plugin; beg = Stream_GetPosition(s); @@ -873,7 +876,6 @@ static int rdpgfx_on_data_received(IWTSVirtualChannelCallback* pChannelCallback, static int rdpgfx_on_open(IWTSVirtualChannelCallback* pChannelCallback) { RDPGFX_CHANNEL_CALLBACK* callback = (RDPGFX_CHANNEL_CALLBACK*) pChannelCallback; - RDPGFX_PLUGIN* gfx = (RDPGFX_PLUGIN*) callback->plugin; WLog_DBG(TAG, "OnOpen"); @@ -885,7 +887,6 @@ static int rdpgfx_on_open(IWTSVirtualChannelCallback* pChannelCallback) static int rdpgfx_on_close(IWTSVirtualChannelCallback* pChannelCallback) { RDPGFX_CHANNEL_CALLBACK* callback = (RDPGFX_CHANNEL_CALLBACK*) pChannelCallback; - RDPGFX_PLUGIN* gfx = (RDPGFX_PLUGIN*) callback->plugin; WLog_DBG(TAG, "OnClose"); diff --git a/channels/rdpgfx/client/rdpgfx_main.h b/channels/rdpgfx/client/rdpgfx_main.h index e5017dce1..39e59db52 100644 --- a/channels/rdpgfx/client/rdpgfx_main.h +++ b/channels/rdpgfx/client/rdpgfx_main.h @@ -69,6 +69,7 @@ struct _RDPGFX_PLUGIN ZGFX_CONTEXT* zgfx; UINT32 UnacknowledgedFrames; UINT32 TotalDecodedFrames; + BOOL suspendFrameAcks; wHashTable* SurfaceTable; diff --git a/client/X11/xf_gfx.c b/client/X11/xf_gfx.c index cce37106f..173bb90c2 100644 --- a/client/X11/xf_gfx.c +++ b/client/X11/xf_gfx.c @@ -353,7 +353,7 @@ int xf_SurfaceCommand_ClearCodec(xfContext* xfc, RdpgfxClientContext* context, R if (status < 0) { - WLog_ERR(TAG, "clear_decompress failure: %d\n", status); + WLog_ERR(TAG, "clear_decompress failure: %d", status); return -1; } @@ -467,7 +467,7 @@ int xf_SurfaceCommand_Alpha(xfContext* xfc, RdpgfxClientContext* context, RDPGFX if (!surface) return -1; - WLog_DBG(TAG, "xf_SurfaceCommand_Alpha: status: %d\n", status); + WLog_DBG(TAG, "xf_SurfaceCommand_Alpha: status: %d", status); /* fill with green for now to distinguish from the rest */ freerdp_image_fill(surface->data, PIXEL_FORMAT_XRGB32, surface->scanline, From 11f785290f3203a1b77cd0467d44bf77921e2e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Tue, 7 Jul 2015 13:36:36 -0400 Subject: [PATCH 04/14] channels/drdynvc: use same status code as mstsc for DVC opening failure --- channels/drdynvc/client/drdynvc_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channels/drdynvc/client/drdynvc_main.c b/channels/drdynvc/client/drdynvc_main.c index bda686976..ce6c774bd 100644 --- a/channels/drdynvc/client/drdynvc_main.c +++ b/channels/drdynvc/client/drdynvc_main.c @@ -781,7 +781,7 @@ static int drdynvc_process_create_request(drdynvcPlugin* drdynvc, int Sp, int cb else { WLog_DBG(TAG, "no listener"); - Stream_Write_UINT32(data_out, 0xFFFFFFFF); + Stream_Write_UINT32(data_out, 0xC0000001); /* same code used by mstsc */ } status = drdynvc_send(drdynvc, data_out); From 23f9b3bbc05fe78b5db635b8d1513bb824281fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 8 Jul 2015 11:17:56 -0400 Subject: [PATCH 05/14] channels/rdpgfx: reset state on channel close --- channels/rdpgfx/client/rdpgfx_main.c | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/channels/rdpgfx/client/rdpgfx_main.c b/channels/rdpgfx/client/rdpgfx_main.c index bed2f61ba..6d6ab10fd 100644 --- a/channels/rdpgfx/client/rdpgfx_main.c +++ b/channels/rdpgfx/client/rdpgfx_main.c @@ -886,12 +886,62 @@ static int rdpgfx_on_open(IWTSVirtualChannelCallback* pChannelCallback) static int rdpgfx_on_close(IWTSVirtualChannelCallback* pChannelCallback) { + int count; + int index; + ULONG_PTR* pKeys = NULL; RDPGFX_CHANNEL_CALLBACK* callback = (RDPGFX_CHANNEL_CALLBACK*) pChannelCallback; + RDPGFX_PLUGIN* gfx = (RDPGFX_PLUGIN*) callback->plugin; + RdpgfxClientContext* context = (RdpgfxClientContext*) gfx->iface.pInterface; WLog_DBG(TAG, "OnClose"); free(callback); + gfx->UnacknowledgedFrames = 0; + gfx->TotalDecodedFrames = 0; + + if (gfx->zgfx) + { + zgfx_context_free(gfx->zgfx); + gfx->zgfx = zgfx_context_new(FALSE); + + if (!gfx->zgfx) + return -1; + } + + count = HashTable_GetKeys(gfx->SurfaceTable, &pKeys); + + for (index = 0; index < count; index++) + { + RDPGFX_DELETE_SURFACE_PDU pdu; + + pdu.surfaceId = ((UINT16) pKeys[index]) - 1; + + if (context && context->DeleteSurface) + { + context->DeleteSurface(context, &pdu); + } + } + + free(pKeys); + + for (index = 0; index < gfx->MaxCacheSlot; index++) + { + if (gfx->CacheSlots[index]) + { + RDPGFX_EVICT_CACHE_ENTRY_PDU pdu; + + pdu.cacheSlot = (UINT16) index; + + if (context && context->EvictCacheEntry) + { + context->EvictCacheEntry(context, &pdu); + } + + gfx->CacheSlots[index] = NULL; + } + } + return 0; } From aae86d61ce2dda5462c88b49b048361881aeb0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 8 Jul 2015 14:16:29 -0400 Subject: [PATCH 06/14] channels/rdpgfx: fix TotalDecodedFrames counter --- channels/rdpgfx/client/rdpgfx_main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/channels/rdpgfx/client/rdpgfx_main.c b/channels/rdpgfx/client/rdpgfx_main.c index 6d6ab10fd..d551b5fcd 100644 --- a/channels/rdpgfx/client/rdpgfx_main.c +++ b/channels/rdpgfx/client/rdpgfx_main.c @@ -370,6 +370,9 @@ int rdpgfx_recv_end_frame_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) context->EndFrame(context, &pdu); } + gfx->UnacknowledgedFrames--; + gfx->TotalDecodedFrames++; + ack.frameId = pdu.frameId; ack.totalFramesDecoded = gfx->TotalDecodedFrames; @@ -377,7 +380,7 @@ int rdpgfx_recv_end_frame_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) { ack.queueDepth = SUSPEND_FRAME_ACKNOWLEDGEMENT; - if (gfx->TotalDecodedFrames == 0) + if (gfx->TotalDecodedFrames == 1) rdpgfx_send_frame_acknowledge_pdu(callback, &ack); } else @@ -386,9 +389,6 @@ int rdpgfx_recv_end_frame_pdu(RDPGFX_CHANNEL_CALLBACK* callback, wStream* s) rdpgfx_send_frame_acknowledge_pdu(callback, &ack); } - gfx->UnacknowledgedFrames--; - gfx->TotalDecodedFrames++; - return 1; } From 9400ccb4bc1af83b67c7ecba8edb35da130673b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Tue, 28 Jul 2015 16:01:34 -0400 Subject: [PATCH 07/14] libfreerdp-codec: add x264 stubs --- CMakeLists.txt | 5 + cmake/Findx264.cmake | 33 ++++++ libfreerdp/CMakeLists.txt | 6 ++ libfreerdp/codec/h264.c | 214 +++++++++++++++++++++++++++++++------- 4 files changed, 222 insertions(+), 36 deletions(-) create mode 100644 cmake/Findx264.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b050a3a1d..3cff6b7fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -537,6 +537,10 @@ set(JPEG_FEATURE_TYPE "OPTIONAL") set(JPEG_FEATURE_PURPOSE "codec") set(JPEG_FEATURE_DESCRIPTION "use JPEG library") +set(X264_FEATURE_TYPE "OPTIONAL") +set(X264_FEATURE_PURPOSE "codec") +set(X264_FEATURE_DESCRIPTION "use x264 library") + set(OPENH264_FEATURE_TYPE "OPTIONAL") set(OPENH264_FEATURE_PURPOSE "codec") set(OPENH264_FEATURE_DESCRIPTION "use OpenH264 library") @@ -620,6 +624,7 @@ find_feature(GStreamer_0_10 ${GSTREAMER_0_10_FEATURE_TYPE} ${GSTREAMER_0_10_FEAT find_feature(GStreamer_1_0 ${GSTREAMER_1_0_FEATURE_TYPE} ${GSTREAMER_1_0_FEATURE_PURPOSE} ${GSTREAMER_1_0_FEATURE_DESCRIPTION}) find_feature(JPEG ${JPEG_FEATURE_TYPE} ${JPEG_FEATURE_PURPOSE} ${JPEG_FEATURE_DESCRIPTION}) +find_feature(x264 ${X264_FEATURE_TYPE} ${X264_FEATURE_PURPOSE} ${X264_FEATURE_DESCRIPTION}) find_feature(OpenH264 ${OPENH264_FEATURE_TYPE} ${OPENH264_FEATURE_PURPOSE} ${OPENH264_FEATURE_DESCRIPTION}) find_feature(GSM ${GSM_FEATURE_TYPE} ${GSM_FEATURE_PURPOSE} ${GSM_FEATURE_DESCRIPTION}) diff --git a/cmake/Findx264.cmake b/cmake/Findx264.cmake new file mode 100644 index 000000000..f8df4754e --- /dev/null +++ b/cmake/Findx264.cmake @@ -0,0 +1,33 @@ + +if (X264_INCLUDE_DIR AND X264_LIBRARY) + set(X264_FIND_QUIETLY TRUE) +endif (X264_INCLUDE_DIR AND X264_LIBRARY) + +find_path(X264_INCLUDE_DIR NAMES x264.h + PATH_SUFFIXES include + HINTS ${X264_ROOT}) +find_library(X264_LIBRARY + NAMES x264 + PATH_SUFFIXES lib + HINTS ${X264_ROOT}) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(x264 DEFAULT_MSG X264_LIBRARY X264_INCLUDE_DIR) + +if (X264_INCLUDE_DIR AND X264_LIBRARY) + set(X264_FOUND TRUE) + set(X264_LIBRARIES ${X264_LIBRARY}) +endif (X264_INCLUDE_DIR AND X264_LIBRARY) + +if (X264_FOUND) + if (NOT X264_FIND_QUIETLY) + message(STATUS "Found x264: ${X264_LIBRARIES}") + endif (NOT X264_FIND_QUIETLY) +else (X264_FOUND) + if (X264_FIND_REQUIRED) + message(FATAL_ERROR "x264 was not found") + endif(X264_FIND_REQUIRED) +endif (X264_FOUND) + +mark_as_advanced(X264_INCLUDE_DIR X264_LIBRARY) + diff --git a/libfreerdp/CMakeLists.txt b/libfreerdp/CMakeLists.txt index 112b75a0c..7e75370a9 100644 --- a/libfreerdp/CMakeLists.txt +++ b/libfreerdp/CMakeLists.txt @@ -165,6 +165,12 @@ if(WITH_JPEG) freerdp_library_add(${JPEG_LIBRARIES}) endif() +if(WITH_X264) + freerdp_definition_add(-DWITH_X264) + freerdp_include_directory_add(${X264_INCLUDE_DIR}) + freerdp_library_add(${X264_LIBRARIES}) +endif() + if(WITH_OPENH264) freerdp_definition_add(-DWITH_OPENH264) freerdp_include_directory_add(${OPENH264_INCLUDE_DIR}) diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index b170ca32d..15a21a128 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -58,6 +58,121 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_dummy = dummy_decompress }; +/** + * x264 subsystem + */ + +#ifdef WITH_X264 + +#define NAL_UNKNOWN X264_NAL_UNKNOWN +#define NAL_SLICE X264_NAL_SLICE +#define NAL_SLICE_DPA X264_NAL_SLICE_DPA +#define NAL_SLICE_DPB X264_NAL_SLICE_DPB +#define NAL_SLICE_DPC X264_NAL_SLICE_DPC +#define NAL_SLICE_IDR X264_NAL_SLICE_IDR +#define NAL_SEI X264_NAL_SEI +#define NAL_SPS X264_NAL_SPS +#define NAL_PPS X264_NAL_PPS +#define NAL_AUD X264_NAL_AUD +#define NAL_FILLER X264_NAL_FILLER + +#define NAL_PRIORITY_DISPOSABLE X264_NAL_PRIORITY_DISPOSABLE +#define NAL_PRIORITY_LOW X264_NAL_PRIORITY_LOW +#define NAL_PRIORITY_HIGH X264_NAL_PRIORITY_HIGH +#define NAL_PRIORITY_HIGHEST X264_NAL_PRIORITY_HIGHEST + +#include +#include + +struct _H264_CONTEXT_X264 +{ + void* dummy; +}; +typedef struct _H264_CONTEXT_X264 H264_CONTEXT_X264; + +static int x264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) +{ + //H264_CONTEXT_X264* sys = (H264_CONTEXT_X264*) h264->pSystemData; + + return 1; +} + +static int x264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize) +{ + //H264_CONTEXT_X264* sys = (H264_CONTEXT_X264*) h264->pSystemData; + + return 1; +} + +static void x264_uninit(H264_CONTEXT* h264) +{ + H264_CONTEXT_X264* sys = (H264_CONTEXT_X264*) h264->pSystemData; + + if (sys) + { + free(sys); + h264->pSystemData = NULL; + } +} + +static BOOL x264_init(H264_CONTEXT* h264) +{ + H264_CONTEXT_X264* sys; + + sys = (H264_CONTEXT_X264*) calloc(1, sizeof(H264_CONTEXT_X264)); + + if (!sys) + { + goto EXCEPTION; + } + + h264->pSystemData = (void*) sys; + + if (h264->Compressor) + { + + } + else + { + + } + + return TRUE; + +EXCEPTION: + x264_uninit(h264); + + return FALSE; +} + +static H264_CONTEXT_SUBSYSTEM g_Subsystem_x264 = +{ + "x264", + x264_init, + x264_uninit, + x264_decompress, + x264_compress +}; + +#undef NAL_UNKNOWN +#undef NAL_SLICE +#undef NAL_SLICE_DPA +#undef NAL_SLICE_DPB +#undef NAL_SLICE_DPC +#undef NAL_SLICE_IDR +#undef NAL_SEI +#undef NAL_SPS +#undef NAL_PPS +#undef NAL_AUD +#undef NAL_FILLER + +#undef NAL_PRIORITY_DISPOSABLE +#undef NAL_PRIORITY_LOW +#undef NAL_PRIORITY_HIGH +#undef NAL_PRIORITY_HIGHEST + +#endif + /** * OpenH264 subsystem */ @@ -90,7 +205,7 @@ static int openh264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSiz H264_CONTEXT_OPENH264* sys = (H264_CONTEXT_OPENH264*) h264->pSystemData; if (!sys->pDecoder) - return -1; + return -2001; /* * Decompress the image. The RDP host only seems to send I420 format. @@ -102,22 +217,38 @@ static int openh264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSiz ZeroMemory(&sBufferInfo, sizeof(sBufferInfo)); - state = (*sys->pDecoder)->DecodeFrame2( - sys->pDecoder, - pSrcData, - SrcSize, - h264->pYUVData, - &sBufferInfo); - - /** - * Calling DecodeFrame2 twice apparently works around Openh264 issue #1136: - * https://github.com/cisco/openh264/issues/1136 - * - * This is a hack, but it works and it is only necessary for the first frame. - */ + state = (*sys->pDecoder)->DecodeFrame2(sys->pDecoder, pSrcData, SrcSize, h264->pYUVData, &sBufferInfo); if (sBufferInfo.iBufferStatus != 1) - state = (*sys->pDecoder)->DecodeFrame2(sys->pDecoder, NULL, 0, h264->pYUVData, &sBufferInfo); + { + if (state == dsNoParamSets) + { + /* this happens on the first frame due to missing parameter sets */ + state = (*sys->pDecoder)->DecodeFrame2(sys->pDecoder, NULL, 0, h264->pYUVData, &sBufferInfo); + } + else if (state == dsErrorFree) + { + /* call DecodeFrame2 again to decode without delay */ + state = (*sys->pDecoder)->DecodeFrame2(sys->pDecoder, NULL, 0, h264->pYUVData, &sBufferInfo); + } + else + { + WLog_WARN(TAG, "DecodeFrame2 state: 0x%02X iBufferStatus: %d", state, sBufferInfo.iBufferStatus); + return -2002; + } + } + + if (sBufferInfo.iBufferStatus != 1) + { + WLog_WARN(TAG, "DecodeFrame2 iBufferStatus: %d", sBufferInfo.iBufferStatus); + return 0; + } + + if (state != dsErrorFree) + { + WLog_WARN(TAG, "DecodeFrame2 state: 0x%02X", state); + return -2003; + } pSystemBuffer = &sBufferInfo.UsrData.sSystemBuffer; @@ -128,17 +259,11 @@ static int openh264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSiz pSystemBuffer->iStride[0], pSystemBuffer->iStride[1]); #endif - if (state != 0) - return -1; - - if (sBufferInfo.iBufferStatus != 1) - return -2; - if (pSystemBuffer->iFormat != videoFormatI420) - return -1; + return -2004; if (!h264->pYUVData[0] || !h264->pYUVData[1] || !h264->pYUVData[2]) - return -1; + return -2005; h264->iStride[0] = pSystemBuffer->iStride[0]; h264->iStride[1] = pSystemBuffer->iStride[1]; @@ -165,8 +290,7 @@ static int openh264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstS if (!h264->pYUVData[0] || !h264->pYUVData[1] || !h264->pYUVData[2]) return -1; - if (sys->EncParamExt.iPicWidth != h264->width || - sys->EncParamExt.iPicHeight != h264->height) + if ((sys->EncParamExt.iPicWidth != h264->width) || (sys->EncParamExt.iPicHeight != h264->height)) { status = (*sys->pEncoder)->GetDefaultParams(sys->pEncoder, &sys->EncParamExt); @@ -190,6 +314,7 @@ static int openh264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstS sys->EncParamExt.sSpatialLayers[0].iVideoWidth = sys->EncParamExt.iPicWidth; sys->EncParamExt.sSpatialLayers[0].iVideoHeight = sys->EncParamExt.iPicHeight; sys->EncParamExt.sSpatialLayers[0].iMaxSpatialBitrate = sys->EncParamExt.iMaxBitrate; + switch (h264->RateControlMode) { case H264_RATECONTROL_VBR: @@ -197,6 +322,7 @@ static int openh264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstS sys->EncParamExt.iTargetBitrate = h264->BitRate; sys->EncParamExt.sSpatialLayers[0].iSpatialBitrate = sys->EncParamExt.iTargetBitrate; break; + case H264_RATECONTROL_CQP: sys->EncParamExt.iRCMode = RC_OFF_MODE; sys->EncParamExt.sSpatialLayers[0].iDLayerQp = h264->QP; @@ -259,6 +385,7 @@ static int openh264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstS } } break; + case H264_RATECONTROL_CQP: if (sys->EncParamExt.sSpatialLayers[0].iDLayerQp != h264->QP) { @@ -296,8 +423,10 @@ static int openh264_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstS WLog_ERR(TAG, "Failed to encode frame (status=%ld)", status); return status; } + *ppDstData = info.sLayerInfo[0].pBsBuf; *pDstSize = 0; + for (i = 0; i < info.iLayerNum; i++) { for (j = 0; j < info.sLayerInfo[i].iNalCount; j++) @@ -375,7 +504,7 @@ static BOOL openh264_init(H264_CONTEXT* h264) ZeroMemory(&sDecParam, sizeof(sDecParam)); sDecParam.eOutputColorFormat = videoFormatI420; sDecParam.eEcActiveIdc = ERROR_CON_FRAME_COPY; - sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_DEFAULT; + sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; status = (*sys->pDecoder)->Initialize(sys->pDecoder, &sDecParam); @@ -618,10 +747,10 @@ int h264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize, int width, height; BYTE* pYUVPoint[3]; RDPGFX_RECT16* rect; - primitives_t *prims = primitives_get(); + primitives_t* prims = primitives_get(); if (!h264) - return -1; + return -1001; #if 0 WLog_INFO(TAG, "h264_decompress: pSrcData=%p, SrcSize=%u, pDstData=%p, nDstStep=%d, nDstHeight=%d, numRegionRects=%d", @@ -629,9 +758,14 @@ int h264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize, #endif if (!(pDstData = *ppDstData)) - return -1; + return -1002; - if ((status = h264->subsystem->Decompress(h264, pSrcData, SrcSize)) < 0) + status = h264->subsystem->Decompress(h264, pSrcData, SrcSize); + + if (status == 0) + return 1; + + if (status < 0) return status; pYUVData = h264->pYUVData; @@ -641,17 +775,17 @@ int h264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize, { rect = &(regionRects[index]); - /* Check, if the ouput rectangle is valid in decoded h264 frame. */ + /* Check, if the output rectangle is valid in decoded h264 frame. */ if ((rect->right > h264->width) || (rect->left > h264->width)) - return -1; + return -1003; if ((rect->top > h264->height) || (rect->bottom > h264->height)) - return -1; + return -1004; /* Check, if the output rectangle is valid in destination buffer. */ if ((rect->right > nDstWidth) || (rect->left > nDstWidth)) - return -1; + return -1005; if ((rect->bottom > nDstHeight) || (rect->top > nDstHeight)) - return -1; + return -1006; width = rect->right - rect->left; height = rect->bottom - rect->top; @@ -682,7 +816,7 @@ int h264_compress(H264_CONTEXT* h264, BYTE* pSrcData, DWORD SrcFormat, int status = -1; prim_size_t roi; int nWidth, nHeight; - primitives_t *prims = primitives_get(); + primitives_t* prims = primitives_get(); if (!h264) return -1; @@ -743,6 +877,14 @@ BOOL h264_context_init(H264_CONTEXT* h264) } #endif +#ifdef WITH_X264 + if (g_Subsystem_x264.Init(h264)) + { + h264->subsystem = &g_Subsystem_x264; + return TRUE; + } +#endif + return FALSE; } From 99228ee254b66776ad690bf8d7f00fba20fc340c Mon Sep 17 00:00:00 2001 From: Clive Stevens Date: Wed, 29 Jul 2015 14:08:31 +0100 Subject: [PATCH 08/14] Fix race condition in freerdp_channels_client_load If g_pInterface is set to NULL outside the critical section in one thread while another thread is in the critical section, the channel client context allocated in the VirtualChannelEntry won't end up getting freed. The channel client context is normally loaded from g_pInterface in FreeRDP_VirtualChannelInit and stored in the pChannelInitData, then copied from there onto the pChannelOpenData structure in FreeRDP_VirtualChannelOpen and finally freed in freerdp_channels_free. --- libfreerdp/core/client.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libfreerdp/core/client.c b/libfreerdp/core/client.c index 31e363a5c..8e20787c7 100644 --- a/libfreerdp/core/client.c +++ b/libfreerdp/core/client.c @@ -720,7 +720,6 @@ int freerdp_channels_client_load(rdpChannels* channels, rdpSettings* settings, P EntryPoints.pVirtualChannelClose = FreeRDP_VirtualChannelClose; EntryPoints.pVirtualChannelWrite = FreeRDP_VirtualChannelWrite; - g_pInterface = NULL; EntryPoints.MagicNumber = FREERDP_CHANNEL_MAGIC_NUMBER; EntryPoints.ppInterface = &g_pInterface; EntryPoints.pExtendedData = data; @@ -731,6 +730,7 @@ int freerdp_channels_client_load(rdpChannels* channels, rdpSettings* settings, P EnterCriticalSection(&g_channels_lock); + g_pInterface = NULL; g_ChannelInitData.channels = channels; status = pChannelClientData->entry((PCHANNEL_ENTRY_POINTS) &EntryPoints); From 4dc47d48676c315546b0a8b93fdb2d036b228ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 29 Jul 2015 16:33:04 -0400 Subject: [PATCH 09/14] libfreerdp-codec: start H.264 MediaFoundation support --- libfreerdp/codec/h264.c | 247 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index 15a21a128..85a052a08 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -58,6 +58,245 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_dummy = dummy_decompress }; +/** + * Media Foundation subsystem + */ + +#ifdef _WIN32 + +#include +#include +#include +#include + +#undef DEFINE_GUID +#define INITGUID +#include + +DEFINE_GUID(CLSID_CMSH264DecoderMFT,0x62CE7E72,0x4C71,0x4d20,0xB1,0x5D,0x45,0x28,0x31,0xA8,0x7D,0x9D); +DEFINE_GUID(IID_IMFTransform,0xbf94c121,0x5b05,0x4e6f,0x80,0x00,0xba,0x59,0x89,0x61,0x41,0x4d); +DEFINE_GUID(MF_MT_MAJOR_TYPE,0x48eba18e,0xf8c9,0x4687,0xbf,0x11,0x0a,0x74,0xc9,0xf9,0x6a,0x8f); +DEFINE_GUID(MF_MT_SUBTYPE,0xf7e34c9a,0x42e8,0x4714,0xb7,0x4b,0xcb,0x29,0xd7,0x2c,0x35,0xe5); +DEFINE_GUID(MFMediaType_Video,0x73646976,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); +DEFINE_GUID(IID_ICodecAPI,0x901db4c7,0x31ce,0x41a2,0x85,0xdc,0x8f,0xa0,0xbf,0x41,0xb8,0xda); +DEFINE_GUID(CODECAPI_AVLowLatencyMode,0x9c27891a,0xed7a,0x40e1,0x88,0xe8,0xb2,0x27,0x27,0xa0,0x24,0xee); + +#ifndef FCC +#define FCC(ch4) ((((DWORD)(ch4) & 0xFF) << 24) | \ + (((DWORD)(ch4) & 0xFF00) << 8) | \ + (((DWORD)(ch4) & 0xFF0000) >> 8) | \ + (((DWORD)(ch4) & 0xFF000000) >> 24)) +#endif + +DEFINE_GUID(MFVideoFormat_H264, FCC('H264'), 0x0000,0x0010,0x80,0x00,0x00,0xaa,0x00,0x38,0x9b,0x71); +DEFINE_GUID(MFVideoFormat_IYUV, FCC('IYUV'), 0x0000,0x0010,0x80,0x00,0x00,0xaa,0x00,0x38,0x9b,0x71); + +#if 0 +typedef void* IMFSample; +typedef void* IMFTransform; +typedef void* IMFMediaBuffer; +typedef void* IMFMediaType; +typedef void* IMFDXGIDeviceManager; +#endif + +typedef HRESULT (__stdcall * pfnMFStartup)(ULONG Version, DWORD dwFlags); +typedef HRESULT (__stdcall * pfnMFShutdown)(void); +typedef HRESULT (__stdcall * pfnMFCreateSample)(IMFSample** ppIMFSample); +typedef HRESULT (__stdcall * pfnMFCreateMemoryBuffer)(DWORD cbMaxLength, IMFMediaBuffer** ppBuffer); +typedef HRESULT (__stdcall * pfnMFCreateMediaType)(IMFMediaType** ppMFType); +typedef HRESULT (__stdcall * pfnMFCreateDXGIDeviceManager)(UINT* pResetToken, IMFDXGIDeviceManager** ppDXVAManager); + +struct _H264_CONTEXT_MF +{ + ICodecAPI* codecApi; + IMFTransform* transform; + IMFMediaType* inputType; + IMFMediaType* outputType; + HMODULE mfplat; + pfnMFStartup MFStartup; + pfnMFShutdown MFShutdown; + pfnMFCreateSample MFCreateSample; + pfnMFCreateMemoryBuffer MFCreateMemoryBuffer; + pfnMFCreateMediaType MFCreateMediaType; + pfnMFCreateDXGIDeviceManager MFCreateDXGIDeviceManager; +}; +typedef struct _H264_CONTEXT_MF H264_CONTEXT_MF; + +static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) +{ + H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; + + return 1; +} + +static int mf_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize) +{ + H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; + + return 1; +} + +static HRESULT mf_find_output_type(H264_CONTEXT_MF* sys, const GUID* guid, IMFMediaType** ppMediaType) +{ + DWORD idx = 0; + GUID mediaGuid; + HRESULT hr = S_OK; + IMFMediaType* pMediaType = NULL; + + while (1) + { + hr = sys->transform->lpVtbl->GetOutputAvailableType(sys->transform, 0, idx, &pMediaType); + + if (FAILED(hr)) + break; + + pMediaType->lpVtbl->GetGUID(pMediaType, &MF_MT_SUBTYPE, &mediaGuid); + + if (IsEqualGUID(&mediaGuid, guid)) + { + *ppMediaType = pMediaType; + return S_OK; + } + + idx++; + } + + return hr; +} + +static void mf_uninit(H264_CONTEXT* h264) +{ + H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; + + if (sys) + { + if (sys->transform) + { + sys->transform->lpVtbl->Release(sys->transform); + sys->transform = NULL; + } + + if (sys->mfplat) + { + FreeLibrary(sys->mfplat); + sys->mfplat = NULL; + } + + free(sys); + h264->pSystemData = NULL; + } +} + +static BOOL mf_init(H264_CONTEXT* h264) +{ + HRESULT hr; + H264_CONTEXT_MF* sys; + + sys = (H264_CONTEXT_MF*) calloc(1, sizeof(H264_CONTEXT_MF)); + + if (!sys) + goto EXCEPTION; + + h264->pSystemData = (void*) sys; + + /* http://decklink-sdk-delphi.googlecode.com/svn/trunk/Blackmagic%20DeckLink%20SDK%209.7/Win/Samples/Streaming/StreamingPreview/DecoderMF.cpp */ + + sys->mfplat = LoadLibraryA("mfplat.dll"); + + if (!sys->mfplat) + goto EXCEPTION; + + sys->MFStartup = (pfnMFStartup) GetProcAddress(sys->mfplat, "MFStartup"); + sys->MFShutdown = (pfnMFShutdown) GetProcAddress(sys->mfplat, "MFShutdown"); + sys->MFCreateSample = (pfnMFCreateSample) GetProcAddress(sys->mfplat, "MFCreateSample"); + sys->MFCreateMemoryBuffer = (pfnMFCreateMemoryBuffer) GetProcAddress(sys->mfplat, "MFCreateMemoryBuffer"); + sys->MFCreateMediaType = (pfnMFCreateMediaType) GetProcAddress(sys->mfplat, "MFCreateMediaType"); + sys->MFCreateDXGIDeviceManager = (pfnMFCreateDXGIDeviceManager) GetProcAddress(sys->mfplat, "MFCreateDXGIDeviceManager"); + + if (!sys->MFStartup || !sys->MFShutdown || !sys->MFCreateSample || + !sys->MFCreateMemoryBuffer || !sys->MFCreateMediaType || !sys->MFCreateDXGIDeviceManager) + goto EXCEPTION; + + CoInitializeEx(NULL, 0); + + if (h264->Compressor) + { + + } + else + { + VARIANT var = { 0 }; + + hr = CoCreateInstance(&CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, &IID_IMFTransform, (void**) &sys->transform); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = sys->transform->lpVtbl->QueryInterface(sys->transform, &IID_ICodecAPI, (void**) &sys->codecApi); + + if (FAILED(hr)) + goto EXCEPTION; + + var.vt = VT_UI4; + var.ulVal = 1; + + hr = sys->codecApi->lpVtbl->SetValue(sys->codecApi, &CODECAPI_AVLowLatencyMode, &var); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = sys->MFCreateMediaType(&sys->inputType); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = sys->inputType->lpVtbl->SetGUID(sys->inputType, &MF_MT_MAJOR_TYPE, &MFMediaType_Video); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = sys->inputType->lpVtbl->SetGUID(sys->inputType, &MF_MT_SUBTYPE, &MFVideoFormat_H264); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = sys->transform->lpVtbl->SetInputType(sys->transform, 0, sys->inputType, 0); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = mf_find_output_type(sys, &MFVideoFormat_IYUV, &sys->outputType); + + if (FAILED(hr)) + goto EXCEPTION; + + hr = sys->transform->lpVtbl->SetOutputType(sys->transform, 0, sys->outputType, 0); + + if (FAILED(hr)) + goto EXCEPTION; + } + + fprintf(stderr, "H264 success\n"); + return TRUE; + +EXCEPTION: + fprintf(stderr, "H264 failure\n"); + mf_uninit(h264); + + return FALSE; +} + +static H264_CONTEXT_SUBSYSTEM g_Subsystem_MF = +{ + "MediaFoundation", + mf_init, + mf_uninit, + mf_decompress, + mf_compress +}; + +#endif + /** * x264 subsystem */ @@ -861,6 +1100,14 @@ error_1: BOOL h264_context_init(H264_CONTEXT* h264) { +#ifdef _WIN32 + if (g_Subsystem_MF.Init(h264)) + { + h264->subsystem = &g_Subsystem_MF; + return TRUE; + } +#endif + #ifdef WITH_LIBAVCODEC if (g_Subsystem_libavcodec.Init(h264)) { From 008d9f3b79d1729b523d718a82792bd3009438a2 Mon Sep 17 00:00:00 2001 From: Martin Haimberger Date: Thu, 30 Jul 2015 07:02:36 -0700 Subject: [PATCH 10/14] fixed misc *_free functions to accept NULL Following types of collections support now NULL in the free call: * ArrayList * BitStream * ContdownEvent * Dictionary * KeyValuePair * MessageQueue --- winpr/libwinpr/utils/collections/ArrayList.c | 3 +++ winpr/libwinpr/utils/collections/BitStream.c | 3 +++ winpr/libwinpr/utils/collections/CountdownEvent.c | 3 +++ winpr/libwinpr/utils/collections/Dictionary.c | 3 +++ winpr/libwinpr/utils/collections/KeyValuePair.c | 3 +++ winpr/libwinpr/utils/collections/MessageQueue.c | 3 +++ 6 files changed, 18 insertions(+) diff --git a/winpr/libwinpr/utils/collections/ArrayList.c b/winpr/libwinpr/utils/collections/ArrayList.c index c30cc96f0..6d507ca1d 100644 --- a/winpr/libwinpr/utils/collections/ArrayList.c +++ b/winpr/libwinpr/utils/collections/ArrayList.c @@ -470,6 +470,9 @@ out_free: void ArrayList_Free(wArrayList *arrayList) { + if (!arrayList) + return; + ArrayList_Clear(arrayList); DeleteCriticalSection(&arrayList->lock); free(arrayList->array); diff --git a/winpr/libwinpr/utils/collections/BitStream.c b/winpr/libwinpr/utils/collections/BitStream.c index 26e739cd5..8dfb776d9 100644 --- a/winpr/libwinpr/utils/collections/BitStream.c +++ b/winpr/libwinpr/utils/collections/BitStream.c @@ -348,5 +348,8 @@ wBitStream* BitStream_New() void BitStream_Free(wBitStream* bs) { + if (!bs) + return; + free(bs); } diff --git a/winpr/libwinpr/utils/collections/CountdownEvent.c b/winpr/libwinpr/utils/collections/CountdownEvent.c index a29121a88..fabaa8d3d 100644 --- a/winpr/libwinpr/utils/collections/CountdownEvent.c +++ b/winpr/libwinpr/utils/collections/CountdownEvent.c @@ -182,6 +182,9 @@ fail_critical_section: void CountdownEvent_Free(wCountdownEvent* countdown) { + if (!countdown) + return; + DeleteCriticalSection(&countdown->lock); CloseHandle(countdown->event); diff --git a/winpr/libwinpr/utils/collections/Dictionary.c b/winpr/libwinpr/utils/collections/Dictionary.c index 0cb5b8bcd..8f932782c 100644 --- a/winpr/libwinpr/utils/collections/Dictionary.c +++ b/winpr/libwinpr/utils/collections/Dictionary.c @@ -128,6 +128,9 @@ wDictionary* Dictionary_New(BOOL synchronized) void Dictionary_Free(wDictionary* dictionary) { + if (!dictionary) + return; + free(dictionary); } diff --git a/winpr/libwinpr/utils/collections/KeyValuePair.c b/winpr/libwinpr/utils/collections/KeyValuePair.c index 488583101..19a211e73 100644 --- a/winpr/libwinpr/utils/collections/KeyValuePair.c +++ b/winpr/libwinpr/utils/collections/KeyValuePair.c @@ -46,5 +46,8 @@ wKeyValuePair* KeyValuePair_New(void* key, void* value) void KeyValuePair_Free(wKeyValuePair* keyValuePair) { + if (!keyValuePair) + return; + free(keyValuePair); } diff --git a/winpr/libwinpr/utils/collections/MessageQueue.c b/winpr/libwinpr/utils/collections/MessageQueue.c index a1fc39b84..e902562d9 100644 --- a/winpr/libwinpr/utils/collections/MessageQueue.c +++ b/winpr/libwinpr/utils/collections/MessageQueue.c @@ -223,6 +223,9 @@ error_array: void MessageQueue_Free(wMessageQueue* queue) { + if (!queue) + return; + CloseHandle(queue->event); DeleteCriticalSection(&queue->lock); From 32d1053abe49d5d4334d12b005694dd6e5004a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Thu, 30 Jul 2015 14:07:35 -0400 Subject: [PATCH 11/14] libfreerdp-codec: add more Media Foundation H.264 code --- libfreerdp/codec/h264.c | 84 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index 85a052a08..874360cf1 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -112,6 +112,7 @@ struct _H264_CONTEXT_MF IMFTransform* transform; IMFMediaType* inputType; IMFMediaType* outputType; + IMFSample* sample; HMODULE mfplat; pfnMFStartup MFStartup; pfnMFShutdown MFShutdown; @@ -124,9 +125,58 @@ typedef struct _H264_CONTEXT_MF H264_CONTEXT_MF; static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) { + HRESULT hr; + BYTE* pbBuffer = NULL; + DWORD cbMaxLength = 0; + DWORD cbCurrentLength = 0; + IMFMediaBuffer* mediaBuffer = NULL; H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; + hr = sys->MFCreateMemoryBuffer(SrcSize, &mediaBuffer); + + if (FAILED(hr)) + goto error; + + hr = mediaBuffer->lpVtbl->Lock(mediaBuffer, &pbBuffer, &cbMaxLength, &cbCurrentLength); + + if (FAILED(hr)) + goto error; + + CopyMemory(pbBuffer, pSrcData, SrcSize); + + hr = mediaBuffer->lpVtbl->SetCurrentLength(mediaBuffer, SrcSize); + + if (FAILED(hr)) + goto error; + + hr = mediaBuffer->lpVtbl->Unlock(mediaBuffer); + + if (FAILED(hr)) + goto error; + + hr = sys->MFCreateSample(&sys->sample); + + if (FAILED(hr)) + goto error; + + sys->sample->lpVtbl->AddBuffer(sys->sample, mediaBuffer); + + if (FAILED(hr)) + goto error; + + hr = sys->transform->lpVtbl->ProcessInput(sys->transform, 0, sys->sample, 0); + + if (FAILED(hr)) + goto error; + + //mediaBuffer->lpVtbl->Release(mediaBuffer); + //sys->sample->lpVtbl->Release(sys->sample); + return 1; + +error: + fprintf(stderr, "mf_decompress error\n"); + return -1; } static int mf_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize) @@ -172,16 +222,36 @@ static void mf_uninit(H264_CONTEXT* h264) { if (sys->transform) { - sys->transform->lpVtbl->Release(sys->transform); + //sys->transform->lpVtbl->Release(sys->transform); sys->transform = NULL; } + if (sys->codecApi) + { + //sys->codecApi->lpVtbl->Release(sys->codecApi); + sys->codecApi = NULL; + } + + if (sys->inputType) + { + sys->inputType->lpVtbl->Release(sys->inputType); + sys->inputType = NULL; + } + + if (sys->outputType) + { + sys->outputType->lpVtbl->Release(sys->outputType); + sys->outputType = NULL; + } + if (sys->mfplat) { FreeLibrary(sys->mfplat); sys->mfplat = NULL; } + sys->MFShutdown(); + free(sys); h264->pSystemData = NULL; } @@ -217,7 +287,7 @@ static BOOL mf_init(H264_CONTEXT* h264) !sys->MFCreateMemoryBuffer || !sys->MFCreateMediaType || !sys->MFCreateDXGIDeviceManager) goto EXCEPTION; - CoInitializeEx(NULL, 0); + CoInitializeEx(NULL, COINIT_APARTMENTTHREADED); if (h264->Compressor) { @@ -227,6 +297,11 @@ static BOOL mf_init(H264_CONTEXT* h264) { VARIANT var = { 0 }; + hr = sys->MFStartup(MF_VERSION, 0); + + if (FAILED(hr)) + goto EXCEPTION; + hr = CoCreateInstance(&CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, &IID_IMFTransform, (void**) &sys->transform); if (FAILED(hr)) @@ -275,14 +350,11 @@ static BOOL mf_init(H264_CONTEXT* h264) if (FAILED(hr)) goto EXCEPTION; } - - fprintf(stderr, "H264 success\n"); return TRUE; EXCEPTION: - fprintf(stderr, "H264 failure\n"); + fprintf(stderr, "mf_init failure\n"); mf_uninit(h264); - return FALSE; } From d9525022856334eb9a45fe606b3a3a920b5808de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Mon, 3 Aug 2015 16:21:48 -0400 Subject: [PATCH 12/14] libfreerdp-codec: initial Media Foundation H.264 decoder support --- libfreerdp/codec/h264.c | 368 +++++++++++++++++++++++++++++++--------- 1 file changed, 289 insertions(+), 79 deletions(-) diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index 874360cf1..eee22b85f 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -65,6 +65,7 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_dummy = #ifdef _WIN32 #include +#include #include #include #include @@ -74,12 +75,20 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_dummy = #include DEFINE_GUID(CLSID_CMSH264DecoderMFT,0x62CE7E72,0x4C71,0x4d20,0xB1,0x5D,0x45,0x28,0x31,0xA8,0x7D,0x9D); +DEFINE_GUID(CLSID_VideoProcessorMFT,0x88753b26,0x5b24,0x49bd,0xb2,0xe7,0x0c,0x44,0x5c,0x78,0xc9,0x82); DEFINE_GUID(IID_IMFTransform,0xbf94c121,0x5b05,0x4e6f,0x80,0x00,0xba,0x59,0x89,0x61,0x41,0x4d); DEFINE_GUID(MF_MT_MAJOR_TYPE,0x48eba18e,0xf8c9,0x4687,0xbf,0x11,0x0a,0x74,0xc9,0xf9,0x6a,0x8f); +DEFINE_GUID(MF_MT_FRAME_SIZE,0x1652c33d,0xd6b2,0x4012,0xb8,0x34,0x72,0x03,0x08,0x49,0xa3,0x7d); +DEFINE_GUID(MF_MT_DEFAULT_STRIDE,0x644b4e48,0x1e02,0x4516,0xb0,0xeb,0xc0,0x1c,0xa9,0xd4,0x9a,0xc6); DEFINE_GUID(MF_MT_SUBTYPE,0xf7e34c9a,0x42e8,0x4714,0xb7,0x4b,0xcb,0x29,0xd7,0x2c,0x35,0xe5); +DEFINE_GUID(MF_XVP_DISABLE_FRC,0x2c0afa19,0x7a97,0x4d5a,0x9e,0xe8,0x16,0xd4,0xfc,0x51,0x8d,0x8c); DEFINE_GUID(MFMediaType_Video,0x73646976,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); +DEFINE_GUID(MFVideoFormat_RGB32,22,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); +DEFINE_GUID(MFVideoFormat_ARGB32,21,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); DEFINE_GUID(IID_ICodecAPI,0x901db4c7,0x31ce,0x41a2,0x85,0xdc,0x8f,0xa0,0xbf,0x41,0xb8,0xda); DEFINE_GUID(CODECAPI_AVLowLatencyMode,0x9c27891a,0xed7a,0x40e1,0x88,0xe8,0xb2,0x27,0x27,0xa0,0x24,0xee); +DEFINE_GUID(CODECAPI_AVDecVideoMaxCodedWidth,0x5ae557b8,0x77af,0x41f5,0x9f,0xa6,0x4d,0xb2,0xfe,0x1d,0x4b,0xca); +DEFINE_GUID(CODECAPI_AVDecVideoMaxCodedHeight,0x7262a16a,0xd2dc,0x4e75,0x9b,0xa8,0x65,0xc0,0xc6,0xd3,0x2b,0x13); #ifndef FCC #define FCC(ch4) ((((DWORD)(ch4) & 0xFF) << 24) | \ @@ -113,6 +122,8 @@ struct _H264_CONTEXT_MF IMFMediaType* inputType; IMFMediaType* outputType; IMFSample* sample; + UINT32 frameWidth; + UINT32 frameHeight; HMODULE mfplat; pfnMFStartup MFStartup; pfnMFShutdown MFShutdown; @@ -123,69 +134,6 @@ struct _H264_CONTEXT_MF }; typedef struct _H264_CONTEXT_MF H264_CONTEXT_MF; -static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) -{ - HRESULT hr; - BYTE* pbBuffer = NULL; - DWORD cbMaxLength = 0; - DWORD cbCurrentLength = 0; - IMFMediaBuffer* mediaBuffer = NULL; - H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; - - hr = sys->MFCreateMemoryBuffer(SrcSize, &mediaBuffer); - - if (FAILED(hr)) - goto error; - - hr = mediaBuffer->lpVtbl->Lock(mediaBuffer, &pbBuffer, &cbMaxLength, &cbCurrentLength); - - if (FAILED(hr)) - goto error; - - CopyMemory(pbBuffer, pSrcData, SrcSize); - - hr = mediaBuffer->lpVtbl->SetCurrentLength(mediaBuffer, SrcSize); - - if (FAILED(hr)) - goto error; - - hr = mediaBuffer->lpVtbl->Unlock(mediaBuffer); - - if (FAILED(hr)) - goto error; - - hr = sys->MFCreateSample(&sys->sample); - - if (FAILED(hr)) - goto error; - - sys->sample->lpVtbl->AddBuffer(sys->sample, mediaBuffer); - - if (FAILED(hr)) - goto error; - - hr = sys->transform->lpVtbl->ProcessInput(sys->transform, 0, sys->sample, 0); - - if (FAILED(hr)) - goto error; - - //mediaBuffer->lpVtbl->Release(mediaBuffer); - //sys->sample->lpVtbl->Release(sys->sample); - - return 1; - -error: - fprintf(stderr, "mf_decompress error\n"); - return -1; -} - -static int mf_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize) -{ - H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; - - return 1; -} - static HRESULT mf_find_output_type(H264_CONTEXT_MF* sys, const GUID* guid, IMFMediaType** ppMediaType) { DWORD idx = 0; @@ -214,6 +162,264 @@ static HRESULT mf_find_output_type(H264_CONTEXT_MF* sys, const GUID* guid, IMFMe return hr; } +static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) +{ + HRESULT hr; + BYTE* pbBuffer = NULL; + DWORD cbMaxLength = 0; + DWORD cbCurrentLength = 0; + DWORD outputStatus = 0; + IMFSample* inputSample = NULL; + IMFSample* outputSample = NULL; + IMFMediaBuffer* inputBuffer = NULL; + IMFMediaBuffer* outputBuffer = NULL; + MFT_OUTPUT_STREAM_INFO streamInfo; + MFT_OUTPUT_DATA_BUFFER outputDataBuffer; + H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; + + hr = sys->MFCreateMemoryBuffer(SrcSize, &inputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateMemoryBuffer failure: 0x%04X", hr); + goto error; + } + + hr = inputBuffer->lpVtbl->Lock(inputBuffer, &pbBuffer, &cbMaxLength, &cbCurrentLength); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "Lock failure: 0x%04X", hr); + goto error; + } + + CopyMemory(pbBuffer, pSrcData, SrcSize); + + hr = inputBuffer->lpVtbl->SetCurrentLength(inputBuffer, SrcSize); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "SetCurrentLength failure: 0x%04X", hr); + goto error; + } + + hr = inputBuffer->lpVtbl->Unlock(inputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "Unlock failure: 0x%04X", hr); + goto error; + } + + hr = sys->MFCreateSample(&inputSample); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateSample failure: 0x%04X", hr); + goto error; + } + + inputSample->lpVtbl->AddBuffer(inputSample, inputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "AddBuffer failure: 0x%04X", hr); + goto error; + } + + hr = sys->transform->lpVtbl->ProcessInput(sys->transform, 0, inputSample, 0); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "ProcessInput failure: 0x%04X", hr); + goto error; + } + + hr = sys->MFCreateSample(&outputSample); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateSample failure: 0x%04X", hr); + goto error; + } + + hr = sys->transform->lpVtbl->GetOutputStreamInfo(sys->transform, 0, &streamInfo); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "GetOutputStreamInfo failure: 0x%04X", hr); + goto error; + } + + hr = sys->MFCreateMemoryBuffer(streamInfo.cbSize, &outputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateMemoryBuffer failure: 0x%04X", hr); + goto error; + } + + outputSample->lpVtbl->AddBuffer(outputSample, outputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "AddBuffer failure: 0x%04X", hr); + goto error; + } + + outputDataBuffer.dwStreamID = 0; + outputDataBuffer.dwStatus = 0; + outputDataBuffer.pEvents = NULL; + outputDataBuffer.pSample = outputSample; + + hr = sys->transform->lpVtbl->ProcessOutput(sys->transform, 0, 1, &outputDataBuffer, &outputStatus); + + if (hr == MF_E_TRANSFORM_STREAM_CHANGE) + { + BYTE* pYUVData; + int offset = 0; + UINT32 stride = 0; + UINT64 frameSize = 0; + IMFAttributes* attributes = NULL; + + fprintf(stderr, "MF_E_TRANSFORM_STREAM_CHANGE\n"); + + hr = mf_find_output_type(sys, &MFVideoFormat_IYUV, &sys->outputType); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "mf_find_output_type failure: 0x%04X", hr); + goto error; + } + + hr = sys->transform->lpVtbl->SetOutputType(sys->transform, 0, sys->outputType, 0); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "SetOutputType failure: 0x%04X", hr); + goto error; + } + + hr = sys->outputType->lpVtbl->GetUINT64(sys->outputType, &MF_MT_FRAME_SIZE, &frameSize); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "GetUINT64(MF_MT_FRAME_SIZE) failure: 0x%04X", hr); + goto error; + } + + sys->frameWidth = (UINT32) (frameSize >> 32); + sys->frameHeight = (UINT32) frameSize; + + hr = sys->outputType->lpVtbl->GetUINT32(sys->outputType, &MF_MT_DEFAULT_STRIDE, &stride); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "GetUINT32(MF_MT_DEFAULT_STRIDE) failure: 0x%04X", hr); + goto error; + } + + fprintf(stderr, "frame width: %d height: %d stride: %d\n", sys->frameWidth, sys->frameHeight, stride); + + h264->iStride[0] = stride; + h264->iStride[1] = stride / 2; + h264->iStride[2] = stride / 2; + + pYUVData = (BYTE*) calloc(1, 2 * stride * sys->frameHeight); + + h264->pYUVData[0] = &pYUVData[offset]; + pYUVData += h264->iStride[0] * sys->frameHeight; + + h264->pYUVData[1] = &pYUVData[offset]; + pYUVData += h264->iStride[1] * (sys->frameHeight / 2); + + h264->pYUVData[2] = &pYUVData[offset]; + pYUVData += h264->iStride[2] * (sys->frameHeight / 2); + + h264->width = sys->frameWidth; + h264->height = sys->frameHeight; + } + else if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) + { + fprintf(stderr, "MF_E_TRANSFORM_NEED_MORE_INPUT\n"); + } + else if (FAILED(hr)) + { + WLog_ERR(TAG, "ProcessOutput failure: 0x%04X", hr); + goto error; + } + else + { + int offset = 0; + BYTE* buffer = NULL; + DWORD bufferCount = 0; + DWORD cbMaxLength = 0; + DWORD cbCurrentLength = 0; + + hr = outputSample->lpVtbl->GetBufferCount(outputSample, &bufferCount); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "GetBufferCount failure: 0x%04X", hr); + goto error; + } + + hr = outputSample->lpVtbl->ConvertToContiguousBuffer(outputSample, &outputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "ConvertToContiguousBuffer failure: 0x%04X", hr); + goto error; + } + + hr = outputBuffer->lpVtbl->Lock(outputBuffer, &buffer, &cbMaxLength, &cbCurrentLength); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "Lock failure: 0x%04X", hr); + goto error; + } + + CopyMemory(h264->pYUVData[0], &buffer[offset], h264->iStride[0] * sys->frameHeight); + offset += h264->iStride[0] * sys->frameHeight; + + CopyMemory(h264->pYUVData[1], &buffer[offset], h264->iStride[1] * (sys->frameHeight / 2)); + offset += h264->iStride[1] * (sys->frameHeight / 2); + + CopyMemory(h264->pYUVData[2], &buffer[offset], h264->iStride[2] * (sys->frameHeight / 2)); + offset += h264->iStride[2] * (sys->frameHeight / 2); + + hr = outputBuffer->lpVtbl->Unlock(outputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "Unlock failure: 0x%04X", hr); + goto error; + } + } + + inputSample->lpVtbl->DeleteAllItems(inputSample); + inputSample->lpVtbl->Release(inputSample); + inputBuffer->lpVtbl->Release(inputBuffer); + + outputSample->lpVtbl->DeleteAllItems(outputSample); + outputSample->lpVtbl->Release(outputSample); + outputBuffer->lpVtbl->Release(outputBuffer); + + return 1; + +error: + fprintf(stderr, "mf_decompress error\n"); + return -1; +} + +static int mf_compress(H264_CONTEXT* h264, BYTE** ppDstData, UINT32* pDstSize) +{ + H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; + + return 1; +} + static void mf_uninit(H264_CONTEXT* h264) { H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; @@ -250,6 +456,10 @@ static void mf_uninit(H264_CONTEXT* h264) sys->mfplat = NULL; } + free(h264->pYUVData[0]); + h264->pYUVData[0] = h264->pYUVData[1] = h264->pYUVData[2] = NULL; + h264->iStride[0] = h264->iStride[1] = h264->iStride[2] = 0; + sys->MFShutdown(); free(sys); @@ -265,7 +475,7 @@ static BOOL mf_init(H264_CONTEXT* h264) sys = (H264_CONTEXT_MF*) calloc(1, sizeof(H264_CONTEXT_MF)); if (!sys) - goto EXCEPTION; + goto error; h264->pSystemData = (void*) sys; @@ -274,7 +484,7 @@ static BOOL mf_init(H264_CONTEXT* h264) sys->mfplat = LoadLibraryA("mfplat.dll"); if (!sys->mfplat) - goto EXCEPTION; + goto error; sys->MFStartup = (pfnMFStartup) GetProcAddress(sys->mfplat, "MFStartup"); sys->MFShutdown = (pfnMFShutdown) GetProcAddress(sys->mfplat, "MFShutdown"); @@ -283,9 +493,9 @@ static BOOL mf_init(H264_CONTEXT* h264) sys->MFCreateMediaType = (pfnMFCreateMediaType) GetProcAddress(sys->mfplat, "MFCreateMediaType"); sys->MFCreateDXGIDeviceManager = (pfnMFCreateDXGIDeviceManager) GetProcAddress(sys->mfplat, "MFCreateDXGIDeviceManager"); - if (!sys->MFStartup || !sys->MFShutdown || !sys->MFCreateSample || - !sys->MFCreateMemoryBuffer || !sys->MFCreateMediaType || !sys->MFCreateDXGIDeviceManager) - goto EXCEPTION; + if (!sys->MFStartup || !sys->MFShutdown || !sys->MFCreateSample || !sys->MFCreateMemoryBuffer || + !sys->MFCreateMediaType || !sys->MFCreateDXGIDeviceManager) + goto error; CoInitializeEx(NULL, COINIT_APARTMENTTHREADED); @@ -300,17 +510,17 @@ static BOOL mf_init(H264_CONTEXT* h264) hr = sys->MFStartup(MF_VERSION, 0); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = CoCreateInstance(&CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, &IID_IMFTransform, (void**) &sys->transform); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = sys->transform->lpVtbl->QueryInterface(sys->transform, &IID_ICodecAPI, (void**) &sys->codecApi); if (FAILED(hr)) - goto EXCEPTION; + goto error; var.vt = VT_UI4; var.ulVal = 1; @@ -318,41 +528,41 @@ static BOOL mf_init(H264_CONTEXT* h264) hr = sys->codecApi->lpVtbl->SetValue(sys->codecApi, &CODECAPI_AVLowLatencyMode, &var); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = sys->MFCreateMediaType(&sys->inputType); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = sys->inputType->lpVtbl->SetGUID(sys->inputType, &MF_MT_MAJOR_TYPE, &MFMediaType_Video); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = sys->inputType->lpVtbl->SetGUID(sys->inputType, &MF_MT_SUBTYPE, &MFVideoFormat_H264); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = sys->transform->lpVtbl->SetInputType(sys->transform, 0, sys->inputType, 0); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = mf_find_output_type(sys, &MFVideoFormat_IYUV, &sys->outputType); if (FAILED(hr)) - goto EXCEPTION; + goto error; hr = sys->transform->lpVtbl->SetOutputType(sys->transform, 0, sys->outputType, 0); if (FAILED(hr)) - goto EXCEPTION; + goto error; } return TRUE; -EXCEPTION: +error: fprintf(stderr, "mf_init failure\n"); mf_uninit(h264); return FALSE; From 8fb9535e4a583613a580e72e0bee13167bf97b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Tue, 4 Aug 2015 10:26:13 -0400 Subject: [PATCH 13/14] libfreerdp-codec: fix leaks in H.264 Media Foundation decoder --- libfreerdp/codec/h264.c | 169 +++++++++++++++++++++++++++++----------- 1 file changed, 125 insertions(+), 44 deletions(-) diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index eee22b85f..24c428bbf 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -124,6 +124,8 @@ struct _H264_CONTEXT_MF IMFSample* sample; UINT32 frameWidth; UINT32 frameHeight; + IMFSample* outputSample; + IMFMediaBuffer* outputBuffer; HMODULE mfplat; pfnMFStartup MFStartup; pfnMFShutdown MFShutdown; @@ -156,12 +158,63 @@ static HRESULT mf_find_output_type(H264_CONTEXT_MF* sys, const GUID* guid, IMFMe return S_OK; } + pMediaType->lpVtbl->Release(pMediaType); + idx++; } return hr; } +static HRESULT mf_create_output_sample(H264_CONTEXT_MF* sys) +{ + HRESULT hr = S_OK; + MFT_OUTPUT_STREAM_INFO streamInfo; + + if (sys->outputSample) + { + sys->outputSample->lpVtbl->Release(sys->outputSample); + sys->outputSample = NULL; + } + + hr = sys->MFCreateSample(&sys->outputSample); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateSample failure: 0x%04X", hr); + goto error; + } + + hr = sys->transform->lpVtbl->GetOutputStreamInfo(sys->transform, 0, &streamInfo); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "GetOutputStreamInfo failure: 0x%04X", hr); + goto error; + } + + hr = sys->MFCreateMemoryBuffer(streamInfo.cbSize, &sys->outputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateMemoryBuffer failure: 0x%04X", hr); + goto error; + } + + sys->outputSample->lpVtbl->AddBuffer(sys->outputSample, sys->outputBuffer); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "AddBuffer failure: 0x%04X", hr); + goto error; + } + + sys->outputBuffer->lpVtbl->Release(sys->outputBuffer); + +error: + return hr; +} + static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) { HRESULT hr; @@ -170,10 +223,8 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) DWORD cbCurrentLength = 0; DWORD outputStatus = 0; IMFSample* inputSample = NULL; - IMFSample* outputSample = NULL; IMFMediaBuffer* inputBuffer = NULL; IMFMediaBuffer* outputBuffer = NULL; - MFT_OUTPUT_STREAM_INFO streamInfo; MFT_OUTPUT_DATA_BUFFER outputDataBuffer; H264_CONTEXT_MF* sys = (H264_CONTEXT_MF*) h264->pSystemData; @@ -227,6 +278,8 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) goto error; } + inputBuffer->lpVtbl->Release(inputBuffer); + hr = sys->transform->lpVtbl->ProcessInput(sys->transform, 0, inputSample, 0); if (FAILED(hr)) @@ -235,42 +288,18 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) goto error; } - hr = sys->MFCreateSample(&outputSample); + hr = mf_create_output_sample(sys); if (FAILED(hr)) { - WLog_ERR(TAG, "MFCreateSample failure: 0x%04X", hr); - goto error; - } - - hr = sys->transform->lpVtbl->GetOutputStreamInfo(sys->transform, 0, &streamInfo); - - if (FAILED(hr)) - { - WLog_ERR(TAG, "GetOutputStreamInfo failure: 0x%04X", hr); - goto error; - } - - hr = sys->MFCreateMemoryBuffer(streamInfo.cbSize, &outputBuffer); - - if (FAILED(hr)) - { - WLog_ERR(TAG, "MFCreateMemoryBuffer failure: 0x%04X", hr); - goto error; - } - - outputSample->lpVtbl->AddBuffer(outputSample, outputBuffer); - - if (FAILED(hr)) - { - WLog_ERR(TAG, "AddBuffer failure: 0x%04X", hr); + WLog_ERR(TAG, "mf_create_output_sample failure: 0x%04X", hr); goto error; } outputDataBuffer.dwStreamID = 0; outputDataBuffer.dwStatus = 0; outputDataBuffer.pEvents = NULL; - outputDataBuffer.pSample = outputSample; + outputDataBuffer.pSample = sys->outputSample; hr = sys->transform->lpVtbl->ProcessOutput(sys->transform, 0, 1, &outputDataBuffer, &outputStatus); @@ -282,7 +311,11 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) UINT64 frameSize = 0; IMFAttributes* attributes = NULL; - fprintf(stderr, "MF_E_TRANSFORM_STREAM_CHANGE\n"); + if (sys->outputType) + { + sys->outputType->lpVtbl->Release(sys->outputType); + sys->outputType = NULL; + } hr = mf_find_output_type(sys, &MFVideoFormat_IYUV, &sys->outputType); @@ -300,6 +333,14 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) goto error; } + hr = mf_create_output_sample(sys); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "mf_create_output_sample failure: 0x%04X", hr); + goto error; + } + hr = sys->outputType->lpVtbl->GetUINT64(sys->outputType, &MF_MT_FRAME_SIZE, &frameSize); if (FAILED(hr)) @@ -319,8 +360,6 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) goto error; } - fprintf(stderr, "frame width: %d height: %d stride: %d\n", sys->frameWidth, sys->frameHeight, stride); - h264->iStride[0] = stride; h264->iStride[1] = stride / 2; h264->iStride[2] = stride / 2; @@ -341,7 +380,7 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) } else if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) { - fprintf(stderr, "MF_E_TRANSFORM_NEED_MORE_INPUT\n"); + } else if (FAILED(hr)) { @@ -356,7 +395,7 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) DWORD cbMaxLength = 0; DWORD cbCurrentLength = 0; - hr = outputSample->lpVtbl->GetBufferCount(outputSample, &bufferCount); + hr = sys->outputSample->lpVtbl->GetBufferCount(sys->outputSample, &bufferCount); if (FAILED(hr)) { @@ -364,11 +403,11 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) goto error; } - hr = outputSample->lpVtbl->ConvertToContiguousBuffer(outputSample, &outputBuffer); + hr = sys->outputSample->lpVtbl->GetBufferByIndex(sys->outputSample, 0, &outputBuffer); if (FAILED(hr)) { - WLog_ERR(TAG, "ConvertToContiguousBuffer failure: 0x%04X", hr); + WLog_ERR(TAG, "GetBufferByIndex failure: 0x%04X", hr); goto error; } @@ -396,15 +435,11 @@ static int mf_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize) WLog_ERR(TAG, "Unlock failure: 0x%04X", hr); goto error; } + + outputBuffer->lpVtbl->Release(outputBuffer); } - inputSample->lpVtbl->DeleteAllItems(inputSample); inputSample->lpVtbl->Release(inputSample); - inputBuffer->lpVtbl->Release(inputBuffer); - - outputSample->lpVtbl->DeleteAllItems(outputSample); - outputSample->lpVtbl->Release(outputSample); - outputBuffer->lpVtbl->Release(outputBuffer); return 1; @@ -428,13 +463,13 @@ static void mf_uninit(H264_CONTEXT* h264) { if (sys->transform) { - //sys->transform->lpVtbl->Release(sys->transform); + sys->transform->lpVtbl->Release(sys->transform); sys->transform = NULL; } if (sys->codecApi) { - //sys->codecApi->lpVtbl->Release(sys->codecApi); + sys->codecApi->lpVtbl->Release(sys->codecApi); sys->codecApi = NULL; } @@ -450,6 +485,12 @@ static void mf_uninit(H264_CONTEXT* h264) sys->outputType = NULL; } + if (sys->outputSample) + { + sys->outputSample->lpVtbl->Release(sys->outputSample); + sys->outputSample = NULL; + } + if (sys->mfplat) { FreeLibrary(sys->mfplat); @@ -461,6 +502,8 @@ static void mf_uninit(H264_CONTEXT* h264) h264->iStride[0] = h264->iStride[1] = h264->iStride[2] = 0; sys->MFShutdown(); + + CoUninitialize(); free(sys); h264->pSystemData = NULL; @@ -510,17 +553,26 @@ static BOOL mf_init(H264_CONTEXT* h264) hr = sys->MFStartup(MF_VERSION, 0); if (FAILED(hr)) + { + WLog_ERR(TAG, "MFStartup failure: 0x%04X", hr); goto error; + } hr = CoCreateInstance(&CLSID_CMSH264DecoderMFT, NULL, CLSCTX_INPROC_SERVER, &IID_IMFTransform, (void**) &sys->transform); if (FAILED(hr)) + { + WLog_ERR(TAG, "CoCreateInstance(CLSID_CMSH264DecoderMFT) failure: 0x%04X", hr); goto error; + } hr = sys->transform->lpVtbl->QueryInterface(sys->transform, &IID_ICodecAPI, (void**) &sys->codecApi); if (FAILED(hr)) + { + WLog_ERR(TAG, "QueryInterface(IID_ICodecAPI) failure: 0x%04X", hr); goto error; + } var.vt = VT_UI4; var.ulVal = 1; @@ -528,37 +580,66 @@ static BOOL mf_init(H264_CONTEXT* h264) hr = sys->codecApi->lpVtbl->SetValue(sys->codecApi, &CODECAPI_AVLowLatencyMode, &var); if (FAILED(hr)) + { + WLog_ERR(TAG, "SetValue(CODECAPI_AVLowLatencyMode) failure: 0x%04X", hr); goto error; + } hr = sys->MFCreateMediaType(&sys->inputType); if (FAILED(hr)) + { + WLog_ERR(TAG, "MFCreateMediaType failure: 0x%04X", hr); goto error; + } hr = sys->inputType->lpVtbl->SetGUID(sys->inputType, &MF_MT_MAJOR_TYPE, &MFMediaType_Video); if (FAILED(hr)) + { + WLog_ERR(TAG, "SetGUID(MF_MT_MAJOR_TYPE) failure: 0x%04X", hr); goto error; + } hr = sys->inputType->lpVtbl->SetGUID(sys->inputType, &MF_MT_SUBTYPE, &MFVideoFormat_H264); if (FAILED(hr)) + { + WLog_ERR(TAG, "SetGUID(MF_MT_SUBTYPE) failure: 0x%04X", hr); goto error; + } hr = sys->transform->lpVtbl->SetInputType(sys->transform, 0, sys->inputType, 0); if (FAILED(hr)) + { + WLog_ERR(TAG, "SetInputType failure: 0x%04X", hr); goto error; + } hr = mf_find_output_type(sys, &MFVideoFormat_IYUV, &sys->outputType); if (FAILED(hr)) + { + WLog_ERR(TAG, "mf_find_output_type failure: 0x%04X", hr); goto error; + } hr = sys->transform->lpVtbl->SetOutputType(sys->transform, 0, sys->outputType, 0); if (FAILED(hr)) + { + WLog_ERR(TAG, "SetOutputType failure: 0x%04X", hr); goto error; + } + + hr = mf_create_output_sample(sys); + + if (FAILED(hr)) + { + WLog_ERR(TAG, "mf_create_output_sample failure: 0x%04X", hr); + goto error; + } } return TRUE; From 162411dc67a2aeca84f0166f4ea3bc25c8a2a04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Tue, 4 Aug 2015 11:55:03 -0400 Subject: [PATCH 14/14] libfreerdp-codec: fix Windows XP build for H.264 decoder --- libfreerdp/codec/h264.c | 50 +++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/libfreerdp/codec/h264.c b/libfreerdp/codec/h264.c index 24c428bbf..765a5c82c 100644 --- a/libfreerdp/codec/h264.c +++ b/libfreerdp/codec/h264.c @@ -64,9 +64,11 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_dummy = #ifdef _WIN32 +#include +#include + #include #include -#include #include #include @@ -85,28 +87,42 @@ DEFINE_GUID(MF_XVP_DISABLE_FRC,0x2c0afa19,0x7a97,0x4d5a,0x9e,0xe8,0x16,0xd4,0xfc DEFINE_GUID(MFMediaType_Video,0x73646976,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); DEFINE_GUID(MFVideoFormat_RGB32,22,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); DEFINE_GUID(MFVideoFormat_ARGB32,21,0x0000,0x0010,0x80,0x00,0x00,0xAA,0x00,0x38,0x9B,0x71); +DEFINE_GUID(MFVideoFormat_H264,0x34363248,0x0000,0x0010,0x80,0x00,0x00,0xaa,0x00,0x38,0x9b,0x71); +DEFINE_GUID(MFVideoFormat_IYUV,0x56555949,0x0000,0x0010,0x80,0x00,0x00,0xaa,0x00,0x38,0x9b,0x71); DEFINE_GUID(IID_ICodecAPI,0x901db4c7,0x31ce,0x41a2,0x85,0xdc,0x8f,0xa0,0xbf,0x41,0xb8,0xda); DEFINE_GUID(CODECAPI_AVLowLatencyMode,0x9c27891a,0xed7a,0x40e1,0x88,0xe8,0xb2,0x27,0x27,0xa0,0x24,0xee); DEFINE_GUID(CODECAPI_AVDecVideoMaxCodedWidth,0x5ae557b8,0x77af,0x41f5,0x9f,0xa6,0x4d,0xb2,0xfe,0x1d,0x4b,0xca); DEFINE_GUID(CODECAPI_AVDecVideoMaxCodedHeight,0x7262a16a,0xd2dc,0x4e75,0x9b,0xa8,0x65,0xc0,0xc6,0xd3,0x2b,0x13); -#ifndef FCC -#define FCC(ch4) ((((DWORD)(ch4) & 0xFF) << 24) | \ - (((DWORD)(ch4) & 0xFF00) << 8) | \ - (((DWORD)(ch4) & 0xFF0000) >> 8) | \ - (((DWORD)(ch4) & 0xFF000000) >> 24)) -#endif +#ifndef __IMFDXGIDeviceManager_FWD_DEFINED__ +#define __IMFDXGIDeviceManager_FWD_DEFINED__ +typedef interface IMFDXGIDeviceManager IMFDXGIDeviceManager; +#endif /* __IMFDXGIDeviceManager_FWD_DEFINED__ */ -DEFINE_GUID(MFVideoFormat_H264, FCC('H264'), 0x0000,0x0010,0x80,0x00,0x00,0xaa,0x00,0x38,0x9b,0x71); -DEFINE_GUID(MFVideoFormat_IYUV, FCC('IYUV'), 0x0000,0x0010,0x80,0x00,0x00,0xaa,0x00,0x38,0x9b,0x71); +#ifndef __IMFDXGIDeviceManager_INTERFACE_DEFINED__ +#define __IMFDXGIDeviceManager_INTERFACE_DEFINED__ -#if 0 -typedef void* IMFSample; -typedef void* IMFTransform; -typedef void* IMFMediaBuffer; -typedef void* IMFMediaType; -typedef void* IMFDXGIDeviceManager; -#endif +typedef struct IMFDXGIDeviceManagerVtbl +{ + HRESULT (STDMETHODCALLTYPE * QueryInterface)(IMFDXGIDeviceManager* This, REFIID riid, void** ppvObject); + ULONG (STDMETHODCALLTYPE * AddRef)(IMFDXGIDeviceManager* This); + ULONG (STDMETHODCALLTYPE * Release)(IMFDXGIDeviceManager* This); + HRESULT (STDMETHODCALLTYPE * CloseDeviceHandle)(IMFDXGIDeviceManager* This, HANDLE hDevice); + HRESULT (STDMETHODCALLTYPE * GetVideoService)(IMFDXGIDeviceManager* This, HANDLE hDevice, REFIID riid, void** ppService); + HRESULT (STDMETHODCALLTYPE * LockDevice)(IMFDXGIDeviceManager* This, HANDLE hDevice, REFIID riid, void** ppUnkDevice, BOOL fBlock); + HRESULT (STDMETHODCALLTYPE * OpenDeviceHandle)(IMFDXGIDeviceManager* This, HANDLE* phDevice); + HRESULT (STDMETHODCALLTYPE * ResetDevice)(IMFDXGIDeviceManager* This, IUnknown* pUnkDevice, UINT resetToken); + HRESULT (STDMETHODCALLTYPE * TestDevice)(IMFDXGIDeviceManager* This, HANDLE hDevice); + HRESULT (STDMETHODCALLTYPE * UnlockDevice)(IMFDXGIDeviceManager* This, HANDLE hDevice, BOOL fSaveState); +} +IMFDXGIDeviceManagerVtbl; + +interface IMFDXGIDeviceManager +{ + CONST_VTBL struct IMFDXGIDeviceManagerVtbl* lpVtbl; +}; + +#endif /* __IMFDXGIDeviceManager_INTERFACE_DEFINED__ */ typedef HRESULT (__stdcall * pfnMFStartup)(ULONG Version, DWORD dwFlags); typedef HRESULT (__stdcall * pfnMFShutdown)(void); @@ -644,7 +660,7 @@ static BOOL mf_init(H264_CONTEXT* h264) return TRUE; error: - fprintf(stderr, "mf_init failure\n"); + WLog_ERR(TAG, "mf_init failure"); mf_uninit(h264); return FALSE; }