diff --git a/libfreerdp/codec/h264_openh264.c b/libfreerdp/codec/h264_openh264.c index fde84593a..6c679a046 100644 --- a/libfreerdp/codec/h264_openh264.c +++ b/libfreerdp/codec/h264_openh264.c @@ -22,13 +22,15 @@ #include "config.h" #endif +#include +#include + #include #include -#include -#include "wels/codec_def.h" -#include "wels/codec_api.h" -#include "wels/codec_ver.h" +#include +#include +#include #include "h264.h" @@ -77,11 +79,23 @@ static void openh264_trace_callback(H264_CONTEXT* h264, int level, const char* m static int openh264_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 SrcSize) { DECODING_STATE state; - SBufferInfo sBufferInfo; + SBufferInfo sBufferInfo = { 0 }; SSysMEMBuffer* pSystemBuffer; - H264_CONTEXT_OPENH264* sys = (H264_CONTEXT_OPENH264*)h264->pSystemData; - UINT32* iStride = h264->iStride; - BYTE** pYUVData = h264->pYUVData; + H264_CONTEXT_OPENH264* sys; + UINT32* iStride; + BYTE** pYUVData; + + WINPR_ASSERT(h264); + WINPR_ASSERT(pSrcData || (SrcSize == 0)); + + sys = (H264_CONTEXT_OPENH264*)h264->pSystemData; + WINPR_ASSERT(sys); + + iStride = h264->iStride; + WINPR_ASSERT(iStride); + + pYUVData = h264->pYUVData; + WINPR_ASSERT(pYUVData); if (!sys->pDecoder) return -2001; @@ -92,7 +106,8 @@ static int openh264_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 pYUVData[0] = NULL; pYUVData[1] = NULL; pYUVData[2] = NULL; - ZeroMemory(&sBufferInfo, sizeof(sBufferInfo)); + + WINPR_ASSERT(sys->pDecoder); state = (*sys->pDecoder)->DecodeFrame2(sys->pDecoder, pSrcData, SrcSize, pYUVData, &sBufferInfo); @@ -116,6 +131,21 @@ static int openh264_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT32 } } + if (state != dsErrorFree) + { + WLog_Print(h264->log, WLOG_WARN, "DecodeFrame2 state: 0x%02X", state); + return -2003; + } + +#if OPENH264_MAJOR >= 2 + state = (*sys->pDecoder)->FlushFrame(sys->pDecoder, pYUVData, &sBufferInfo); + if (state != dsErrorFree) + { + WLog_Print(h264->log, WLOG_WARN, "FlushFrame state: 0x%02X", state); + return -2003; + } +#endif + pSystemBuffer = &sBufferInfo.UsrData.sSystemBuffer; iStride[0] = pSystemBuffer->iStride[0]; iStride[1] = pSystemBuffer->iStride[1]; @@ -156,11 +186,19 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI { int i, j; int status; - SFrameBSInfo info; - SSourcePicture pic; - SBitrateInfo bitrate; + SFrameBSInfo info = { 0 }; + SSourcePicture pic = { 0 }; + H264_CONTEXT_OPENH264* sys; + + WINPR_ASSERT(h264); + WINPR_ASSERT(pYUVData); + WINPR_ASSERT(iStride); + WINPR_ASSERT(ppDstData); + WINPR_ASSERT(pDstSize); + sys = &((H264_CONTEXT_OPENH264*)h264->pSystemData)[0]; + WINPR_ASSERT(sys); if (!sys->pEncoder) return -1; @@ -175,9 +213,11 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI (h264->BitRate > INT_MAX) || (h264->QP > INT_MAX)) return -1; + WINPR_ASSERT(sys->pEncoder); if ((sys->EncParamExt.iPicWidth != (int)h264->width) || (sys->EncParamExt.iPicHeight != (int)h264->height)) { + WINPR_ASSERT((*sys->pEncoder)->GetDefaultParams); status = (*sys->pEncoder)->GetDefaultParams(sys->pEncoder, &sys->EncParamExt); if (status < 0) @@ -226,6 +266,7 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI #endif } + WINPR_ASSERT((*sys->pEncoder)->InitializeExt); status = (*sys->pEncoder)->InitializeExt(sys->pEncoder, &sys->EncParamExt); if (status < 0) @@ -235,6 +276,7 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI return status; } + WINPR_ASSERT((*sys->pEncoder)->GetOption); status = (*sys->pEncoder) ->GetOption(sys->pEncoder, ENCODER_OPTION_SVC_ENCODE_PARAM_EXT, &sys->EncParamExt); @@ -253,9 +295,13 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI case H264_RATECONTROL_VBR: if (sys->EncParamExt.iTargetBitrate != (int)h264->BitRate) { + SBitrateInfo bitrate = { 0 }; + sys->EncParamExt.iTargetBitrate = (int)h264->BitRate; bitrate.iLayer = SPATIAL_LAYER_ALL; bitrate.iBitrate = (int)h264->BitRate; + + WINPR_ASSERT((*sys->pEncoder)->SetOption); status = (*sys->pEncoder) ->SetOption(sys->pEncoder, ENCODER_OPTION_BITRATE, &bitrate); @@ -270,6 +316,8 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI if (sys->EncParamExt.fMaxFrameRate != (int)h264->FrameRate) { sys->EncParamExt.fMaxFrameRate = (int)h264->FrameRate; + + WINPR_ASSERT((*sys->pEncoder)->SetOption); status = (*sys->pEncoder) ->SetOption(sys->pEncoder, ENCODER_OPTION_FRAME_RATE, &sys->EncParamExt.fMaxFrameRate); @@ -288,6 +336,8 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI if (sys->EncParamExt.sSpatialLayers[0].iDLayerQp != (int)h264->QP) { sys->EncParamExt.sSpatialLayers[0].iDLayerQp = (int)h264->QP; + + WINPR_ASSERT((*sys->pEncoder)->SetOption); status = (*sys->pEncoder) ->SetOption(sys->pEncoder, ENCODER_OPTION_SVC_ENCODE_PARAM_EXT, &sys->EncParamExt); @@ -304,8 +354,6 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI } } - memset(&info, 0, sizeof(SFrameBSInfo)); - memset(&pic, 0, sizeof(SSourcePicture)); pic.iPicWidth = (int)h264->width; pic.iPicHeight = (int)h264->height; pic.iColorFormat = videoFormatI420; @@ -315,6 +363,8 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI pic.pData[0] = (unsigned char*)pYUVData[0]; pic.pData[1] = (unsigned char*)pYUVData[1]; pic.pData[2] = (unsigned char*)pYUVData[2]; + + WINPR_ASSERT((*sys->pEncoder)->EncodeFrame); status = (*sys->pEncoder)->EncodeFrame(sys->pEncoder, &pic, &info); if (status < 0) @@ -340,7 +390,11 @@ static int openh264_compress(H264_CONTEXT* h264, const BYTE** pYUVData, const UI static void openh264_uninit(H264_CONTEXT* h264) { UINT32 x; - H264_CONTEXT_OPENH264* sysContexts = (H264_CONTEXT_OPENH264*)h264->pSystemData; + H264_CONTEXT_OPENH264* sysContexts; + + WINPR_ASSERT(h264); + + sysContexts = (H264_CONTEXT_OPENH264*)h264->pSystemData; if (sysContexts) { @@ -377,6 +431,8 @@ static BOOL openh264_load_functionpointers(H264_CONTEXT* h264, const char* name) { H264_CONTEXT_OPENH264* sysContexts; + WINPR_ASSERT(name); + if (!h264) return FALSE; @@ -439,13 +495,16 @@ static BOOL openh264_init(H264_CONTEXT* h264) #endif UINT32 x; long status; - SDecodingParam sDecParam; + SDecodingParam sDecParam = { 0 }; H264_CONTEXT_OPENH264* sysContexts; static int traceLevel = WELS_LOG_DEBUG; #if (OPENH264_MAJOR == 1) && (OPENH264_MINOR <= 5) static EVideoFormatType videoFormat = videoFormatI420; #endif static WelsTraceCallback traceCallback = (WelsTraceCallback)openh264_trace_callback; + + WINPR_ASSERT(h264); + h264->numSystemData = 1; sysContexts = (H264_CONTEXT_OPENH264*)calloc(h264->numSystemData, sizeof(H264_CONTEXT_OPENH264));