From 20f09b2d34f79cf06fc4c911b597bc929606ba67 Mon Sep 17 00:00:00 2001
From: akallabeth <akallabeth@posteo.net>
Date: Mon, 20 Nov 2023 11:02:21 +0100
Subject: [PATCH] [codec,aac] add NaN filter

ffmpeg format conversion from int16 to float sometimes produces values
that are NaN or Infinity.
Replace these values as these input values break the AAC encoder
---
 libfreerdp/codec/dsp_ffmpeg.c | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/libfreerdp/codec/dsp_ffmpeg.c b/libfreerdp/codec/dsp_ffmpeg.c
index 7f385e7c7..ccac94710 100644
--- a/libfreerdp/codec/dsp_ffmpeg.c
+++ b/libfreerdp/codec/dsp_ffmpeg.c
@@ -430,9 +430,29 @@ static BOOL ffmpeg_resample_frame(AVAudioResampleContext* context, AVFrame* in,
 static BOOL ffmpeg_encode_frame(AVCodecContext* context, AVFrame* in, AVPacket* packet,
                                 wStream* out)
 {
-	int ret;
+	if (in->format == AV_SAMPLE_FMT_FLTP)
+	{
+		uint8_t** pp = in->extended_data;
+		for (int y = 0; y < in->channels; y++)
+		{
+			float* data = pp[y];
+			for (int x = 0; x < in->nb_samples; x++)
+			{
+				const float val1 = data[x];
+				if (isnan(val1))
+					data[x] = 0.0f;
+				else if (isinf(val1))
+				{
+					if (val1 < 0.0f)
+						data[x] = -1.0f;
+					else
+						data[x] = 1.0f;
+				}
+			}
+		}
+	}
 	/* send the packet with the compressed data to the encoder */
-	ret = avcodec_send_frame(context, in);
+	int ret = avcodec_send_frame(context, in);
 
 	if (ret < 0)
 	{