add h265 support

This commit is contained in:
F. Duncanh
2024-09-17 18:12:40 -04:00
parent 0a2dbaa9e2
commit 57bd7555fa
9 changed files with 519 additions and 260 deletions

View File

@@ -195,7 +195,10 @@ raop_rtp_mirror_thread(void *arg)
uint64_t ntp_timestamp_local = 0;
unsigned char nal_start_code[4] = { 0x00, 0x00, 0x00, 0x01 };
bool logger_debug = (logger_get_level(raop_rtp_mirror->logger) >= LOGGER_DEBUG);
bool h265_video_detected = false;
bool h265_video = false;
video_codec_t codec;
const char h264[] = "h264";
const char h265[] = "h265";
while (1) {
fd_set rfds;
@@ -377,9 +380,9 @@ raop_rtp_mirror_thread(void *arg)
uint64_t ntp_now = raop_ntp_get_local_time(raop_rtp_mirror->ntp);
int64_t latency = ((int64_t) ntp_now) - ((int64_t) ntp_timestamp_local);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp video: now = %8.6f, ntp = %8.6f, latency = %8.6f, ts = %8.6f, %s",
"raop_rtp video: now = %8.6f, ntp = %8.6f, latency = %8.6f, ts = %8.6f, %s %s",
(double) ntp_now / SEC, (double) ntp_timestamp_local / SEC, (double) latency / SEC,
(double) ntp_timestamp_remote / SEC, packet_description);
(double) ntp_timestamp_remote / SEC, packet_description, h265_video ? h265 : h264);
}
unsigned char* payload_out;
@@ -442,98 +445,93 @@ raop_rtp_mirror_thread(void *arg)
valid_data = false;
break;
}
int nalu_type = payload_decrypted[nalu_size] & 0x1f;
int ref_idc = (payload_decrypted[nalu_size] >> 5);
/* check for unsupported h265 video (sometimes sent by macOS in high-def screen mirroring) */
if (payload_decrypted[nalu_size + 1] == 0x01) {
switch (payload_decrypted[nalu_size]) {
case 0x28: // h265 IDR type 20 NAL
case 0x02: // h265 non-IDR type 1 NAL
ref_idc = 0;
h265_video_detected = true;
int nalu_type;
if (h265_video) {
nalu_type = payload_decrypted[nalu_size] & 0x7e >> 1;;
//logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG," h265 video, NALU type %d, size %d", nalu_type, nc_len);
} else {
nalu_type = payload_decrypted[nalu_size] & 0x1f;
int ref_idc = (payload_decrypted[nalu_size] >> 5);
switch (nalu_type) {
case 14: /* Prefix NALu , seen before all VCL Nalu's in AirMyPc */
case 5: /*IDR, slice_layer_without_partitioning */
case 1: /*non-IDR, slice_layer_without_partitioning */
break;
case 2: /* slice data partition A */
case 3: /* slice data partition B */
case 4: /* slice data partition C */
logger_log(raop_rtp_mirror->logger, LOGGER_INFO,
"unexpected partitioned VCL NAL unit: nalu_type = %d, ref_idc = %d, nalu_size = %d,"
"processed bytes %d, payloadsize = %d nalus_count = %d",
nalu_type, ref_idc, nc_len, nalu_size, payload_size, nalus_count);
break;
case 6:
if (logger_debug) {
char *str = utils_data_to_string(payload_decrypted + nalu_size, nc_len, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror SEI NAL size = %d", nc_len);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp_mirror h264 Supplemental Enhancement Information:\n%s", str);
free(str);
}
break;
case 7:
if (logger_debug) {
char *str = utils_data_to_string(payload_decrypted + nalu_size, nc_len, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror SPS NAL size = %d", nc_len);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp_mirror h264 Sequence Parameter Set:\n%s", str);
free(str);
}
break;
case 8:
if (logger_debug) {
char *str = utils_data_to_string(payload_decrypted + nalu_size, nc_len, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror PPS NAL size = %d", nc_len);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp_mirror h264 Picture Parameter Set :\n%s", str);
free(str);
}
break;
default:
break;
}
if (h265_video_detected) {
break;
}
}
switch (nalu_type) {
case 14: /* Prefix NALu , seen before all VCL Nalu's in AirMyPc */
case 5: /*IDR, slice_layer_without_partitioning */
case 1: /*non-IDR, slice_layer_without_partitioning */
break;
case 2: /* slice data partition A */
case 3: /* slice data partition B */
case 4: /* slice data partition C */
logger_log(raop_rtp_mirror->logger, LOGGER_INFO,
"unexpected partitioned VCL NAL unit: nalu_type = %d, ref_idc = %d, nalu_size = %d,"
"processed bytes %d, payloadsize = %d nalus_count = %d",
nalu_type, ref_idc, nc_len, nalu_size, payload_size, nalus_count);
break;
case 6:
if (logger_debug) {
char *str = utils_data_to_string(payload_decrypted + nalu_size, nc_len, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror SEI NAL size = %d", nc_len);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp_mirror h264 Supplemental Enhancement Information:\n%s", str);
free(str);
}
break;
case 7:
if (logger_debug) {
char *str = utils_data_to_string(payload_decrypted + nalu_size, nc_len, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror SPS NAL size = %d", nc_len);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp_mirror h264 Sequence Parameter Set:\n%s", str);
free(str);
}
break;
case 8:
if (logger_debug) {
char *str = utils_data_to_string(payload_decrypted + nalu_size, nc_len, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror PPS NAL size = %d", nc_len);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG,
"raop_rtp_mirror h264 Picture Parameter Set :\n%s", str);
free(str);
}
break;
default:
logger_log(raop_rtp_mirror->logger, LOGGER_INFO,
"unexpected non-VCL NAL unit: nalu_type = %d, ref_idc = %d, nalu_size = %d,"
"processed bytes %d, payloadsize = %d nalus_count = %d",
nalu_type, ref_idc, nc_len, nalu_size, payload_size, nalus_count);
break;
logger_log(raop_rtp_mirror->logger, LOGGER_INFO,
"unexpected non-VCL NAL unit: nalu_type = %d, ref_idc = %d, nalu_size = %d,"
"processed bytes %d, payloadsize = %d nalus_count = %d",
nalu_type, ref_idc, nc_len, nalu_size, payload_size, nalus_count);
break;
}
}
nalu_size += nc_len;
}
if (h265_video_detected) {
logger_log(raop_rtp_mirror->logger, LOGGER_ERR,
"unsupported h265 video detected");
free (payload_out);
break;
}
if (nalu_size != payload_size) valid_data = false;
if(!valid_data) {
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "nalu marked as invalid");
payload_out[0] = 1; /* mark video data as invalid h264 (failed decryption) */
}
payload_decrypted = NULL;
h264_decode_struct h264_data;
h264_data.ntp_time_local = ntp_timestamp_local;
h264_data.ntp_time_remote = ntp_timestamp_remote;
h264_data.nal_count = nalus_count; /*nal_count will be the number of nal units in the packet */
h264_data.data_len = payload_size;
h264_data.data = payload_out;
video_decode_struct video_data;
video_data.is_h265 = h265_video;
video_data.ntp_time_local = ntp_timestamp_local;
video_data.ntp_time_remote = ntp_timestamp_remote;
video_data.nal_count = nalus_count; /*nal_count will be the number of nal units in the packet */
video_data.data_len = payload_size;
video_data.data = payload_out;
if (prepend_sps_pps) {
h264_data.data_len += sps_pps_len;
h264_data.nal_count += 2;
prepend_sps_pps = false;
video_data.data_len += sps_pps_len;
video_data.nal_count += 2;
if (h265_video) {
video_data.nal_count++;
}
prepend_sps_pps = false;
}
//char *str3 = utils_data_to_string(payload_out, video_data.data_len, 16);
//printf("%s\n", str3);
//free (str3);
raop_rtp_mirror->callbacks.video_resume(raop_rtp_mirror->callbacks.cls);
raop_rtp_mirror->callbacks.video_process(raop_rtp_mirror->callbacks.cls, raop_rtp_mirror->ntp, &h264_data);
raop_rtp_mirror->callbacks.video_process(raop_rtp_mirror->callbacks.cls, raop_rtp_mirror->ntp, &video_data);
free(payload_out);
break;
case 0x01:
@@ -542,10 +540,13 @@ raop_rtp_mirror_thread(void *arg)
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "\nReceived unencrypted codec packet from client:"
" payload_size %d header %s ts_client = %8.6f",
payload_size, packet_description, (double) ntp_timestamp_remote / SEC);
if (payload_size == 0) {
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror, discard type 0x01 packet with no payload");
break;
}
//char *str1 = utils_data_to_string(payload, payload_size, 16);
//printf("unencrypted payload, size %d\n", payload_size);
//printf("%s\n", str1);
//free (str1);
codec = VIDEO_CODEC_UNKNOWN;
assert (raop_rtp_mirror->callbacks.video_set_codec);
ntp_timestamp_nal = ntp_timestamp_raw;
float width = byteutils_get_float(packet, 16);
float height = byteutils_get_float(packet, 20);
@@ -565,55 +566,140 @@ raop_rtp_mirror_thread(void *arg)
}
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror width_source = %f height_source = %f width = %f height = %f",
width_source, height_source, width, height);
short sps_size = byteutils_get_short_be(payload,6);
unsigned char *sequence_parameter_set = payload + 8;
short pps_size = byteutils_get_short_be(payload, sps_size + 9);
unsigned char *picture_parameter_set = payload + sps_size + 11;
int data_size = 6;
if (logger_debug) {
char *str = utils_data_to_string(payload, data_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror: SPS+PPS header size = %d", data_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror h264 SPS+PPS header:\n%s", str);
free(str);
str = utils_data_to_string(sequence_parameter_set, sps_size,16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror SPS NAL size = %d", sps_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror h264 Sequence Parameter Set:\n%s", str);
free(str);
str = utils_data_to_string(picture_parameter_set, pps_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror PPS NAL size = %d", pps_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror h264 Picture Parameter Set:\n%s", str);
free(str);
if (payload_size == 0) {
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror, received 0x01 packet with no payload (h265 video?)");
raop_rtp_mirror->callbacks.video_pause(raop_rtp_mirror->callbacks.cls);
break;
}
data_size = payload_size - sps_size - pps_size - 11;
if (data_size > 0 && logger_debug) {
char *str = utils_data_to_string (picture_parameter_set + pps_size, data_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "remainder size = %d", data_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "remainder of SPS+PPS packet:\n%s", str);
free(str);
} else if (data_size < 0) {
logger_log(raop_rtp_mirror->logger, LOGGER_ERR, " pps_sps error: packet remainder size = %d < 0", data_size);
}
// Copy the sps and pps into a buffer to prepend to the next NAL unit.
if (sps_pps) {
free(sps_pps);
sps_pps = NULL;
}
sps_pps_len = sps_size + pps_size + 8;
sps_pps = (unsigned char*) malloc(sps_pps_len);
assert(sps_pps);
memcpy(sps_pps, nal_start_code, 4);
memcpy(sps_pps + 4, sequence_parameter_set, sps_size);
memcpy(sps_pps + sps_size + 4, nal_start_code, 4);
memcpy(sps_pps + sps_size + 8, payload + sps_size + 11, pps_size);
prepend_sps_pps = true;
/* test for a H265 VPS/SPs/PPS */
unsigned char hvc1[] = { 0x68, 0x76, 0x63, 0x31 };
uint64_t ntp_offset = 0;
ntp_offset = raop_ntp_convert_remote_time(raop_rtp_mirror->ntp, ntp_offset);
if (!ntp_offset) {
logger_log(raop_rtp_mirror->logger, LOGGER_WARNING, "ntp synchronization has not yet started: synchronized video may fail");
if (!memcmp(payload + 4, hvc1, 4)) {
/* hvc1 HECV detected */
codec = VIDEO_CODEC_H265;
printf("h265 detected\n");
h265_video = true;
raop_rtp_mirror->callbacks.video_set_codec(raop_rtp_mirror->callbacks.cls, codec);
unsigned char vps_code[] = { 0xa0, 0x00, 0x01, 0x00 };
unsigned char sps_code[] = { 0xa1, 0x00, 0x01, 0x00 };
unsigned char pps_code[] = { 0xa2, 0x00, 0x01, 0x00 };
unsigned char *vps;
short vps_size;
unsigned char *sps;
short sps_size;
unsigned char *pps;
short pps_size;
unsigned char * ptr = payload + 0x75;
if (memcmp(ptr, vps_code, 4)) {
logger_log(raop_rtp_mirror->logger, LOGGER_ERR, "non-conforming HEVC VPS/SPS/PPS payload (VPS)");
raop_rtp_mirror->callbacks.video_pause(raop_rtp_mirror->callbacks.cls);
break;
}
vps_size = byteutils_get_short_be(ptr, 3);
ptr += 5;
vps = ptr;
if (logger_debug) {
char *str = utils_data_to_string(vps, vps_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "h265 vps size %d\n%s",vps_size, str);
free(str);
}
ptr += vps_size;
if (memcmp(ptr, sps_code, 4)) {
logger_log(raop_rtp_mirror->logger, LOGGER_ERR, "non-conforming HEVC VPS/SPS/PPS payload (SPS)");
raop_rtp_mirror->callbacks.video_pause(raop_rtp_mirror->callbacks.cls);
break;
}
sps_size = byteutils_get_short_be(ptr, 3);
ptr += 5;
sps = ptr;
if (logger_debug) {
char *str = utils_data_to_string(sps, sps_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "h265 sps size %d\n%s",vps_size, str);
free(str);
}
ptr += sps_size;
if (memcmp(ptr, pps_code, 4)) {
logger_log(raop_rtp_mirror->logger, LOGGER_ERR, "non-conforming HEVC VPS/SPS/PPS payload (PPS)");
raop_rtp_mirror->callbacks.video_pause(raop_rtp_mirror->callbacks.cls);
break;
}
pps_size = byteutils_get_short_be(ptr, 3);
ptr += 5;
pps = ptr;
if (logger_debug) {
char *str = utils_data_to_string(pps, pps_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "h265 pps size %d\n%s",pps_size, str);
free(str);
}
sps_pps_len = vps_size + sps_size + pps_size + 12;
sps_pps = (unsigned char*) malloc(sps_pps_len);
assert(sps_pps);
ptr = sps_pps;
memcpy(ptr, nal_start_code, 4);
ptr += 4;
memcpy(ptr, vps, vps_size);
ptr += vps_size;
memcpy(ptr, nal_start_code, 4);
ptr += 4;
memcpy(ptr, sps, sps_size);
ptr += sps_size;
memcpy(ptr, nal_start_code, 4);
ptr += 4;
memcpy(ptr, pps, pps_size);
// printf (" HEVC (hvc1) vps + sps + pps NALU\n");
//char *str = utils_data_to_string(sps_pps, sps_pps_len, 16);
//printf("%s\n", str);
//free (str);
} else {
codec = VIDEO_CODEC_H264;
h265_video = false;
raop_rtp_mirror->callbacks.video_set_codec(raop_rtp_mirror->callbacks.cls, codec);
short sps_size = byteutils_get_short_be(payload,6);
unsigned char *sequence_parameter_set = payload + 8;
short pps_size = byteutils_get_short_be(payload, sps_size + 9);
unsigned char *picture_parameter_set = payload + sps_size + 11;
int data_size = 6;
if (logger_debug) {
char *str = utils_data_to_string(payload, data_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror: SPS+PPS header size = %d", data_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror h264 SPS+PPS header:\n%s", str);
free(str);
str = utils_data_to_string(sequence_parameter_set, sps_size,16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror SPS NAL size = %d", sps_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror h264 Sequence Parameter Set:\n%s", str);
free(str);
str = utils_data_to_string(picture_parameter_set, pps_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror PPS NAL size = %d", pps_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "raop_rtp_mirror h264 Picture Parameter Set:\n%s", str);
free(str);
}
data_size = payload_size - sps_size - pps_size - 11;
if (data_size > 0 && logger_debug) {
char *str = utils_data_to_string (picture_parameter_set + pps_size, data_size, 16);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "remainder size = %d", data_size);
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "remainder of SPS+PPS packet:\n%s", str);
free(str);
} else if (data_size < 0) {
logger_log(raop_rtp_mirror->logger, LOGGER_ERR, " pps_sps error: packet remainder size = %d < 0", data_size);
}
// Copy the sps and pps into a buffer to prepend to the next NAL unit.
sps_pps_len = sps_size + pps_size + 8;
sps_pps = (unsigned char*) malloc(sps_pps_len);
assert(sps_pps);
memcpy(sps_pps, nal_start_code, 4);
memcpy(sps_pps + 4, sequence_parameter_set, sps_size);
memcpy(sps_pps + sps_size + 4, nal_start_code, 4);
memcpy(sps_pps + sps_size + 8, payload + sps_size + 11, pps_size);
}
prepend_sps_pps = true;
// h264codec_t h264;
// h264.version = payload[0];
// h264.profile_high = payload[1];
@@ -628,7 +714,6 @@ raop_rtp_mirror_thread(void *arg)
// h264.pps_size = pps_size;
// h264.picture_parameter_set = malloc(h264.pps_size);
// memcpy(h264.picture_parameter_set, picture_parameter_set, pps_size);
raop_rtp_mirror->callbacks.video_pause(raop_rtp_mirror->callbacks.cls);
break;
case 0x02:
logger_log(raop_rtp_mirror->logger, LOGGER_DEBUG, "\nReceived old-protocol once-per-second packet from client:"