libavformat: add mbedTLS based TLS

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

Changelog

@@ -24,6 +24,7 @@ version <next>:
 - Haivision SRT protocol via libsrt
 - Dropped support for building for Windows XP. The minimum supported Windows
   version is Windows Vista.
+- support mbedTLS-based TLS
 
 
 version 12:

configure

@@ -229,6 +229,7 @@ External library support:
   --enable-libxcb-shm        X11 shm communication [auto]
   --enable-libxcb-xfixes     X11 mouse rendering [auto]
   --enable-libxvid           MPEG-4 ASP video encoding
+  --enable-mbedtls           crypto
   --enable-openssl           crypto
   --enable-zlib              compression [autodetect]
 
@@ -1343,6 +1344,7 @@ EXTERNAL_LIBRARY_VERSION3_LIST="
     libopencore_amrwb
     libvo_aacenc
     libvo_amrwbenc
+    mbedtls
 "
 
 EXTERNAL_LIBRARY_LIST="
@@ -2506,7 +2508,7 @@ xcbgrab_indev_suggest="libxcb_shm libxcb_xfixes"
 
 # protocols
 ffrtmpcrypt_protocol_conflict="librtmp_protocol"
-ffrtmpcrypt_protocol_deps_any="gmp openssl"
+ffrtmpcrypt_protocol_deps_any="gmp mbedtls openssl"
 ffrtmpcrypt_protocol_select="tcp_protocol"
 ffrtmphttp_protocol_conflict="librtmp_protocol"
 ffrtmphttp_protocol_select="http_protocol"
@@ -2539,7 +2541,7 @@ sctp_protocol_deps="struct_sctp_event_subscribe"
 sctp_protocol_select="network"
 srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
-tls_protocol_deps_any="gnutls openssl"
+tls_protocol_deps_any="gnutls mbedtls openssl"
 tls_protocol_select="tcp_protocol"
 udp_protocol_select="network"
 unix_protocol_deps="sys_un_h"
@@ -2959,6 +2961,12 @@ fi
 enabled_all gnutls openssl &&
     die "GnuTLS and OpenSSL must not be enabled at the same time."
 
+enabled_all gnutls mbedtls &&
+    die "GnuTLS and mbedTLS must not be enabled at the same time."
+
+enabled_all openssl mbedtls &&
+    die "OpenSSL and mbedTLS must not be enabled at the same time."
+
 # Disable all the library-specific components if the library itself
 # is disabled, see AVCODEC_LIST and following _LIST variables.
 
@@ -4710,6 +4718,7 @@ enabled libx265           && require_pkg_config libx265 x265 x265.h x265_api_get
                              require_cpp_condition libx265 x265.h "X265_BUILD >= 57"
 enabled libxavs           && require libxavs "stdint.h xavs.h" xavs_encoder_encode -lxavs
 enabled libxvid           && require libxvid xvid.h xvid_global -lxvidcore
+enabled mbedtls           && require mbedtls mbedtls/ssl.h mbedtls_ssl_init -lmbedtls -lmbedcrypto -lmbedx509
 enabled mmal              && { check_lib mmal interface/mmal/mmal.h mmal_port_connect -lmmal_core -lmmal_util -lmmal_vc_client -lbcm_host ||
                                { ! enabled cross_compile &&
                                  add_cflags -isystem/opt/vc/include/ -isystem/opt/vc/include/interface/vmcs_host/linux -isystem/opt/vc/include/interface/vcos/pthreads -fgnu89-inline &&

libavformat/Makefile

@@ -408,6 +408,7 @@ OBJS-$(CONFIG_SCTP_PROTOCOL)             += sctp.o
 OBJS-$(CONFIG_SRTP_PROTOCOL)             += srtpproto.o srtp.o
 OBJS-$(CONFIG_TCP_PROTOCOL)              += tcp.o
 TLS-OBJS-$(CONFIG_GNUTLS)                += tls_gnutls.o
+TLS-OBJS-$(CONFIG_MBEDTLS)               += tls_mbedtls.o
 TLS-OBJS-$(CONFIG_OPENSSL)               += tls_openssl.o
 OBJS-$(CONFIG_TLS_PROTOCOL)              += tls.o $(TLS-OBJS-yes)
 OBJS-$(CONFIG_UDP_PROTOCOL)              += udp.o

libavformat/rtmpdh.c

@@ -132,6 +132,56 @@ static int bn_modexp(FFBigNum bn, FFBigNum y, FFBigNum q, FFBigNum p)
     BN_CTX_free(ctx);
     return 0;
 }
+#elif CONFIG_MBEDTLS
+#define bn_new(bn)                      \
+    do {                                \
+        bn = av_malloc(sizeof(*bn));    \
+        if (bn)                         \
+            mbedtls_mpi_init(bn);       \
+    } while (0)
+#define bn_free(bn)                     \
+    do {                                \
+        mbedtls_mpi_free(bn);           \
+        av_free(bn);                    \
+    } while (0)
+#define bn_set_word(bn, w)          mbedtls_mpi_lset(bn, w)
+#define bn_cmp(a, b)                mbedtls_mpi_cmp_mpi(a, b)
+#define bn_copy(to, from)           mbedtls_mpi_copy(to, from)
+#define bn_sub_word(bn, w)          mbedtls_mpi_sub_int(bn, bn, w)
+#define bn_cmp_1(bn)                mbedtls_mpi_cmp_int(bn, 1)
+#define bn_num_bytes(bn)            (mbedtls_mpi_bitlen(bn) + 7) / 8
+#define bn_bn2bin(bn, buf, len)     mbedtls_mpi_write_binary(bn, buf, len)
+#define bn_bin2bn(bn, buf, len)                     \
+    do {                                            \
+        bn_new(bn);                                 \
+        if (bn)                                     \
+            mbedtls_mpi_read_binary(bn, buf, len);  \
+    } while (0)
+#define bn_hex2bn(bn, buf, ret)                     \
+    do {                                            \
+        bn_new(bn);                                 \
+        if (bn)                                     \
+            ret = (mbedtls_mpi_read_string(bn, 16, buf) == 0);  \
+        else                                        \
+            ret = 1;                                \
+    } while (0)
+#define bn_random(bn, num_bits)                     \
+    do {                                            \
+        mbedtls_entropy_context entropy_ctx;        \
+        mbedtls_ctr_drbg_context ctr_drbg_ctx;      \
+                                                    \
+        mbedtls_entropy_init(&entropy_ctx);         \
+        mbedtls_ctr_drbg_init(&ctr_drbg_ctx);       \
+        mbedtls_ctr_drbg_seed(&ctr_drbg_ctx,        \
+                              mbedtls_entropy_func, \
+                              &entropy_ctx,         \
+                              NULL, 0);             \
+        mbedtls_mpi_fill_random(bn, (num_bits + 7) / 8, mbedtls_ctr_drbg_random, &ctr_drbg_ctx); \
+        mbedtls_ctr_drbg_free(&ctr_drbg_ctx);       \
+        mbedtls_entropy_free(&entropy_ctx);         \
+    } while (0)
+#define bn_modexp(bn, y, q, p)      mbedtls_mpi_exp_mod(bn, y, q, p, 0)
+
 #endif
 
 #define MAX_BYTES 18000

libavformat/rtmpdh.h

@@ -35,6 +35,12 @@ typedef mpz_ptr FFBigNum;
 #include <openssl/dh.h>
 
 typedef BIGNUM *FFBigNum;
+#elif CONFIG_MBEDTLS
+#include <mbedtls/bignum.h>
+#include <mbedtls/ctr_drbg.h>
+#include <mbedtls/entropy.h>
+
+typedef mbedtls_mpi *FFBigNum;
 #endif
 
 typedef struct FF_DH {

libavformat/version.h

@@ -30,7 +30,7 @@
 #include "libavutil/version.h"
 
 #define LIBAVFORMAT_VERSION_MAJOR 58
-#define LIBAVFORMAT_VERSION_MINOR  1
+#define LIBAVFORMAT_VERSION_MINOR  2
 #define LIBAVFORMAT_VERSION_MICRO  0
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \