avformat: Add max_streams option

This allows user apps to stop OOM due to excessive number of streams
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 1296f844)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

doc/formats.texi

@@ -205,6 +205,10 @@ For example to separate the fields with newlines and indention:
 ffprobe -dump_separator "
                           "  -i ~/videos/matrixbench_mpeg2.mpg
 @end example
+
+@item max_streams @var{integer} (@emph{input})
+Specifies the maximum number of streams. This can be used to reject files that
+would require too many resources due to a large number of streams.
 @end table
 
 @c man end FORMAT OPTIONS

libavformat/avformat.h

@@ -1884,6 +1884,13 @@ typedef struct AVFormatContext {
      * - decoding: set by user through AVOptions (NO direct access)
      */
     char *protocol_blacklist;
+
+    /**
+     * The maximum number of streams.
+     * - encoding: unused
+     * - decoding: set by user through AVOptions (NO direct access)
+     */
+    int max_streams;
 } AVFormatContext;
 
 int av_format_get_probe_score(const AVFormatContext *s);

libavformat/options_table.h

@@ -103,6 +103,7 @@ static const AVOption avformat_options[] = {
 {"format_whitelist", "List of demuxers that are allowed to be used", OFFSET(format_whitelist), AV_OPT_TYPE_STRING, { .str = NULL },  CHAR_MIN, CHAR_MAX, D },
 {"protocol_whitelist", "List of protocols that are allowed to be used", OFFSET(protocol_whitelist), AV_OPT_TYPE_STRING, { .str = NULL },  CHAR_MIN, CHAR_MAX, D },
 {"protocol_blacklist", "List of protocols that are not allowed to be used", OFFSET(protocol_blacklist), AV_OPT_TYPE_STRING, { .str = NULL },  CHAR_MIN, CHAR_MAX, D },
+{"max_streams", "maximum number of streams", OFFSET(max_streams), AV_OPT_TYPE_INT, { .i64 = INT_MAX }, 0, INT_MAX, D },
 {NULL},
 };
 

libavformat/utils.c

@@ -4087,7 +4087,7 @@ AVStream *avformat_new_stream(AVFormatContext *s, const AVCodec *c)
     int i;
     AVStream **streams;
 
-    if (s->nb_streams >= INT_MAX/sizeof(*streams))
+    if (s->nb_streams >= FFMIN(s->max_streams, INT_MAX/sizeof(*streams)))
         return NULL;
     streams = av_realloc_array(s->streams, s->nb_streams + 1, sizeof(*streams));
     if (!streams)