avformat/matroskaenc: Check ff_vorbiscomment_length in put_flac_codecpriv()

Its currently guaranteed to be smaller but its safer to check anyway
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 66f26b3e)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/matroskaenc.c

@@ -543,12 +543,15 @@ static int put_flac_codecpriv(AVFormatContext *s,
                              "Lavf" : LIBAVFORMAT_IDENT;
         AVDictionary *dict = NULL;
         uint8_t buf[32], *data, *p;
-        int len;
+        int64_t len;
 
         snprintf(buf, sizeof(buf), "0x%"PRIx64, codec->channel_layout);
         av_dict_set(&dict, "WAVEFORMATEXTENSIBLE_CHANNEL_MASK", buf, 0);
 
         len = ff_vorbiscomment_length(dict, vendor);
+        if (len >= ((1<<24) - 4))
+            return AVERROR(EINVAL);
+
         data = av_malloc(len + 4);
         if (!data) {
             av_dict_free(&dict);