avcodec/dnxhd_parser: Do not return invalid value from dnxhd_find_frame_end() on error

Fixes: Null pointer dereference

Fixes: CVE-2017-9608
Found-by: Yihan Lian
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 611b3562)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/dnxhd_parser.c

@@ -87,16 +87,18 @@ static int dnxhd_find_frame_end(DNXHDParserContext *dctx,
                 dctx->w = (state >> 32) & 0xFFFF;
             } else if (dctx->cur_byte == 42) {
                 int cid = (state >> 32) & 0xFFFFFFFF;
+                int remaining;
 
                 if (cid <= 0)
                     continue;
 
-                dctx->remaining = avpriv_dnxhd_get_frame_size(cid);
-                if (dctx->remaining <= 0) {
-                    dctx->remaining = dnxhd_get_hr_frame_size(cid, dctx->w, dctx->h);
-                    if (dctx->remaining <= 0)
-                        return dctx->remaining;
+                remaining = avpriv_dnxhd_get_frame_size(cid);
+                if (remaining <= 0) {
+                    remaining = dnxhd_get_hr_frame_size(cid, dctx->w, dctx->h);
+                    if (remaining <= 0)
+                        continue;
                 }
+                dctx->remaining = remaining;
                 if (buf_size - i >= dctx->remaining && (!dctx->interlaced || dctx->cur_field)) {
                     int remaining = dctx->remaining;