vorbis: fix overflows in floor1[] vector and inverse db table index.

libavcodec/vorbis.c

@@ -152,7 +152,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     }
 }
 
-static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
+static inline void render_line_unrolled(intptr_t x, int y, int x1,
                                         intptr_t sy, int ady, int adx,
                                         float *buf)
 {
@@ -164,30 +164,30 @@ static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
         if (err >= 0) {
             err += ady - adx;
             y   += sy;
-            buf[x++] = ff_vorbis_floor1_inverse_db_table[y];
+            buf[x++] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
         }
-        buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+        buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
     }
     if (x <= 0) {
         if (err + ady >= 0)
             y += sy;
-        buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+        buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
     }
 }
 
-static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
+static void render_line(int x0, int y0, int x1, int y1, float *buf)
 {
     int dy  = y1 - y0;
     int adx = x1 - x0;
     int ady = FFABS(dy);
     int sy  = dy < 0 ? -1 : 1;
-    buf[x0] = ff_vorbis_floor1_inverse_db_table[y0];
+    buf[x0] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y0)];
     if (ady*2 <= adx) { // optimized common case
         render_line_unrolled(x0, y0, x1, sy, ady, adx, buf);
     } else {
         int base  = dy / adx;
         int x     = x0;
-        uint8_t y = y0;
+        int y     = y0;
         int err   = -adx;
         ady -= FFABS(base) * adx;
         while (++x < x1) {
@@ -197,7 +197,7 @@ static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
                 err -= adx;
                 y   += sy;
             }
-            buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+            buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
         }
     }
 }
@@ -206,8 +206,7 @@ void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
                                   uint16_t *y_list, int *flag,
                                   int multiplier, float *out, int samples)
 {
-    int lx, i;
-    uint8_t ly;
+    int lx, ly, i;
     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {

libavcodec/vorbisdec.c

@@ -1244,20 +1244,20 @@ static int vorbis_floor1_decode(vorbis_context *vc,
             floor1_flag[i]               = 1;
             if (val >= room) {
                 if (highroom > lowroom) {
-                    floor1_Y_final[i] = val - lowroom + predicted;
+                    floor1_Y_final[i] = av_clip_uint16(val - lowroom + predicted);
                 } else {
-                    floor1_Y_final[i] = predicted - val + highroom - 1;
+                    floor1_Y_final[i] = av_clip_uint16(predicted - val + highroom - 1);
                 }
             } else {
                 if (val & 1) {
-                    floor1_Y_final[i] = predicted - (val + 1) / 2;
+                    floor1_Y_final[i] = av_clip_uint16(predicted - (val + 1) / 2);
                 } else {
-                    floor1_Y_final[i] = predicted + val / 2;
+                    floor1_Y_final[i] = av_clip_uint16(predicted + val / 2);
                 }
             }
         } else {
             floor1_flag[i]    = 0;
-            floor1_Y_final[i] = predicted;
+            floor1_Y_final[i] = av_clip_uint16(predicted);
         }
 
         av_dlog(NULL, " Decoded floor(%d) = %u / val %u\n",