imgcodesc: fix code problems with integer overflow / address arithmetic / UB

modules/highgui/src/grfmt_bmp.cpp

@@ -185,7 +185,7 @@ bool  BmpDecoder::readHeader()
 bool  BmpDecoder::readData( Mat& img )
 {
     uchar* data = img.data;
-    int step = (int)img.step;
+    int step = validateToInt(img.step);
     bool color = img.channels() > 1;
     uchar  gray_palette[256];
     bool   result = false;
@@ -198,7 +198,7 @@ bool  BmpDecoder::readData( Mat& img )
 
     if( m_origin == IPL_ORIGIN_BL )
     {
-        data += (m_height - 1)*step;
+        data += (m_height - 1)*(size_t)step;
         step = -step;
     }
 
@@ -522,7 +522,7 @@ bool  BmpEncoder::write( const Mat& img, const vector<int>& )
     int  bitmapHeaderSize = 40;
     int  paletteSize = channels > 1 ? 0 : 1024;
     int  headerSize = 14 /* fileheader */ + bitmapHeaderSize + paletteSize;
-    int  fileSize = fileStep*height + headerSize;
+    size_t fileSize = (size_t)fileStep*height + headerSize;
     PaletteEntry palette[256];
 
     if( m_buf )
@@ -532,7 +532,7 @@ bool  BmpEncoder::write( const Mat& img, const vector<int>& )
     strm.putBytes( fmtSignBmp, (int)strlen(fmtSignBmp) );
 
     // write file header
-    strm.putDWord( fileSize ); // file size
+    strm.putDWord( validateToInt(fileSize) ); // file size
     strm.putDWord( 0 );
     strm.putDWord( headerSize );
 

modules/highgui/src/grfmt_exr.cpp

@@ -188,7 +188,7 @@ bool  ExrDecoder::readData( Mat& img )
     bool color = img.channels() > 1;
 
     uchar* data = img.data;
-    int step = img.step;
+    size_t step = img.step;
     bool justcopy = m_native_depth;
     bool chromatorgb = false;
     bool rgbtogray = false;
@@ -196,8 +196,8 @@ bool  ExrDecoder::readData( Mat& img )
     FrameBuffer frame;
     int xsample[3] = {1, 1, 1};
     char *buffer;
-    int xstep;
-    int ystep;
+    size_t xstep = 0;
+    size_t ystep = 0;
 
     xstep = m_native_depth ? 4 : 1;
 
@@ -589,7 +589,7 @@ bool  ExrEncoder::write( const Mat& img, const vector<int>& )
     bool isfloat = depth == CV_32F || depth == CV_64F;
     depth = CV_ELEM_SIZE1(depth)*8;
     uchar* data = img.data;
-    int step = img.step;
+    size_t step = img.step;
 
     Header header( width, height );
     Imf::PixelType type;
@@ -619,7 +619,7 @@ bool  ExrEncoder::write( const Mat& img, const vector<int>& )
     FrameBuffer frame;
 
     char *buffer;
-    int bufferstep;
+    size_t bufferstep;
     int size;
     if( type == FLOAT && depth == 32 )
     {

modules/highgui/src/grfmt_jpeg.cpp

@@ -392,7 +392,7 @@ int my_jpeg_load_dht (struct jpeg_decompress_struct *info, unsigned char *dht,
 bool  JpegDecoder::readData( Mat& img )
 {
     bool result = false;
-    int step = (int)img.step;
+    size_t step = img.step;
     bool color = img.channels() > 1;
 
     if( m_state && m_width && m_height )

modules/highgui/src/grfmt_jpeg2000.cpp

@@ -156,7 +156,7 @@ bool  Jpeg2KDecoder::readData( Mat& img )
     bool result = false;
     int color = img.channels() > 1;
     uchar* data = img.data;
-    int step = (int)img.step;
+    size_t step = img.step;
     jas_stream_t* stream = (jas_stream_t*)m_stream;
     jas_image_t* image = (jas_image_t*)m_image;
 
@@ -252,9 +252,9 @@ bool  Jpeg2KDecoder::readData( Mat& img )
                         if( !jas_image_readcmpt( image, cmptlut[i], 0, 0, xend / xstep, yend / ystep, buffer ))
                         {
                             if( img.depth() == CV_8U )
-                                result = readComponent8u( data + i, buffer, step, cmptlut[i], maxval, offset, ncmpts );
+                                result = readComponent8u( data + i, buffer, validateToInt(step), cmptlut[i], maxval, offset, ncmpts );
                             else
-                                result = readComponent16u( ((unsigned short *)data) + i, buffer, step / 2, cmptlut[i], maxval, offset, ncmpts );
+                                result = readComponent16u( ((unsigned short *)data) + i, buffer, validateToInt(step / 2), cmptlut[i], maxval, offset, ncmpts );
                             if( !result )
                             {
                                 i = ncmpts;

modules/highgui/src/grfmt_sunras.cpp

@@ -156,7 +156,7 @@ bool  SunRasterDecoder::readData( Mat& img )
 {
     int color = img.channels() > 1;
     uchar* data = img.data;
-    int step = (int)img.step;
+    size_t step = img.step;
     uchar  gray_palette[256];
     bool   result = false;
     int  src_pitch = ((m_width*m_bpp + 7)/8 + 1) & -2;
@@ -304,11 +304,11 @@ bad_decoding_1bpp:
                         code = m_strm.getByte();
 
                         if( color )
-                            data = FillUniColor( data, line_end, step, width3,
+                            data = FillUniColor( data, line_end, validateToInt(step), width3,
                                                  y, m_height, len,
                                                  m_palette[code] );
                         else
-                            data = FillUniGray( data, line_end, step, width3,
+                            data = FillUniGray( data, line_end, validateToInt(step), width3,
                                                 y, m_height, len,
                                                 gray_palette[code] );
                         if( y >= m_height )

modules/highgui/src/utils.cpp

@@ -42,6 +42,13 @@
 #include "precomp.hpp"
 #include "utils.hpp"
 
+int validateToInt(size_t sz)
+{
+    int valueInt = (int)sz;
+    CV_Assert((size_t)valueInt == sz);
+    return valueInt;
+}
+
 #define  SCALE  14
 #define  cR  (int)(0.299*(1 << SCALE) + 0.5)
 #define  cG  (int)(0.587*(1 << SCALE) + 0.5)
@@ -537,23 +544,25 @@ uchar* FillColorRow1( uchar* data, uchar* indices, int len, PaletteEntry* palett
 {
     uchar* end = data + len*3;
 
+    const PaletteEntry p0 = palette[0], p1 = palette[1];
+
     while( (data += 24) < end )
     {
         int idx = *indices++;
-        *((PaletteEntry*)(data - 24)) = palette[(idx & 128) != 0];
-        *((PaletteEntry*)(data - 21)) = palette[(idx & 64) != 0];
-        *((PaletteEntry*)(data - 18)) = palette[(idx & 32) != 0];
-        *((PaletteEntry*)(data - 15)) = palette[(idx & 16) != 0];
-        *((PaletteEntry*)(data - 12)) = palette[(idx & 8) != 0];
-        *((PaletteEntry*)(data - 9)) = palette[(idx & 4) != 0];
-        *((PaletteEntry*)(data - 6)) = palette[(idx & 2) != 0];
-        *((PaletteEntry*)(data - 3)) = palette[(idx & 1) != 0];
+        *((PaletteEntry*)(data - 24)) = (idx & 128) ? p1 : p0;
+        *((PaletteEntry*)(data - 21)) = (idx & 64) ? p1 : p0;
+        *((PaletteEntry*)(data - 18)) = (idx & 32) ? p1 : p0;
+        *((PaletteEntry*)(data - 15)) = (idx & 16) ? p1 : p0;
+        *((PaletteEntry*)(data - 12)) = (idx & 8) ? p1 : p0;
+        *((PaletteEntry*)(data - 9)) = (idx & 4) ? p1 : p0;
+        *((PaletteEntry*)(data - 6)) = (idx & 2) ? p1 : p0;
+        *((PaletteEntry*)(data - 3)) = (idx & 1) ? p1 : p0;
     }
 
-    int idx = indices[0] << 24;
+    int idx = indices[0];
     for( data -= 24; data < end; data += 3, idx += idx )
     {
-        PaletteEntry clr = palette[idx < 0];
+        const PaletteEntry clr = (idx & 128) ? p1 : p0;
         WRITE_PIX( data, clr );
     }
 
@@ -565,23 +574,25 @@ uchar* FillGrayRow1( uchar* data, uchar* indices, int len, uchar* palette )
 {
     uchar* end = data + len;
 
+    const uchar p0 = palette[0], p1 = palette[1];
+
     while( (data += 8) < end )
     {
         int idx = *indices++;
-        *((uchar*)(data - 8)) = palette[(idx & 128) != 0];
-        *((uchar*)(data - 7)) = palette[(idx & 64) != 0];
-        *((uchar*)(data - 6)) = palette[(idx & 32) != 0];
-        *((uchar*)(data - 5)) = palette[(idx & 16) != 0];
-        *((uchar*)(data - 4)) = palette[(idx & 8) != 0];
-        *((uchar*)(data - 3)) = palette[(idx & 4) != 0];
-        *((uchar*)(data - 2)) = palette[(idx & 2) != 0];
-        *((uchar*)(data - 1)) = palette[(idx & 1) != 0];
+        *((uchar*)(data - 8)) = (idx & 128) ? p1 : p0;
+        *((uchar*)(data - 7)) = (idx & 64) ? p1 : p0;
+        *((uchar*)(data - 6)) = (idx & 32) ? p1 : p0;
+        *((uchar*)(data - 5)) = (idx & 16) ? p1 : p0;
+        *((uchar*)(data - 4)) = (idx & 8) ? p1 : p0;
+        *((uchar*)(data - 3)) = (idx & 4) ? p1 : p0;
+        *((uchar*)(data - 2)) = (idx & 2) ? p1 : p0;
+        *((uchar*)(data - 1)) = (idx & 1) ? p1 : p0;
     }
 
-    int idx = indices[0] << 24;
+    int idx = indices[0];
     for( data -= 8; data < end; data++, idx += idx )
     {
-        data[0] = palette[idx < 0];
+        data[0] = (idx & 128) ? p1 : p0;
     }
 
     return data;

modules/highgui/src/utils.hpp

@@ -42,6 +42,8 @@
 #ifndef _UTILS_H_
 #define _UTILS_H_
 
+int validateToInt(size_t step);
+
 struct PaletteEntry
 {
     unsigned char b, g, r, a;